LfsStore.java

  1. /*
  2.  * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> and others
  3.  *
  4.  * This program and the accompanying materials are made available under the
  5.  * terms of the Eclipse Distribution License v. 1.0 which is available at
  6.  * https://www.eclipse.org/org/documents/edl-v10.php.
  7.  *
  8.  * SPDX-License-Identifier: BSD-3-Clause
  9.  */

  11. package org.eclipse.jgit.pgm.debug;

  13. import java.io.File;
  14. import java.io.IOException;
  15. import java.net.InetAddress;
  16. import java.net.URI;
  17. import java.net.URISyntaxException;
  18. import java.net.UnknownHostException;
  19. import java.nio.file.Path;
  20. import java.nio.file.Paths;
  21. import java.text.MessageFormat;

  23. import org.eclipse.jetty.server.Connector;
  24. import org.eclipse.jetty.server.HttpConfiguration;
  25. import org.eclipse.jetty.server.HttpConnectionFactory;
  26. import org.eclipse.jetty.server.Server;
  27. import org.eclipse.jetty.server.ServerConnector;
  28. import org.eclipse.jetty.server.handler.ContextHandlerCollection;
  29. import org.eclipse.jetty.servlet.ServletContextHandler;
  30. import org.eclipse.jetty.servlet.ServletHolder;
  31. import org.eclipse.jgit.errors.ConfigInvalidException;
  32. import org.eclipse.jgit.lfs.server.LargeFileRepository;
  33. import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
  34. import org.eclipse.jgit.lfs.server.fs.FileLfsRepository;
  35. import org.eclipse.jgit.lfs.server.fs.FileLfsServlet;
  36. import org.eclipse.jgit.lfs.server.s3.S3Config;
  37. import org.eclipse.jgit.lfs.server.s3.S3Repository;
  38. import org.eclipse.jgit.pgm.Command;
  39. import org.eclipse.jgit.pgm.TextBuiltin;
  40. import org.eclipse.jgit.pgm.internal.CLIText;
  41. import org.eclipse.jgit.storage.file.FileBasedConfig;
  42. import org.eclipse.jgit.util.FS;
  43. import org.kohsuke.args4j.Argument;
  44. import org.kohsuke.args4j.Option;

  46. @Command(common = true, usage = "usage_runLfsStore")
  47. class LfsStore extends TextBuiltin {

  49.     /**
  50.      * Tiny web application server for testing
  51.      */
  52.     static class AppServer {

  54.         private final Server server;

  56.         private final ServerConnector connector;

  58.         private final ContextHandlerCollection contexts;

  60.         private URI uri;

  62.         AppServer(int port) {
  63.             server = new Server();

  65.             HttpConfiguration http_config = new HttpConfiguration();
  66.             http_config.setOutputBufferSize(32768);

  68.             connector = new ServerConnector(server,
  69.                     new HttpConnectionFactory(http_config));
  70.             connector.setPort(port);
  71.             try {
  72.                 String host = InetAddress.getByName("localhost") //$NON-NLS-1$
  73.                         .getHostAddress();
  74.                 connector.setHost(host);
  75.                 if (host.contains(":") && !host.startsWith("[")) //$NON-NLS-1$ //$NON-NLS-2$
  76.                     host = "[" + host + "]"; //$NON-NLS-1$//$NON-NLS-2$
  77.                 uri = new URI("http://" + host + ":" + port); //$NON-NLS-1$ //$NON-NLS-2$
  78.             } catch (UnknownHostException e) {
  79.                 throw new RuntimeException("Cannot find localhost", e); //$NON-NLS-1$
  80.             } catch (URISyntaxException e) {
  81.                 throw new RuntimeException("Unexpected URI error on " + uri, e); //$NON-NLS-1$
  82.             }

  84.             contexts = new ContextHandlerCollection();
  85.             server.setHandler(contexts);
  86.             server.setConnectors(new Connector[] { connector });
  87.         }

  89.         /**
  90.          * Create a new servlet context within the server.
  91.          * <p>
  92.          * This method should be invoked before the server is started, once for
  93.          * each context the caller wants to register.
  94.          *
  95.          * @param path
  96.          *            path of the context; use "/" for the root context if
  97.          *            binding to the root is desired.
  98.          * @return the context to add servlets into.
  99.          */
  100.         ServletContextHandler addContext(String path) {
  101.             assertNotRunning();
  102.             if ("".equals(path)) //$NON-NLS-1$
  103.                 path = "/"; //$NON-NLS-1$

  105.             ServletContextHandler ctx = new ServletContextHandler();
  106.             ctx.setContextPath(path);
  107.             contexts.addHandler(ctx);

  109.             return ctx;
  110.         }

  112.         void start() throws Exception {
  113.             server.start();
  114.         }

  116.         void stop() throws Exception {
  117.             server.stop();
  118.         }

  120.         URI getURI() {
  121.             return uri;
  122.         }

  124.         private void assertNotRunning() {
  125.             if (server.isRunning()) {
  126.                 throw new IllegalStateException("server is running"); //$NON-NLS-1$
  127.             }
  128.         }
  129.     }

  131.     private enum StoreType {
  132.         FS, S3;
  133.     }

  135.     private enum StorageClass {
  136.         REDUCED_REDUNDANCY, STANDARD
  137.     }

  139.     private static final String OBJECTS = "objects/"; //$NON-NLS-1$

  141.     private static final String STORE_PATH = "/" + OBJECTS + "*"; //$NON-NLS-1$//$NON-NLS-2$

  143.     private static final String PROTOCOL_PATH = "/lfs/objects/batch"; //$NON-NLS-1$

  145.     @Option(name = "--port", aliases = {"-p" },
  146.             metaVar = "metaVar_port", usage = "usage_LFSPort")
  147.     int port;

  149.     @Option(name = "--store", metaVar = "metaVar_lfsStorage", usage = "usage_LFSRunStore")
  150.     StoreType storeType;

  152.     @Option(name = "--store-url", aliases = {"-u" }, metaVar = "metaVar_url",
  153.             usage = "usage_LFSStoreUrl")
  154.     String storeUrl;

  156.     @Option(name = "--region", aliases = {"-r" },
  157.             metaVar = "metaVar_s3Region", usage = "usage_S3Region")
  158.     String region; // $NON-NLS-1$

  160.     @Option(name = "--bucket", aliases = {"-b" },
  161.             metaVar = "metaVar_s3Bucket", usage = "usage_S3Bucket")
  162.     String bucket; // $NON-NLS-1$

  164.     @Option(name = "--storage-class", aliases = {"-c" },
  165.             metaVar = "metaVar_s3StorageClass", usage = "usage_S3StorageClass")
  166.     StorageClass storageClass = StorageClass.REDUCED_REDUNDANCY;

  168.     @Option(name = "--expire", aliases = {"-e" },
  169.             metaVar = "metaVar_seconds", usage = "usage_S3Expiration")
  170.     int expirationSeconds = 600;

  172.     @Option(name = "--no-ssl-verify", usage = "usage_S3NoSslVerify")
  173.     boolean disableSslVerify = false;

  175.     @Argument(required = false, metaVar = "metaVar_directory", usage = "usage_LFSDirectory")
  176.     String directory;

  178.     String protocolUrl;

  180.     String accessKey;

  182.     String secretKey;

  184.     /** {@inheritDoc} */
  185.     @Override
  186.     protected boolean requiresRepository() {
  187.         return false;
  188.     }

  190.     /** {@inheritDoc} */
  191.     @Override
  192.     protected void run() throws Exception {
  193.         AppServer server = new AppServer(port);
  194.         URI baseURI = server.getURI();
  195.         ServletContextHandler app = server.addContext("/"); //$NON-NLS-1$

  197.         final LargeFileRepository repository;
  198.         switch (storeType) {
  199.         case FS:
  200.             Path dir = Paths.get(directory);
  201.             FileLfsRepository fsRepo = new FileLfsRepository(
  202.                     getStoreUrl(baseURI), dir);
  203.             FileLfsServlet content = new FileLfsServlet(fsRepo, 30000);
  204.             app.addServlet(new ServletHolder(content), STORE_PATH);
  205.             repository = fsRepo;
  206.             break;

  208.         case S3:
  209.             readAWSKeys();
  210.             checkOptions();
  211.             S3Config config = new S3Config(region, bucket,
  212.                     storageClass.toString(), accessKey, secretKey,
  213.                     expirationSeconds, disableSslVerify);
  214.             repository = new S3Repository(config);
  215.             break;
  216.         default:
  217.             throw new IllegalArgumentException(MessageFormat
  218.                     .format(CLIText.get().lfsUnknownStoreType, storeType));
  219.         }

  221.         LfsProtocolServlet protocol = new LfsProtocolServlet() {

  223.             private static final long serialVersionUID = 1L;

  225.             @Override
  226.             protected LargeFileRepository getLargeFileRepository(
  227.                     LfsRequest request, String path, String auth) {
  228.                 return repository;
  229.             }
  230.         };
  231.         app.addServlet(new ServletHolder(protocol), PROTOCOL_PATH);

  233.         server.start();

  235.         outw.println(MessageFormat.format(CLIText.get().lfsProtocolUrl,
  236.                 getProtocolUrl(baseURI)));
  237.         if (storeType == StoreType.FS) {
  238.             outw.println(MessageFormat.format(CLIText.get().lfsStoreDirectory,
  239.                     directory));
  240.             outw.println(MessageFormat.format(CLIText.get().lfsStoreUrl,
  241.                     getStoreUrl(baseURI)));
  242.         }
  243.     }

  245.     private void checkOptions() {
  246.         if (bucket == null || bucket.length() == 0) {
  247.             throw die(MessageFormat.format(CLIText.get().s3InvalidBucket,
  248.                     bucket));
  249.         }
  250.     }

  252.     private void readAWSKeys() throws IOException, ConfigInvalidException {
  253.         String credentialsPath = System.getProperty("user.home") //$NON-NLS-1$
  254.                 + "/.aws/credentials"; //$NON-NLS-1$
  255.         FileBasedConfig c = new FileBasedConfig(new File(credentialsPath),
  256.                 FS.DETECTED);
  257.         c.load();
  258.         accessKey = c.getString("default", null, "accessKey"); //$NON-NLS-1$//$NON-NLS-2$
  259.         secretKey = c.getString("default", null, "secretKey"); //$NON-NLS-1$ //$NON-NLS-2$
  260.         if (accessKey == null || accessKey.isEmpty()) {
  261.             throw die(MessageFormat.format(CLIText.get().lfsNoAccessKey,
  262.                     credentialsPath));
  263.         }
  264.         if (secretKey == null || secretKey.isEmpty()) {
  265.             throw die(MessageFormat.format(CLIText.get().lfsNoSecretKey,
  266.                     credentialsPath));
  267.         }
  268.     }

  270.     private String getStoreUrl(URI baseURI) {
  271.         if (storeUrl == null) {
  272.             if (storeType == StoreType.FS) {
  273.                 storeUrl = baseURI + "/" + OBJECTS; //$NON-NLS-1$
  274.             } else {
  275.                 die("Local store not running and no --store-url specified"); //$NON-NLS-1$
  276.             }
  277.         }
  278.         return storeUrl;
  279.     }

  281.     private String getProtocolUrl(URI baseURI) {
  282.         if (protocolUrl == null) {
  283.             protocolUrl = baseURI + PROTOCOL_PATH;
  284.         }
  285.         return protocolUrl;
  286.     }
  287. }
Created with JaCoCo 0.8.7.202105040129