Add git url support and fix some errors

README.md

@@ -6,20 +6,6 @@ Simple project for convert old repository for using git-lfs feature.
 
 # How to use
 
-## How to get LFS server URL
-
-You can get correct LFS server URL by command like:
-
-```
-ssh git@github.com git-lfs-authenticate bozaro/git-lfs-migrate upload
-```
-
-For GitHub LFS server URL looks like:
-
-```
-https://bozaro:*****@api.github.com/lfs/bozaro/git-lfs-migrate
-```
-
 ## Run from binaries
 
 For quick run you need:
@@ -41,23 +27,28 @@ git clone --mirror git@github.com:bozaro/git-lfs-migrate.git
 # Convert repository with moving .md and .jar file to LFS
 #
 # Usage: <main class> [options] LFS file suffixes
-#  Options:
-#  * -d, --destination
-#       Destination repository
-#    -h, --help
-#       Show help
-#       Default: false
-#    -l, --lfs
-#       LFS URL
-#  * -s, --source
-#       Source repository
-#    -t, --threads
-#       Thread count
-#       Default: 8
+#   Options:
+#     -c, --cache
+#        Source repository
+#        Default: .
+#   * -d, --destination
+#        Destination repository
+#     -g, --git
+#       GIT repository url (ignored with --lfs parameter)
+#     -h, --help
+#        Show help
+#        Default: false
+#     -l, --lfs
+#        LFS server url (can be determinated by --git paramter)
+#   * -s, --source
+#        Source repository
+#     -t, --threads
+#        Thread count
+#        Default: CPU core count
 java -jar git-lfs-migrate.jar \
      -s git-lfs-migrate.git \
      -d git-lfs-migrate-converted.git \
-     -l https://bozaro:*****@api.github.com/lfs/bozaro/git-lfs-migrate-converted \
+     -g git@github.com:bozaro/git-lfs-migrate-converted.git \
      .md \
      .jar
 
@@ -76,6 +67,14 @@ After that you with have:
 *.jar   filter=lfs diff=lfs merge=lfs -crlf
 ```
 
+Supported Git url formats:
+
+ * https://user:passw0rd@github.com/foo/bar.git
+ * http://user:passw0rd@github.com/foo/bar.git
+ * git://user:passw0rd@github.com/foo/bar.git
+ * ssh://git@github.com/foo/bar.git
+ * git@github.com:foo/bar.git
+
 ## Build from sources
 
 To build from sources you need install JDK 1.8 or later and run build script.

build.gradle

@@ -23,8 +23,8 @@ dependencies {
   compile "org.jgrapht:jgrapht-core:0.9.1"
   compile "com.beust:jcommander:1.35"
 
-  compile "ru.bozaro.gitlfs:gitlfs-pointer:0.1.0"
-  compile "ru.bozaro.gitlfs:gitlfs-client:0.1.0"
+  compile "ru.bozaro.gitlfs:gitlfs-pointer:0.3.0"
+  compile "ru.bozaro.gitlfs:gitlfs-client:0.3.0"
 
   testCompile "org.testng:testng:6.8.8"
 }

src/main/java/git/lfs/migrate/GitConverter.java

@@ -9,12 +9,11 @@
 import org.mapdb.DB;
 import org.mapdb.DBMaker;
 import org.mapdb.HTreeMap;
-import ru.bozaro.gitlfs.common.client.BasicAuthProvider;
-import ru.bozaro.gitlfs.common.client.Client;
+import ru.bozaro.gitlfs.client.Client;
+import ru.bozaro.gitlfs.client.auth.AuthProvider;
 import ru.bozaro.gitlfs.pointer.Pointer;
 
 import java.io.*;
-import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
@@ -28,7 +27,7 @@ public class GitConverter implements AutoCloseable {
   @NotNull
   private static final String GIT_ATTRIBUTES = ".gitattributes";
   @Nullable
-  private final URI lfs;
+  private final AuthProvider auth;
   @NotNull
   private final String[] suffixes;
   @NotNull
@@ -40,10 +39,10 @@ public class GitConverter implements AutoCloseable {
   @NotNull
   private final HTreeMap<String, String> cacheSha256;
 
-  public GitConverter(@NotNull File cachePath, @NotNull File basePath, @Nullable URI lfs, @NotNull String[] suffixes) {
+  public GitConverter(@NotNull File cachePath, @NotNull File basePath, @Nullable AuthProvider auth, @NotNull String[] suffixes) {
     this.basePath = basePath;
     this.suffixes = suffixes.clone();
-    this.lfs = lfs;
+    this.auth = auth;
 
     tempPath = new File(basePath, "lfs/tmp");
     tempPath.mkdirs();
@@ -259,7 +258,7 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv
           inserter.insert(loader.getType(), loader.getBytes());
           return id;
         }
-        final String hash = (lfs == null) ? createLocalFile(id, loader) : createRemoteFile(id, loader, lfs);
+        final String hash = (auth == null) ? createLocalFile(id, loader) : createRemoteFile(id, loader, auth);
         // Create pointer.
         StringWriter pointer = new StringWriter();
         pointer.write("version https://git-lfs.github.com/spec/v1\n");
@@ -272,7 +271,7 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv
   }
 
   @NotNull
-  private String createRemoteFile(@NotNull ObjectId id, @NotNull ObjectLoader loader, @NotNull URI lfs) throws IOException {
+  private String createRemoteFile(@NotNull ObjectId id, @NotNull ObjectLoader loader, @NotNull AuthProvider auth) throws IOException {
     // Create LFS stream.
     final String hash;
     final String cached = cacheSha256.get(id.name());
@@ -294,7 +293,7 @@ private String createRemoteFile(@NotNull ObjectId id, @NotNull ObjectLoader load
     } else {
       hash = cached;
     }
-    final Client client = new Client(new BasicAuthProvider(lfs));
+    final Client client = new Client(auth);
     client.putObject(loader::openStream, hash, size);
     return hash;
   }

src/main/java/git/lfs/migrate/Main.java

@@ -10,10 +10,12 @@
 import org.jgrapht.graph.SimpleDirectedGraph;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import ru.bozaro.gitlfs.client.AuthHelper;
+import ru.bozaro.gitlfs.client.auth.AuthProvider;
+import ru.bozaro.gitlfs.client.auth.BasicAuthProvider;
 
 import java.io.File;
 import java.io.IOException;
-import java.net.MalformedURLException;
 import java.net.URI;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
@@ -43,18 +45,19 @@ public static void main(@NotNull String[] args) throws IOException, InterruptedE
       return;
     }
     final long time = System.currentTimeMillis();
-    processRepository(cmd.src, cmd.dst, cmd.cache, prepareUrl(cmd.lfs), cmd.threads, cmd.suffixes.toArray(new String[cmd.suffixes.size()]));
+    final AuthProvider auth;
+    if (cmd.lfs != null) {
+      auth = new BasicAuthProvider(URI.create(cmd.lfs));
+    } else if (cmd.git != null) {
+      auth = AuthHelper.create(cmd.git);
+    } else {
+      auth = null;
+    }
+    processRepository(cmd.src, cmd.dst, cmd.cache, auth, cmd.threads, cmd.suffixes.toArray(new String[cmd.suffixes.size()]));
     log.info("Convert time: {}", System.currentTimeMillis() - time);
   }
 
-  @Nullable
-  private static URI prepareUrl(@Nullable String url) throws MalformedURLException {
-    if (url == null) return null;
-    if (url.endsWith("/")) return URI.create(url);
-    return URI.create(url + "/");
-  }
-
-  public static void processRepository(@NotNull File srcPath, @NotNull File dstPath, @NotNull File cachePath, @Nullable URI lfs, int threads, @NotNull String... suffixes) throws IOException, InterruptedException {
+  public static void processRepository(@NotNull File srcPath, @NotNull File dstPath, @NotNull File cachePath, @Nullable AuthProvider auth, int threads, @NotNull String... suffixes) throws IOException, InterruptedException {
     removeDirectory(dstPath);
     dstPath.mkdirs();
 
@@ -65,7 +68,7 @@ public static void processRepository(@NotNull File srcPath, @NotNull File dstPat
         .setMustExist(false)
         .setGitDir(dstPath).build();
 
-    final GitConverter converter = new GitConverter(cachePath, dstPath, lfs, suffixes);
+    final GitConverter converter = new GitConverter(cachePath, dstPath, auth, suffixes);
     try {
       dstRepo.create(true);
       // Load all revision list.
@@ -280,7 +283,10 @@ public static class CmdArgs {
     @Parameter(names = {"-c", "--cache"}, description = "Source repository", required = false)
     @NotNull
     private File cache = new File(".");
-    @Parameter(names = {"-l", "--lfs"}, description = "LFS URL", required = false)
+    @Parameter(names = {"-g", "--git"}, description = "GIT repository url (ignored with --lfs parameter)", required = false)
+    @Nullable
+    private String git;
+    @Parameter(names = {"-l", "--lfs"}, description = "LFS server url (can be determinated by --git paramter)", required = false)
     @Nullable
     private String lfs;
     @Parameter(names = {"-t", "--threads"}, description = "Thread count", required = false)