Add already uploaded file SHA256 hash caching

.gitignore

@@ -1,5 +1,6 @@
 *.class
 *.local
+*.mapdb*
 
 # Mobile Tools for Java (J2ME)
 .mtj.tmp/

build.gradle

@@ -18,6 +18,7 @@ apply plugin: "java"
 dependencies {
   compile "org.jetbrains:annotations:13.0"
   compile "org.eclipse.jgit:org.eclipse.jgit:3.6.2.201501210735-r"
+  compile "org.mapdb:mapdb:1.0.6"
   compile "org.slf4j:slf4j-simple:1.7.7"
   compile "org.jgrapht:jgrapht-core:0.9.1"
   compile "com.beust:jcommander:1.35"

src/main/java/git/lfs/migrate/GitConverter.java

@@ -16,6 +16,9 @@
 import org.eclipse.jgit.revwalk.*;
 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
 import org.jetbrains.annotations.NotNull;
+import org.mapdb.DB;
+import org.mapdb.DBMaker;
+import org.mapdb.HTreeMap;
 
 import java.io.*;
 import java.net.URL;
@@ -28,7 +31,7 @@
  * Converter for git objects.
  * Created by bozaro on 09.06.15.
  */
-public class GitConverter {
+public class GitConverter implements AutoCloseable {
   @NotNull
   private static final String GIT_ATTRIBUTES = ".gitattributes";
   private static final int PASS_COUNT = 3;
@@ -38,15 +41,33 @@ public class GitConverter {
   private final String[] suffixes;
   @NotNull
   private final File basePath;
+  @NotNull
   private final File tempPath;
+  @NotNull
+  private final DB cache;
+  @NotNull
+  private final HTreeMap<String, String> cacheSha256;
 
-  public GitConverter(@NotNull File basePath, @Nullable URL lfs, @NotNull String[] suffixes) {
+  public GitConverter(@NotNull File cachePath, @NotNull File basePath, @Nullable URL lfs, @NotNull String[] suffixes) {
     this.basePath = basePath;
     this.suffixes = suffixes.clone();
     this.lfs = lfs;
 
     tempPath = new File(basePath, "lfs/tmp");
     tempPath.mkdirs();
+
+    cachePath.mkdirs();
+    cache = DBMaker.newFileDB(new File(cachePath, "git-lfs-migrate.mapdb"))
+        .asyncWriteEnable()
+        .mmapFileEnable()
+        .cacheSoftRefEnable()
+        .make();
+    cacheSha256 = cache.getHashMap("sha256");
+  }
+
+  @Override
+  public void close() throws IOException {
+    cache.close();
   }
 
   @NotNull
@@ -246,7 +267,7 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv
           inserter.insert(loader.getType(), loader.getBytes());
           return id;
         }
-        final String hash = (lfs == null) ? createLocalFile(loader) : createRemoteFile(loader, lfs);
+        final String hash = (lfs == null) ? createLocalFile(id, loader) : createRemoteFile(id, loader, lfs);
         // Create pointer.
         StringWriter pointer = new StringWriter();
         pointer.write("version https://git-lfs.github.com/spec/v1\n");
@@ -259,24 +280,32 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv
   }
 
   @NotNull
-  private String createRemoteFile(@NotNull ObjectLoader loader, @NotNull URL lfs) throws IOException {
+  private String createRemoteFile(@NotNull ObjectId id, @NotNull ObjectLoader loader, @NotNull URL lfs) throws IOException {
     // Create LFS stream.
-    final MessageDigest md = createSha256();
-    try (InputStream istream = loader.openStream()) {
-      byte[] buffer = new byte[0x10000];
-      while (true) {
-        int size = istream.read(buffer);
-        if (size <= 0) break;
-        md.update(buffer, 0, size);
+    final String hash;
+    final String cached = cacheSha256.get(id.name());
+    if (cached == null) {
+      final MessageDigest md = createSha256();
+      try (InputStream istream = loader.openStream()) {
+        byte[] buffer = new byte[0x10000];
+        while (true) {
+          int size = istream.read(buffer);
+          if (size <= 0) break;
+          md.update(buffer, 0, size);
+        }
       }
+      hash = new String(Hex.encodeHex(md.digest(), true));
+      cacheSha256.put(id.name(), hash);
+      cache.commit();
+    } else {
+      hash = cached;
     }
-    final String hash = new String(Hex.encodeHex(md.digest(), true));
     reply(() -> checkAndUpload(loader, lfs, hash));
     return hash;
   }
 
   @NotNull
-  private String createLocalFile(@NotNull ObjectLoader loader) throws IOException {
+  private String createLocalFile(@NotNull ObjectId id, @NotNull ObjectLoader loader) throws IOException {
     // Create LFS stream.
     final File tmpFile = new File(tempPath, UUID.randomUUID().toString());
     final MessageDigest md = createSha256();
@@ -290,7 +319,9 @@ private String createLocalFile(@NotNull ObjectLoader loader) throws IOException
         md.update(buffer, 0, size);
       }
     }
-    String hash = new String(Hex.encodeHex(md.digest(), true));
+    final String hash = new String(Hex.encodeHex(md.digest(), true));
+    cacheSha256.putIfAbsent(id.name(), hash);
+    cache.commit();
     // Rename file.
     final File lfsFile = new File(basePath, "lfs/objects/" + hash.substring(0, 2) + "/" + hash.substring(2, 4) + "/" + hash);
     lfsFile.getParentFile().mkdirs();

src/main/java/git/lfs/migrate/Main.java

@@ -43,7 +43,7 @@ public static void main(@NotNull String[] args) throws IOException, InterruptedE
       return;
     }
     final long time = System.currentTimeMillis();
-    processRepository(cmd.src, cmd.dst, prepareUrl(cmd.lfs), cmd.threads, cmd.suffixes.toArray(new String[cmd.suffixes.size()]));
+    processRepository(cmd.src, cmd.dst, cmd.cache, prepareUrl(cmd.lfs), cmd.threads, cmd.suffixes.toArray(new String[cmd.suffixes.size()]));
     log.info("Convert time: {}", System.currentTimeMillis() - time);
   }
 
@@ -54,7 +54,7 @@ private static URL prepareUrl(@Nullable String url) throws MalformedURLException
     return new URL(url + "/");
   }
 
-  public static void processRepository(@NotNull File srcPath, @NotNull File dstPath, @Nullable URL lfs, int threads, @NotNull String... suffixes) throws IOException, InterruptedException {
+  public static void processRepository(@NotNull File srcPath, @NotNull File dstPath, @NotNull File cachePath, @Nullable URL lfs, int threads, @NotNull String... suffixes) throws IOException, InterruptedException {
     removeDirectory(dstPath);
     dstPath.mkdirs();
 
@@ -64,7 +64,8 @@ public static void processRepository(@NotNull File srcPath, @NotNull File dstPat
     final Repository dstRepo = new FileRepositoryBuilder()
         .setMustExist(false)
         .setGitDir(dstPath).build();
-    final GitConverter converter = new GitConverter(dstPath, lfs, suffixes);
+
+    final GitConverter converter = new GitConverter(cachePath, dstPath, lfs, suffixes);
     try {
       dstRepo.create(true);
       // Load all revision list.
@@ -276,6 +277,9 @@ public static class CmdArgs {
     @Parameter(names = {"-d", "--destination"}, description = "Destination repository", required = true)
     @NotNull
     private File dst;
+    @Parameter(names = {"-c", "--cache"}, description = "Source repository", required = false)
+    @NotNull
+    private File cache = new File(".");
     @Parameter(names = {"-l", "--lfs"}, description = "LFS URL", required = false)
     @Nullable
     private String lfs;