Merge pull request #9 from bozaro/glob

Support an arbitrary glob, not just a suffix

README.md

@@ -15,7 +15,7 @@ For quick run you need:
  * After unpacking archive you can run server executing:
 
    ```bash
-   java -jar build/deploy/git-lfs-migrate.jar -s source-repo.git -d target-repo.git -l http://test:test@lfs-server/ .psd .zip .bin
+   java -jar build/deploy/git-lfs-migrate.jar -s source-repo.git -d target-repo.git -l http://test:test@lfs-server/ "*.psd" "*.zip" "*.bin"
    ```
 
 For example, you can convert bozaro/git-lfs-migrate to bozaro/git-lfs-migrate-converted by commands:
@@ -27,7 +27,7 @@ git clone --mirror git@github.com:bozaro/git-lfs-migrate.git
 
 # Convert repository with moving .md and .jar file to LFS
 #
-# Usage: <main class> [options] LFS file suffixes
+# Usage: <main class> [options] LFS file glob patterns
 #   Options:
 #     -c, --cache
 #        Source repository
@@ -56,8 +56,8 @@ java -jar git-lfs-migrate.jar \
      -s git-lfs-migrate.git \
      -d git-lfs-migrate-converted.git \
      -g git@github.com:bozaro/git-lfs-migrate-converted.git \
-     .md \
-     .jar
+     "*.md" \
+     "*.jar"
 
 # Push coverted repository to new repository
 cd git-lfs-migrate-converted.git
@@ -101,5 +101,5 @@ call gradlew.bat deployZip
 When build completes you can convert repository executing:
 
 ```bash
-java -jar build/deploy/git-lfs-migrate.jar -s source-repo.git -d target-repo.git -l http://test:test@lfs-server/ .psd .zip .bin
+java -jar build/deploy/git-lfs-migrate.jar -s source-repo.git -d target-repo.git -l http://test:test@lfs-server/ "*.psd" "*.zip" "*.bin"
 ```

src/main/java/git/lfs/migrate/GitConverter.java

@@ -1,6 +1,8 @@
 package git.lfs.migrate;
 
 import org.apache.commons.codec.binary.Hex;
+import org.eclipse.jgit.errors.InvalidPatternException;
+import org.eclipse.jgit.fnmatch.FileNameMatcher;
 import org.eclipse.jgit.lib.*;
 import org.eclipse.jgit.revwalk.*;
 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
@@ -31,7 +33,7 @@ public class GitConverter implements AutoCloseable {
   @NotNull
   private static final String GIT_ATTRIBUTES = ".gitattributes";
   @NotNull
-  private final String[] suffixes;
+  private final String[] globs;
   @NotNull
   private final File basePath;
   @NotNull
@@ -41,9 +43,14 @@ public class GitConverter implements AutoCloseable {
   @NotNull
   private final HTreeMap<String, String> cacheSha256;
 
-  public GitConverter(@NotNull File cachePath, @NotNull File basePath, @NotNull String[] suffixes) throws IOException {
+  public GitConverter(@NotNull File cachePath, @NotNull File basePath, @NotNull String[] globs) throws IOException, InvalidPatternException {
     this.basePath = basePath;
-    this.suffixes = suffixes.clone();
+    this.globs = globs.clone();
+    Arrays.sort(globs);
+
+    for (String glob : globs) {
+      new FileNameMatcher(glob, '/');
+    }
 
     tempPath = new File(basePath, "lfs/tmp");
     makeParentDirs(tempPath);
@@ -73,7 +80,7 @@ public ConvertTask convertTask(@NotNull ObjectReader reader, @NotNull TaskKey ke
           return convertCommitTask((RevCommit) revObject);
         }
         if (revObject instanceof RevTree) {
-          return convertTreeTask(reader, revObject, false);
+          return convertTreeTask(reader, revObject, Objects.requireNonNull(key.getPath()));
         }
         if (revObject instanceof RevBlob) {
           return copyTask(reader, revObject);
@@ -83,13 +90,6 @@ public ConvertTask convertTask(@NotNull ObjectReader reader, @NotNull TaskKey ke
         }
         throw new IllegalStateException("Unsupported object type: " + key + " (" + revObject.getClass().getName() + ")");
       }
-      case Root: {
-        final RevObject revObject = new RevWalk(reader).parseAny(key.getObjectId());
-        if (revObject instanceof RevTree) {
-          return convertTreeTask(reader, revObject, true);
-        }
-        throw new IllegalStateException("Unsupported object type: " + key + " (" + revObject.getClass().getName() + ")");
-      }
       case Attribute:
         return createAttributesTask(reader, key.getObjectId());
       case UploadLfs:
@@ -122,14 +122,14 @@ private ConvertTask convertTagTask(@NotNull RevTag revObject) throws IOException
       @Override
       public Iterable<TaskKey> depends() {
         return Collections.singletonList(
-            new TaskKey(TaskType.Simple, revObject.getObject())
+            new TaskKey(TaskType.Simple, "", revObject.getObject())
         );
       }
 
       @NotNull
       @Override
       public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolver resolver, @Nullable Uploader uploader) throws IOException {
-        final ObjectId id = resolver.resolve(TaskType.Simple, revObject.getObject());
+        final ObjectId id = resolver.resolve(TaskType.Simple, "", revObject.getObject());
         final TagBuilder builder = new TagBuilder();
         builder.setMessage(revObject.getFullMessage());
         builder.setTag(revObject.getTagName());
@@ -148,9 +148,9 @@ private ConvertTask convertCommitTask(@NotNull RevCommit revObject) throws IOExc
       public Iterable<TaskKey> depends() {
         List<TaskKey> result = new ArrayList<>();
         for (RevCommit parent : revObject.getParents()) {
-          result.add(new TaskKey(TaskType.Simple, parent));
+          result.add(new TaskKey(TaskType.Simple, "", parent));
         }
-        result.add(new TaskKey(TaskType.Root, revObject.getTree()));
+        result.add(new TaskKey(TaskType.Simple, "", revObject.getTree()));
         return result;
       }
 
@@ -164,39 +164,43 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv
         builder.setMessage(revObject.getFullMessage());
         // Set parents
         for (RevCommit oldParent : revObject.getParents()) {
-          builder.addParentId(resolver.resolve(TaskType.Simple, oldParent));
+          builder.addParentId(resolver.resolve(TaskType.Simple, "", oldParent));
         }
         // Set tree
-        builder.setTreeId(resolver.resolve(TaskType.Root, revObject.getTree()));
+        builder.setTreeId(resolver.resolve(TaskType.Simple, "", revObject.getTree()));
         return inserter.insert(builder);
       }
     };
   }
 
   @NotNull
-  private ConvertTask convertTreeTask(@NotNull ObjectReader reader, @NotNull ObjectId id, boolean rootTree) {
+  private ConvertTask convertTreeTask(@NotNull ObjectReader reader, @NotNull ObjectId id, @NotNull String path) {
     return new ConvertTask() {
       @NotNull
       private List<GitTreeEntry> getEntries() throws IOException {
         final List<GitTreeEntry> entries = new ArrayList<>();
         final CanonicalTreeParser treeParser = new CanonicalTreeParser(null, reader, id);
-        boolean needAttributes = rootTree;
+        boolean needAttributes = path.isEmpty();
         while (!treeParser.eof()) {
           final FileMode fileMode = treeParser.getEntryFileMode();
           final TaskType blobTask;
+          final String pathTask;
           if (needAttributes && treeParser.getEntryPathString().equals(GIT_ATTRIBUTES)) {
             blobTask = TaskType.Attribute;
+            pathTask = null;
             needAttributes = false;
-          } else if (isFile(fileMode) && matchFilename(treeParser.getEntryPathString())) {
+          } else if (isFile(fileMode) && matchFilename(path + "/" + treeParser.getEntryPathString())) {
             blobTask = TaskType.UploadLfs;
+            pathTask = null;
           } else {
             blobTask = TaskType.Simple;
+            pathTask = path + "/" + treeParser.getEntryPathString();
           }
-          entries.add(new GitTreeEntry(fileMode, new TaskKey(blobTask, treeParser.getEntryObjectId()), treeParser.getEntryPathString()));
+          entries.add(new GitTreeEntry(fileMode, new TaskKey(blobTask, pathTask, treeParser.getEntryObjectId()), treeParser.getEntryPathString()));
           treeParser.next();
         }
-        if (needAttributes && suffixes.length > 0) {
-          entries.add(new GitTreeEntry(FileMode.REGULAR_FILE, new TaskKey(TaskType.Attribute, ObjectId.zeroId()), GIT_ATTRIBUTES));
+        if (needAttributes && globs.length > 0) {
+          entries.add(new GitTreeEntry(FileMode.REGULAR_FILE, new TaskKey(TaskType.Attribute, null, ObjectId.zeroId()), GIT_ATTRIBUTES));
         }
         return entries;
       }
@@ -228,12 +232,21 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv
   }
 
   private boolean matchFilename(@NotNull String fileName) {
-    for (String suffix : suffixes) {
-      if (fileName.endsWith(suffix)) {
-        return true;
+    if (!fileName.startsWith("/")) {
+      throw new IllegalStateException("Unexpected file name: " + fileName);
+    }
+    try {
+      for (String glob : globs) {
+        final FileNameMatcher matcher = new FileNameMatcher(glob, null);
+        matcher.append(fileName.substring(1));
+        if (matcher.isMatch()) {
+          return true;
+        }
       }
+      return false;
+    } catch (InvalidPatternException e) {
+      throw new IllegalArgumentException(e);
     }
-    return false;
   }
 
   @NotNull
@@ -358,8 +371,8 @@ public Iterable<TaskKey> depends() throws IOException {
       @Override
       public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolver resolver, @Nullable Uploader uploader) throws IOException {
         final Set<String> attributes = new TreeSet<>();
-        for (String suffix : suffixes) {
-          attributes.add("*" + suffix + "\tfilter=lfs diff=lfs merge=lfs -crlf");
+        for (String glob : globs) {
+          attributes.add(glob + "\tfilter=lfs diff=lfs merge=lfs -crlf");
         }
         final ByteArrayOutputStream blob = new ByteArrayOutputStream();
         try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(openAttributes(reader, id), StandardCharsets.UTF_8))) {
@@ -410,16 +423,28 @@ private InputStream openAttributes(@NotNull ObjectReader reader, @Nullable Objec
   }
 
   public enum TaskType {
-    Simple, Root, Attribute, UploadLfs,
+    Simple(true),
+    Attribute(false),
+    UploadLfs(false);
+
+    TaskType(boolean needPath) {
+      this.needPath = needPath;
+    }
+
+    private final boolean needPath;
+
+    public boolean needPath() {
+      return needPath;
+    }
   }
 
   public interface ConvertResolver {
     @NotNull
     ObjectId resolve(@NotNull TaskKey key);
 
     @NotNull
-    default ObjectId resolve(@NotNull TaskType type, @NotNull ObjectId objectId) {
-      return resolve(new TaskKey(type, objectId));
+    default ObjectId resolve(@NotNull TaskType type, @Nullable String path, @NotNull ObjectId objectId) {
+      return resolve(new TaskKey(type, path, objectId));
     }
   }
 

src/main/java/git/lfs/migrate/Main.java

@@ -3,6 +3,7 @@
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
 import org.apache.commons.httpclient.HttpStatus;
+import org.eclipse.jgit.errors.InvalidPatternException;
 import org.eclipse.jgit.lib.*;
 import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
 import org.jetbrains.annotations.NotNull;
@@ -43,7 +44,7 @@ public class Main {
   @NotNull
   private static final Logger log = LoggerFactory.getLogger(Main.class);
 
-  public static void main(@NotNull String[] args) throws IOException, InterruptedException, ExecutionException {
+  public static void main(@NotNull String[] args) throws IOException, InterruptedException, ExecutionException, InvalidPatternException {
     final CmdArgs cmd = new CmdArgs();
     final JCommander jc = new JCommander(cmd);
     jc.parse(args);
@@ -69,7 +70,7 @@ public static void main(@NotNull String[] args) throws IOException, InterruptedE
       }
       return;
     }
-    processRepository(cmd.src, cmd.dst, cmd.cache, auth, cmd.writeThreads, cmd.uploadThreads, cmd.suffixes.toArray(new String[cmd.suffixes.size()]));
+    processRepository(cmd.src, cmd.dst, cmd.cache, auth, cmd.writeThreads, cmd.uploadThreads, cmd.globs.toArray(new String[cmd.globs.size()]));
     log.info("Convert time: {}", System.currentTimeMillis() - time);
   }
 
@@ -112,7 +113,7 @@ private static boolean checkLfsAuthenticate(@Nullable AuthProvider auth) throws
     return false;
   }
 
-  public static void processRepository(@NotNull File srcPath, @NotNull File dstPath, @NotNull File cachePath, @Nullable AuthProvider auth, int writeThreads, int uploadThreads, @NotNull String... suffixes) throws IOException, InterruptedException, ExecutionException {
+  public static void processRepository(@NotNull File srcPath, @NotNull File dstPath, @NotNull File cachePath, @Nullable AuthProvider auth, int writeThreads, int uploadThreads, @NotNull String... globs) throws IOException, InterruptedException, ExecutionException, InvalidPatternException {
     removeDirectory(dstPath);
     dstPath.mkdirs();
 
@@ -123,7 +124,7 @@ public static void processRepository(@NotNull File srcPath, @NotNull File dstPat
         .setMustExist(false)
         .setGitDir(dstPath).build();
 
-    final GitConverter converter = new GitConverter(cachePath, dstPath, suffixes);
+    final GitConverter converter = new GitConverter(cachePath, dstPath, globs);
     try {
       dstRepo.create(true);
       // Load all revision list.
@@ -150,7 +151,7 @@ public static void processRepository(@NotNull File srcPath, @NotNull File dstPat
       for (Map.Entry<String, Ref> ref : srcRepo.getAllRefs().entrySet()) {
         RefUpdate refUpdate = dstRepo.updateRef(ref.getKey());
         final ObjectId oldId = ref.getValue().getObjectId();
-        final ObjectId newId = converted.get(new TaskKey(GitConverter.TaskType.Simple, oldId));
+        final ObjectId newId = converted.get(new TaskKey(GitConverter.TaskType.Simple, "", oldId));
         refUpdate.setNewObjectId(newId);
         refUpdate.update();
         log.info("  convert ref: {} -> {} ({})", oldId.getName(), newId.getName(), ref.getKey());
@@ -269,7 +270,7 @@ private static SimpleDirectedGraph<TaskKey, DefaultEdge> loadTaskGraph(@NotNull
       final Deque<TaskKey> queue = new ArrayDeque<>();
       // Heads
       for (Ref ref : refs.values()) {
-        final TaskKey taskKey = new TaskKey(GitConverter.TaskType.Simple, ref.getObjectId());
+        final TaskKey taskKey = new TaskKey(GitConverter.TaskType.Simple, "", ref.getObjectId());
         if (graph.addVertex(taskKey)) {
           queue.add(taskKey);
           reporter.increment();
@@ -420,9 +421,9 @@ public static class CmdArgs {
     @Parameter(names = {"--check-lfs"}, description = "Check LFS server settings and exit")
     private boolean checkLfs = false;
 
-    @Parameter(description = "LFS file suffixes")
+    @Parameter(description = "LFS file glob patterns")
     @NotNull
-    private List<String> suffixes = new ArrayList<>();
+    private List<String> globs = new ArrayList<>();
     @Parameter(names = {"-h", "--help"}, description = "Show help", help = true)
     private boolean help = false;
   }

src/main/java/git/lfs/migrate/TaskKey.java

@@ -2,6 +2,9 @@
 
 import org.eclipse.jgit.lib.ObjectId;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Objects;
 
 /**
  * Key of converter task.
@@ -14,10 +17,16 @@ public final class TaskKey {
   private final GitConverter.TaskType type;
   @NotNull
   private final ObjectId objectId;
+  @Nullable
+  private final String path;
 
-  public TaskKey(@NotNull GitConverter.TaskType type, @NotNull ObjectId objectId) {
+  public TaskKey(@NotNull GitConverter.TaskType type, @Nullable String path, @NotNull ObjectId objectId) {
     this.type = type;
+    this.path = path;
     this.objectId = objectId.copy();
+    if (type.needPath() == (path == null)) {
+      throw new IllegalStateException();
+    }
   }
 
   @NotNull
@@ -30,6 +39,11 @@ public ObjectId getObjectId() {
     return objectId;
   }
 
+  @Nullable
+  public String getPath() {
+    return path;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;
@@ -38,19 +52,22 @@ public boolean equals(Object o) {
     TaskKey taskKey = (TaskKey) o;
 
     return (type == taskKey.type)
-        && objectId.equals(taskKey.objectId);
-
+        && objectId.equals(taskKey.objectId)
+        && Objects.equals(path, taskKey.path);
   }
 
   @Override
   public int hashCode() {
     int result = type.hashCode();
     result = 31 * result + objectId.hashCode();
+    if (path != null) {
+      result = 31 * result + path.hashCode();
+    }
     return result;
   }
 
   @Override
   public String toString() {
-    return type + ":" + objectId.name();
+    return type + ":" + objectId.name() + (path == null ? "" : " (" + path + ")");
   }
 }