diff --git a/.gitignore b/.gitignore index feb1717..100205e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.orig *.class *.local *.mapdb* diff --git a/CHANGELOG.md b/CHANGELOG.md index d06d4f6..392f80c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changes +## 0.2.4 + + * Fix non-suffix pattern matching (#20, thanks to @wesyq). + ## 0.2.3 * Allow load globs from file (#19, thanks to @leth). diff --git a/build.gradle b/build.gradle index 2f84c9b..d35385d 100644 --- a/build.gradle +++ b/build.gradle @@ -23,16 +23,17 @@ apply plugin: "com.github.ben-manes.versions" dependencies { compile "org.jetbrains:annotations:15.0" - compile "org.eclipse.jgit:org.eclipse.jgit:4.1.1.201511131810-r" - compile "org.mapdb:mapdb:1.0.8" - compile "org.slf4j:slf4j-simple:1.7.13" - compile "org.jgrapht:jgrapht-core:0.9.1" - compile "com.beust:jcommander:1.48" + compile "org.eclipse.jgit:org.eclipse.jgit:4.5.0.201609210915-r" + compile "org.mapdb:mapdb:3.0.2" + compile "org.slf4j:slf4j-simple:1.7.21" + compile "org.jgrapht:jgrapht-core:1.0.0" + compile "com.beust:jcommander:1.58" compile "ru.bozaro.gitlfs:gitlfs-pointer:0.10.0" compile "ru.bozaro.gitlfs:gitlfs-client:0.10.0" - testCompile "org.testng:testng:6.9.10" + testCompile "com.google.jimfs:jimfs:1.1" + testCompile "org.testng:testng:6.9.13.6" } sourceCompatibility = JavaVersion.VERSION_1_8 diff --git a/src/main/java/git/lfs/migrate/GitConverter.java b/src/main/java/git/lfs/migrate/GitConverter.java index 30ed2e2..f4b88d8 100644 --- a/src/main/java/git/lfs/migrate/GitConverter.java +++ b/src/main/java/git/lfs/migrate/GitConverter.java @@ -1,5 +1,7 @@ package git.lfs.migrate; +import git.path.PathMatcher; +import git.path.WildcardHelper; import org.apache.commons.codec.binary.Hex; import org.eclipse.jgit.errors.InvalidPatternException; import org.eclipse.jgit.fnmatch.FileNameMatcher; @@ -9,8 +11,9 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.mapdb.DB; -import org.mapdb.DBMaker; import org.mapdb.HTreeMap; +import org.mapdb.Serializer; +import org.mapdb.serializer.SerializerJava; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ru.bozaro.gitlfs.common.data.Meta; @@ -18,6 +21,9 @@ import java.io.*; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.*; @@ -27,7 +33,7 @@ * Converter for git objects. * Created by bozaro on 09.06.15. */ -public class GitConverter implements AutoCloseable { +public class GitConverter { @NotNull private static final Logger log = LoggerFactory.getLogger(GitConverter.class); @NotNull @@ -35,37 +41,34 @@ public class GitConverter implements AutoCloseable { @NotNull private final String[] globs; @NotNull - private final File basePath; - @NotNull - private final File tempPath; + private final PathMatcher[] matchers; @NotNull private final DB cache; @NotNull + private final Path basePath; + @NotNull + private final Path tempPath; + @NotNull private final HTreeMap cacheMeta; - public GitConverter(@NotNull File cachePath, @NotNull File basePath, @NotNull String[] globs) throws IOException, InvalidPatternException { + public GitConverter(@NotNull DB cache, @NotNull Path basePath, @NotNull String[] globs) throws IOException, InvalidPatternException { this.basePath = basePath; + this.cache = cache; this.globs = globs.clone(); + this.matchers = convertGlobs(globs); Arrays.sort(globs); for (String glob : globs) { new FileNameMatcher(glob, '/'); } - tempPath = new File(basePath, "lfs/tmp"); - makeParentDirs(tempPath); - makeParentDirs(cachePath); - cache = DBMaker.newFileDB(new File(cachePath, "git-lfs-migrate.mapdb")) - .asyncWriteEnable() - .mmapFileEnable() - .cacheSoftRefEnable() - .make(); - cacheMeta = cache.getHashMap("meta"); - } - - @Override - public void close() throws IOException { - cache.close(); + tempPath = basePath.resolve("lfs/tmp"); + Files.createDirectories(tempPath); + //noinspection unchecked + cacheMeta = cache.hashMap("meta") + .keySerializer(Serializer.STRING) + .valueSerializer(new SerializerJava()) + .createOrOpen(); } @NotNull @@ -231,22 +234,29 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv }; } - private boolean matchFilename(@NotNull String fileName) { + @NotNull + private static PathMatcher[] convertGlobs(String[] globs) throws InvalidPatternException { + final PathMatcher[] matchers = new PathMatcher[globs.length]; + for (int i = 0; i < globs.length; ++i) { + String glob = globs[i]; + if (!glob.contains("/")) { + glob = "**/" + glob; + } + matchers[i] = WildcardHelper.createMatcher(glob, true); + } + return matchers; + } + + public boolean matchFilename(@NotNull String fileName) { if (!fileName.startsWith("/")) { throw new IllegalStateException("Unexpected file name: " + fileName); } - try { - for (String glob : globs) { - final FileNameMatcher matcher = new FileNameMatcher(glob, null); - matcher.append(fileName.substring(1)); - if (matcher.isMatch()) { - return true; - } + for (PathMatcher matcher : matchers) { + if (WildcardHelper.isMatch(matcher, fileName)) { + return true; } - return false; - } catch (InvalidPatternException e) { - throw new IllegalArgumentException(e); } + return false; } @NotNull @@ -320,11 +330,11 @@ private String createRemoteFile(@NotNull ObjectId id, @NotNull ObjectLoader load @NotNull private String createLocalFile(@NotNull ObjectId id, @NotNull ObjectLoader loader) throws IOException { // Create LFS stream. - final File tmpFile = new File(tempPath, UUID.randomUUID().toString()); + final Path tmpFile = tempPath.resolve(UUID.randomUUID().toString()); final MessageDigest md = createSha256(); int size = 0; try (InputStream istream = loader.openStream(); - OutputStream ostream = new FileOutputStream(tmpFile)) { + OutputStream ostream = Files.newOutputStream(tmpFile)) { byte[] buffer = new byte[0x10000]; while (true) { int read = istream.read(buffer); @@ -338,24 +348,20 @@ private String createLocalFile(@NotNull ObjectId id, @NotNull ObjectLoader loade cacheMeta.putIfAbsent(id.name(), new MetaData(hash, size)); cache.commit(); // Rename file. - final File lfsFile = new File(basePath, "lfs/objects/" + hash.substring(0, 2) + "/" + hash.substring(2, 4) + "/" + hash); - makeParentDirs(lfsFile.getParentFile()); - if (lfsFile.exists()) { - if (!tmpFile.delete()) { - log.warn("Can't delete temporary file: {}", lfsFile.getAbsolutePath()); + final Path lfsFile = basePath.resolve("lfs/objects/" + hash.substring(0, 2) + "/" + hash.substring(2, 4) + "/" + hash); + Files.createDirectories(lfsFile.getParent()); + if (Files.exists(lfsFile)) { + try { + Files.delete(tmpFile); + } catch (IOException e) { + log.warn("Can't delete temporary file: {}", lfsFile.toAbsolutePath()); } - } else if (!tmpFile.renameTo(lfsFile)) { - throw new IOException("Can't rename file: " + tmpFile + " -> " + lfsFile); + } else { + Files.move(tmpFile, lfsFile, StandardCopyOption.ATOMIC_MOVE); } return hash; } - private void makeParentDirs(@NotNull File path) throws IOException { - if (!path.mkdirs() && !path.exists()) { - throw new IOException("Can't create directory: " + path.getAbsolutePath()); - } - } - @NotNull private static MessageDigest createSha256() { // Prepare for hash calculation diff --git a/src/main/java/git/lfs/migrate/Main.java b/src/main/java/git/lfs/migrate/Main.java index c5757bd..c6b1da8 100644 --- a/src/main/java/git/lfs/migrate/Main.java +++ b/src/main/java/git/lfs/migrate/Main.java @@ -13,6 +13,8 @@ import org.jetbrains.annotations.Nullable; import org.jgrapht.graph.DefaultEdge; import org.jgrapht.graph.SimpleDirectedGraph; +import org.mapdb.DB; +import org.mapdb.DBMaker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ru.bozaro.gitlfs.client.AuthHelper; @@ -25,13 +27,9 @@ import ru.bozaro.gitlfs.common.data.*; import ru.bozaro.gitlfs.common.data.Error; -import java.io.File; import java.io.IOException; import java.net.URI; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; +import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.security.GeneralSecurityException; import java.util.*; @@ -79,7 +77,7 @@ public static void main(@NotNull String[] args) throws Exception { String[] globs = cmd.globs.toArray(new String[cmd.globs.size()]); if (cmd.globFile != null) { globs = Stream.concat(Arrays.stream(globs), - Files.lines(cmd.globFile.toPath()) + Files.lines(cmd.globFile) .map(String::trim) .filter(s -> !s.isEmpty()) ).toArray(String[]::new); @@ -147,19 +145,21 @@ private static boolean checkLfsAuthenticate(@Nullable Client client) throws IOEx return false; } - public static void processRepository(@NotNull File srcPath, @NotNull File dstPath, @NotNull File cachePath, @Nullable Client client, int writeThreads, int uploadThreads, @NotNull String... globs) throws IOException, InterruptedException, ExecutionException, InvalidPatternException { + public static void processRepository(@NotNull Path srcPath, @NotNull Path dstPath, @NotNull Path cachePath, @Nullable Client client, int writeThreads, int uploadThreads, @NotNull String... globs) throws IOException, InterruptedException, ExecutionException, InvalidPatternException { removeDirectory(dstPath); - dstPath.mkdirs(); + Files.createDirectories(dstPath); final Repository srcRepo = new FileRepositoryBuilder() .setMustExist(true) - .setGitDir(srcPath).build(); + .setGitDir(srcPath.toFile()).build(); final Repository dstRepo = new FileRepositoryBuilder() .setMustExist(false) - .setGitDir(dstPath).build(); + .setGitDir(dstPath.toFile()).build(); - final GitConverter converter = new GitConverter(cachePath, dstPath, globs); - try { + try (DB cache = DBMaker.fileDB(cachePath.resolve("git-lfs-migrate.mapdb").toFile()) + .fileMmapEnableIfSupported() + .make()) { + final GitConverter converter = new GitConverter(cache, dstPath, globs); dstRepo.create(true); // Load all revision list. log.info("Reading full objects list..."); @@ -279,9 +279,9 @@ private static void processSingleThread(@NotNull GitConverter converter, @NotNul } } - private static void removeDirectory(@NotNull File path) throws IOException { - if (path.exists()) { - Files.walkFileTree(path.toPath(), new SimpleFileVisitor() { + private static void removeDirectory(@NotNull Path path) throws IOException { + if (Files.exists(path)) { + Files.walkFileTree(path, new SimpleFileVisitor() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { Files.delete(file); @@ -435,13 +435,13 @@ private void print(long current) { public static class CmdArgs { @Parameter(names = {"-s", "--source"}, description = "Source repository", required = true) @NotNull - private File src; + private Path src; @Parameter(names = {"-d", "--destination"}, description = "Destination repository", required = true) @NotNull - private File dst; + private Path dst; @Parameter(names = {"-c", "--cache"}, description = "Source repository", required = false) @NotNull - private File cache = new File("."); + private Path cache = FileSystems.getDefault().getPath("."); @Parameter(names = {"-g", "--git"}, description = "GIT repository url (ignored with --lfs parameter)", required = false) @Nullable private String git; @@ -457,7 +457,7 @@ public static class CmdArgs { @Parameter(names = {"--no-check-certificate"}, description = "Don't check the server certificate against the available certificate authorities") private boolean noCheckCertificate = false; @Parameter(names = {"--glob-file"}, description = "File containing glob patterns") - private File globFile = null; + private Path globFile = null; @Parameter(description = "LFS file glob patterns") @NotNull diff --git a/src/main/java/git/path/NameMatcher.java b/src/main/java/git/path/NameMatcher.java new file mode 100644 index 0000000..89d1bcd --- /dev/null +++ b/src/main/java/git/path/NameMatcher.java @@ -0,0 +1,14 @@ +package git.path; + +import org.jetbrains.annotations.NotNull; + +/** + * Interface for matching name of path. + * + * @author Artem V. Navrotskiy + */ +public interface NameMatcher { + boolean isMatch(@NotNull String name, boolean isDir); + + boolean isRecursive(); +} diff --git a/src/main/java/git/path/PathMatcher.java b/src/main/java/git/path/PathMatcher.java new file mode 100644 index 0000000..4d71a90 --- /dev/null +++ b/src/main/java/git/path/PathMatcher.java @@ -0,0 +1,16 @@ +package git.path; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Interface for path matching. + * + * @author Artem V. Navrotskiy + */ +public interface PathMatcher { + @Nullable + PathMatcher createChild(@NotNull String name, boolean isDir); + + boolean isMatch(); +} diff --git a/src/main/java/git/path/WildcardHelper.java b/src/main/java/git/path/WildcardHelper.java new file mode 100644 index 0000000..d566d51 --- /dev/null +++ b/src/main/java/git/path/WildcardHelper.java @@ -0,0 +1,194 @@ +package git.path; + +import git.path.matcher.name.ComplexMatcher; +import git.path.matcher.name.EqualsMatcher; +import git.path.matcher.name.RecursiveMatcher; +import git.path.matcher.name.SimpleMatcher; +import git.path.matcher.path.AlwaysMatcher; +import git.path.matcher.path.RecursivePathMatcher; +import org.eclipse.jgit.errors.InvalidPatternException; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.List; + +/** + * Git wildcard mask. + *

+ * Pattern format: http://git-scm.com/docs/gitignore + * + * @author Artem V. Navrotskiy + */ +public class WildcardHelper { + public static final char PATH_SEPARATOR = '/'; + + @Nullable + public static PathMatcher createMatcher(@NotNull String pattern, boolean exact) throws InvalidPatternException { + final NameMatcher[] nameMatchers = createNameMatchers(pattern); + if (nameMatchers.length > 0) { + return new RecursivePathMatcher(nameMatchers, exact); + } else { + return exact ? null : AlwaysMatcher.INSTANCE; + } + } + + private static NameMatcher[] createNameMatchers(@NotNull String pattern) throws InvalidPatternException { + final List tokens = WildcardHelper.splitPattern(pattern); + WildcardHelper.normalizePattern(tokens); + final NameMatcher[] result = new NameMatcher[tokens.size() - 1]; + for (int i = 0; i < result.length; ++i) { + result[i] = WildcardHelper.nameMatcher(tokens.get(i + 1)); + } + return result; + } + + @NotNull + private static NameMatcher nameMatcher(@NotNull String mask) throws InvalidPatternException { + if (mask.equals("**/")) { + return RecursiveMatcher.INSTANCE; + } + final boolean dirOnly = mask.endsWith("/"); + final String nameMask = tryRemoveBackslashes(dirOnly ? mask.substring(0, mask.length() - 1) : mask); + if ((nameMask.indexOf('[') < 0) && (nameMask.indexOf(']') < 0) && (nameMask.indexOf('\\') < 0)) { + // Subversion compatible mask. + if (nameMask.indexOf('?') < 0) { + int asterisk = nameMask.indexOf('*'); + if (asterisk < 0) { + return new EqualsMatcher(nameMask, dirOnly); + } else if (mask.indexOf('*', asterisk + 1) < 0) { + return new SimpleMatcher(nameMask.substring(0, asterisk), nameMask.substring(asterisk + 1), dirOnly); + } + } + return new ComplexMatcher(nameMask, dirOnly, true); + } else { + return new ComplexMatcher(nameMask, dirOnly, false); + } + } + + @NotNull + static String tryRemoveBackslashes(@NotNull String pattern) { + final StringBuilder result = new StringBuilder(pattern.length()); + int start = 0; + while (true) { + int next = pattern.indexOf('\\', start); + if (next == -1) { + if (start < pattern.length()) { + result.append(pattern, start, pattern.length()); + } + break; + } + if (next == pattern.length() - 1) { + // Return original string. + return pattern; + } + switch (pattern.charAt(next + 1)) { + case ' ': + case '#': + case '!': + result.append(pattern, start, next); + start = next + 1; + break; + default: + return pattern; + } + } + return result.toString(); + } + + /** + * Split pattern with saving slashes. + * + * @param pattern Path pattern. + * @return Path pattern items. + */ + @NotNull + public static List splitPattern(@NotNull String pattern) { + final List result = new ArrayList<>(count(pattern, PATH_SEPARATOR) + 1); + int start = 0; + while (true) { + int next = pattern.indexOf(PATH_SEPARATOR, start); + if (next == -1) { + if (start < pattern.length()) { + result.add(pattern.substring(start)); + } + break; + } + result.add(pattern.substring(start, next + 1)); + start = next + 1; + } + return result; + } + + /** + * Remove redundant pattern parts and make patterns more simple. + * + * @param tokens Original modifiable list. + * @return Return tokens, + */ + @NotNull + public static List normalizePattern(@NotNull List tokens) { + // By default without slashes using mask for files in all subdirectories + if ((tokens.size() == 1) && !tokens.get(0).startsWith("/")) { + tokens.add(0, "**/"); + } + // Normalized pattern always starts with "/" + if (tokens.size() == 0 || !tokens.get(0).equals("/")) { + tokens.add(0, "/"); + } + // Replace: + // * "**/*/" to "*/**/" + // * "**/**/" to "**/" + // * "**.foo" to "**/*.foo" + int index = 1; + while (index < tokens.size()) { + final String thisToken = tokens.get(index); + final String prevToken = tokens.get(index - 1); + if (thisToken.equals("/")) { + tokens.remove(index); + continue; + } + if (thisToken.equals("**/") && prevToken.equals("**/")) { + tokens.remove(index); + continue; + } + if ((!thisToken.equals("**/")) && thisToken.startsWith("**")) { + tokens.add(index, "**/"); + tokens.set(index + 1, thisToken.substring(1)); + continue; + } + if (thisToken.equals("*/") && prevToken.equals("**/")) { + tokens.set(index - 1, "*/"); + tokens.set(index, "**/"); + index--; + continue; + } + index++; + } + return tokens; + } + + private static int count(@NotNull String s, char c) { + int start = 0; + int count = 0; + while (true) { + start = s.indexOf(c, start); + if (start == -1) + break; + count++; + start++; + } + return count; + } + + public static boolean isMatch(@Nullable PathMatcher matcher, @NotNull String fileName) { + List items = splitPattern(fileName.substring(1)); + PathMatcher m = matcher; + for (String item : items) { + if (m == null) break; + final boolean dir = item.endsWith(String.valueOf(PATH_SEPARATOR)); + m = m.createChild(dir ? item.substring(0, item.length() - 1) : item, dir); + } + return (m != null) && m.isMatch(); + } +} diff --git a/src/main/java/git/path/matcher/name/ComplexMatcher.java b/src/main/java/git/path/matcher/name/ComplexMatcher.java new file mode 100644 index 0000000..0889181 --- /dev/null +++ b/src/main/java/git/path/matcher/name/ComplexMatcher.java @@ -0,0 +1,65 @@ +package git.path.matcher.name; + +import git.path.NameMatcher; +import org.eclipse.jgit.errors.InvalidPatternException; +import org.eclipse.jgit.ignore.internal.IMatcher; +import org.eclipse.jgit.ignore.internal.PathMatcher; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Objects; + +/** + * Simple matcher for regexp compare. + * + * @author Artem V. Navrotskiy + */ +public final class ComplexMatcher implements NameMatcher { + @Nullable + private final String pattern; + @NotNull + private final IMatcher matcher; + private final boolean dirOnly; + private final boolean svnMask; + + public ComplexMatcher(@NotNull String pattern, boolean dirOnly, boolean svnMask) throws InvalidPatternException { + this.pattern = pattern; + this.dirOnly = dirOnly; + this.svnMask = svnMask; + this.matcher = PathMatcher.createPathMatcher(dirOnly ? pattern.substring(0, pattern.length() - 1) : pattern, null, dirOnly); + } + + @Override + public boolean isMatch(@NotNull String name, boolean isDir) { + return matcher.matches(name, isDir); + } + + @Override + public boolean isRecursive() { + return false; + } + + @Override + public boolean equals(@Nullable Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + final ComplexMatcher that = (ComplexMatcher) o; + + return (dirOnly == that.dirOnly) + && Objects.equals(pattern, that.pattern); + } + + @Override + public int hashCode() { + int result = pattern != null ? pattern.hashCode() : 0; + result = 31 * result + (dirOnly ? 1 : 0); + return result; + } + + @Override + @NotNull + public String toString() { + return pattern + (dirOnly ? "/" : ""); + } +} diff --git a/src/main/java/git/path/matcher/name/EqualsMatcher.java b/src/main/java/git/path/matcher/name/EqualsMatcher.java new file mode 100644 index 0000000..0a321fc --- /dev/null +++ b/src/main/java/git/path/matcher/name/EqualsMatcher.java @@ -0,0 +1,55 @@ +package git.path.matcher.name; + +import git.path.NameMatcher; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Simple matcher for equals compare. + * + * @author Artem V. Navrotskiy + */ +public class EqualsMatcher implements NameMatcher { + @NotNull + private final String name; + private final boolean dirOnly; + + public EqualsMatcher(@NotNull String name, boolean dirOnly) { + this.name = name; + this.dirOnly = dirOnly; + } + + @Override + public boolean isMatch(@NotNull String name, boolean isDir) { + return (!dirOnly || isDir) && this.name.equals(name); + } + + @Override + public boolean isRecursive() { + return false; + } + + @Override + public boolean equals(@Nullable Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + final EqualsMatcher that = (EqualsMatcher) o; + + return (dirOnly == that.dirOnly) + && name.equals(that.name); + } + + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + (dirOnly ? 1 : 0); + return result; + } + + @Override + @NotNull + public String toString() { + return name + (dirOnly ? "/" : ""); + } +} diff --git a/src/main/java/git/path/matcher/name/RecursiveMatcher.java b/src/main/java/git/path/matcher/name/RecursiveMatcher.java new file mode 100644 index 0000000..60caac5 --- /dev/null +++ b/src/main/java/git/path/matcher/name/RecursiveMatcher.java @@ -0,0 +1,32 @@ +package git.path.matcher.name; + +import git.path.NameMatcher; +import org.jetbrains.annotations.NotNull; + +/** + * Recursive directory matcher like "**". + * + * @author Artem V. Navrotskiy + */ +public class RecursiveMatcher implements NameMatcher { + public static final RecursiveMatcher INSTANCE = new RecursiveMatcher(); + + private RecursiveMatcher() { + } + + @Override + public boolean isMatch(@NotNull String name, boolean isDir) { + return true; + } + + @Override + public boolean isRecursive() { + return true; + } + + @Override + @NotNull + public String toString() { + return "**/"; + } +} diff --git a/src/main/java/git/path/matcher/name/SimpleMatcher.java b/src/main/java/git/path/matcher/name/SimpleMatcher.java new file mode 100644 index 0000000..b415046 --- /dev/null +++ b/src/main/java/git/path/matcher/name/SimpleMatcher.java @@ -0,0 +1,60 @@ +package git.path.matcher.name; + +import git.path.NameMatcher; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Simple matcher for mask with only one asterisk. + * + * @author Artem V. Navrotskiy + */ +public class SimpleMatcher implements NameMatcher { + @NotNull + private final String prefix; + @NotNull + private final String suffix; + private final boolean dirOnly; + + public SimpleMatcher(@NotNull String prefix, @NotNull String suffix, boolean dirOnly) { + this.prefix = prefix; + this.suffix = suffix; + this.dirOnly = dirOnly; + } + + @Override + public boolean isMatch(@NotNull String name, boolean isDir) { + return (!dirOnly || isDir) && (name.length() >= prefix.length() + suffix.length()) && name.startsWith(prefix) && name.endsWith(suffix); + } + + @Override + public boolean isRecursive() { + return false; + } + + @Override + public boolean equals(@Nullable Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + final SimpleMatcher that = (SimpleMatcher) o; + + return (dirOnly == that.dirOnly) + && (prefix.equals(that.prefix)) + && suffix.equals(that.suffix); + } + + @Override + public int hashCode() { + int result = prefix.hashCode(); + result = 31 * result + suffix.hashCode(); + result = 31 * result + (dirOnly ? 1 : 0); + return result; + } + + @Override + @NotNull + public String toString() { + return prefix + "*" + suffix + (dirOnly ? "/" : ""); + } +} diff --git a/src/main/java/git/path/matcher/path/AlwaysMatcher.java b/src/main/java/git/path/matcher/path/AlwaysMatcher.java new file mode 100644 index 0000000..97fea10 --- /dev/null +++ b/src/main/java/git/path/matcher/path/AlwaysMatcher.java @@ -0,0 +1,29 @@ +package git.path.matcher.path; + +import git.path.PathMatcher; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Matches with any path. + * + * @author Artem V. Navrotskiy + */ +public class AlwaysMatcher implements PathMatcher { + @NotNull + public final static AlwaysMatcher INSTANCE = new AlwaysMatcher(); + + private AlwaysMatcher() { + } + + @Nullable + @Override + public PathMatcher createChild(@NotNull String name, boolean isDir) { + return this; + } + + @Override + public boolean isMatch() { + return true; + } +} diff --git a/src/main/java/git/path/matcher/path/RecursivePathMatcher.java b/src/main/java/git/path/matcher/path/RecursivePathMatcher.java new file mode 100644 index 0000000..3afde3c --- /dev/null +++ b/src/main/java/git/path/matcher/path/RecursivePathMatcher.java @@ -0,0 +1,132 @@ +package git.path.matcher.path; + +import git.path.NameMatcher; +import git.path.PathMatcher; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Arrays; +import java.util.Objects; + +/** + * Complex full-feature pattern matcher. + * + * @author Artem V. Navrotskiy + */ +public final class RecursivePathMatcher implements PathMatcher { + @NotNull + private final static int[] START_ARRAY = {0}; + @NotNull + private final int[] indexes; + @NotNull + private final NameMatcher[] nameMatchers; + private final boolean exact; + + public RecursivePathMatcher(@NotNull NameMatcher[] nameMatchers, boolean exact) { + this(nameMatchers, exact, START_ARRAY); + } + + private RecursivePathMatcher(@NotNull NameMatcher[] nameMatchers, boolean exact, @NotNull int[] indexes) { + this.nameMatchers = nameMatchers; + this.exact = exact; + this.indexes = indexes; + } + + @Nullable + @Override + public PathMatcher createChild(@NotNull String name, boolean isDir) { + final int[] childs = new int[indexes.length * 2]; + boolean changed = false; + int count = 0; + for (int index : indexes) { + if (index < nameMatchers.length && nameMatchers[index].isMatch(name, isDir)) { + if (nameMatchers[index].isRecursive()) { + childs[count++] = index; + if (index + 1 == nameMatchers.length) { + if (!exact) { + return AlwaysMatcher.INSTANCE; + } + if (isDir) { + childs[count++] = index + 1; + } + } else if (nameMatchers[index + 1].isMatch(name, isDir)) { + if (index + 2 == nameMatchers.length) { + if (!exact || !isDir) { + return AlwaysMatcher.INSTANCE; + } + } + childs[count++] = index + 2; + } + changed = true; + } else { + if (index + 1 == nameMatchers.length) { + if (!exact || !isDir) { + return AlwaysMatcher.INSTANCE; + } + } + childs[count++] = index + 1; + changed = true; + } + } else { + changed = true; + } + } + if (!isDir) { + return null; + } + if (!changed) { + return this; + } + return count == 0 ? null : new RecursivePathMatcher(nameMatchers, exact, Arrays.copyOf(childs, count)); + } + + @Override + public boolean isMatch() { + for (int index : indexes) { + if (index == nameMatchers.length) { + return true; + } + } + return false; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + RecursivePathMatcher that = (RecursivePathMatcher) o; + + if (indexes.length != that.indexes.length) return false; + final int offset = indexes[0]; + final int thatOffset = that.indexes[0]; + if (nameMatchers.length - offset != that.nameMatchers.length - thatOffset) return false; + + final int shift = thatOffset - offset; + for (int i = offset; i < indexes.length; ++i) { + if (indexes[i] != that.indexes[i + shift]) { + return false; + } + } + for (int i = offset; i < nameMatchers.length; ++i) { + if (!Objects.equals(nameMatchers[i], that.nameMatchers[i + shift])) { + return false; + } + } + return true; + } + + @Override + public int hashCode() { + int offset = indexes[0]; + int result = 0; + for (int index : indexes) { + result = 31 * (index - offset); + assert (offset <= index); + } + for (int i = offset; i < nameMatchers.length; ++i) { + result = 31 * result + nameMatchers[i].hashCode(); + } + return result; + } +} diff --git a/src/test/java/git/lfs/migrate/GitConverterTest.java b/src/test/java/git/lfs/migrate/GitConverterTest.java new file mode 100644 index 0000000..4af9ed3 --- /dev/null +++ b/src/test/java/git/lfs/migrate/GitConverterTest.java @@ -0,0 +1,60 @@ +package git.lfs.migrate; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import org.eclipse.jgit.errors.InvalidPatternException; +import org.jetbrains.annotations.NotNull; +import org.mapdb.DBMaker; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; + +/** + * Full LFS convert. + * + * @author Artem V. Navrotskiy + */ +public class GitConverterTest { + @DataProvider + public Object[][] matchFilenameProvider() { + return new Object[][]{ + new Object[]{"/LICENSE", true}, + new Object[]{"/foo/bar/LICENSE", true}, + new Object[]{"/LICENSE/foo/bar", false}, + new Object[]{"/foo/LICENSE/bar", false}, + new Object[]{"/dist.zip", true}, + new Object[]{"/foo/bar/dist.zip", true}, + new Object[]{"/dist.zip/foo/bar", false}, + new Object[]{"/foo/dist.zip/bar", false}, + new Object[]{"/.some", true}, + new Object[]{"/foo/bar/.some", true}, + new Object[]{"/.some/foo/bar", false}, + new Object[]{"/foo/.some/bar", false}, + new Object[]{"/test_some", true}, + new Object[]{"/foo/bar/test_some", true}, + new Object[]{"/test_some/foo/bar", false}, + new Object[]{"/root", true}, + new Object[]{"/root/data", false}, + new Object[]{"/some/data", true}, + new Object[]{"/some/data/data", false}, + new Object[]{"/qwerty/some/data", false}, + }; + } + + @Test(dataProvider = "matchFilenameProvider") + public void matchFilenameTest(@NotNull String path, boolean expected) throws IOException, InvalidPatternException { + FileSystem fs = Jimfs.newFileSystem(Configuration.unix()); + GitConverter converter = new GitConverter(DBMaker.memoryDB().make(), fs.getPath("/tmp/migrate"), new String[]{ + "*.zip", + ".*", + "LICENSE", + "test*", + "/root", + "some/data", + }); + Assert.assertEquals(converter.matchFilename(path), expected); + } +} diff --git a/src/test/java/git/path/DeleteTreeVisitor.java b/src/test/java/git/path/DeleteTreeVisitor.java new file mode 100644 index 0000000..d16d8f5 --- /dev/null +++ b/src/test/java/git/path/DeleteTreeVisitor.java @@ -0,0 +1,25 @@ +package git.path; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +/** + * @author Artem V. Navrotskiy + */ +class DeleteTreeVisitor extends SimpleFileVisitor { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } +} diff --git a/src/test/java/git/path/WildcardTest.java b/src/test/java/git/path/WildcardTest.java new file mode 100644 index 0000000..eff8c5f --- /dev/null +++ b/src/test/java/git/path/WildcardTest.java @@ -0,0 +1,214 @@ +package git.path; + +import com.google.common.io.ByteStreams; +import org.eclipse.jgit.errors.InvalidPatternException; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.testng.Assert; +import org.testng.SkipException; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +/** + * Test wildcard parsing. + * + * @author Artem V. Navrotskiy + */ +public class WildcardTest { + @DataProvider + public static Object[][] splitPatternData() { + return new Object[][]{ + new Object[]{"foo", new String[]{"foo"}}, + new Object[]{"foo/", new String[]{"foo/"}}, + new Object[]{"/bar", new String[]{"/", "bar"}}, + new Object[]{"/foo/bar/**", new String[]{"/", "foo/", "bar/", "**"}}, + }; + } + + @Test(dataProvider = "splitPatternData") + public static void splitPatternTest(@NotNull String pattern, @NotNull String[] expected) { + final List actual = WildcardHelper.splitPattern(pattern); + Assert.assertEquals(actual.toArray(new String[actual.size()]), expected, pattern); + } + + @DataProvider + public static Object[][] normalizePatternData() { + return new Object[][]{ + // Simple mask + new Object[]{"/", new String[0]}, + new Object[]{"*/", new String[]{"*/", "**/"}}, + new Object[]{"*", new String[]{"**/", "*"}}, + new Object[]{"**", new String[]{"**/", "*"}}, + new Object[]{"**/", new String[]{"**/"}}, + new Object[]{"foo", new String[]{"**/", "foo"}}, + new Object[]{"foo/", new String[]{"**/", "foo/"}}, + new Object[]{"/foo", new String[]{"foo"}}, + + // Convert path file mask + new Object[]{"foo/**.bar", new String[]{"foo/", "**/", "*.bar"}}, + new Object[]{"foo/***.bar", new String[]{"foo/", "**/", "*.bar"}}, + + // Collapse and reorder adjacent masks + new Object[]{"foo/*/bar", new String[]{"foo/", "*/", "bar"}}, + new Object[]{"foo/**/bar", new String[]{"foo/", "**/", "bar"}}, + new Object[]{"foo/*/*/bar", new String[]{"foo/", "*/", "*/", "bar"}}, + new Object[]{"foo/**/*/bar", new String[]{"foo/", "*/", "**/", "bar"}}, + new Object[]{"foo/*/**/bar", new String[]{"foo/", "*/", "**/", "bar"}}, + new Object[]{"foo/*/**.bar", new String[]{"foo/", "*/", "**/", "*.bar"}}, + new Object[]{"foo/**/**/bar", new String[]{"foo/", "**/", "bar"}}, + new Object[]{"foo/**/**.bar", new String[]{"foo/", "**/", "*.bar"}}, + new Object[]{"foo/**/*/**/*/bar", new String[]{"foo/", "*/", "*/", "**/", "bar"}}, + new Object[]{"foo/**/*/**/*/**.bar", new String[]{"foo/", "*/", "*/", "**/", "*.bar"}}, + + // Collapse trailing masks + new Object[]{"foo/**", new String[]{"foo/", "**/", "*"}}, + new Object[]{"foo/**/*", new String[]{"foo/", "**/", "*"}}, + new Object[]{"foo/**/*/*", new String[]{"foo/", "*/", "**/", "*"}}, + new Object[]{"foo/**/", new String[]{"foo/", "**/"}}, + new Object[]{"foo/**/*/", new String[]{"foo/", "*/", "**/"}}, + new Object[]{"foo/**/*/*/", new String[]{"foo/", "*/", "*/", "**/"}}, + }; + } + + @Test(dataProvider = "normalizePatternData") + public static void normalizePatternTest(@NotNull String pattern, @NotNull String[] expected) { + final List actual = WildcardHelper.normalizePattern(WildcardHelper.splitPattern(pattern)); + Assert.assertTrue(actual.size() > 0); + Assert.assertEquals(actual.remove(0), "/"); + Assert.assertEquals(actual.toArray(new String[actual.size()]), expected, pattern); + } + + @DataProvider + public static Object[][] pathMatcherData() { + return new Object[][]{ + // Simple pattern + new Object[]{"/", "foo/bar", true, null}, + new Object[]{"*", "foo/bar", true, true}, + new Object[]{"*/", "foo/bar", true, null}, + new Object[]{"/", "foo/bar/", true, null}, + new Object[]{"*", "foo/bar/", true, true}, + new Object[]{"*/", "foo/bar/", true, true}, + new Object[]{"**/", "foo/bar/", true, true}, + new Object[]{"foo/**/", "foo/bar/", true, true}, + new Object[]{"foo/**/", "foo/bar/xxx", true, null}, + new Object[]{"foo/**/", "foo/bar/xxx/", true, true}, + new Object[]{"f*o", "foo/bar", true, null}, + new Object[]{"/f*o", "foo/bar", true, null}, + new Object[]{"f*o/", "foo/bar", true, null}, + new Object[]{"foo/", "foo/bar", true, null}, + new Object[]{"/foo/", "foo/bar", true, null}, + new Object[]{"/foo", "foo/", true, true}, + new Object[]{"foo", "foo/", true, true}, + new Object[]{"foo/", "foo/", true, true}, + new Object[]{"foo/", "foo", null, null}, + new Object[]{"bar", "foo/bar", true, true}, + new Object[]{"b*r", "foo/bar", true, true}, + new Object[]{"/bar", "foo/bar", null, null}, + new Object[]{"bar/", "foo/bar", null, null}, + new Object[]{"b*r/", "foo/bar", null, null}, + new Object[]{"bar/", "foo/bar/", true, true}, + new Object[]{"b*r/", "foo/bar/", true, true}, + new Object[]{"b[a-z]r", "foo/bar", true, true}, + new Object[]{"b[a-z]r", "foo/b0r", null, null}, + new Object[]{"b[a-z]r", "foo/b0r/", false, false}, + new Object[]{"/t*e*t", "test", true, true}, + // More complex pattern + new Object[]{"foo/*/bar/", "foo/bar/", false, false}, + new Object[]{"foo/*/bar/", "bar/", null, null}, + new Object[]{"foo/*/bar/", "foo/a/bar/", true, true}, + new Object[]{"foo/*/bar/", "foo/a/b/bar/", null, null}, + new Object[]{"foo/*/*/bar/", "foo/a/b/bar/", true, true}, + + new Object[]{"foo/**/bar/a/", "foo/bar/b/bar/a/", true, true}, + new Object[]{"foo/**/bar/a/", "foo/bar/bar/bar/a/", true, true}, + new Object[]{"foo/**/bar/a/", "foo/bar/bar/b/a/", false, false}, + new Object[]{"foo/**/bar/", "foo/bar/", true, true}, + new Object[]{"foo/**/bar/", "bar/", null, null}, + new Object[]{"foo/**/bar/", "foo/a/bar/", true, true}, + new Object[]{"foo/**/bar/", "foo/a/b/bar/", true, true}, + new Object[]{"foo/*/**/*/bar/", "foo/a/bar/", false, false}, + new Object[]{"foo/*/**/*/bar/", "foo/a/b/bar/", true, true}, + new Object[]{"foo/*/**/*/bar/", "foo/a/b/c/bar/", true, true}, + new Object[]{"foo/**/xxx/**/bar/", "foo/xxx/bar/", true, true}, + new Object[]{"foo/**/xxx/**/bar/", "foo/xxx/b/c/bar/", true, true}, + new Object[]{"foo/**/xxx/**/bar/", "foo/a/xxx/c/bar/", true, true}, + new Object[]{"foo/**/xxx/**/bar/", "foo/a/c/xxx/bar/", true, true}, + new Object[]{"foo/**/xxx/**/bar/", "foo/bar/xxx/", false, false}, + new Object[]{"foo/**/xxx/**/bar/", "foo/bar/xxx/bar/", true, true}, + new Object[]{"foo/**/xxx/**/bar/", "foo/bar/xxx/xxx/bar/", true, true}, + }; + } + + @Test(dataProvider = "pathMatcherData") + public static void nativeMatcherExactTest(@NotNull String pattern, @NotNull String path, @Nullable Boolean ignored, @Nullable Boolean expectedMatch) throws InvalidPatternException, IOException, InterruptedException { + Path temp = Files.createTempDirectory("git-matcher"); + try { + if (new ProcessBuilder() + .directory(temp.toFile()) + .command("git", "init", ".") + .start() + .waitFor() != 0) { + throw new SkipException("Can't find git"); + } + Files.write(temp.resolve(".gitattributes"), (pattern + " test\n").getBytes(StandardCharsets.UTF_8)); + byte[] output = ByteStreams.toByteArray( + new ProcessBuilder() + .directory(temp.toFile()) + .command("git", "check-attr", "-a", "--", path) + .start() + .getInputStream() + ); + Assert.assertEquals(output.length > 0, expectedMatch == Boolean.TRUE); + } finally { + Files.walkFileTree(temp, new DeleteTreeVisitor()); + } + } + + @Test(dataProvider = "pathMatcherData") + public static void pathMatcherPrefixTest(@NotNull String pattern, @NotNull String path, @Nullable Boolean expectedMatch, @Nullable Boolean ignored) throws InvalidPatternException { + pathMatcherCheck(pattern, path, false, expectedMatch); + } + + @Test(dataProvider = "pathMatcherData") + public static void pathMatcherExactTest(@NotNull String pattern, @NotNull String path, @Nullable Boolean ignored, @Nullable Boolean expectedMatch) throws InvalidPatternException { + pathMatcherCheck(pattern, path, true, expectedMatch); + } + + private static void pathMatcherCheck(@NotNull String pattern, @NotNull String path, boolean exact, @Nullable Boolean expectedMatch) throws InvalidPatternException { + PathMatcher matcher = WildcardHelper.createMatcher(pattern, exact); + for (String name : WildcardHelper.splitPattern(path)) { + if (matcher == null) break; + boolean isDir = name.endsWith("/"); + matcher = matcher.createChild(isDir ? name.substring(0, name.length() - 1) : name, isDir); + } + if (expectedMatch == null) { + Assert.assertNull(matcher); + } else { + Assert.assertNotNull(matcher); + Assert.assertEquals(matcher.isMatch(), expectedMatch.booleanValue()); + } + } + + @DataProvider + public static Object[][] tryRemoveBackslashesData() { + return new Object[][]{ + new Object[]{"test", "test"}, + new Object[]{"test\\n", "test\\n"}, + new Object[]{"space\\ ", "space "}, + new Object[]{"foo\\!bar\\ ", "foo!bar "}, + new Object[]{"\\#some", "#some"}, + new Object[]{"foo\\[bar", "foo\\[bar"}, + }; + } + + @Test(dataProvider = "tryRemoveBackslashesData") + public static void tryRemoveBackslashesTest(@NotNull String pattern, @NotNull String expected) { + Assert.assertEquals(WildcardHelper.tryRemoveBackslashes(pattern), expected); + } +}