Skip to content

Commit

Permalink
Don't convert pointers for already uploaded LFS files
Browse files Browse the repository at this point in the history
  • Loading branch information
bozaro committed Aug 23, 2015
1 parent d2e19aa commit ba89922
Show file tree
Hide file tree
Showing 2 changed files with 144 additions and 2 deletions.
20 changes: 18 additions & 2 deletions src/main/java/git/lfs/migrate/GitConverter.java
Expand Up @@ -175,7 +175,7 @@ private List<GitTreeEntry> getEntries() throws IOException {
if (needAttributes && treeParser.getEntryPathString().equals(GIT_ATTRIBUTES)) {
blobTask = TaskType.Attribute;
needAttributes = false;
} else if ((fileMode.getObjectType() == Constants.OBJ_BLOB) && ((fileMode.getBits() & FileMode.TYPE_MASK) == FileMode.TYPE_FILE) && matchFilename(treeParser.getEntryPathString())) {
} else if (isFile(fileMode) && matchFilename(treeParser.getEntryPathString())) {
blobTask = TaskType.UploadLfs;
} else {
blobTask = TaskType.Simple;
Expand All @@ -189,6 +189,10 @@ private List<GitTreeEntry> getEntries() throws IOException {
return entries;
}

private boolean isFile(@NotNull FileMode fileMode) {
return (fileMode.getObjectType() == Constants.OBJ_BLOB) && ((fileMode.getBits() & FileMode.TYPE_MASK) == FileMode.TYPE_FILE);
}

@NotNull
@Override
public Iterable<TaskKey> depends() throws IOException {
Expand Down Expand Up @@ -236,6 +240,13 @@ public Iterable<TaskKey> depends() throws IOException {
@NotNull
@Override
public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolver resolver) throws IOException {
final ObjectLoader loader = reader.open(id, Constants.OBJ_BLOB);
// Is object already converted?
if (isLfsPointer(loader)) {
inserter.insert(loader.getType(), loader.getBytes());
return id;
}
// Prepare for hash calculation
final MessageDigest md;
try {
md = MessageDigest.getInstance("SHA-256");
Expand All @@ -244,7 +255,6 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv
}
// Create LFS stream.
final File tmpFile = new File(tempPath, id.getName());
final ObjectLoader loader = reader.open(id, Constants.OBJ_BLOB);
try (InputStream istream = loader.openStream();
OutputStream ostream = new FileOutputStream(tmpFile)) {
byte[] buffer = new byte[0x10000];
Expand Down Expand Up @@ -277,6 +287,12 @@ public ObjectId convert(@NotNull ObjectInserter inserter, @NotNull ConvertResolv
};
}

private boolean isLfsPointer(@NotNull ObjectLoader loader) {
if (loader.getSize() > LfsPointer.POINTER_MAX_SIZE) return false;
if (LfsPointer.parsePointer(loader.getBytes()) == null) return false;
return true;
}

private void checkAndUpload(@NotNull ObjectLoader loader, @NotNull String hash) throws IOException {
if (lfs == null) {
return;
Expand Down
126 changes: 126 additions & 0 deletions src/main/java/git/lfs/migrate/LfsPointer.java
@@ -0,0 +1,126 @@
package git.lfs.migrate;

import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.TreeMap;

/**
* Class for read/writer pointer blobs.
* https://github.com/github/git-lfs/blob/master/docs/spec.md
*
* @author Artem V. Navrotskiy <bozaro@users.noreply.github.com>
*/
public class LfsPointer {
public static final int POINTER_MAX_SIZE = 1024;
@NotNull
public static final String OID = "oid";
@NotNull
public static final String SIZE = "size";

@NotNull
private static final String[] REQUIRED = {
OID,
SIZE,
};
@NotNull
private static final byte[] PREFIX = "version ".getBytes(StandardCharsets.UTF_8);

/**
* Read pointer data.
*
* @param blob Blob data.
* @return Return pointer info or null if blob is not a pointer data.
*/
@Nullable
public static Map<String, String> parsePointer(@NotNull byte[] blob) {
return parsePointer(blob, 0, blob.length);
}

/**
* Read pointer data.
*
* @param blob Blob data.
* @return Return pointer info or null if blob is not a pointer data.
*/
@Nullable
public static Map<String, String> parsePointer(@NotNull byte[] blob, final int offset, final int length) {
// Check prefix
if (length < PREFIX.length) return null;
for (int i = 0; i < PREFIX.length; ++i) {
if (blob[i] != PREFIX[i]) return null;
}
// Reading key value pairs
final TreeMap<String, String> result = new TreeMap<>();
final CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder();
String lastKey = null;
int keyOffset = offset;
int required = 0;
while (true) {
if (keyOffset >= length) {
break;
}
int valueOffset = keyOffset;
// Key
while (true) {
valueOffset++;
if (valueOffset < length) {
final byte c = blob[valueOffset];
if (c == ' ') break;
// Keys MUST only use the characters [a-z] [0-9] . -.
if (c >= 'a' && c <= 'z') continue;
if (c >= '0' && c <= '9') continue;
if (c == '.' || c == '-') continue;
}
// Found invalid character.
return null;
}
int endOffset = valueOffset;
// Value
while (true) {
endOffset++;
if (endOffset >= length) return null;
// Values MUST NOT contain return or newline characters.
if (blob[endOffset] == '\n') break;
}
final String key = new String(blob, keyOffset, valueOffset - keyOffset, StandardCharsets.UTF_8);

final String value;
try {
value = decoder.decode(ByteBuffer.wrap(blob, valueOffset + 1, endOffset - valueOffset - 1)).toString();
} catch (CharacterCodingException e) {
return null;
}
if (required < REQUIRED.length && REQUIRED[required].equals(key)) {
required++;
}
if (keyOffset > offset) {
if (lastKey != null && key.compareTo(lastKey) <= 0) {
return null;
}
lastKey = key;
}
if (result.put(key, value) != null) {
return null;
}
keyOffset = endOffset + 1;
}
// Not found all required fields.
if (required < REQUIRED.length) {
return null;
}
try {
if (Long.parseLong(result.get(SIZE)) < 0) {
return null;
}
} catch (NumberFormatException ignored) {
return null;
}
return result;
}
}

0 comments on commit ba89922

Please sign in to comment.