Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

introduce chunked backups #11900

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

introduce chunked backups #11900

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
14 changes: 10 additions & 4 deletions app/src/main/java/org/thoughtcrime/securesms/backup/FullBackupBase.java
View file
Open in desktop
Expand Up @@ -15,9 +15,9 @@ public abstract class FullBackupBase {
private static final int DIGEST_ROUNDS = 250_000;

static class BackupStream {
static @NonNull byte[] getBackupKey(@NonNull String passphrase, @Nullable byte[] salt) {
static @NonNull byte[] getBackupKey(@NonNull String passphrase, @Nullable byte[] salt, boolean isChunked) {
try {
EventBus.getDefault().post(new BackupEvent(BackupEvent.Type.PROGRESS, 0, 0));
EventBus.getDefault().post(new BackupEvent(BackupEvent.Type.PROGRESS, isChunked, 0, 0));

MessageDigest digest = MessageDigest.getInstance("SHA-512");
byte[] input = passphrase.replace(" ", "").getBytes();
Expand All @@ -26,7 +26,7 @@ static class BackupStream {
if (salt != null) digest.update(salt);

for (int i = 0; i < DIGEST_ROUNDS; i++) {
if (i % 1000 == 0) EventBus.getDefault().post(new BackupEvent(BackupEvent.Type.PROGRESS, 0, 0));
if (i % 1000 == 0) EventBus.getDefault().post(new BackupEvent(BackupEvent.Type.PROGRESS, isChunked, 0, 0));
digest.update(hash);
hash = digest.digest(input);
}
Expand All @@ -45,11 +45,13 @@ public enum Type {
}

private final Type type;
private final boolean isChunked;
private final long count;
private final long estimatedTotalCount;

BackupEvent(Type type, long count, long estimatedTotalCount) {
BackupEvent(Type type, boolean isChunked, long count, long estimatedTotalCount) {
this.type = type;
this.isChunked = isChunked;
this.count = count;
this.estimatedTotalCount = estimatedTotalCount;
}
Expand All @@ -58,6 +60,10 @@ public Type getType() {
return type;
}

public boolean isChunked() {
return isChunked;
}

public long getCount() {
return count;
}
Expand Down
92 changes: 64 additions & 28 deletions app/src/main/java/org/thoughtcrime/securesms/backup/FullBackupExporter.java
View file
Open in desktop

Large diffs are not rendered by default.

31 changes: 23 additions & 8 deletions app/src/main/java/org/thoughtcrime/securesms/backup/FullBackupImporter.java
View file
Open in desktop
Expand Up @@ -50,6 +50,7 @@
import java.security.InvalidKeyException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
Expand Down Expand Up @@ -86,12 +87,21 @@ public static void importFile(@NonNull Context context, @NonNull AttachmentSecre
throws IOException
{
try (InputStream is = getInputStream(context, uri)) {
importFile(context, attachmentSecret, db, is, passphrase);
importFile(context, attachmentSecret, db, is, false, passphrase);
}
}

public static void importFile(@NonNull Context context, @NonNull AttachmentSecret attachmentSecret,
@NonNull SQLiteDatabase db, @NonNull InputStream is, @NonNull String passphrase)
@NonNull SQLiteDatabase db, @NonNull Uri[] uris, @NonNull String passphrase)
throws IOException
{
try (InputStream is = getInputStream(context, uris)) {
importFile(context, attachmentSecret, db, is, true, passphrase);
}
}

public static void importFile(@NonNull Context context, @NonNull AttachmentSecret attachmentSecret,
@NonNull SQLiteDatabase db, @NonNull InputStream is, boolean isChunked, @NonNull String passphrase)
throws IOException
{
int count = 0;
Expand All @@ -101,14 +111,14 @@ public static void importFile(@NonNull Context context, @NonNull AttachmentSecre
db.beginTransaction();
keyValueDatabase.beginTransaction();
try {
BackupRecordInputStream inputStream = new BackupRecordInputStream(is, passphrase);
BackupRecordInputStream inputStream = new BackupRecordInputStream(is, passphrase, isChunked);

dropAllTables(db);

BackupFrame frame;

while (!(frame = inputStream.readFrame()).getEnd()) {
if (count % 100 == 0) EventBus.getDefault().post(new BackupEvent(BackupEvent.Type.PROGRESS, count, 0));
if (count % 100 == 0) EventBus.getDefault().post(new BackupEvent(BackupEvent.Type.PROGRESS, isChunked, count, 0));
count++;

if (frame.hasVersion()) processVersion(db, frame.getVersion());
Expand All @@ -128,17 +138,22 @@ public static void importFile(@NonNull Context context, @NonNull AttachmentSecre
keyValueDatabase.endTransaction();
}

EventBus.getDefault().post(new BackupEvent(BackupEvent.Type.FINISHED, count, 0));
EventBus.getDefault().post(new BackupEvent(BackupEvent.Type.FINISHED, isChunked, count, 0));
}

private static @NonNull InputStream getInputStream(@NonNull Context context, @NonNull Uri uri) throws IOException{
if (BackupUtil.isUserSelectionRequired(context) || uri.getScheme().equals("content")) {
return Objects.requireNonNull(context.getContentResolver().openInputStream(uri));
} else {
return new FileInputStream(new File(Objects.requireNonNull(uri.getPath())));
return new FileInputStream(Objects.requireNonNull(uri.getPath()));
}
}

private static @NonNull InputStream getInputStream(@NonNull Context context, @NonNull Uri[] uris) throws IOException{
Arrays.sort(uris);
return new MultiFileInputStream(context.getContentResolver(), uris);
}

private static void processVersion(@NonNull SQLiteDatabase db, DatabaseVersion version) throws IOException {
if (version.getVersion() > db.getVersion()) {
throw new DatabaseDowngradeException(db.getVersion(), version.getVersion());
Expand Down Expand Up @@ -313,7 +328,7 @@ private static class BackupRecordInputStream extends BackupStream {
private byte[] iv;
private int counter;

private BackupRecordInputStream(@NonNull InputStream in, @NonNull String passphrase) throws IOException {
private BackupRecordInputStream(@NonNull InputStream in, @NonNull String passphrase, boolean isChunked) throws IOException {
try {
this.in = in;

Expand All @@ -338,7 +353,7 @@ private BackupRecordInputStream(@NonNull InputStream in, @NonNull String passphr
throw new IOException("Invalid IV length!");
}

byte[] key = getBackupKey(passphrase, header.hasSalt() ? header.getSalt().toByteArray() : null);
byte[] key = getBackupKey(passphrase, header.hasSalt() ? header.getSalt().toByteArray() : null, isChunked);
byte[] derived = new HKDFv3().deriveSecrets(key, "Backup Export".getBytes(), 64);
byte[][] split = ByteUtil.split(derived, 32, 32);

Expand Down
71 changes: 71 additions & 0 deletions app/src/main/java/org/thoughtcrime/securesms/backup/MultiFileInputStream.java
View file
Open in desktop
@@ -0,0 +1,71 @@
package org.thoughtcrime.securesms.backup;

import android.content.ContentResolver;
import android.net.Uri;

import java.io.InputStream;
import java.io.IOException;

public class MultiFileInputStream extends InputStream {
private int currentChunk = -1;
private InputStream currentInputStream = null;
ContentResolver contentResolver;
Uri[] uris;
boolean noFilesLeft;

public MultiFileInputStream(ContentResolver contentResolver,
Uri[] uris) {
this.contentResolver = contentResolver;
this.uris = uris;
this.noFilesLeft = (uris.length == 0);
}

public void close() throws IOException {
if (currentInputStream != null) {
currentInputStream.close();
}
}

public void swapFiles() throws IOException, SecurityException {
if (currentInputStream != null) {
currentInputStream.close();
}
currentChunk++;
Uri uri = uris[currentChunk];
currentInputStream = contentResolver.openInputStream(uri);
}

public int read() throws IOException {
byte[] b = new byte[1];
if (read(b) == -1) {
return -1;
}
return b[0];
}

public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}

public int read(byte[] b, int off, int len) throws IOException {
if (currentInputStream == null) {
if (noFilesLeft) {
return -1;
}
swapFiles();
}
while (true) {
int bytesRead = currentInputStream.read(b, off, len);
boolean eofRead = (bytesRead == -1);
if (eofRead) {
if (noFilesLeft) {
return -1;
} else {
swapFiles();
}
} else {
return bytesRead;
}
}
}
}
83 changes: 83 additions & 0 deletions app/src/main/java/org/thoughtcrime/securesms/backup/MultiFileOutputStream.java
View file
Open in desktop
@@ -0,0 +1,83 @@
package org.thoughtcrime.securesms.backup;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;

public class MultiFileOutputStream extends OutputStream {
private int currentChunk = -1;
private int bytesRemaining = 0;
private OutputStream currentOutputStream = null;
private final File dir;
private final String prefix;
private final String suffix;
private final List<File> files;

public MultiFileOutputStream(File dir,
String prefix,
String suffix) {
this.dir = dir;
this.prefix = prefix;
this.suffix = suffix;
this.files = new ArrayList<>();
}

public void close() throws IOException {
if (currentOutputStream != null) {
currentOutputStream.close();
}
}

public void flush() throws IOException {
if (currentOutputStream != null) {
currentOutputStream.flush();
}
}

public void write(int b) throws IOException {
byte[] c = new byte[1];
c[0] = (byte) b;
write(c);
}

public void write(byte[] b) throws IOException {
write(b, 0, b.length);
}

private void swapFiles() throws IOException {
if (currentOutputStream != null) {
currentOutputStream.close();
}
currentChunk++;
String filename = String.format(Locale.ENGLISH, "%s.%03d%s", prefix, currentChunk, suffix);
File f = new File(dir, filename);
files.add(f);
currentOutputStream = new FileOutputStream(f);
bytesRemaining = Util.MAX_BYTES_PER_FILE;
}

public void write(byte[] b, int offset, int len) throws IOException {
if (bytesRemaining == 0) {
swapFiles();
}
int lLen = len;
int lOffset = offset;
while (bytesRemaining < lLen) {
int bytesToWrite = bytesRemaining;
currentOutputStream.write(b, lOffset, bytesToWrite);
swapFiles();
lOffset += bytesToWrite;
lLen -= bytesToWrite;
}
currentOutputStream.write(b, lOffset, lLen);
bytesRemaining -= lLen;
}

public List<File> getFiles() {
return files;
}
}
89 changes: 89 additions & 0 deletions app/src/main/java/org/thoughtcrime/securesms/backup/MultiFileOutputStream29.java
View file
Open in desktop
@@ -0,0 +1,89 @@
package org.thoughtcrime.securesms.backup;

import android.content.Context;

import androidx.documentfile.provider.DocumentFile;

import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Objects;

public class MultiFileOutputStream29 extends OutputStream {
private int currentChunk = -1;
private int bytesRemaining = 0;
private OutputStream currentOutputStream = null;
private final DocumentFile dir;
private final String prefix;
private final String suffix;
private final List<DocumentFile> files;
private final Context context;

public MultiFileOutputStream29(Context context,
DocumentFile dir,
String prefix,
String suffix) {
this.context = context;
this.dir = dir;
this.prefix = prefix;
this.files = new ArrayList<>();
this.suffix = suffix;
}

public void close() throws IOException {
if (currentOutputStream != null) {
currentOutputStream.close();
}
}

public void flush() throws IOException {
if (currentOutputStream != null) {
currentOutputStream.flush();
}
}

public void write(int b) throws IOException {
byte[] c = new byte[1];
c[0] = (byte) b;
write(c);
}

public void write(byte[] b) throws IOException {
write(b, 0, b.length);
}

private void swapFiles() throws IOException {
if (currentOutputStream != null) {
currentOutputStream.close();
}
currentChunk++;
String filename = String.format(Locale.ENGLISH, "%s.%03d%s", prefix, currentChunk, suffix);
DocumentFile f = dir.createFile("application/octet-stream", filename);
files.add(f);
currentOutputStream = Objects.requireNonNull(context.getContentResolver().openOutputStream(f.getUri()));
bytesRemaining = Util.MAX_BYTES_PER_FILE;
}

public void write(byte[] b, int offset, int len) throws IOException {
if (bytesRemaining == 0) {
swapFiles();
}
int lLen = len;
int lOffset = offset;
while (bytesRemaining < lLen) {
int bytesToWrite = bytesRemaining;
currentOutputStream.write(b, lOffset, bytesToWrite);
swapFiles();
lOffset += bytesToWrite;
lLen -= bytesToWrite;
}
currentOutputStream.write(b, lOffset, lLen);
bytesRemaining -= lLen;
}

public List<DocumentFile> getFiles() {
return files;
}
}