diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 2453e2c6d..da962147e 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -16,7 +16,6 @@ */ package com.netflix.priam; -import com.netflix.priam.backup.BackupService; import com.netflix.priam.backupv2.BackupV2Service; import com.netflix.priam.cluster.management.ClusterManagementService; import com.netflix.priam.config.IConfiguration; @@ -46,7 +45,6 @@ public class PriamServer implements IService { private final ICassandraProcess cassProcess; private final RestoreContext restoreContext; private final IService backupV2Service; - private final IService backupService; private final IService cassandraTunerService; private final IService clusterManagementService; private static final int CASSANDRA_MONITORING_INITIAL_DELAY = 10; @@ -60,7 +58,6 @@ public PriamServer( Sleeper sleeper, ICassandraProcess cassProcess, RestoreContext restoreContext, - BackupService backupService, BackupV2Service backupV2Service, CassandraTunerService cassandraTunerService, ClusterManagementService clusterManagementService) { @@ -70,7 +67,6 @@ public PriamServer( this.sleeper = sleeper; this.cassProcess = cassProcess; this.restoreContext = restoreContext; - this.backupService = backupService; this.backupV2Service = backupV2Service; this.cassandraTunerService = cassandraTunerService; this.clusterManagementService = clusterManagementService; @@ -130,9 +126,6 @@ public void scheduleService() throws Exception { PriamConfigurationPersister.class, PriamConfigurationPersister.getTimer(config)); - // Set up V1 Snapshot Service - backupService.scheduleService(); - // Set up V2 Snapshot Service backupV2Service.scheduleService(); } diff --git a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java index 2eb1fd09f..ef8a6234c 100644 --- a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java @@ -19,19 +19,15 @@ import com.google.api.client.util.Lists; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.compress.CompressionType; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.utils.DateUtil; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; -import java.util.Arrays; import java.util.Date; import java.util.List; -import java.util.Optional; import javax.inject.Inject; /** @@ -40,18 +36,13 @@ * this instance. */ public class RemoteBackupPath extends AbstractBackupPath { - private static final ImmutableSet V2_ONLY_FILE_TYPES = - ImmutableSet.of( - BackupFileType.META_V2, - BackupFileType.SST_V2, - BackupFileType.SECONDARY_INDEX_V2); @Inject public RemoteBackupPath(IConfiguration config, InstanceIdentity factory) { super(config, factory); } - private ImmutableList.Builder getV2Prefix() { + private ImmutableList.Builder getPrefix() { ImmutableList.Builder prefix = ImmutableList.builder(); prefix.add(baseDir, prependHash(clusterName), token); return prefix; @@ -82,8 +73,8 @@ private String removeHash(String appNameWithHash) { * Another major difference w.r.t. V1 is having no distinction between SNAP and SST files as we upload SSTables only * once to remote file system. */ - private String getV2Location() { - ImmutableList.Builder parts = getV2Prefix(); + private String getLocation() { + ImmutableList.Builder parts = getPrefix(); // JDK-8177809 truncate to seconds to ensure consistent behavior with our old method of // getting lastModified time (File::lastModified) in Java 8. long lastModified = getLastModified().toEpochMilli() / 1_000L * 1_000L; @@ -98,7 +89,25 @@ private String getV2Location() { return toPath(parts.build()).toString(); } - private void parseV2Location(Path remotePath) { + private Path toPath(ImmutableList parts) { + return Paths.get(parts.get(0), parts.subList(1, parts.size()).toArray(new String[0])); + } + + /** + * Format of backup path: 1. For old style backups: + * BASE/REGION/CLUSTER/TOKEN/[SNAPSHOTTIME]/[SST|SNAP|META]/KEYSPACE/COLUMNFAMILY/FILE + * + *

2. For new style backups (SnapshotMetaService) + * BASE/[cluster_name_hash]_cluster/TOKEN//[META_V2|SST_V2]/KEYSPACE/COLUMNFAMILY/[last_modified_time_ms]/FILE.compression + */ + @Override + public String getRemotePath() { + return getLocation(); + } + + @Override + public void parseRemote(String remoteFilepath) { + Path remotePath = Paths.get(remoteFilepath); Preconditions.checkArgument( remotePath.getNameCount() >= 8, String.format("%s has fewer than %d parts", remotePath, 8)); @@ -128,36 +137,9 @@ private void parseV2Location(Path remotePath) { Paths.get(config.getDataFileLocation(), parts.toArray(new String[] {})).toFile(); } - private String getV1Location() { - ImmutableList.Builder parts = ImmutableList.builder(); - String timeString = DateUtil.formatyyyyMMddHHmm(time); - parts.add(baseDir, region, clusterName, token, timeString, type.toString()); - if (BackupFileType.isDataFile(type)) { - parts.add(keyspace, columnFamily); - } - parts.add(fileName); - return toPath(parts.build()).toString(); - } - - private Path toPath(ImmutableList parts) { - return Paths.get(parts.get(0), parts.subList(1, parts.size()).toArray(new String[0])); - } - - private void parseV1Location(Path remotePath) { - Preconditions.checkArgument( - remotePath.getNameCount() >= 7, - String.format("%s has fewer than %d parts", remotePath, 7)); - parseV1Prefix(remotePath); - time = DateUtil.getDate(remotePath.getName(4).toString()); - type = BackupFileType.valueOf(remotePath.getName(5).toString()); - if (BackupFileType.isDataFile(type)) { - keyspace = remotePath.getName(6).toString(); - columnFamily = remotePath.getName(7).toString(); - } - fileName = remotePath.getName(remotePath.getNameCount() - 1).toString(); - } - - private void parseV1Prefix(Path remotePath) { + @Override + public void parsePartialPrefix(String remoteFilePath) { + Path remotePath = Paths.get(remoteFilePath); Preconditions.checkArgument( remotePath.getNameCount() >= 4, String.format("%s needs %d parts to parse prefix", remotePath, 4)); @@ -167,38 +149,6 @@ private void parseV1Prefix(Path remotePath) { token = remotePath.getName(3).toString(); } - /** - * Format of backup path: 1. For old style backups: - * BASE/REGION/CLUSTER/TOKEN/[SNAPSHOTTIME]/[SST|SNAP|META]/KEYSPACE/COLUMNFAMILY/FILE - * - *

2. For new style backups (SnapshotMetaService) - * BASE/[cluster_name_hash]_cluster/TOKEN//[META_V2|SST_V2]/KEYSPACE/COLUMNFAMILY/[last_modified_time_ms]/FILE.compression - */ - @Override - public String getRemotePath() { - return V2_ONLY_FILE_TYPES.contains(type) ? getV2Location() : getV1Location(); - } - - @Override - public void parseRemote(String remotePath) { - // Hack to determine type in advance of parsing. Will disappear once v1 is retired - Optional inferredType = - Arrays.stream(BackupFileType.values()) - .filter(bft -> remotePath.contains(PATH_SEP + bft.toString() + PATH_SEP)) - .findAny() - .filter(V2_ONLY_FILE_TYPES::contains); - if (inferredType.isPresent()) { - parseV2Location(Paths.get(remotePath)); - } else { - parseV1Location(Paths.get(remotePath)); - } - } - - @Override - public void parsePartialPrefix(String remoteFilePath) { - parseV1Prefix(Paths.get(remoteFilePath)); - } - @Override public String remotePrefix(Date start, Date end, String location) { return PATH_JOINER.join( @@ -217,7 +167,7 @@ public Path remoteV2Prefix(Path location, BackupFileType fileType) { clusterName = removeHash(location.getName(2).toString()); } token = instanceIdentity.getInstance().getToken(); - ImmutableList.Builder parts = getV2Prefix(); + ImmutableList.Builder parts = getPrefix(); parts.add(fileType.toString()); return toPath(parts.build()); } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java deleted file mode 100755 index 5651ebb45..000000000 --- a/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.aws; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; -import com.netflix.priam.aws.auth.IS3Credential; -import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.identity.config.InstanceInfo; -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Singleton; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/* - * A version of S3FileSystem which allows it api access across different AWS accounts. - * - * *Note: ideally, this object should extend S3FileSystem but could not be done because: - * - S3FileSystem is a singleton and it uses DI. To follow the DI pattern, the best way to get this singleton is via injection. - * - S3FileSystem registers a MBean to JMX which must be only once per JVM. If not, you get - * java.lang.RuntimeException: javax.management.InstanceAlreadyExistsException: com.priam.aws.S3FileSystemMBean:name=S3FileSystemMBean - * - - */ -@Singleton -public class S3CrossAccountFileSystem { - private static final Logger logger = LoggerFactory.getLogger(S3CrossAccountFileSystem.class); - - private AmazonS3 s3Client; - private final S3FileSystem s3fs; - private final IConfiguration config; - private final IS3Credential s3Credential; - private final InstanceInfo instanceInfo; - - @Inject - public S3CrossAccountFileSystem( - @Named("backup") IBackupFileSystem fs, - @Named("awss3roleassumption") IS3Credential s3Credential, - IConfiguration config, - InstanceInfo instanceInfo) { - - this.s3fs = (S3FileSystem) fs; - this.config = config; - this.s3Credential = s3Credential; - this.instanceInfo = instanceInfo; - } - - public IBackupFileSystem getBackupFileSystem() { - return this.s3fs; - } - - public AmazonS3 getCrossAcctS3Client() { - if (this.s3Client == null) { - - synchronized (this) { - if (this.s3Client == null) { - - try { - - this.s3Client = - AmazonS3Client.builder() - .withCredentials(s3Credential.getAwsCredentialProvider()) - .withRegion(instanceInfo.getRegion()) - .build(); - - } catch (Exception e) { - throw new IllegalStateException( - "Exception in getting handle to s3 client. Msg: " - + e.getLocalizedMessage(), - e); - } - - // Lets leverage the IBackupFileSystem behaviors except we want it to use our - // amazon S3 client which has cross AWS account api capability. - this.s3fs.setS3Client(s3Client); - } - } - } - - return this.s3Client; - } -} diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java deleted file mode 100755 index 098dcb62a..000000000 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java +++ /dev/null @@ -1,197 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.aws; - -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.PartETag; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.BackupRestoreException; -import com.netflix.priam.backup.DynamicRateLimiter; -import com.netflix.priam.backup.RangeReadInputStream; -import com.netflix.priam.compress.ChunkedStream; -import com.netflix.priam.compress.ICompression; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredential; -import com.netflix.priam.cryptography.IFileCryptography; -import com.netflix.priam.identity.config.InstanceInfo; -import com.netflix.priam.merics.BackupMetrics; -import com.netflix.priam.notification.BackupNotificationMgr; -import java.io.*; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Provider; -import javax.inject.Singleton; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Implementation of IBackupFileSystem for S3. The upload/download will work with ciphertext. */ -@Singleton -public class S3EncryptedFileSystem extends S3FileSystemBase { - - private static final Logger logger = LoggerFactory.getLogger(S3EncryptedFileSystem.class); - private final IFileCryptography encryptor; - private final DynamicRateLimiter dynamicRateLimiter; - - @Inject - public S3EncryptedFileSystem( - Provider pathProvider, - ICompression compress, - final IConfiguration config, - ICredential cred, - @Named("filecryptoalgorithm") IFileCryptography fileCryptography, - BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr, - InstanceInfo instanceInfo, - DynamicRateLimiter dynamicRateLimiter) { - - super(pathProvider, compress, config, backupMetrics, backupNotificationMgr); - this.encryptor = fileCryptography; - this.dynamicRateLimiter = dynamicRateLimiter; - super.s3Client = - AmazonS3Client.builder() - .withCredentials(cred.getAwsCredentialProvider()) - .withRegion(instanceInfo.getRegion()) - .build(); - } - - @Override - protected void downloadFileImpl(AbstractBackupPath path, String suffix) - throws BackupRestoreException { - String remotePath = path.getRemotePath(); - Path localPath = Paths.get(path.newRestoreFile().getAbsolutePath() + suffix); - try (OutputStream os = new FileOutputStream(localPath.toFile()); - RangeReadInputStream rris = - new RangeReadInputStream( - s3Client, getShard(), super.getFileSize(remotePath), remotePath)) { - /* - * To handle use cases where decompression should be done outside of the download. For example, the file have been compressed and then encrypted. - * Hence, decompressing it here would compromise the decryption. - */ - IOUtils.copyLarge(rris, os); - } catch (Exception e) { - throw new BackupRestoreException( - "Exception encountered downloading " - + remotePath - + " from S3 bucket " - + getShard() - + ", Msg: " - + e.getMessage(), - e); - } - } - - @Override - protected long uploadFileImpl(AbstractBackupPath path, Instant target) - throws BackupRestoreException { - Path localPath = Paths.get(path.getBackupFile().getAbsolutePath()); - String remotePath = path.getRemotePath(); - - long chunkSize = getChunkSize(localPath); - // initialize chunking request to aws - InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(config.getBackupPrefix(), remotePath); - // Fetch the aws generated upload id for this chunking request - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - DataPart part = - new DataPart(config.getBackupPrefix(), remotePath, initResponse.getUploadId()); - // Metadata on number of parts to be uploaded - List partETags = Collections.synchronizedList(new ArrayList<>()); - - // Read chunks from src, compress it, and write to temp file - File compressedDstFile = new File(localPath.toString() + ".compressed"); - if (logger.isDebugEnabled()) - logger.debug( - "Compressing {} with chunk size {}", - compressedDstFile.getAbsolutePath(), - chunkSize); - - try (InputStream in = new FileInputStream(localPath.toFile()); - BufferedOutputStream compressedBos = - new BufferedOutputStream(new FileOutputStream(compressedDstFile))) { - Iterator compressedChunks = - new ChunkedStream(in, chunkSize, path.getCompression()); - while (compressedChunks.hasNext()) { - byte[] compressedChunk = compressedChunks.next(); - compressedBos.write(compressedChunk); - } - } catch (Exception e) { - String message = - "Exception in compressing the input data during upload to EncryptedStore Msg: " - + e.getMessage(); - logger.error(message, e); - throw new BackupRestoreException(message); - } - - // == Read compressed data, encrypt each chunk, upload it to aws - try (BufferedInputStream compressedBis = - new BufferedInputStream(new FileInputStream(compressedDstFile))) { - Iterator chunks = this.encryptor.encryptStream(compressedBis, remotePath); - - // identifies this part position in the object we are uploading - int partNum = 0; - long encryptedFileSize = 0; - - while (chunks.hasNext()) { - byte[] chunk = chunks.next(); - // throttle upload to endpoint - rateLimiter.acquire(chunk.length); - dynamicRateLimiter.acquire(path, target, chunk.length); - - DataPart dp = - new DataPart( - ++partNum, - chunk, - config.getBackupPrefix(), - remotePath, - initResponse.getUploadId()); - S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags); - encryptedFileSize += chunk.length; - executor.submit(partUploader); - } - - executor.sleepTillEmpty(); - if (partNum != partETags.size()) { - throw new BackupRestoreException( - "Number of parts(" - + partNum - + ") does not match the expected number of uploaded parts(" - + partETags.size() - + ")"); - } - - // complete the aws chunking upload by providing to aws the ETag that uniquely - // identifies the combined object datav - CompleteMultipartUploadResult resultS3MultiPartUploadComplete = - new S3PartUploader(s3Client, part, partETags).completeUpload(); - checkSuccessfulUpload(resultS3MultiPartUploadComplete, localPath); - return encryptedFileSize; - } catch (Exception e) { - new S3PartUploader(s3Client, part, partETags).abortUpload(); - throw new BackupRestoreException("Error uploading file: " + localPath, e); - } finally { - if (compressedDstFile.exists()) compressedDstFile.delete(); - } - } -} diff --git a/priam/src/main/java/com/netflix/priam/aws/UpdateCleanupPolicy.java b/priam/src/main/java/com/netflix/priam/aws/UpdateCleanupPolicy.java deleted file mode 100644 index fc0e32438..000000000 --- a/priam/src/main/java/com/netflix/priam/aws/UpdateCleanupPolicy.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2013 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.aws; - -import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.scheduler.SimpleTimer; -import com.netflix.priam.scheduler.Task; -import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.utils.RetryableCallable; -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Singleton; - -/** Updates the cleanup policy for the bucket */ -@Singleton -public class UpdateCleanupPolicy extends Task { - public static final String JOBNAME = "UpdateCleanupPolicy"; - private final IBackupFileSystem fs; - - @Inject - public UpdateCleanupPolicy(IConfiguration config, @Named("backup") IBackupFileSystem fs) { - super(config); - this.fs = fs; - } - - @Override - public void execute() throws Exception { - // Set cleanup policy of retention is specified - new RetryableCallable() { - @Override - public Void retriableCall() throws Exception { - fs.cleanup(); - return null; - } - }.call(); - } - - @Override - public String getName() { - return JOBNAME; - } - - public static TaskTimer getTimer() { - return new SimpleTimer(JOBNAME); - } -} diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index e5c9a69fa..7e61ccada 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -48,28 +48,18 @@ public abstract class AbstractBackupPath implements Comparable DATA_FILE_TYPES = - ImmutableSet.of(SECONDARY_INDEX_V2, SNAP, SST, SST_V2); - - private static ImmutableSet V2_FILE_TYPES = - ImmutableSet.of(SECONDARY_INDEX_V2, SST_V2, META_V2); + ImmutableSet.of(SECONDARY_INDEX_V2, SST_V2); public static boolean isDataFile(BackupFileType type) { return DATA_FILE_TYPES.contains(type); } - public static boolean isV2(BackupFileType type) { - return V2_FILE_TYPES.contains(type); - } - public static BackupFileType fromString(String s) throws BackupRestoreException { try { return BackupFileType.valueOf(s); @@ -88,7 +78,6 @@ public static BackupFileType fromString(String s) throws BackupRestoreException protected String token; protected String region; protected String indexDir; - protected Date time; private long size; // uncompressed file size private long compressedFileSize = 0; protected final InstanceIdentity instanceIdentity; @@ -136,8 +125,6 @@ public void parseLocal(File file, BackupFileType type) { if (BackupFileType.isDataFile(type)) { this.keyspace = parts[0]; this.columnFamily = parts[1]; - } - if (BackupFileType.isDataFile(type)) { Optional folder = BackupFolder.fromName(parts[2]); this.isIncremental = folder.filter(BackupFolder.BACKUPS::equals).isPresent(); if (type == BackupFileType.SECONDARY_INDEX_V2) { @@ -146,16 +133,6 @@ public void parseLocal(File file, BackupFileType type) { this.indexDir = parts[index]; } } - - /* - 1. For old style snapshots, make this value to time at which backup was executed. - 2. This is to ensure that all the files from the snapshot are uploaded under single directory in remote file system. - 3. For META files we always override the time field via @link{Metadata#decorateMetaJson} - */ - this.time = - type == BackupFileType.SNAP - ? DateUtil.getDate(parts[3]) - : new Date(lastModified.toEpochMilli()); } /** Given a date range, find a common string prefix Eg: 20120212, 20120213 = 2012021 */ @@ -180,7 +157,6 @@ public File newRestoreFile() { PATH_JOINER.join(dataDir, keyspace, columnFamily, indexDir, fileName); return_ = new File(restoreFileName); break; - case META: case META_V2: return_ = new File(PATH_JOINER.join(config.getDataFileLocation(), fileName)); break; @@ -254,14 +230,6 @@ public String getRegion() { return region; } - public Date getTime() { - return time; - } - - public void setTime(Date time) { - this.time = time; - } - /* @return original, uncompressed file size */ diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index ba7f94865..a0a3eccb6 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -44,7 +44,6 @@ import java.util.concurrent.*; import javax.inject.Inject; import javax.inject.Provider; -import org.apache.commons.collections4.iterators.FilterIterator; import org.apache.commons.collections4.iterators.TransformIterator; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -308,29 +307,6 @@ public Iterator listPrefixes(Date date) { }); } - @Override - public Iterator list(String path, Date start, Date till) { - String prefix = pathProvider.get().remotePrefix(start, till, path); - Iterator fileIterator = listFileSystem(prefix, null, null); - - @SuppressWarnings("unchecked") - TransformIterator transformIterator = - new TransformIterator( - fileIterator, - remotePath -> { - AbstractBackupPath abstractBackupPath = pathProvider.get(); - abstractBackupPath.parseRemote(remotePath.toString()); - return abstractBackupPath; - }); - - return new FilterIterator<>( - transformIterator, - abstractBackupPath -> - (abstractBackupPath.getTime().after(start) - && abstractBackupPath.getTime().before(till)) - || abstractBackupPath.getTime().equals(start)); - } - @Override public int getUploadTasksQueued() { return tasksQueued.size(); diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupFileSystemContext.java b/priam/src/main/java/com/netflix/priam/backup/BackupFileSystemContext.java deleted file mode 100755 index 22c5b381b..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/BackupFileSystemContext.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup; - -import com.netflix.priam.config.IConfiguration; -import javax.inject.Inject; -import javax.inject.Named; - -public class BackupFileSystemContext implements IFileSystemContext { - - private IBackupFileSystem fs = null, encryptedFs = null; - - @Inject - public BackupFileSystemContext( - @Named("backup") IBackupFileSystem fs, - @Named("encryptedbackup") IBackupFileSystem encryptedFs) { - - this.fs = fs; - this.encryptedFs = encryptedFs; - } - - public BackupFileSystemContext(@Named("backup") IBackupFileSystem fs) { - this.fs = fs; - } - - public IBackupFileSystem getFileStrategy(IConfiguration config) { - - if (!config.isEncryptBackupEnabled()) { - - return this.fs; - - } else { - - return this.encryptedFs; - } - } -} diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupHelperImpl.java b/priam/src/main/java/com/netflix/priam/backup/BackupHelperImpl.java index 9f13891f7..8b469f296 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupHelperImpl.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupHelperImpl.java @@ -28,12 +28,10 @@ public class BackupHelperImpl implements BackupHelper { @Inject public BackupHelperImpl( - IConfiguration config, - IFileSystemContext backupFileSystemCtx, - Provider pathFactory) { + IConfiguration config, IBackupFileSystem fs, Provider pathFactory) { this.config = config; this.pathFactory = pathFactory; - this.fs = backupFileSystemCtx.getFileStrategy(config); + this.fs = fs; } /** @@ -88,9 +86,7 @@ public ImmutableSet getBackupPaths( private CompressionType getCorrectCompressionAlgorithm( AbstractBackupPath path, Set compressedFiles) { - if (!AbstractBackupPath.BackupFileType.isV2(path.getType()) - || path.getLastModified().toEpochMilli() - < config.getCompressionTransitionEpochMillis()) { + if (path.getLastModified().toEpochMilli() < config.getCompressionTransitionEpochMillis()) { return CompressionType.SNAPPY; } String file = path.getFileName(); diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java b/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java index b34220dca..fadc433be 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java @@ -31,16 +31,14 @@ public final class BackupMetadata implements Serializable { private Status status; private boolean cassandraSnapshotSuccess; private Date lastValidated; - private BackupVersion backupVersion; private String snapshotLocation; - public BackupMetadata(BackupVersion backupVersion, String token, Date start) { + public BackupMetadata(String token, Date start) { if (start == null || token == null || StringUtils.isEmpty(token)) throw new IllegalArgumentException( String.format( "Invalid Input: Token: %s or start date: %s is null or empty.", token, start)); - this.backupVersion = backupVersion; this.snapshotDate = DateUtil.formatyyyyMMdd(start); this.token = token; this.start = start; @@ -57,8 +55,7 @@ public boolean equals(Object o) { return this.snapshotDate.equals(that.snapshotDate) && this.token.equals(that.token) - && this.start.equals(that.start) - && this.backupVersion.equals(that.backupVersion); + && this.start.equals(that.start); } @Override @@ -66,7 +63,6 @@ public int hashCode() { int result = this.snapshotDate.hashCode(); result = 31 * result + this.token.hashCode(); result = 31 * result + this.start.hashCode(); - result = 31 * result + this.backupVersion.hashCode(); return result; } @@ -171,15 +167,6 @@ public void setCassandraSnapshotSuccess(boolean cassandraSnapshotSuccess) { this.cassandraSnapshotSuccess = cassandraSnapshotSuccess; } - /** - * Get the backup version for the snapshot. - * - * @return backup version of the snapshot. - */ - public BackupVersion getBackupVersion() { - return backupVersion; - } - /** * Return the last validation timestamp of this backup metadata. Validation of backup implies * finding if all the files are successfully stored in remote file system. diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java index c97ba72e3..5c8d01202 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java @@ -19,7 +19,6 @@ import com.google.common.collect.ImmutableMap; import com.netflix.priam.backupv2.IMetaProxy; -import com.netflix.priam.backupv2.MetaV2Proxy; import com.netflix.priam.utils.DateUtil; import java.nio.file.Path; import java.time.Instant; @@ -84,8 +83,7 @@ public static List getIncrementalPaths( DateUtil.DateRange dateRange, IMetaProxy metaProxy) { Instant snapshotTime; - if (metaProxy instanceof MetaV2Proxy) snapshotTime = latestValidMetaFile.getLastModified(); - else snapshotTime = latestValidMetaFile.getTime().toInstant(); + snapshotTime = latestValidMetaFile.getLastModified(); DateUtil.DateRange incrementalDateRange = new DateUtil.DateRange(snapshotTime, dateRange.getEndTime()); List incrementalPaths = new ArrayList<>(); diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupService.java b/priam/src/main/java/com/netflix/priam/backup/BackupService.java deleted file mode 100644 index 701b1e02c..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/BackupService.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2019 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backup; - -import com.netflix.priam.aws.UpdateCleanupPolicy; -import com.netflix.priam.config.IBackupRestoreConfig; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.defaultimpl.IService; -import com.netflix.priam.scheduler.PriamScheduler; -import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.tuner.CassandraTunerService; -import javax.inject.Inject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Encapsulate the backup service 1.0 - Execute all the tasks required to run backup service. - * - *

Created by aagrawal on 3/9/19. - */ -public class BackupService implements IService { - private final PriamScheduler scheduler; - private final IConfiguration config; - private final IBackupRestoreConfig backupRestoreConfig; - private final CassandraTunerService cassandraTunerService; - private static final Logger logger = LoggerFactory.getLogger(BackupService.class); - - @Inject - public BackupService( - IConfiguration config, - IBackupRestoreConfig backupRestoreConfig, - PriamScheduler priamScheduler, - CassandraTunerService cassandraTunerService) { - this.config = config; - this.backupRestoreConfig = backupRestoreConfig; - this.scheduler = priamScheduler; - this.cassandraTunerService = cassandraTunerService; - } - - @Override - public void scheduleService() throws Exception { - // Start the snapshot backup schedule - Always run this. (If you want to - // set it off, set backup hour to -1) or set backup cron to "-1" - TaskTimer snapshotTimer = SnapshotBackup.getTimer(config); - scheduleTask(scheduler, SnapshotBackup.class, snapshotTimer); - - if (snapshotTimer != null) { - // Set cleanup - scheduleTask(scheduler, UpdateCleanupPolicy.class, UpdateCleanupPolicy.getTimer()); - // Schedule commit log task - scheduleTask( - scheduler, CommitLogBackupTask.class, CommitLogBackupTask.getTimer(config)); - } - - // Start the Incremental backup schedule if enabled - scheduleTask( - scheduler, - IncrementalBackup.class, - IncrementalBackup.getTimer(config, backupRestoreConfig)); - } - - @Override - public void updateServicePre() throws Exception { - // Run the task to tune Cassandra - cassandraTunerService.onChangeUpdateService(); - } - - @Override - public void updateServicePost() throws Exception {} -} diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java index 40c3bf695..891a8a727 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java @@ -186,8 +186,7 @@ public void failed(BackupMetadata backupMetadata) { */ protected abstract LinkedList fetch(String snapshotDate); - public List getLatestBackupMetadata( - BackupVersion backupVersion, DateUtil.DateRange dateRange) { + public List getLatestBackupMetadata(DateUtil.DateRange dateRange) { Instant startDay = dateRange.getStartTime().truncatedTo(ChronoUnit.DAYS); Instant endDay = dateRange.getEndTime().truncatedTo(ChronoUnit.DAYS); @@ -210,7 +209,6 @@ public List getLatestBackupMetadata( .stream() .filter(Objects::nonNull) .filter(backupMetadata -> backupMetadata.getStatus() == Status.FINISHED) - .filter(backupMetadata -> backupMetadata.getBackupVersion().equals(backupVersion)) .filter( backupMetadata -> backupMetadata diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 67b01b086..f292bc4f0 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -39,7 +39,6 @@ public class BackupVerification { private static final Logger logger = LoggerFactory.getLogger(BackupVerification.class); - private final IMetaProxy metaV1Proxy; private final IMetaProxy metaV2Proxy; private final IBackupStatusMgr backupStatusMgr; private final Provider abstractBackupPathProvider; @@ -47,33 +46,18 @@ public class BackupVerification { @Inject public BackupVerification( - @Named("v1") IMetaProxy metaV1Proxy, @Named("v2") IMetaProxy metaV2Proxy, IBackupStatusMgr backupStatusMgr, Provider abstractBackupPathProvider) { - this.metaV1Proxy = metaV1Proxy; this.metaV2Proxy = metaV2Proxy; this.backupStatusMgr = backupStatusMgr; this.abstractBackupPathProvider = abstractBackupPathProvider; } - public IMetaProxy getMetaProxy(BackupVersion backupVersion) { - switch (backupVersion) { - case SNAPSHOT_BACKUP: - return metaV1Proxy; - case SNAPSHOT_META_SERVICE: - return metaV2Proxy; - } - - return null; - } - - public Optional verifyLatestBackup( - BackupVersion backupVersion, boolean force, DateRange dateRange) + public Optional verifyLatestBackup(boolean force, DateRange dateRange) throws IllegalArgumentException { - IMetaProxy metaProxy = getMetaProxy(backupVersion); - for (BackupMetadata backupMetadata : - backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange)) { + IMetaProxy metaProxy = metaV2Proxy; + for (BackupMetadata backupMetadata : backupStatusMgr.getLatestBackupMetadata(dateRange)) { if (backupMetadata.getLastValidated() == null || force) { Optional result = verifyBackup(metaProxy, backupMetadata); if (result.isPresent()) { @@ -88,12 +72,11 @@ public Optional verifyLatestBackup( return Optional.empty(); } - public List verifyBackupsInRange( - BackupVersion backupVersion, DateRange dateRange) throws IllegalArgumentException { - IMetaProxy metaProxy = getMetaProxy(backupVersion); + public List verifyBackupsInRange(DateRange dateRange) + throws IllegalArgumentException { + IMetaProxy metaProxy = metaV2Proxy; List results = new ArrayList<>(); - for (BackupMetadata backupMetadata : - backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange)) { + for (BackupMetadata backupMetadata : backupStatusMgr.getLatestBackupMetadata(dateRange)) { if (backupMetadata.getLastValidated() != null || verifyBackup(metaProxy, backupMetadata).isPresent()) { results.add(backupMetadata); diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVersion.java b/priam/src/main/java/com/netflix/priam/backup/BackupVersion.java deleted file mode 100644 index 84ea3ad36..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVersion.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2019 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backup; - -import com.netflix.priam.scheduler.UnsupportedTypeException; -import java.util.HashMap; -import java.util.Map; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Enum to capture backup versions. Possible version are V1 and V2. Created by aagrawal on 1/29/19. - */ -public enum BackupVersion { - SNAPSHOT_BACKUP(1), - SNAPSHOT_META_SERVICE(2); - - private static final Logger logger = LoggerFactory.getLogger(BackupVersion.class); - - private final int backupVersion; - private static Map map = new HashMap<>(); - - static { - for (BackupVersion backupVersion : BackupVersion.values()) { - map.put(backupVersion.getBackupVersion(), backupVersion); - } - } - - BackupVersion(int backupVersion) { - this.backupVersion = backupVersion; - } - - public static BackupVersion lookup(int backupVersion, boolean acceptIllegalValue) - throws UnsupportedTypeException { - BackupVersion backupVersionResolved = map.get(backupVersion); - if (backupVersionResolved == null) { - String message = - String.format( - "%s is not a supported BackupVersion. Supported values are %s", - backupVersion, getSupportedValues()); - - if (acceptIllegalValue) { - message = - message - + ". Since acceptIllegalValue is set to True, returning NULL instead."; - logger.error(message); - return null; - } - - logger.error(message); - throw new UnsupportedTypeException(message); - } - return backupVersionResolved; - } - - private static String getSupportedValues() { - StringBuilder supportedValues = new StringBuilder(); - boolean first = true; - for (BackupVersion type : BackupVersion.values()) { - if (!first) { - supportedValues.append(","); - } - supportedValues.append(type); - first = false; - } - - return supportedValues.toString(); - } - - public static BackupVersion lookup(int backupVersion) throws UnsupportedTypeException { - return lookup(backupVersion, false); - } - - public int getBackupVersion() { - return backupVersion; - } -} diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java deleted file mode 100644 index a7f8f588e..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup; - -import com.google.common.collect.Lists; -import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.utils.DateUtil; -import java.io.File; -import java.util.List; -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Provider; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -// Providing this if we want to use it outside Quart -public class CommitLogBackup { - private static final Logger logger = LoggerFactory.getLogger(CommitLogBackup.class); - private final Provider pathFactory; - private final List clRemotePaths = Lists.newArrayList(); - private final IBackupFileSystem fs; - - @Inject - public CommitLogBackup( - Provider pathFactory, @Named("backup") IBackupFileSystem fs) { - this.pathFactory = pathFactory; - this.fs = fs; - } - - public List upload(String archivedDir, final String snapshotName) - throws Exception { - logger.info("Inside upload CommitLog files"); - - if (StringUtils.isBlank(archivedDir)) { - throw new IllegalArgumentException("The archived commitlog director is blank or null"); - } - - File archivedCommitLogDir = new File(archivedDir); - if (!archivedCommitLogDir.exists()) { - throw new IllegalArgumentException( - "The archived commitlog director does not exist: " + archivedDir); - } - - if (logger.isDebugEnabled()) { - logger.debug("Scanning for backup in: {}", archivedCommitLogDir.getAbsolutePath()); - } - List bps = Lists.newArrayList(); - for (final File file : archivedCommitLogDir.listFiles()) { - logger.debug("Uploading commit log {} for backup", file.getCanonicalFile()); - try { - AbstractBackupPath bp = pathFactory.get(); - bp.parseLocal(file, BackupFileType.CL); - - if (snapshotName != null) bp.time = DateUtil.getDate(snapshotName); - - fs.uploadAndDelete(bp, false /* async */); - bps.add(bp); - addToRemotePath(bp.getRemotePath()); - } catch (Exception e) { - logger.error( - "Failed to upload local file {}. Ignoring to continue with rest of backup.", - file, - e); - } - } - return bps; - } - - private void addToRemotePath(String remotePath) { - this.clRemotePaths.add(remotePath); - } -} diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java deleted file mode 100644 index d5cecdca0..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup; - -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.scheduler.SimpleTimer; -import com.netflix.priam.scheduler.TaskTimer; -import java.io.File; -import javax.inject.Inject; -import javax.inject.Singleton; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -// Provide this to be run as a Quart job -@Singleton -public class CommitLogBackupTask extends AbstractBackup { - public static final String JOBNAME = "CommitLogBackup"; - - private static final Logger logger = LoggerFactory.getLogger(CommitLogBackupTask.class); - private final CommitLogBackup clBackup; - - @Inject - public CommitLogBackupTask(IConfiguration config, CommitLogBackup clBackup) { - super(config); - this.clBackup = clBackup; - } - - @Override - public void execute() throws Exception { - try { - logger.debug("Checking for any archived commitlogs"); - // double-check the permission - if (config.isBackingUpCommitLogs()) - clBackup.upload(config.getCommitLogBackupRestoreFromDirs(), null); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - @Override - public String getName() { - return JOBNAME; - } - - public static TaskTimer getTimer(IConfiguration config) { - if (config.isBackingUpCommitLogs()) - return new SimpleTimer(JOBNAME, 60L * 1000); // every 1 min - else return null; - } - - @Override - protected void processColumnFamily(File columnFamilyDirectory) { - // Do nothing. - } -} diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index e33ff3bfc..84a75b4f4 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -106,17 +106,6 @@ default String getShard() { */ Path getPrefix(); - /** - * List all files in the backup location for the specified time range. - * - * @param path This is used as the `prefix` for listing files in the filesystem. All the files - * that start with this prefix will be returned. - * @param start Start date of the file upload. - * @param till End date of the file upload. - * @return Iterator of the AbstractBackupPath matching the criteria. - */ - Iterator list(String path, Date start, Date till); - /** Get a list of prefixes for the cluster available in backup for the specified date */ Iterator listPrefixes(Date date); diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java index 95b69b007..de7b53469 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java @@ -93,11 +93,9 @@ public interface IBackupStatusMgr { * Get the list of backup metadata which are finished and have started in the daterange * provided, in reverse chronological order of start date. * - * @param backupVersion backup version of the backups to search. * @param dateRange time period in which snapshot should have started. Finish time may be after * the endTime in input. * @return list of backup metadata which satisfies the input criteria */ - List getLatestBackupMetadata( - BackupVersion backupVersion, DateUtil.DateRange dateRange); + List getLatestBackupMetadata(DateUtil.DateRange dateRange); } diff --git a/priam/src/main/java/com/netflix/priam/backup/IFileSystemContext.java b/priam/src/main/java/com/netflix/priam/backup/IFileSystemContext.java deleted file mode 100755 index 380a98103..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/IFileSystemContext.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup; - -import com.google.inject.ImplementedBy; -import com.netflix.priam.config.IConfiguration; - -@ImplementedBy(BackupFileSystemContext.class) -public interface IFileSystemContext { - IBackupFileSystem getFileStrategy(IConfiguration config); -} diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index 8e2a9646e..c757db36e 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -90,10 +90,8 @@ public static boolean isEnabled( // Once backup 1.0 is gone, we should not check for enableV2Backups. enabled = (configuration.isIncrementalBackupEnabled() - && (SnapshotBackup.isBackupEnabled(configuration) - || (backupRestoreConfig.enableV2Backups() - && SnapshotMetaTask.isBackupEnabled( - backupRestoreConfig)))); + && (backupRestoreConfig.enableV2Backups() + && SnapshotMetaTask.isBackupEnabled(backupRestoreConfig))); logger.info("Incremental backups are enabled: {}", enabled); if (!enabled) { @@ -115,21 +113,18 @@ public String getName() { @Override protected void processColumnFamily(File backupDir) throws Exception { - BackupFileType fileType = - backupRestoreConfig.enableV2Backups() ? BackupFileType.SST_V2 : BackupFileType.SST; - // upload SSTables and components ImmutableList> futures = backupHelper.uploadAndDeleteAllFiles( - backupDir, fileType, config.enableAsyncIncremental()); + backupDir, BackupFileType.SST_V2, config.enableAsyncIncremental()); Futures.whenAllComplete(futures).call(() -> null, MoreExecutors.directExecutor()); // Next, upload secondary indexes - fileType = BackupFileType.SECONDARY_INDEX_V2; + boolean async = config.enableAsyncIncremental(); for (File directory : getSecondaryIndexDirectories(backupDir)) { futures = backupHelper.uploadAndDeleteAllFiles( - directory, fileType, config.enableAsyncIncremental()); + directory, BackupFileType.SECONDARY_INDEX_V2, async); if (futures.stream().allMatch(ListenableFuture::isDone)) { deleteIfEmpty(directory); } else { diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java deleted file mode 100644 index 5a716062c..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2013 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.backup; - -import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.utils.DateUtil; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.List; -import javax.inject.Inject; -import javax.inject.Provider; -import org.apache.commons.io.FileUtils; -import org.json.simple.JSONArray; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Class to create a meta data file with a list of snapshot files. Also list the contents of a meta - * data file. - */ -public class MetaData { - private static final Logger logger = LoggerFactory.getLogger(MetaData.class); - private final Provider pathFactory; - private final List metaRemotePaths = new ArrayList<>(); - private final IBackupFileSystem fs; - - @Inject - public MetaData( - Provider pathFactory, - IFileSystemContext backupFileSystemCtx, - IConfiguration config) { - - this.pathFactory = pathFactory; - this.fs = backupFileSystemCtx.getFileStrategy(config); - } - - public AbstractBackupPath set(List bps, String snapshotName) - throws Exception { - File metafile = createTmpMetaFile(); - try (FileWriter fr = new FileWriter(metafile)) { - JSONArray jsonObj = new JSONArray(); - for (AbstractBackupPath filePath : bps) jsonObj.add(filePath.getRemotePath()); - fr.write(jsonObj.toJSONString()); - } - AbstractBackupPath backupfile = decorateMetaJson(metafile, snapshotName); - fs.uploadAndDelete(backupfile, false /* async */); - addToRemotePath(backupfile.getRemotePath()); - return backupfile; - } - - /* - From the meta.json to be created, populate its meta data for the backup file. - */ - public AbstractBackupPath decorateMetaJson(File metafile, String snapshotName) - throws ParseException { - AbstractBackupPath backupfile = pathFactory.get(); - backupfile.parseLocal(metafile, BackupFileType.META); - backupfile.setTime(DateUtil.getDate(snapshotName)); - return backupfile; - } - - /* - * Determines the existence of the backup meta file. This meta file could be snapshot (meta.json) or - * incrementals (meta_keyspace_cf..json). - * - * @param backup meta file to search - * @return true if backup meta file exist, false otherwise. - */ - public Boolean doesExist(final AbstractBackupPath meta) { - try { - fs.downloadFile(meta, "" /* suffix */, 5 /* retries */); - } catch (Exception e) { - logger.error("Error downloading the Meta data try with a different date...", e); - } - - return meta.newRestoreFile().exists(); - } - - public File createTmpMetaFile() throws IOException { - File metafile = File.createTempFile("meta", ".json"); - File destFile = new File(metafile.getParent(), "meta.json"); - if (destFile.exists()) destFile.delete(); - FileUtils.moveFile(metafile, destFile); - return destFile; - } - - private void addToRemotePath(String remotePath) { - metaRemotePaths.add(remotePath); - } -} diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java deleted file mode 100644 index ff82758ab..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright 2013 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.backup; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ListenableFuture; -import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.backupv2.ForgottenFilesManager; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.connection.CassandraOperations; -import com.netflix.priam.health.CassandraMonitor; -import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.scheduler.CronTimer; -import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.utils.DateUtil; -import com.netflix.priam.utils.ThreadSleeper; -import java.io.File; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.Future; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import javax.inject.Inject; -import javax.inject.Singleton; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Task for running daily snapshots */ -@Singleton -public class SnapshotBackup extends AbstractBackup { - private static final Logger logger = LoggerFactory.getLogger(SnapshotBackup.class); - public static final String JOBNAME = "SnapshotBackup"; - private final MetaData metaData; - private final ThreadSleeper sleeper = new ThreadSleeper(); - private static final long WAIT_TIME_MS = 60 * 1000 * 10; - private final InstanceIdentity instanceIdentity; - private final IBackupStatusMgr snapshotStatusMgr; - private final BackupRestoreUtil backupRestoreUtil; - private final ForgottenFilesManager forgottenFilesManager; - private String snapshotName = null; - private Instant snapshotInstant = DateUtil.getInstant(); - private List abstractBackupPaths = null; - private final CassandraOperations cassandraOperations; - private final BackupHelper backupHelper; - private static final Lock lock = new ReentrantLock(); - - @Inject - public SnapshotBackup( - IConfiguration config, - BackupHelper backupHelper, - MetaData metaData, - IBackupStatusMgr snapshotStatusMgr, - InstanceIdentity instanceIdentity, - CassandraOperations cassandraOperations, - ForgottenFilesManager forgottenFilesManager) { - super(config); - this.backupHelper = backupHelper; - this.metaData = metaData; - this.snapshotStatusMgr = snapshotStatusMgr; - this.instanceIdentity = instanceIdentity; - this.cassandraOperations = cassandraOperations; - backupRestoreUtil = - new BackupRestoreUtil( - config.getSnapshotIncludeCFList(), config.getSnapshotExcludeCFList()); - this.forgottenFilesManager = forgottenFilesManager; - } - - @Override - public void execute() throws Exception { - // If Cassandra is started then only start Snapshot Backup - while (!CassandraMonitor.hasCassadraStarted()) { - logger.debug( - "Cassandra has not yet started, hence Snapshot Backup will start after [" - + WAIT_TIME_MS / 1000 - + "] secs ..."); - sleeper.sleep(WAIT_TIME_MS); - } - - // Do not allow more than one snapshot to run at the same time. This is possible as this - // happens on CRON. - if (!lock.tryLock()) { - logger.warn("Snapshot Operation is already running! Try again later."); - throw new Exception("Snapshot Operation already running"); - } - - try { - // Clean up all the backup directories, if any. - cleanOldBackups(config); - executeSnapshot(); - } finally { - lock.unlock(); - } - } - - private void executeSnapshot() throws Exception { - Date startTime = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime(); - snapshotName = DateUtil.formatyyyyMMddHHmm(startTime); - snapshotInstant = DateUtil.getInstant(); - String token = instanceIdentity.getInstance().getToken(); - - // Save start snapshot status - BackupMetadata backupMetadata = - new BackupMetadata(BackupVersion.SNAPSHOT_BACKUP, token, startTime); - snapshotStatusMgr.start(backupMetadata); - - try { - logger.info("Starting snapshot {}", snapshotName); - cassandraOperations.takeSnapshot(snapshotName); - backupMetadata.setCassandraSnapshotSuccess(true); - - // Collect all snapshot dir's under keyspace dir's - abstractBackupPaths = Lists.newArrayList(); - // Try to upload all the files as part of snapshot. If there is any error, there will be - // an exception and snapshot will be considered as failure. - initiateBackup(SNAPSHOT_FOLDER, backupRestoreUtil); - - // All the files are uploaded successfully as part of snapshot. - // pre condition notify of meta.json upload - File tmpMetaFile = metaData.createTmpMetaFile(); - // Note: no need to remove this temp as it is done within createTmpMetaFile() - AbstractBackupPath metaJsonAbp = metaData.decorateMetaJson(tmpMetaFile, snapshotName); - - // Upload meta file - AbstractBackupPath metaJson = metaData.set(abstractBackupPaths, snapshotName); - - logger.info("Snapshot upload complete for {}", snapshotName); - backupMetadata.setSnapshotLocation( - config.getBackupPrefix() + File.separator + metaJson.getRemotePath()); - snapshotStatusMgr.finish(backupMetadata); - } catch (Exception e) { - logger.error( - "Exception occurred while taking snapshot: {}. Exception: {}", - snapshotName, - e.getLocalizedMessage()); - snapshotStatusMgr.failed(backupMetadata); - throw e; - } finally { - try { - cassandraOperations.clearSnapshot(snapshotName); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - } - - private File getValidSnapshot(File snpDir, String snapshotName) { - for (File snapshotDir : snpDir.listFiles()) - if (snapshotDir.getName().matches(snapshotName)) return snapshotDir; - return null; - } - - @Override - public String getName() { - return JOBNAME; - } - - public static boolean isBackupEnabled(IConfiguration config) throws Exception { - return (getTimer(config) != null); - } - - public static TaskTimer getTimer(IConfiguration config) throws Exception { - TaskTimer timer = CronTimer.getCronTimer(JOBNAME, config.getBackupCronExpression()); - if (timer == null) { - // Clean up all the backup directories, if any. - cleanOldBackups(config); - } - return timer; - } - - private static void cleanOldBackups(IConfiguration configuration) throws Exception { - Set backupPaths = AbstractBackup.getBackupDirectories(configuration, SNAPSHOT_FOLDER); - for (Path backupDirPath : backupPaths) - try (DirectoryStream directoryStream = - Files.newDirectoryStream(backupDirPath, path -> Files.isDirectory(path))) { - for (Path backupDir : directoryStream) { - if (isValidBackupDir(backupDir)) { - FileUtils.deleteDirectory(backupDir.toFile()); - } - } - } - } - - @Override - protected void processColumnFamily(File backupDir) throws Exception { - File snapshotDir = getValidSnapshot(backupDir, snapshotName); - - if (snapshotDir == null) { - logger.warn("{} folder does not contain {} snapshots", backupDir, snapshotName); - return; - } - - forgottenFilesManager.findAndMoveForgottenFiles(snapshotInstant, snapshotDir); - // Add files to this dir - - ImmutableList> futures = - backupHelper.uploadAndDeleteAllFiles( - snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot()); - for (Future future : futures) { - abstractBackupPaths.add(future.get()); - } - } - - private static boolean isValidBackupDir(Path backupDir) { - String backupDirName = backupDir.toFile().getName(); - // Check if it of format yyyyMMddHHmm - return (DateUtil.getDate(backupDirName) != null); - } -} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index 467334984..28282e789 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -75,7 +75,7 @@ public BackupTTLTask( IConfiguration configuration, IBackupRestoreConfig backupRestoreConfig, @Named("v2") IMetaProxy metaProxy, - IFileSystemContext backupFileSystemCtx, + IBackupFileSystem filesystem, Provider abstractBackupPathProvider, TokenRetriever tokenRetriever, InstanceState instanceState) @@ -83,7 +83,7 @@ public BackupTTLTask( super(configuration); this.backupRestoreConfig = backupRestoreConfig; this.metaProxy = metaProxy; - this.fileSystem = backupFileSystemCtx.getFileStrategy(configuration); + this.fileSystem = filesystem; this.abstractBackupPathProvider = abstractBackupPathProvider; this.instanceState = instanceState; this.maxWaitMillis = diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 21c3f758b..56df6ec43 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -86,9 +86,7 @@ public void execute() throws Exception { Instant slo = now.minus(backupRestoreConfig.getBackupVerificationSLOInHours(), ChronoUnit.HOURS); DateRange dateRange = new DateRange(slo, now); - List verifiedBackups = - backupVerification.verifyBackupsInRange( - BackupVersion.SNAPSHOT_META_SERVICE, dateRange); + List verifiedBackups = backupVerification.verifyBackupsInRange(dateRange); verifiedBackups .stream() diff --git a/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java b/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java deleted file mode 100644 index 1aa75b5c0..000000000 --- a/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright 2019 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backupv2; - -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.merics.BackupMetrics; -import com.netflix.priam.utils.DateUtil; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Collection; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import javax.inject.Inject; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.filefilter.FileFilterUtils; -import org.apache.commons.io.filefilter.IOFileFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Created by aagrawal on 1/1/19. */ -public class ForgottenFilesManager { - private static final Logger logger = LoggerFactory.getLogger(ForgottenFilesManager.class); - - private BackupMetrics backupMetrics; - private IConfiguration config; - private static final String TMP_EXT = ".tmp"; - - private static final Pattern tmpFilePattern = - Pattern.compile("^((.*)\\-(.*)\\-)?tmp(link)?\\-((?:l|k).)\\-(\\d)*\\-(.*)$"); - - protected static final String LOST_FOUND = "lost+found"; - - @Inject - public ForgottenFilesManager(IConfiguration configuration, BackupMetrics backupMetrics) { - this.config = configuration; - this.backupMetrics = backupMetrics; - } - - public void findAndMoveForgottenFiles(Instant snapshotInstant, File snapshotDir) { - try { - Collection snapshotFiles = - FileUtils.listFiles(snapshotDir, FileFilterUtils.fileFileFilter(), null); - File columnfamilyDir = snapshotDir.getParentFile().getParentFile(); - Collection columnfamilyFiles = - getColumnfamilyFiles(snapshotInstant, columnfamilyDir); - - // Remove the SSTable(s) which are part of snapshot from the CF file list. - // This cannot be a simple removeAll as snapshot files have "different" file folder - // prefix. - for (File file : snapshotFiles) { - // Get its parent directory file based on this file. - File originalFile = new File(columnfamilyDir, file.getName()); - columnfamilyFiles.remove(originalFile); - } - - // If there are no "extra" SSTables in CF data folder, we are done. - if (columnfamilyFiles.size() == 0) return; - - logger.warn( - "# of potential forgotten files: {} found for CF: {}", - columnfamilyFiles.size(), - columnfamilyDir.getName()); - - // Move the files to lost_found directory if configured. - moveForgottenFiles(columnfamilyDir, columnfamilyFiles); - - } catch (Exception e) { - // Eat the exception, if there, for any reason. This should not stop the snapshot for - // any reason. - logger.error( - "Exception occurred while trying to find forgottenFile. Ignoring the error and continuing with remaining backup", - e); - e.printStackTrace(); - } - } - - protected Collection getColumnfamilyFiles(Instant snapshotInstant, File columnfamilyDir) { - // Find all the files in columnfamily folder which is : - // 1. Not a temp file. - // 2. Is a file. (we don't care about directories) - // 3. Is older than snapshot time, as new files keep getting created after taking a - // snapshot. - IOFileFilter tmpFileFilter1 = FileFilterUtils.suffixFileFilter(TMP_EXT); - IOFileFilter tmpFileFilter2 = - FileFilterUtils.asFileFilter( - pathname -> tmpFilePattern.matcher(pathname.getName()).matches()); - IOFileFilter tmpFileFilter = FileFilterUtils.or(tmpFileFilter1, tmpFileFilter2); - /* - Here we are allowing files which were more than - @link{IConfiguration#getGracePeriodDaysForCompaction}. We do this to allow cassandra - to have files which were generated as part of long running compaction. - Refer to https://issues.apache.org/jira/browse/CASSANDRA-6756 and - https://issues.apache.org/jira/browse/CASSANDRA-7066 - for more information. - */ - IOFileFilter ageFilter = - FileFilterUtils.ageFileFilter( - snapshotInstant - .minus(config.getGracePeriodDaysForCompaction(), ChronoUnit.DAYS) - .toEpochMilli()); - IOFileFilter fileFilter = - FileFilterUtils.and( - FileFilterUtils.notFileFilter(tmpFileFilter), - FileFilterUtils.fileFileFilter(), - ageFilter); - - return FileUtils.listFiles(columnfamilyDir, fileFilter, null); - } - - protected void moveForgottenFiles(File columnfamilyDir, Collection columnfamilyFiles) - throws IOException { - // This is a list of potential forgotten file(s). Note that C* might still be using - // files as part of read, so we really do not want to move them until we meet the - // @link{IConfiguration#getForgottenFileGracePeriodDaysForRead} window elapses. - - final Path destDir = Paths.get(columnfamilyDir.getAbsolutePath(), LOST_FOUND); - FileUtils.forceMkdir(destDir.toFile()); - final Collection columnfamilyPaths = - columnfamilyFiles - .parallelStream() - .map(file -> Paths.get(file.getAbsolutePath())) - .collect(Collectors.toList()); - - for (Path file : columnfamilyPaths) { - try { - final Path symbolic_link = - Paths.get(destDir.toFile().getAbsolutePath(), file.toFile().getName()); - // Lets see if there is a symbolic link to this file already? - if (!Files.exists(symbolic_link)) { - // If not, lets create one and work on next file. - Files.createSymbolicLink(symbolic_link, file); - continue; - } else if (Files.isSymbolicLink(symbolic_link)) { - // Symbolic link exists, is it older than our timeframe? - Instant last_modified_time = - Files.getLastModifiedTime(symbolic_link, LinkOption.NOFOLLOW_LINKS) - .toInstant(); - if (DateUtil.getInstant() - .isAfter( - last_modified_time.plus( - config.getForgottenFileGracePeriodDaysForRead(), - ChronoUnit.DAYS))) { - // Eligible for move. - logger.info( - "Eligible for move: Forgotten file: {} found for CF: {}", - file, - columnfamilyDir.getName()); - backupMetrics.incrementForgottenFiles(1); - if (config.isForgottenFileMoveEnabled()) { - try { - // Remove our symbolic link. Note that deletion of symbolic link - // does not remove the original file. - Files.delete(symbolic_link); - FileUtils.moveFileToDirectory( - file.toFile(), destDir.toFile(), true); - logger.warn( - "Successfully moved forgotten file: {} found for CF: {}", - file, - columnfamilyDir.getName()); - } catch (IOException e) { - logger.error( - "Exception occurred while trying to move forgottenFile: {}. Ignoring the error and continuing with remaining backup/forgotten files.", - file); - e.printStackTrace(); - } - } - } - } - - } catch (IOException e) { - logger.error("Forgotten file: Error while trying to process the file: {}", file); - e.printStackTrace(); - } - } - - // Clean LOST_FOUND directory of any previous symbolic link files which are not considered - // lost any more. - for (File file : FileUtils.listFiles(destDir.toFile(), null, false)) { - Path filePath = Paths.get(file.getAbsolutePath()); - if (Files.isSymbolicLink(filePath)) { - Path originalFile = Files.readSymbolicLink(filePath); - if (!columnfamilyPaths.contains(originalFile)) { - Files.delete(filePath); - logger.info( - "Deleting the symbolic link as it is not considered as lost anymore. filePath: {}", - filePath); - } - } - } - } -} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index a13c96853..ccc94a86d 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -22,7 +22,6 @@ import com.google.gson.stream.JsonWriter; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.backup.IFileSystemContext; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; import java.io.FileWriter; @@ -97,10 +96,10 @@ public MetaFileWriter( IConfiguration configuration, InstanceIdentity instanceIdentity, Provider pathFactory, - IFileSystemContext backupFileSystemCtx, + IBackupFileSystem fileSystem, @Named("v2") IMetaProxy metaProxy) { this.pathFactory = pathFactory; - this.backupFileSystem = backupFileSystemCtx.getFileStrategy(configuration); + this.backupFileSystem = fileSystem; this.metaProxy = metaProxy; List backupIdentifier = new ArrayList<>(); backupIdentifier.add(instanceIdentity.getInstance().getToken()); diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java deleted file mode 100644 index a5991c880..000000000 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backupv2; - -import com.google.common.collect.Lists; -import com.netflix.priam.backup.*; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.utils.DateUtil; -import java.io.FileReader; -import java.nio.file.InvalidPathException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.temporal.ChronoUnit; -import java.util.*; -import javax.inject.Inject; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.iterators.FilterIterator; -import org.apache.commons.io.FileUtils; -import org.json.simple.parser.JSONParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Created by aagrawal on 12/18/18. */ -public class MetaV1Proxy implements IMetaProxy { - private static final Logger logger = LoggerFactory.getLogger(MetaV1Proxy.class); - private final IBackupFileSystem fs; - - @Inject - public MetaV1Proxy(IConfiguration configuration, IFileSystemContext backupFileSystemCtx) { - fs = backupFileSystemCtx.getFileStrategy(configuration); - } - - @Override - public Path getLocalMetaFileDirectory() { - return null; - } - - @Override - public String getMetaPrefix(DateUtil.DateRange dateRange) { - return null; - } - - @Override - public List findMetaFiles(DateUtil.DateRange dateRange) { - Date startTime = new Date(dateRange.getStartTime().toEpochMilli()); - Date endTime = new Date(dateRange.getEndTime().toEpochMilli()); - String restorePrefix = fs.getPrefix().toString(); - logger.debug("Looking for snapshot meta file within restore prefix: {}", restorePrefix); - List metas = Lists.newArrayList(); - - Iterator backupfiles = fs.list(restorePrefix, startTime, endTime); - - while (backupfiles.hasNext()) { - AbstractBackupPath path = backupfiles.next(); - if (path.getType() == AbstractBackupPath.BackupFileType.META) - // Since there are now meta file for incrementals as well as snapshot, we need to - // find the correct one (i.e. the snapshot meta file (meta.json)) - if (path.getFileName().equalsIgnoreCase("meta.json")) { - metas.add(path); - } - } - - metas.sort(Collections.reverseOrder()); - - if (metas.size() == 0) { - logger.info( - "No meta v1 file found on remote file system for the time period: {}", - dateRange); - } - - return metas; - } - - @Override - public BackupVerificationResult isMetaFileValid(AbstractBackupPath metaBackupPath) { - BackupVerificationResult result = new BackupVerificationResult(); - result.remotePath = metaBackupPath.getRemotePath(); - result.snapshotInstant = metaBackupPath.getTime().toInstant(); - - try { - // Download the meta file. - Path metaFile = downloadMetaFile(metaBackupPath); - // Read the local meta file. - List metaFileList = getSSTFilesFromMeta(metaFile); - FileUtils.deleteQuietly(metaFile.toFile()); - result.manifestAvailable = true; - - // List the remote file system to validate the backup. - String prefix = fs.getPrefix().toString(); - Date strippedMsSnapshotTime = - new Date(result.snapshotInstant.truncatedTo(ChronoUnit.MINUTES).toEpochMilli()); - Iterator backupfiles = - fs.list(prefix, strippedMsSnapshotTime, strippedMsSnapshotTime); - - // Return validation fail if backup filesystem listing failed. - if (!backupfiles.hasNext()) { - logger.warn( - "ERROR: No files available while doing backup filesystem listing. Declaring the verification failed."); - return result; - } - - // Convert the remote listing to String. - List remoteListing = new ArrayList<>(); - while (backupfiles.hasNext()) { - AbstractBackupPath path = backupfiles.next(); - if (path.getType() == AbstractBackupPath.BackupFileType.SNAP) - remoteListing.add(path.getRemotePath()); - } - - if (metaFileList.isEmpty() && remoteListing.isEmpty()) { - logger.info( - "Uncommon Scenario: Both meta file and backup filesystem listing is empty. Considering this as success"); - result.valid = true; - return result; - } - - ArrayList filesMatched = - (ArrayList) CollectionUtils.intersection(metaFileList, remoteListing); - result.filesMatched = filesMatched.size(); - result.filesInMetaOnly = metaFileList; - result.filesInMetaOnly.removeAll(filesMatched); - - // There could be a scenario that backupfilesystem has more files than meta file. e.g. - // some leftover objects - result.valid = (result.filesInMetaOnly.isEmpty()); - } catch (Exception e) { - logger.error( - "Error while processing meta file: " + metaBackupPath, e.getLocalizedMessage()); - e.printStackTrace(); - } - - return result; - } - - @Override - public Path downloadMetaFile(AbstractBackupPath meta) throws BackupRestoreException { - fs.downloadFile(meta, ".download" /* suffix */, 10 /* retries */); - return Paths.get(meta.newRestoreFile().getAbsolutePath() + ".download"); - } - - @Override - public List getSSTFilesFromMeta(Path localMetaPath) throws Exception { - if (localMetaPath.toFile().isDirectory() || !localMetaPath.toFile().exists()) - throw new InvalidPathException( - localMetaPath.toString(), "Input path is either directory or do not exist"); - - List result = new ArrayList<>(); - JSONParser jsonParser = new JSONParser(); - org.json.simple.JSONArray fileList = - (org.json.simple.JSONArray) - jsonParser.parse(new FileReader(localMetaPath.toFile())); - fileList.forEach(entry -> result.add(entry.toString())); - return result; - } - - @Override - public Iterator getIncrementals(DateUtil.DateRange dateRange) { - String prefix = fs.getPrefix().toString(); - Iterator iterator = - fs.list( - prefix, - new Date(dateRange.getStartTime().toEpochMilli()), - new Date(dateRange.getEndTime().toEpochMilli())); - return new FilterIterator<>( - iterator, - abstractBackupPath -> - abstractBackupPath.getType() == AbstractBackupPath.BackupFileType.SST); - } - - @Override - public void cleanupOldMetaFiles() {} -} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java index b7904167a..68f13daaa 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java @@ -17,7 +17,10 @@ package com.netflix.priam.backupv2; -import com.netflix.priam.backup.*; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BackupRestoreException; +import com.netflix.priam.backup.BackupVerificationResult; +import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; import java.io.File; @@ -47,9 +50,9 @@ public class MetaV2Proxy implements IMetaProxy { @Inject public MetaV2Proxy( IConfiguration configuration, - IFileSystemContext backupFileSystemCtx, + IBackupFileSystem fileSystem, Provider abstractBackupPathProvider) { - fs = backupFileSystemCtx.getFileStrategy(configuration); + this.fs = fileSystem; this.abstractBackupPathProvider = abstractBackupPathProvider; metaFileDirectory = Paths.get(configuration.getDataFileLocation()); } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index a886b5a1a..bc05c1874 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -205,10 +205,7 @@ public void execute() throws Exception { Instant snapshotInstant = clock.instant(); String token = instanceIdentity.getInstance().getToken(); BackupMetadata backupMetadata = - new BackupMetadata( - BackupVersion.SNAPSHOT_META_SERVICE, - token, - new Date(snapshotInstant.toEpochMilli())); + new BackupMetadata(token, new Date(snapshotInstant.toEpochMilli())); snapshotStatusMgr.start(backupMetadata); try { diff --git a/priam/src/main/java/com/netflix/priam/cli/Backuper.java b/priam/src/main/java/com/netflix/priam/cli/Backuper.java deleted file mode 100644 index 1b4bb374c..000000000 --- a/priam/src/main/java/com/netflix/priam/cli/Backuper.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.cli; - -import com.netflix.priam.backup.SnapshotBackup; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class Backuper { - private static final Logger logger = LoggerFactory.getLogger(Backuper.class); - - public static void main(String[] args) { - try { - Application.initialize(); - SnapshotBackup backuper = Application.getInjector().getInstance(SnapshotBackup.class); - try { - backuper.execute(); - } catch (Exception e) { - logger.error("Unable to backup: ", e); - } - } finally { - Application.shutdownAdditionalThreads(); - } - } -} diff --git a/priam/src/main/java/com/netflix/priam/cli/IncrementalBackuper.java b/priam/src/main/java/com/netflix/priam/cli/IncrementalBackuper.java deleted file mode 100644 index 5075e4f58..000000000 --- a/priam/src/main/java/com/netflix/priam/cli/IncrementalBackuper.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.cli; - -import com.netflix.priam.backup.IncrementalBackup; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class IncrementalBackuper { - private static final Logger logger = LoggerFactory.getLogger(IncrementalBackuper.class); - - public static void main(String[] args) { - try { - Application.initialize(); - IncrementalBackup backuper = - Application.getInjector().getInstance(IncrementalBackup.class); - try { - backuper.execute(); - } catch (Exception e) { - logger.error("Unable to backup: ", e); - } - } finally { - Application.shutdownAdditionalThreads(); - } - } -} diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index aab768b29..405687edb 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -297,18 +297,6 @@ default String getCompactionExcludeCFList() { return null; } - /** - * Cron expression to be used for snapshot backups. - * - * @return Backup cron expression for snapshots - * @see quartz-scheduler - * @see http://www.cronmaker.com To build new cron timer - */ - default String getBackupCronExpression() { - return "0 0 12 1/1 * ? *"; - } - /** * Column Family(ies), comma delimited, to include during snapshot backup. Note 1: The expected * format is keyspace.cfname. If no value is provided then snapshot contains all KS,CF(s) Note @@ -575,40 +563,6 @@ default String getInternodeCompression() { return "all"; } - /** - * Enable/disable backup/restore of commit logs. - * - * @return boolean value true if commit log backup/restore is enabled, false otherwise. Default: - * false. - */ - default boolean isBackingUpCommitLogs() { - return false; - } - - default String getCommitLogBackupPropsFile() { - return getCassHome() + "/conf/commitlog_archiving.properties"; - } - - default String getCommitLogBackupArchiveCmd() { - return "/bin/ln %path /mnt/data/backup/%name"; - } - - default String getCommitLogBackupRestoreCmd() { - return "/bin/mv %from %to"; - } - - default String getCommitLogBackupRestoreFromDirs() { - return "/mnt/data/backup/commitlog/"; - } - - default String getCommitLogBackupRestorePointInTime() { - return StringUtils.EMPTY; - } - - default int maxCommitLogsRestore() { - return 10; - } - default boolean isClientSslEnabled() { return false; } diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 9f77bda5d..9e699f0c9 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -212,11 +212,6 @@ public String getMaxDirectMemory() { (PRIAM_PRE + ".direct.memory.size.") + instanceInfo.getInstanceType(), "50G"); } - @Override - public String getBackupCronExpression() { - return config.get(PRIAM_PRE + ".backup.cron", "0 0 12 1/1 * ? *"); // Backup daily at 12 - } - @Override public GCType getGCType() throws UnsupportedTypeException { String gcType = config.get(PRIAM_PRE + ".gc.type", GCType.CMS.getGcType()); @@ -441,44 +436,6 @@ public String getInternodeCompression() { return config.get(PRIAM_PRE + ".internodeCompression", "all"); } - @Override - public boolean isBackingUpCommitLogs() { - return config.get(PRIAM_PRE + ".clbackup.enabled", false); - } - - @Override - public String getCommitLogBackupPropsFile() { - return config.get( - PRIAM_PRE + ".clbackup.propsfile", - getCassHome() + "/conf/commitlog_archiving.properties"); - } - - @Override - public String getCommitLogBackupArchiveCmd() { - return config.get( - PRIAM_PRE + ".clbackup.archiveCmd", "/bin/ln %path /mnt/data/backup/%name"); - } - - @Override - public String getCommitLogBackupRestoreCmd() { - return config.get(PRIAM_PRE + ".clbackup.restoreCmd", "/bin/mv %from %to"); - } - - @Override - public String getCommitLogBackupRestoreFromDirs() { - return config.get(PRIAM_PRE + ".clbackup.restoreDirs", "/mnt/data/backup/commitlog/"); - } - - @Override - public String getCommitLogBackupRestorePointInTime() { - return config.get(PRIAM_PRE + ".clbackup.restoreTime", ""); - } - - @Override - public int maxCommitLogsRestore() { - return config.get(PRIAM_PRE + ".clrestore.max", 10); - } - public boolean isClientSslEnabled() { return config.get(PRIAM_PRE + ".client.sslEnabled", false); } diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java b/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java index 0f68962ac..19ff006de 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java @@ -18,23 +18,14 @@ import com.google.inject.AbstractModule; import com.google.inject.name.Names; -import com.netflix.priam.aws.S3CrossAccountFileSystem; -import com.netflix.priam.aws.S3EncryptedFileSystem; import com.netflix.priam.aws.S3FileSystem; import com.netflix.priam.aws.auth.EC2RoleAssumptionCredential; import com.netflix.priam.aws.auth.IS3Credential; import com.netflix.priam.aws.auth.S3RoleAssumptionCredential; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.backupv2.IMetaProxy; -import com.netflix.priam.backupv2.MetaV1Proxy; import com.netflix.priam.backupv2.MetaV2Proxy; import com.netflix.priam.cred.ICredential; -import com.netflix.priam.cred.ICredentialGeneric; -import com.netflix.priam.cryptography.IFileCryptography; -import com.netflix.priam.cryptography.pgp.PgpCredential; -import com.netflix.priam.cryptography.pgp.PgpCryptography; -import com.netflix.priam.google.GcsCredential; -import com.netflix.priam.google.GoogleEncryptedFileSystem; import com.netflix.spectator.api.NoopRegistry; import com.netflix.spectator.api.Registry; import org.quartz.SchedulerFactory; @@ -45,32 +36,13 @@ public class PriamGuiceModule extends AbstractModule { protected void configure() { bind(SchedulerFactory.class).to(StdSchedulerFactory.class).asEagerSingleton(); - bind(IBackupFileSystem.class).annotatedWith(Names.named("backup")).to(S3FileSystem.class); - bind(IBackupFileSystem.class) - .annotatedWith(Names.named("encryptedbackup")) - .to(S3EncryptedFileSystem.class); - - bind(S3CrossAccountFileSystem.class); - - bind(IBackupFileSystem.class) - .annotatedWith(Names.named("gcsencryptedbackup")) - .to(GoogleEncryptedFileSystem.class); + bind(IBackupFileSystem.class).to(S3FileSystem.class); bind(IS3Credential.class) .annotatedWith(Names.named("awss3roleassumption")) .to(S3RoleAssumptionCredential.class); bind(ICredential.class) .annotatedWith(Names.named("awsec2roleassumption")) .to(EC2RoleAssumptionCredential.class); - bind(IFileCryptography.class) - .annotatedWith(Names.named("filecryptoalgorithm")) - .to(PgpCryptography.class); - bind(ICredentialGeneric.class) - .annotatedWith(Names.named("gcscredential")) - .to(GcsCredential.class); - bind(ICredentialGeneric.class) - .annotatedWith(Names.named("pgpcredential")) - .to(PgpCredential.class); - bind(IMetaProxy.class).annotatedWith(Names.named("v1")).to(MetaV1Proxy.class); bind(IMetaProxy.class).annotatedWith(Names.named("v2")).to(MetaV2Proxy.class); bind(Registry.class).toInstance(new NoopRegistry()); } diff --git a/priam/src/main/java/com/netflix/priam/google/GcsCredential.java b/priam/src/main/java/com/netflix/priam/google/GcsCredential.java deleted file mode 100755 index f8447c3bb..000000000 --- a/priam/src/main/java/com/netflix/priam/google/GcsCredential.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.google; - -import com.amazonaws.auth.AWSCredentialsProvider; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredentialGeneric; -import javax.inject.Inject; - -/* - * A generic implementation of fetch keys as plaintext. The key values are used with Google Cloud Storage. Users may - * want to provide an implementation where your key(s)' value is decrypted using AES encryption algorithm. - */ -public class GcsCredential implements ICredentialGeneric { - - private final IConfiguration config; - - @Inject - public GcsCredential(IConfiguration config) { - this.config = config; - } - - @Override - public AWSCredentialsProvider getAwsCredentialProvider() { - // TODO Auto-generated method stub - return null; - } - - @Override - public byte[] getValue(KEY key) { - if (key == null) { - throw new NullPointerException("Credential key cannot be null."); - } - - if (key.equals(KEY.GCS_PRIVATE_KEY_LOC)) { - return this.config.getGcsServiceAccountPrivateKeyLoc().getBytes(); - } else if (key.equals(KEY.GCS_SERVICE_ID)) { - return this.config.getGcsServiceAccountId().getBytes(); - } else { - throw new IllegalArgumentException("Key value not supported."); - } - } -} diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java deleted file mode 100755 index edb42ca4f..000000000 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ /dev/null @@ -1,263 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.google; - -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.services.storage.Storage; -import com.google.api.services.storage.StorageScopes; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.AbstractFileSystem; -import com.netflix.priam.backup.BackupRestoreException; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredentialGeneric; -import com.netflix.priam.cred.ICredentialGeneric.KEY; -import com.netflix.priam.merics.BackupMetrics; -import com.netflix.priam.notification.BackupNotificationMgr; -import java.io.*; -import java.nio.file.Path; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Provider; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class GoogleEncryptedFileSystem extends AbstractFileSystem { - - private static final Logger logger = LoggerFactory.getLogger(GoogleEncryptedFileSystem.class); - - private static final String APPLICATION_NAME = "gdl"; - private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); - - private HttpTransport httpTransport; - // represents our "service account" credentials we will use to access GCS - private Credential credential; - private Storage gcsStorageHandle; - private Storage.Objects objectsResoruceHandle = null; - private String srcBucketName; - private final IConfiguration config; - - private final ICredentialGeneric gcsCredential; - private final BackupMetrics backupMetrics; - - @Inject - public GoogleEncryptedFileSystem( - Provider pathProvider, - final IConfiguration config, - @Named("gcscredential") ICredentialGeneric credential, - BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationManager) { - super(config, backupMetrics, backupNotificationManager, pathProvider); - this.backupMetrics = backupMetrics; - this.config = config; - this.gcsCredential = credential; - - try { - this.httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - } catch (Exception e) { - throw new IllegalStateException( - "Unable to create a handle to the Google Http tranport", e); - } - - this.srcBucketName = getShard(); - } - - private Storage.Objects constructObjectResourceHandle() { - if (this.objectsResoruceHandle != null) { - return this.objectsResoruceHandle; - } - - constructGcsStorageHandle(); - this.objectsResoruceHandle = this.gcsStorageHandle.objects(); - return this.objectsResoruceHandle; - } - - /* - * Get a handle to the GCS api to manage our data within their storage. Code derive from - * https://code.google.com/p/google-api-java-client/source/browse/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java?repo=samples - * - * Note: GCS storage will use our credential to do auto-refresh of expired tokens - */ - private Storage constructGcsStorageHandle() { - if (this.gcsStorageHandle != null) { - return this.gcsStorageHandle; - } - - try { - constructGcsCredential(); - } catch (Exception e) { - throw new IllegalStateException("Exception during GCS authorization", e); - } - - this.gcsStorageHandle = - new Storage.Builder(this.httpTransport, JSON_FACTORY, this.credential) - .setApplicationName(APPLICATION_NAME) - .build(); - return this.gcsStorageHandle; - } - - /** - * Authorizes the installed application to access user's protected data, code from - * https://developers.google.com/maps-engine/documentation/oauth/serviceaccount and - * http://javadoc.google-api-java-client.googlecode.com/hg/1.8.0-beta/com/google/api/client/googleapis/auth/oauth2/GoogleCredential.html - */ - private Credential constructGcsCredential() throws Exception { - - if (this.credential != null) { - return this.credential; - } - - synchronized (this) { - if (this.credential == null) { - - String service_acct_email = - new String(this.gcsCredential.getValue(KEY.GCS_SERVICE_ID)); - - if (this.config.getGcsServiceAccountPrivateKeyLoc() == null - || this.config.getGcsServiceAccountPrivateKeyLoc().isEmpty()) { - throw new NullPointerException( - "Fast property for the the GCS private key file is null/empty."); - } - - // Take the encrypted private key, decrypted into an in-transit file which is passed - // to GCS - File gcsPrivateKeyHandle = - new File(this.config.getGcsServiceAccountPrivateKeyLoc() + ".output"); - - ByteArrayOutputStream byteos = new ByteArrayOutputStream(); - - byte[] gcsPrivateKeyPlainText = - this.gcsCredential.getValue(KEY.GCS_PRIVATE_KEY_LOC); - try (BufferedOutputStream bos = - new BufferedOutputStream(new FileOutputStream(gcsPrivateKeyHandle))) { - byteos.write(gcsPrivateKeyPlainText); - byteos.writeTo(bos); - } catch (IOException e) { - throw new IOException( - "Exception when writing decrypted gcs private key value to disk.", e); - } - - Collection scopes = new ArrayList<>(1); - scopes.add(StorageScopes.DEVSTORAGE_READ_ONLY); - // Cryptex decrypted service account key derive from the GCS console - this.credential = - new GoogleCredential.Builder() - .setTransport(this.httpTransport) - .setJsonFactory(JSON_FACTORY) - .setServiceAccountId(service_acct_email) - .setServiceAccountScopes(scopes) - .setServiceAccountPrivateKeyFromP12File(gcsPrivateKeyHandle) - .build(); - } - } - - return this.credential; - } - - @Override - protected void downloadFileImpl(AbstractBackupPath path, String suffix) - throws BackupRestoreException { - String remotePath = path.getRemotePath(); - File localFile = new File(path.newRestoreFile().getAbsolutePath() + suffix); - String objectName = parseObjectname(getPrefix().toString()); - com.google.api.services.storage.Storage.Objects.Get get; - - try { - get = constructObjectResourceHandle().get(this.srcBucketName, remotePath); - } catch (IOException e) { - throw new BackupRestoreException( - "IO error retrieving metadata for: " - + objectName - + " from bucket: " - + this.srcBucketName, - e); - } - - // If you're not using GCS' AppEngine, download the whole thing (instead of chunks) in one - // request, if possible. - get.getMediaHttpDownloader().setDirectDownloadEnabled(true); - try (OutputStream os = new FileOutputStream(localFile); - InputStream is = get.executeMediaAsInputStream()) { - IOUtils.copyLarge(is, os); - } catch (IOException e) { - throw new BackupRestoreException( - "IO error during streaming of object: " - + objectName - + " from bucket: " - + this.srcBucketName, - e); - } catch (Exception ex) { - throw new BackupRestoreException( - "Exception encountered when copying bytes from input to output", ex); - } - - backupMetrics.recordDownloadRate(get.getLastResponseHeaders().getContentLength()); - } - - @Override - protected boolean doesRemoteFileExist(Path remotePath) { - // TODO: Implement based on GCS. Since this is only used for upload, leaving it empty - return false; - } - - @Override - public Iterator listFileSystem(String prefix, String delimiter, String marker) { - return new GoogleFileIterator(constructGcsStorageHandle(), prefix, null); - } - - @Override - public void cleanup() { - // TODO Auto-generated method stub - } - - @Override - public void shutdown() { - // TODO Auto-generated method stub - } - - @Override - protected long uploadFileImpl(AbstractBackupPath path, Instant target) - throws BackupRestoreException { - throw new UnsupportedOperationException(); - } - - @Override - public long getFileSize(String remotePath) throws BackupRestoreException { - return 0; - } - - @Override - public void deleteFiles(List remotePaths) throws BackupRestoreException { - // TODO: Delete implementation - } - - /* - * @param pathPrefix - * @return objectName - */ - static String parseObjectname(String pathPrefix) { - int offset = pathPrefix.lastIndexOf(0x2f); - return pathPrefix.substring(offset + 1); - } -} diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java b/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java deleted file mode 100755 index ac06e5592..000000000 --- a/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.google; - -import com.google.api.services.storage.Storage; -import com.google.api.services.storage.model.StorageObject; -import com.google.common.collect.Lists; -import java.io.IOException; -import java.util.Iterator; -import java.util.List; - -/* - * Represents a list of objects within Google Cloud Storage (GCS) - */ -public class GoogleFileIterator implements Iterator { - private Iterator iterator; - private String bucketName; - private String prefix; - private Storage.Objects objectsResoruceHandle = null; - private Storage.Objects.List listObjectsSrvcHandle = null; - private com.google.api.services.storage.model.Objects objectsContainerHandle = null; - - public GoogleFileIterator(Storage gcsStorageHandle, String bucket, String prefix) { - - this.objectsResoruceHandle = gcsStorageHandle.objects(); - this.bucketName = bucket; - this.prefix = prefix; - - try { // == Get the initial page of results - this.iterator = createIterator(); - } catch (Exception e) { - throw new RuntimeException( - "Exception encountered fetching elements, msg: ." + e.getLocalizedMessage(), e); - } - } - - private void initListing() { - try { - - this.listObjectsSrvcHandle = - objectsResoruceHandle.list(bucketName); // == list objects within bucket - // fetch elements within bucket that matches this prefix - this.listObjectsSrvcHandle.setPrefix(this.prefix); - } catch (IOException e) { - throw new RuntimeException("Unable to get gcslist handle to bucket: " + bucketName, e); - } - } - - /* - * Fetch a page of results - */ - private Iterator createIterator() throws Exception { - if (listObjectsSrvcHandle == null) initListing(); - List temp = Lists.newArrayList(); // a container of results - - // Sends the metadata request to the server and returns the parsed metadata response. - this.objectsContainerHandle = listObjectsSrvcHandle.execute(); - - for (StorageObject object : this.objectsContainerHandle.getItems()) { - // processing a page of results - temp.add(object.getName()); - } - return temp.iterator(); - } - - @Override - public boolean hasNext() { - if (this.iterator.hasNext()) { - return true; - } - - while (this.objectsContainerHandle.getNextPageToken() != null && !iterator.hasNext()) - try { // if here, you have iterated through all elements of the previous page, now, get - // the next page of results - this.listObjectsSrvcHandle.setPageToken(objectsContainerHandle.getNextPageToken()); - this.iterator = createIterator(); - } catch (Exception e) { - throw new RuntimeException( - "Exception encountered fetching elements, see previous messages for details.", - e); - } - - return this.iterator.hasNext(); - } - - @Override - public String next() { - return iterator.next(); - } -} diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java deleted file mode 100644 index 6d8e20cfa..000000000 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.resources; - -import com.netflix.priam.backup.*; -import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.config.IBackupRestoreConfig; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.scheduler.PriamScheduler; -import com.netflix.priam.utils.DateUtil; -import com.netflix.priam.utils.DateUtil.DateRange; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.*; -import java.util.stream.Collectors; -import javax.inject.Inject; -import javax.inject.Named; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Path("/v1/backup") -@Produces(MediaType.APPLICATION_JSON) -public class BackupServlet { - private static final Logger logger = LoggerFactory.getLogger(BackupServlet.class); - - private static final String REST_SUCCESS = "[\"ok\"]"; - private static final String REST_HEADER_RANGE = "daterange"; - private static final String REST_HEADER_FILTER = "filter"; - private final IConfiguration config; - private final IBackupRestoreConfig backupRestoreConfig; - private final IBackupFileSystem backupFs; - private final SnapshotBackup snapshotBackup; - private final BackupVerification backupVerification; - @Inject private PriamScheduler scheduler; - private final IBackupStatusMgr completedBkups; - private final BackupService backupService; - @Inject private MetaData metaData; - - @Inject - public BackupServlet( - IConfiguration config, - IBackupRestoreConfig backupRestoreConfig, - @Named("backup") IBackupFileSystem backupFs, - SnapshotBackup snapshotBackup, - IBackupStatusMgr completedBkups, - BackupVerification backupVerification, - BackupService backupService) { - this.config = config; - this.backupRestoreConfig = backupRestoreConfig; - this.backupFs = backupFs; - this.snapshotBackup = snapshotBackup; - this.completedBkups = completedBkups; - this.backupVerification = backupVerification; - this.backupService = backupService; - } - - @GET - @Path("/do_snapshot") - public Response backup() throws Exception { - snapshotBackup.execute(); - return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); - } - - @GET - @Path("/incremental_backup") - public Response backupIncrementals() throws Exception { - scheduler.addTask( - "IncrementalBackup", - IncrementalBackup.class, - IncrementalBackup.getTimer(config, backupRestoreConfig)); - return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); - } - - @GET - @Path("/updateService") - public Response updateService() throws Exception { - backupService.onChangeUpdateService(); - return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); - } - - @GET - @Path("/list") - /* - * Fetch the list of files for the requested date range. - * - * @param date range - * @param filter. The type of data files fetched. E.g. META will only fetch the daily snapshot meta data file (meta.json). - * @return the list of files in json format as part of the Http response body. - */ - public Response list( - @QueryParam(REST_HEADER_RANGE) String daterange, - @QueryParam(REST_HEADER_FILTER) @DefaultValue("") String filter) - throws Exception { - - logger.info( - "Parameters: {backupPrefix: [{}], daterange: [{}], filter: [{}]}", - config.getBackupPrefix(), - daterange, - filter); - - DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); - - Iterator it = - backupFs.list( - config.getBackupPrefix(), - Date.from(dateRange.getStartTime()), - Date.from(dateRange.getEndTime())); - JSONObject object = new JSONObject(); - object = constructJsonResponse(object, it, filter); - return Response.ok(object.toString(2), MediaType.APPLICATION_JSON).build(); - } - - @GET - @Path("/status") - @Produces(MediaType.APPLICATION_JSON) - public Response status() throws Exception { - JSONObject object = new JSONObject(); - object.put("SnapshotStatus", snapshotBackup.state().toString()); - return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); - } - - /* - * Determines the status of a snapshot for a date. If there was at least one successful snpashot for the date, snapshot - * for the date is considered completed. - * @param date date of the snapshot. Format of date is yyyymmdd - * @return {"Snapshotstatus":false} or {"Snapshotstatus":true} - */ - @GET - @Path("/status/{date}") - @Produces(MediaType.APPLICATION_JSON) - public Response statusByDate(@PathParam("date") String date) throws Exception { - Instant startTime = DateUtil.parseInstant(date); - Optional backupMetadataOptional = - this.completedBkups - .getLatestBackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, - new DateRange( - startTime.truncatedTo(ChronoUnit.DAYS), - startTime - .plus(1, ChronoUnit.DAYS) - .truncatedTo(ChronoUnit.DAYS))) - .stream() - .findFirst(); - - JSONObject object = new JSONObject(); - if (!backupMetadataOptional.isPresent()) { - object.put("Snapshotstatus", false); - } else { - - object.put("Snapshotstatus", true); - object.put("Details", new JSONObject(backupMetadataOptional.get().toString())); - } - return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); - } - - /* - * Determines the status of a snapshot for a date. If there was at least one successful snpashot for the date, snapshot - * for the date is considered completed. - * @param date date of the snapshot. Format of date is yyyymmdd - * @return {"Snapshots":["201606060450","201606060504"]} or "Snapshots":[]} - */ - @GET - @Path("/status/{date}/snapshots") - @Produces(MediaType.APPLICATION_JSON) - public Response snapshotsByDate(@PathParam("date") String date) throws Exception { - List metadata = this.completedBkups.locate(date); - JSONObject object = new JSONObject(); - List snapshots = new ArrayList<>(); - - if (metadata != null && !metadata.isEmpty()) - snapshots.addAll( - metadata.stream() - .filter( - backupMetadata -> - backupMetadata - .getBackupVersion() - .equals(BackupVersion.SNAPSHOT_BACKUP)) - .map( - backupMetadata -> - DateUtil.formatyyyyMMddHHmm(backupMetadata.getStart())) - .collect(Collectors.toList())); - - object.put("Snapshots", snapshots); - return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); - } - - /* - * Determines the validity of the backup by i) Downloading meta.json file ii) Listing of the backup directory - * iii) Find the missing or extra files in backup location. - * This by default takes the latest snapshot of the application. One can provide exact hour and min to check specific backup. - * Input: Daterange in the format of yyyyMMddHHmm,yyyyMMddHHmm OR yyyyMMdd,yyyyMMdd OR default - */ - @GET - @Path("/validate/snapshot/{daterange}") - @Produces(MediaType.APPLICATION_JSON) - public Response validateSnapshotByDate( - @PathParam("daterange") String daterange, - @DefaultValue("false") @QueryParam("force") boolean force) - throws Exception { - DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); - Optional result = - backupVerification.verifyLatestBackup( - BackupVersion.SNAPSHOT_BACKUP, force, dateRange); - if (!result.isPresent()) { - return Response.noContent() - .entity("No valid meta found for provided time range") - .build(); - } - - return Response.ok(result.get().toString()).build(); - } - - /* - * A list of files for requested filter. Currently, the only supported filter is META, all others will be ignore. - * For filter of META, ONLY the daily snapshot meta file (meta.json) are accounted for, not the incremental meta file. - * In addition, we do ONLY list the name of the meta data file, not the list of data files within it. - * - * @param handle to the json response - * @param a list of all files (data (*.db), and meta data file (*.json)) from S3 for requested dates. - * @param backup meta data file filter. Currently, the only supported filter is META, all others will be ignore. - * @return a list of files in Json format. - */ - private JSONObject constructJsonResponse( - JSONObject object, Iterator it, String filter) throws Exception { - int fileCnt = 0; - filter = filter.contains("?") ? filter.substring(0, filter.indexOf("?")) : filter; - - try { - JSONArray jArray = new JSONArray(); - while (it.hasNext()) { - AbstractBackupPath p = it.next(); - if (!filter.isEmpty() && BackupFileType.valueOf(filter) != p.getType()) continue; - JSONObject backupJSON = new JSONObject(); - backupJSON.put("bucket", config.getBackupPrefix()); - backupJSON.put("filename", p.getRemotePath()); - backupJSON.put("app", p.getClusterName()); - backupJSON.put("region", p.getRegion()); - backupJSON.put("token", p.getToken()); - backupJSON.put("ts", DateUtil.formatyyyyMMddHHmm(p.getTime())); - backupJSON.put( - "instance_id", p.getInstanceIdentity().getInstance().getInstanceId()); - backupJSON.put("uploaded_ts", DateUtil.formatyyyyMMddHHmm(p.getUploadedTs())); - if ("meta".equalsIgnoreCase(filter)) { // only check for existence of meta file - p.setFileName( - "meta.json"); // ignore incremental meta files, we are only interested - // in daily snapshot - if (metaData.doesExist(p)) { - // if here, snapshot completed. - fileCnt++; - jArray.put(backupJSON); - backupJSON.put("num_files", "1"); - } - } else { // account for every file (data, and meta) . - fileCnt++; - jArray.put(backupJSON); - } - } - object.put("files", jArray); - object.put("num_files", fileCnt); - } catch (JSONException jse) { - logger.info("Caught JSON Exception --> {}", jse.getMessage()); - } - return object; - } -} diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java index b55a1edef..5d2b6a4f6 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java @@ -62,7 +62,7 @@ public BackupServletV2( SnapshotMetaTask snapshotMetaService, BackupTTLTask backupTTLService, IConfiguration configuration, - IFileSystemContext backupFileSystemCtx, + IBackupFileSystem fileSystem, @Named("v2") IMetaProxy metaV2Proxy, Provider pathProvider, BackupV2Service backupService) { @@ -70,7 +70,7 @@ public BackupServletV2( this.backupVerification = backupVerification; this.snapshotMetaService = snapshotMetaService; this.backupTTLService = backupTTLService; - this.fs = backupFileSystemCtx.getFileStrategy(configuration); + this.fs = fileSystem; this.metaProxy = metaV2Proxy; this.pathProvider = pathProvider; this.backupService = backupService; @@ -110,7 +110,6 @@ public Response info(@PathParam("date") String date) { Instant instant = DateUtil.parseInstant(date); List metadataList = backupStatusMgr.getLatestBackupMetadata( - BackupVersion.SNAPSHOT_META_SERVICE, new DateRange( instant, instant.plus(1, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS))); @@ -125,8 +124,7 @@ public Response validateV2SnapshotByDate( throws Exception { DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); Optional result = - backupVerification.verifyLatestBackup( - BackupVersion.SNAPSHOT_META_SERVICE, force, dateRange); + backupVerification.verifyLatestBackup(force, dateRange); if (!result.isPresent()) { return Response.noContent() .entity("No valid meta found for provided time range") diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index 1074206e5..11a133adf 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -16,10 +16,11 @@ */ package com.netflix.priam.restore; -import com.netflix.priam.backup.*; -import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BackupRestoreUtil; +import com.netflix.priam.backup.IBackupFileSystem; +import com.netflix.priam.backup.Status; import com.netflix.priam.backupv2.IMetaProxy; -import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; @@ -29,7 +30,6 @@ import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.Sleeper; -import com.netflix.priam.utils.SystemUtils; import java.io.File; import java.io.IOException; import java.math.BigInteger; @@ -66,19 +66,12 @@ public abstract class AbstractRestore extends Task implements IRestoreStrategy { private final RestoreTokenSelector tokenSelector; private final ICassandraProcess cassProcess; private final InstanceState instanceState; - private final MetaData metaData; private final IPostRestoreHook postRestoreHook; - @Inject - @Named("v1") - IMetaProxy metaV1Proxy; - @Inject @Named("v2") IMetaProxy metaV2Proxy; - @Inject IBackupRestoreConfig backupRestoreConfig; - public AbstractRestore( IConfiguration config, IBackupFileSystem fs, @@ -88,7 +81,6 @@ public AbstractRestore( InstanceIdentity instanceIdentity, RestoreTokenSelector tokenSelector, ICassandraProcess cassProcess, - MetaData metaData, InstanceState instanceState, IPostRestoreHook postRestoreHook) { super(config); @@ -98,7 +90,6 @@ public AbstractRestore( this.instanceIdentity = instanceIdentity; this.tokenSelector = tokenSelector; this.cassProcess = cassProcess; - this.metaData = metaData; this.instanceState = instanceState; backupRestoreUtil = new BackupRestoreUtil( @@ -149,22 +140,6 @@ private void waitForCompletion(List> futureList) throws Exception { for (Future future : futureList) future.get(); } - private List> downloadCommitLogs( - Iterator fsIterator, int lastN, boolean waitForCompletion) - throws Exception { - if (fsIterator == null) return null; - - BoundedList bl = new BoundedList(lastN); - while (fsIterator.hasNext()) { - AbstractBackupPath temp = fsIterator.next(); - if (temp.getType() == BackupFileType.CL) { - bl.add(temp); - } - } - - return download(bl.iterator(), waitForCompletion); - } - private void stopCassProcess() throws IOException { cassProcess.stop(true); } @@ -195,8 +170,7 @@ public void restore(DateUtil.DateRange dateRange) throws Exception { } Date endTime = new Date(dateRange.getEndTime().toEpochMilli()); - IMetaProxy metaProxy = metaV1Proxy; - if (backupRestoreConfig.enableV2Restore()) metaProxy = metaV2Proxy; + IMetaProxy metaProxy = metaV2Proxy; // Set the restore status. instanceState.getRestoreStatus().resetStatus(); @@ -255,24 +229,6 @@ public void restore(DateUtil.DateRange dateRange) throws Exception { List> futureList = new ArrayList<>(); futureList.addAll(download(allFiles.iterator(), false)); - // Downloading CommitLogs - // Note for Backup V2.0 we do not backup commit logs, as saving them is cost-expensive. - if (config.isBackingUpCommitLogs()) { - logger.info( - "Delete all backuped commitlog files in {}", - config.getBackupCommitLogLocation()); - SystemUtils.cleanupDir(config.getBackupCommitLogLocation(), null); - - logger.info("Delete all commitlog files in {}", config.getCommitLogLocation()); - SystemUtils.cleanupDir(config.getCommitLogLocation(), null); - String prefix = fs.getPrefix().toString(); - Iterator commitLogPathIterator = - fs.list(prefix, latestValidMetaFile.get().getTime(), endTime); - futureList.addAll( - downloadCommitLogs( - commitLogPathIterator, config.maxCommitLogsRestore(), false)); - } - // Wait for all the futures to finish. waitForCompletion(futureList); diff --git a/priam/src/main/java/com/netflix/priam/restore/AwsCrossAccountCryptographyRestoreStrategy.java b/priam/src/main/java/com/netflix/priam/restore/AwsCrossAccountCryptographyRestoreStrategy.java deleted file mode 100755 index 565fe764d..000000000 --- a/priam/src/main/java/com/netflix/priam/restore/AwsCrossAccountCryptographyRestoreStrategy.java +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.restore; - -import com.netflix.priam.aws.S3CrossAccountFileSystem; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.MetaData; -import com.netflix.priam.compress.ICompression; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredentialGeneric; -import com.netflix.priam.cryptography.IFileCryptography; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.health.InstanceState; -import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.scheduler.SimpleTimer; -import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.utils.Sleeper; -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Provider; -import javax.inject.Singleton; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/* - * A strategy to restore from an AWS bucket whose objects are not owned by the current IAM role thus requiring AWS cross account assumption. - * In addition, this strategy will handle data which has been encrypted. - */ - -@Singleton -public class AwsCrossAccountCryptographyRestoreStrategy extends EncryptedRestoreBase { - private static final Logger logger = - LoggerFactory.getLogger(AwsCrossAccountCryptographyRestoreStrategy.class); - public static final String JOBNAME = "AWS_CROSS_ACCT_CRYPTOGRAPHY_RESTORE_JOB"; - - // Note: see javadoc for S3CrossAccountFileSystem for reason why we inject a concrete class - // (S3CrossAccountFileSystem) instead of the inteface IBackupFileSystem - @Inject - public AwsCrossAccountCryptographyRestoreStrategy( - final IConfiguration config, - ICassandraProcess cassProcess, - S3CrossAccountFileSystem crossAcctfs, - Sleeper sleeper, - @Named("filecryptoalgorithm") IFileCryptography fileCryptography, - @Named("pgpcredential") ICredentialGeneric credential, - ICompression compress, - Provider pathProvider, - InstanceIdentity id, - RestoreTokenSelector tokenSelector, - MetaData metaData, - InstanceState instanceState, - IPostRestoreHook postRestoreHook) { - - super( - config, - crossAcctfs.getBackupFileSystem(), - JOBNAME, - sleeper, - cassProcess, - pathProvider, - id, - tokenSelector, - credential, - fileCryptography, - compress, - metaData, - instanceState, - postRestoreHook); - } - - /** @return a timer used by the scheduler to determine when "this" should be run. */ - public static TaskTimer getTimer() { - return new SimpleTimer(JOBNAME); - } -} diff --git a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java deleted file mode 100755 index 59842d456..000000000 --- a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java +++ /dev/null @@ -1,208 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.restore; - -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.backup.MetaData; -import com.netflix.priam.compress.CompressionType; -import com.netflix.priam.compress.ICompression; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredentialGeneric; -import com.netflix.priam.cryptography.IFileCryptography; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.health.InstanceState; -import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.scheduler.NamedThreadPoolExecutor; -import com.netflix.priam.utils.RetryableCallable; -import com.netflix.priam.utils.Sleeper; -import java.io.*; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadPoolExecutor; -import javax.inject.Provider; -import org.bouncycastle.util.io.Streams; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Provides common functionality applicable to all restore strategies */ -public abstract class EncryptedRestoreBase extends AbstractRestore { - private static final Logger logger = LoggerFactory.getLogger(EncryptedRestoreBase.class); - private static final String TMP_SUFFIX = ".tmp"; - - private final String jobName; - private final ICredentialGeneric pgpCredential; - private final IFileCryptography fileCryptography; - private final ICompression compress; - private final ThreadPoolExecutor executor; - - protected EncryptedRestoreBase( - IConfiguration config, - IBackupFileSystem fs, - String jobName, - Sleeper sleeper, - ICassandraProcess cassProcess, - Provider pathProvider, - InstanceIdentity instanceIdentity, - RestoreTokenSelector tokenSelector, - ICredentialGeneric pgpCredential, - IFileCryptography fileCryptography, - ICompression compress, - MetaData metaData, - InstanceState instanceState, - IPostRestoreHook postRestoreHook) { - super( - config, - fs, - jobName, - sleeper, - pathProvider, - instanceIdentity, - tokenSelector, - cassProcess, - metaData, - instanceState, - postRestoreHook); - - this.jobName = jobName; - this.pgpCredential = pgpCredential; - this.fileCryptography = fileCryptography; - this.compress = compress; - executor = new NamedThreadPoolExecutor(config.getRestoreThreads(), jobName); - executor.allowCoreThreadTimeOut(true); - logger.info( - "Trying to restore cassandra cluster with filesystem: {}, RestoreStrategy: {}, Encryption: ON, Compression: {}", - fs.getClass(), - jobName, - compress.getClass()); - } - - @Override - protected final Future downloadFile(final AbstractBackupPath path) throws Exception { - final char[] passPhrase = - new String(this.pgpCredential.getValue(ICredentialGeneric.KEY.PGP_PASSWORD)) - .toCharArray(); - File restoreLocation = path.newRestoreFile(); - File tempFile = new File(restoreLocation.getAbsolutePath() + TMP_SUFFIX); - - return executor.submit( - new RetryableCallable() { - - @Override - public Path retriableCall() throws Exception { - - // == download object from source bucket - try { - // Not retrying to download file here as it is already in RetryCallable. - fs.downloadFile(path, TMP_SUFFIX, 0 /* retries */); - } catch (Exception ex) { - // This behavior is retryable; therefore, lets get to a clean state - // before each retry. - if (tempFile.exists()) { - tempFile.createNewFile(); - } - - throw new Exception( - "Exception downloading file from: " - + path.getRemotePath() - + " to: " - + tempFile.getAbsolutePath(), - ex); - } - - // == object downloaded successfully from source, decrypt it. - File decryptedFile = new File(tempFile.getAbsolutePath() + ".decrypted"); - try (OutputStream fOut = - new BufferedOutputStream( - new FileOutputStream( - decryptedFile)); // destination file after - // decryption) - InputStream in = - new BufferedInputStream( - new FileInputStream(tempFile.getAbsolutePath()))) { - InputStream encryptedDataInputStream = - fileCryptography.decryptStream( - in, passPhrase, tempFile.getAbsolutePath()); - Streams.pipeAll(encryptedDataInputStream, fOut); - logger.info( - "Completed decrypting file: {} to final file dest: {}", - tempFile.getAbsolutePath(), - decryptedFile.getAbsolutePath()); - - } catch (Exception ex) { - // This behavior is retryable; therefore, lets get to a clean state - // before each retry. - if (tempFile.exists()) { - tempFile.createNewFile(); - } - - if (decryptedFile.exists()) { - decryptedFile.createNewFile(); - } - - throw new Exception( - "Exception during decryption file: " - + decryptedFile.getAbsolutePath(), - ex); - } - - // == object is downloaded and decrypted, now uncompress it if necessary - if (path.getCompression() == CompressionType.NONE) { - Files.move(decryptedFile.toPath(), restoreLocation.toPath()); - } else { - logger.info( - "Start uncompressing file: {} to the FINAL destination stream", - decryptedFile.getAbsolutePath()); - - try (InputStream is = - new BufferedInputStream( - new FileInputStream(decryptedFile)); - BufferedOutputStream finalDestination = - new BufferedOutputStream( - new FileOutputStream(restoreLocation))) { - compress.decompressAndClose(is, finalDestination); - } catch (Exception ex) { - throw new Exception( - "Exception uncompressing file: " - + decryptedFile.getAbsolutePath() - + " to the FINAL destination stream", - ex); - } - - logger.info( - "Completed uncompressing file: {} to the FINAL destination stream " - + " current worker: {}", - decryptedFile.getAbsolutePath(), - Thread.currentThread().getName()); - } - // if here, everything was successful for this object, lets remove unneeded - // file(s) - if (tempFile.exists()) tempFile.delete(); - - if (decryptedFile.exists()) { - decryptedFile.delete(); - } - - return Paths.get(path.getRemotePath()); - } - }); - } - - @Override - public String getName() { - return this.jobName; - } -} diff --git a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreStrategy.java b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreStrategy.java deleted file mode 100755 index 48197758c..000000000 --- a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreStrategy.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.restore; - -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.backup.MetaData; -import com.netflix.priam.compress.ICompression; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredentialGeneric; -import com.netflix.priam.cryptography.IFileCryptography; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.health.InstanceState; -import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.scheduler.SimpleTimer; -import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.utils.Sleeper; -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Provider; -import javax.inject.Singleton; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/* - * A strategy to restore encrypted data from a primary AWS account - */ -@Singleton -public class EncryptedRestoreStrategy extends EncryptedRestoreBase { - private static final Logger logger = LoggerFactory.getLogger(EncryptedRestoreStrategy.class); - public static final String JOBNAME = "CRYPTOGRAPHY_RESTORE_JOB"; - - @Inject - public EncryptedRestoreStrategy( - final IConfiguration config, - ICassandraProcess cassProcess, - @Named("encryptedbackup") IBackupFileSystem fs, - Sleeper sleeper, - @Named("filecryptoalgorithm") IFileCryptography fileCryptography, - @Named("pgpcredential") ICredentialGeneric credential, - ICompression compress, - Provider pathProvider, - InstanceIdentity id, - RestoreTokenSelector tokenSelector, - MetaData metaData, - InstanceState instanceState, - IPostRestoreHook postRestoreHook) { - - super( - config, - fs, - JOBNAME, - sleeper, - cassProcess, - pathProvider, - id, - tokenSelector, - credential, - fileCryptography, - compress, - metaData, - instanceState, - postRestoreHook); - } - - /* - * @return a timer used by the scheduler to determine when "this" should be run. - */ - public static TaskTimer getTimer() { - return new SimpleTimer(JOBNAME); - } -} diff --git a/priam/src/main/java/com/netflix/priam/restore/GoogleCryptographyRestoreStrategy.java b/priam/src/main/java/com/netflix/priam/restore/GoogleCryptographyRestoreStrategy.java deleted file mode 100755 index c735d254e..000000000 --- a/priam/src/main/java/com/netflix/priam/restore/GoogleCryptographyRestoreStrategy.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.restore; - -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.backup.MetaData; -import com.netflix.priam.compress.ICompression; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredentialGeneric; -import com.netflix.priam.cryptography.IFileCryptography; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.health.InstanceState; -import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.scheduler.SimpleTimer; -import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.utils.Sleeper; -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Provider; -import javax.inject.Singleton; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Singleton -public class GoogleCryptographyRestoreStrategy extends EncryptedRestoreBase { - private static final Logger logger = - LoggerFactory.getLogger(GoogleCryptographyRestoreStrategy.class); - public static final String JOBNAME = "GOOGLECLOUDSTORAGE_RESTORE_JOB"; - - @Inject - public GoogleCryptographyRestoreStrategy( - final IConfiguration config, - ICassandraProcess cassProcess, - @Named("gcsencryptedbackup") IBackupFileSystem fs, - Sleeper sleeper, - @Named("filecryptoalgorithm") IFileCryptography fileCryptography, - @Named("pgpcredential") ICredentialGeneric credential, - ICompression compress, - Provider pathProvider, - InstanceIdentity id, - RestoreTokenSelector tokenSelector, - MetaData metaData, - InstanceState instanceState, - IPostRestoreHook postRestoreHook) { - super( - config, - fs, - JOBNAME, - sleeper, - cassProcess, - pathProvider, - id, - tokenSelector, - credential, - fileCryptography, - compress, - metaData, - instanceState, - postRestoreHook); - } - - /** @return a timer used by the scheduler to determine when "this" should be run. */ - public static TaskTimer getTimer() { - return new SimpleTimer(JOBNAME); - } -} diff --git a/priam/src/main/java/com/netflix/priam/restore/Restore.java b/priam/src/main/java/com/netflix/priam/restore/Restore.java index 6e7e7b906..6dba06149 100644 --- a/priam/src/main/java/com/netflix/priam/restore/Restore.java +++ b/priam/src/main/java/com/netflix/priam/restore/Restore.java @@ -18,7 +18,6 @@ import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.backup.MetaData; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; @@ -29,7 +28,6 @@ import java.nio.file.Path; import java.util.concurrent.Future; import javax.inject.Inject; -import javax.inject.Named; import javax.inject.Provider; import javax.inject.Singleton; import org.slf4j.Logger; @@ -44,13 +42,12 @@ public class Restore extends AbstractRestore { @Inject public Restore( IConfiguration config, - @Named("backup") IBackupFileSystem fs, + IBackupFileSystem fs, Sleeper sleeper, ICassandraProcess cassProcess, Provider pathProvider, InstanceIdentity instanceIdentity, RestoreTokenSelector tokenSelector, - MetaData metaData, InstanceState instanceState, IPostRestoreHook postRestoreHook) { super( @@ -62,7 +59,6 @@ public Restore( instanceIdentity, tokenSelector, cassProcess, - metaData, instanceState, postRestoreHook); } diff --git a/priam/src/main/java/com/netflix/priam/restore/RestoreContext.java b/priam/src/main/java/com/netflix/priam/restore/RestoreContext.java index 2252acfc9..f88c57ddb 100755 --- a/priam/src/main/java/com/netflix/priam/restore/RestoreContext.java +++ b/priam/src/main/java/com/netflix/priam/restore/RestoreContext.java @@ -15,7 +15,6 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.PriamScheduler; -import com.netflix.priam.scheduler.UnsupportedTypeException; import javax.inject.Inject; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -41,113 +40,10 @@ public boolean isRestoreEnabled() { public void restore() throws Exception { if (!isRestoreEnabled()) return; - - // Restore is required. - if (StringUtils.isEmpty(config.getRestoreSourceType()) && !config.isRestoreEncrypted()) { - // Restore is needed and it will be done from the primary AWS account - scheduler.addTask( - Restore.JOBNAME, - Restore.class, - Restore.getTimer()); // restore from the AWS primary acct - logger.info("Scheduled task " + Restore.JOBNAME); - } else if (config.isRestoreEncrypted()) { - SourceType sourceType = SourceType.lookup(config.getRestoreSourceType(), true, false); - - if (sourceType == null) { - scheduler.addTask( - EncryptedRestoreStrategy.JOBNAME, - EncryptedRestoreStrategy.class, - EncryptedRestoreStrategy.getTimer()); - logger.info("Scheduled task " + Restore.JOBNAME); - return; - } - - switch (sourceType) { - case AWSCROSSACCT: - scheduler.addTask( - AwsCrossAccountCryptographyRestoreStrategy.JOBNAME, - AwsCrossAccountCryptographyRestoreStrategy.class, - AwsCrossAccountCryptographyRestoreStrategy.getTimer()); - logger.info( - "Scheduled task " + AwsCrossAccountCryptographyRestoreStrategy.JOBNAME); - break; - - case GOOGLE: - scheduler.addTask( - GoogleCryptographyRestoreStrategy.JOBNAME, - GoogleCryptographyRestoreStrategy.class, - GoogleCryptographyRestoreStrategy.getTimer()); - logger.info("Scheduled task " + GoogleCryptographyRestoreStrategy.JOBNAME); - break; - } - } - } - - enum SourceType { - AWSCROSSACCT("AWSCROSSACCT"), - GOOGLE("GOOGLE"); - - private static final Logger logger = LoggerFactory.getLogger(SourceType.class); - - private final String sourceType; - - SourceType(String sourceType) { - this.sourceType = sourceType.toUpperCase(); - } - - public static SourceType lookup( - String sourceType, boolean acceptNullOrEmpty, boolean acceptIllegalValue) - throws UnsupportedTypeException { - if (StringUtils.isEmpty(sourceType)) - if (acceptNullOrEmpty) return null; - else { - String message = - String.format( - "%s is not a supported SourceType. Supported values are %s", - sourceType, getSupportedValues()); - logger.error(message); - throw new UnsupportedTypeException(message); - } - - try { - return SourceType.valueOf(sourceType.toUpperCase()); - } catch (IllegalArgumentException ex) { - String message = - String.format( - "%s is not a supported SourceType. Supported values are %s", - sourceType, getSupportedValues()); - - if (acceptIllegalValue) { - message = - message - + ". Since acceptIllegalValue is set to True, returning NULL instead."; - logger.error(message); - return null; - } - - logger.error(message); - throw new UnsupportedTypeException(message, ex); - } - } - - private static String getSupportedValues() { - StringBuilder supportedValues = new StringBuilder(); - boolean first = true; - for (SourceType type : SourceType.values()) { - if (!first) supportedValues.append(","); - supportedValues.append(type); - first = false; - } - - return supportedValues.toString(); - } - - public static SourceType lookup(String sourceType) throws UnsupportedTypeException { - return lookup(sourceType, false, false); - } - - public String getSourceType() { - return sourceType; - } + scheduler.addTask( + Restore.JOBNAME, + Restore.class, + Restore.getTimer()); // restore from the AWS primary acct + logger.info("Scheduled task " + Restore.JOBNAME); } } diff --git a/priam/src/main/java/com/netflix/priam/restore/RestoreTokenSelector.java b/priam/src/main/java/com/netflix/priam/restore/RestoreTokenSelector.java index 016dcfd07..4d6701143 100644 --- a/priam/src/main/java/com/netflix/priam/restore/RestoreTokenSelector.java +++ b/priam/src/main/java/com/netflix/priam/restore/RestoreTokenSelector.java @@ -25,7 +25,6 @@ import java.util.Iterator; import java.util.List; import javax.inject.Inject; -import javax.inject.Named; /** Runs algorithms as finding closest token from a list of token (in a backup) */ public class RestoreTokenSelector { @@ -33,7 +32,7 @@ public class RestoreTokenSelector { private final IBackupFileSystem fs; @Inject - public RestoreTokenSelector(ITokenManager tokenManager, @Named("backup") IBackupFileSystem fs) { + public RestoreTokenSelector(ITokenManager tokenManager, IBackupFileSystem fs) { this.tokenManager = tokenManager; this.fs = fs; diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index 7bf55e89a..b66bed266 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -23,7 +23,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Properties; import javax.inject.Inject; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -156,9 +155,6 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed logger.info(yaml.dump(map)); yaml.dump(map, new FileWriter(yamlFile)); - // TODO: port commit log backups to the PropertiesFileTuner implementation - configureCommitLogBackups(); - PropertiesFileTuner propertyTuner = new PropertiesFileTuner(config); for (String propertyFile : config.getTunablePropertyFiles()) { propertyTuner.updateAndSaveProperties(propertyFile); @@ -225,22 +221,6 @@ protected void configfureSecurity(Map map) { serverEnc.put("internode_encryption", config.getInternodeEncryption()); } - protected void configureCommitLogBackups() { - if (!config.isBackingUpCommitLogs()) return; - Properties props = new Properties(); - props.put("archive_command", config.getCommitLogBackupArchiveCmd()); - props.put("restore_command", config.getCommitLogBackupRestoreCmd()); - props.put("restore_directories", config.getCommitLogBackupRestoreFromDirs()); - props.put("restore_point_in_time", config.getCommitLogBackupRestorePointInTime()); - - try (FileOutputStream fos = - new FileOutputStream(new File(config.getCommitLogBackupPropsFile()))) { - props.store(fos, "cassandra commit log archive props, as written by priam"); - } catch (IOException e) { - logger.error("Could not store commitlog_archiving.properties", e); - } - } - public void updateAutoBootstrap(String yamlFile, boolean autobootstrap) throws IOException { DumperOptions options = new DumperOptions(); options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); diff --git a/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy b/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy deleted file mode 100644 index d5baaf0b1..000000000 --- a/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.backup - -import com.netflix.priam.config.FakeConfiguration -import spock.lang.Specification -import spock.lang.Unroll - -/** - Created by aagrawal on 11/7/17. - */ -@Unroll -class TestBackupScheduler extends Specification { - def "IsBackupEnabled CRON #configCRON is #result"() { - expect: - SnapshotBackup.isBackupEnabled(new BackupConfiguration(configCRON)) == result - - where: - configCRON || result - "-1" || false - "0 0 9 1/1 * ? *" || true - } - - def "Exception for illegal value of Snapshot CRON expression , #configCRON"() { - when: - SnapshotBackup.isBackupEnabled(new BackupConfiguration(configCRON)) - - then: - thrown(expectedException) - - where: - configCRON || expectedException - "abc" || Exception - "0 9 1/1 * ? *"|| Exception - } - - def "Validate CRON for backup CRON #configCRON is #result"() { - expect: - SnapshotBackup.getTimer(new BackupConfiguration(configCRON)).cronExpression == result - - where: - configCRON || result - "0 0 9 1/1 * ? *" || "0 0 9 1/1 * ? *" - } - - private class BackupConfiguration extends FakeConfiguration { - private String backupCronExpression - - BackupConfiguration(String backupCronExpression) { - this.backupCronExpression = backupCronExpression - } - - @Override - String getBackupCronExpression() { - return backupCronExpression - } - } - -} diff --git a/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java b/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java index bc75b66a2..166826059 100644 --- a/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java +++ b/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java @@ -43,93 +43,6 @@ public TestRemoteBackupPath() { Guice.createInjector(new BRTestModule()).getProvider(AbstractBackupPath.class); } - @Test - public void testV1BackupPathsSST() { - Path path = - Paths.get("target/data", "keyspace1", "columnfamily1", "backup", "mc-1234-Data.db"); - AbstractBackupPath abstractBackupPath = pathFactory.get(); - abstractBackupPath.parseLocal(path.toFile(), BackupFileType.SST); - - // Verify parse local - Assert.assertEquals( - 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. - Assert.assertEquals("keyspace1", abstractBackupPath.getKeyspace()); - Assert.assertEquals("columnfamily1", abstractBackupPath.getColumnFamily()); - Assert.assertEquals(BackupFileType.SST, abstractBackupPath.getType()); - Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); - Assert.assertEquals( - 0, - abstractBackupPath - .getTime() - .toInstant() - .toEpochMilli()); // Since file do not exist. - - // Verify toRemote and parseRemote. - String remotePath = abstractBackupPath.getRemotePath(); - logger.info(remotePath); - AbstractBackupPath abstractBackupPath2 = pathFactory.get(); - abstractBackupPath2.parseRemote(remotePath); - validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); - Assert.assertEquals(abstractBackupPath.getTime(), abstractBackupPath2.getTime()); - } - - @Test - public void testV1BackupPathsSnap() { - Path path = - Paths.get( - "target/data", - "keyspace1", - "columnfamily1", - "snapshot", - "201801011201", - "mc-1234-Data.db"); - AbstractBackupPath abstractBackupPath = pathFactory.get(); - abstractBackupPath.parseLocal(path.toFile(), BackupFileType.SNAP); - - // Verify parse local - Assert.assertEquals( - 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. - Assert.assertEquals("keyspace1", abstractBackupPath.getKeyspace()); - Assert.assertEquals("columnfamily1", abstractBackupPath.getColumnFamily()); - Assert.assertEquals(BackupFileType.SNAP, abstractBackupPath.getType()); - Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); - Assert.assertEquals( - "201801011201", DateUtil.formatyyyyMMddHHmm(abstractBackupPath.getTime())); - - // Verify toRemote and parseRemote. - String remotePath = abstractBackupPath.getRemotePath(); - logger.info(remotePath); - - AbstractBackupPath abstractBackupPath2 = pathFactory.get(); - abstractBackupPath2.parseRemote(remotePath); - validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); - Assert.assertEquals(abstractBackupPath.getTime(), abstractBackupPath2.getTime()); - } - - @Test - public void testV1BackupPathsMeta() { - Path path = Paths.get("target/data", "meta.json"); - AbstractBackupPath abstractBackupPath = pathFactory.get(); - abstractBackupPath.parseLocal(path.toFile(), BackupFileType.META); - - // Verify parse local - Assert.assertEquals( - 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. - Assert.assertNull(abstractBackupPath.getKeyspace()); - Assert.assertNull(abstractBackupPath.getColumnFamily()); - Assert.assertEquals(BackupFileType.META, abstractBackupPath.getType()); - Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); - - // Verify toRemote and parseRemote. - String remotePath = abstractBackupPath.getRemotePath(); - logger.info(remotePath); - - AbstractBackupPath abstractBackupPath2 = pathFactory.get(); - abstractBackupPath2.parseRemote(remotePath); - validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); - Assert.assertEquals(abstractBackupPath.getTime(), abstractBackupPath2.getTime()); - } - @Test public void testV2BackupPathSST() { Path path = diff --git a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java index 6a1425f49..0e87702ee 100644 --- a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java +++ b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java @@ -23,15 +23,12 @@ import com.netflix.priam.aws.auth.IS3Credential; import com.netflix.priam.aws.auth.S3RoleAssumptionCredential; import com.netflix.priam.backupv2.IMetaProxy; -import com.netflix.priam.backupv2.MetaV1Proxy; import com.netflix.priam.backupv2.MetaV2Proxy; import com.netflix.priam.config.FakeBackupRestoreConfig; import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cred.ICredential; -import com.netflix.priam.cryptography.IFileCryptography; -import com.netflix.priam.cryptography.pgp.PgpCryptography; import com.netflix.priam.defaultimpl.FakeCassandraProcess; import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.identity.FakeMembership; @@ -68,26 +65,16 @@ protected void configure() { bind(IMembership.class) .toInstance(new FakeMembership(Collections.singletonList("fakeInstance1"))); bind(ICredential.class).to(FakeNullCredential.class).in(Scopes.SINGLETON); - bind(IBackupFileSystem.class) - .annotatedWith(Names.named("backup")) - .to(FakeBackupFileSystem.class) - .in(Scopes.SINGLETON); + bind(IBackupFileSystem.class).to(FakeBackupFileSystem.class).in(Scopes.SINGLETON); bind(Sleeper.class).to(FakeSleeper.class); bind(IS3Credential.class) .annotatedWith(Names.named("awss3roleassumption")) .to(S3RoleAssumptionCredential.class); - bind(IBackupFileSystem.class) - .annotatedWith(Names.named("encryptedbackup")) - .to(NullBackupFileSystem.class); - bind(IFileCryptography.class) - .annotatedWith(Names.named("filecryptoalgorithm")) - .to(PgpCryptography.class); bind(ICassandraProcess.class).to(FakeCassandraProcess.class); bind(IPostRestoreHook.class).to(FakePostRestoreHook.class); bind(Registry.class).toInstance(new DefaultRegistry()); - bind(IMetaProxy.class).annotatedWith(Names.named("v1")).to(MetaV1Proxy.class); bind(IMetaProxy.class).annotatedWith(Names.named("v2")).to(MetaV2Proxy.class); bind(DynamicRateLimiter.class).to(FakeDynamicRateLimiter.class); bind(Clock.class).toInstance(Clock.fixed(Instant.EPOCH, ZoneId.systemDefault())); diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index 0750179fd..91501aa02 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -17,20 +17,18 @@ package com.netflix.priam.backup; -import com.netflix.priam.aws.RemoteBackupPath; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; -import java.io.File; -import java.io.FileWriter; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Instant; import java.util.*; import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; -import org.json.simple.JSONArray; @Singleton public class FakeBackupFileSystem extends AbstractFileSystem { @@ -71,31 +69,6 @@ public void addFile(String file) { flist.add(path); } - @SuppressWarnings("unchecked") - @Override - public Iterator list(String bucket, Date start, Date till) { - String[] paths = bucket.split(String.valueOf(RemoteBackupPath.PATH_SEP)); - - if (paths.length > 1) { - baseDir = paths[1]; - region = paths[2]; - clusterName = paths[3]; - } - - List tmpList = new ArrayList<>(); - for (AbstractBackupPath path : flist) { - - if ((path.time.after(start) && path.time.before(till)) - || path.time.equals(start) - && path.baseDir.equals(baseDir) - && path.clusterName.equals(clusterName) - && path.region.equals(region)) { - tmpList.add(path); - } - } - return tmpList.iterator(); - } - @Override public Iterator listFileSystem(String prefix, String delimiter, String marker) { ArrayList items = new ArrayList<>(); @@ -146,21 +119,21 @@ public void cleanup() { @Override protected void downloadFileImpl(AbstractBackupPath path, String suffix) throws BackupRestoreException { - File localFile = new File(path.newRestoreFile().getAbsolutePath() + suffix); - if (path.getType() == AbstractBackupPath.BackupFileType.META) { - // List all files and generate the file - try (FileWriter fr = new FileWriter(localFile)) { - JSONArray jsonObj = new JSONArray(); - for (AbstractBackupPath filePath : flist) { - if (filePath.type == AbstractBackupPath.BackupFileType.SNAP - && filePath.time.equals(path.time)) { - jsonObj.add(filePath.getRemotePath()); - } + if (path.getType() == AbstractBackupPath.BackupFileType.META_V2) { + Path destination = Paths.get(path.newRestoreFile().getAbsolutePath() + suffix); + if (!destination.toFile().exists()) { + Path origin = + Paths.get( + "src/test/resources/" + + path.getClusterName() + + "_" + + Paths.get(path.getRemotePath()).getFileName()) + .toAbsolutePath(); + try { + Files.copy(origin, destination); + } catch (IOException io) { + throw new BackupRestoreException(io.getMessage(), io); } - fr.write(jsonObj.toJSONString()); - fr.flush(); - } catch (IOException io) { - throw new BackupRestoreException(io.getMessage(), io); } } downloadedFiles.add(path.getRemotePath()); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java index 276ace860..672fa0ab4 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java @@ -21,7 +21,6 @@ import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.Key; -import com.google.inject.name.Names; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; @@ -47,19 +46,20 @@ * @author Praveen Sadhu */ public class TestBackup { - private static Injector injector; - private static FakeBackupFileSystem filesystem; + private Injector injector; + private FakeBackupFileSystem filesystem; private static final Logger logger = LoggerFactory.getLogger(TestBackup.class); private static final Set expectedFiles = new HashSet<>(); @BeforeClass public static void setup() throws InterruptedException, IOException { new MockNodeProbe(); + } + + @Before + public void init() { injector = Guice.createInjector(new BRTestModule()); - filesystem = - (FakeBackupFileSystem) - injector.getInstance( - Key.get(IBackupFileSystem.class, Names.named("backup"))); + filesystem = (FakeBackupFileSystem) injector.getInstance(Key.get(IBackupFileSystem.class)); } @AfterClass @@ -68,29 +68,6 @@ public static void cleanup() throws IOException { FileUtils.deleteQuietly(file); } - @Test - public void testSnapshotBackup() throws Exception { - filesystem.cleanup(); - SnapshotBackup backup = injector.getInstance(SnapshotBackup.class); - - // - // backup.execute(); - // Assert.assertEquals(3, filesystem.uploadedFiles.size()); - // System.out.println("***** "+filesystem.uploadedFiles.size()); - // boolean metafile = false; - // for (String filePath : expectedFiles) - // Assert.assertTrue(filesystem.uploadedFiles.contains(filePath)); - // - // for(String filepath : filesystem.uploadedFiles){ - // if( filepath.endsWith("meta.json")){ - // metafile = true; - // break; - // } - // } - // Assert.assertTrue(metafile); - - } - @Test public void testIncrementalBackup() throws Exception { filesystem.cleanup(); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java index 52478f3f6..97bc60e6e 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java @@ -22,13 +22,8 @@ import com.netflix.priam.aws.RemoteBackupPath; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.utils.DateUtil; -import java.io.BufferedOutputStream; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.sql.Date; -import java.text.ParseException; import org.apache.commons.io.FileUtils; import org.junit.AfterClass; import org.junit.Assert; @@ -42,40 +37,24 @@ public class TestBackupFile { @BeforeClass public static void setup() throws IOException { injector = Guice.createInjector(new BRTestModule()); - File file = - new File("target/data/Keyspace1/Standard1/", "Keyspace1-Standard1-ia-5-Data.db"); - if (!file.exists()) { - File dir1 = new File("target/data/Keyspace1/Standard1/"); - if (!dir1.exists()) dir1.mkdirs(); - byte b = 8; - long oneKB = (1024L); - System.out.println(oneKB); - BufferedOutputStream bos1 = new BufferedOutputStream(new FileOutputStream(file)); - for (long i = 0; i < oneKB; i++) { - bos1.write(b); - } - bos1.flush(); - bos1.close(); - } InstanceIdentity factory = injector.getInstance(InstanceIdentity.class); - factory.getInstance().setToken("1234567"); // Token + factory.getInstance().setToken("1234567"); region = factory.getInstanceInfo().getRegion(); } @AfterClass public static void cleanup() throws IOException { - File file = new File("Keyspace1-Standard1-ia-5-Data.db"); - FileUtils.deleteQuietly(file); + FileUtils.deleteDirectory(new File("target")); } @Test - public void testBackupFileCreation() throws ParseException { - // Test snapshot file - String snapshotfile = - "target/data/Keyspace1/Standard1/snapshots/201108082320/Keyspace1-Standard1-ia-5-Data.db"; + public void testBackupFileCreation() throws IOException { + File file = + createFile( + "target/data/Keyspace1/Standard1/snapshots/201108082320/Keyspace1-Standard1-ia-5-Data.db"); RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); - backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); - Assert.assertEquals(BackupFileType.SNAP, backupfile.type); + backupfile.parseLocal(file, BackupFileType.SST_V2); + Assert.assertEquals(BackupFileType.SST_V2, backupfile.type); Assert.assertEquals("Keyspace1", backupfile.keyspace); Assert.assertEquals("Standard1", backupfile.columnFamily); Assert.assertEquals("1234567", backupfile.token); @@ -83,50 +62,50 @@ public void testBackupFileCreation() throws ParseException { Assert.assertEquals(region, backupfile.region); Assert.assertEquals("casstestbackup", backupfile.baseDir); Assert.assertEquals( - "casstestbackup/" - + region - + "/fake-app/1234567/201108082320/SNAP/Keyspace1/Standard1/Keyspace1-Standard1-ia-5-Data.db", + "casstestbackup/1049_fake-app/1234567/SST_V2/1699206297000/Keyspace1/Standard1/SNAPPY/PLAINTEXT/Keyspace1-Standard1-ia-5-Data.db", backupfile.getRemotePath()); } @Test - public void testIncBackupFileCreation() throws ParseException { - // Test incremental file - File bfile = new File("target/data/Keyspace1/Standard1/Keyspace1-Standard1-ia-5-Data.db"); + public void testIncBackupFileCreation() throws IOException { + File file = + createFile( + "target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-5-Data.db"); RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); - backupfile.parseLocal(bfile, BackupFileType.SST); - Assert.assertEquals(BackupFileType.SST, backupfile.type); + backupfile.parseLocal(file, BackupFileType.SST_V2); + Assert.assertEquals(BackupFileType.SST_V2, backupfile.type); Assert.assertEquals("Keyspace1", backupfile.keyspace); Assert.assertEquals("Standard1", backupfile.columnFamily); Assert.assertEquals("1234567", backupfile.token); Assert.assertEquals("fake-app", backupfile.clusterName); Assert.assertEquals(region, backupfile.region); Assert.assertEquals("casstestbackup", backupfile.baseDir); - String datestr = DateUtil.formatyyyyMMddHHmm(new Date(bfile.lastModified())); Assert.assertEquals( - "casstestbackup/" - + region - + "/fake-app/1234567/" - + datestr - + "/SST/Keyspace1/Standard1/Keyspace1-Standard1-ia-5-Data.db", + "casstestbackup/1049_fake-app/1234567/SST_V2/1699206297000/Keyspace1/Standard1/SNAPPY/PLAINTEXT/Keyspace1-Standard1-ia-5-Data.db", backupfile.getRemotePath()); } @Test - public void testMetaFileCreation() throws ParseException { - // Test snapshot file - String filestr = "cass/data/1234567.meta"; - File bfile = new File(filestr); + public void testMetaFileCreation() throws IOException { + File file = createFile("target/data/1234567.meta"); RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); - backupfile.parseLocal(bfile, BackupFileType.META); - backupfile.setTime(DateUtil.getDate("201108082320")); - Assert.assertEquals(BackupFileType.META, backupfile.type); + backupfile.parseLocal(file, BackupFileType.META_V2); + Assert.assertEquals(BackupFileType.META_V2, backupfile.type); Assert.assertEquals("1234567", backupfile.token); Assert.assertEquals("fake-app", backupfile.clusterName); Assert.assertEquals(region, backupfile.region); Assert.assertEquals("casstestbackup", backupfile.baseDir); Assert.assertEquals( - "casstestbackup/" + region + "/fake-app/1234567/201108082320/META/1234567.meta", + "casstestbackup/1049_fake-app/1234567/META_V2/1699206297000/SNAPPY/PLAINTEXT/1234567.meta", backupfile.getRemotePath()); } + + private File createFile(String path) throws IOException { + File file = new File(path); + File dir = file.getParentFile(); + dir.mkdirs(); + file.createNewFile(); + file.setLastModified(1699206297000L); + return file; + } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupHelperImpl.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupHelperImpl.java index 470defc86..9b378e20d 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupHelperImpl.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupHelperImpl.java @@ -98,10 +98,10 @@ public ParameterizedTests(BackupsToCompress which, String tablePart, Compression FakeConfiguration fakeConfiguration = (FakeConfiguration) injector.getInstance(IConfiguration.class); fakeConfiguration.setFakeConfig("Priam.backupsToCompress", which); - IFileSystemContext context = injector.getInstance(IFileSystemContext.class); + IBackupFileSystem fileSystem = injector.getInstance(IBackupFileSystem.class); Provider pathFactory = injector.getProvider(AbstractBackupPath.class); - backupHelper = new BackupHelperImpl(fakeConfiguration, context, pathFactory); + backupHelper = new BackupHelperImpl(fakeConfiguration, fileSystem, pathFactory); } @Test @@ -148,10 +148,10 @@ public static void tearDown() throws IOException { public ProgrammaticTests() { Injector injector = Guice.createInjector(new BRTestModule()); config = (FakeConfiguration) injector.getInstance(IConfiguration.class); - IFileSystemContext context = injector.getInstance(IFileSystemContext.class); + IBackupFileSystem fileSystem = injector.getInstance(IBackupFileSystem.class); Provider pathFactory = injector.getProvider(AbstractBackupPath.class); - backupHelper = new BackupHelperImpl(config, context, pathFactory); + backupHelper = new BackupHelperImpl(config, fileSystem, pathFactory); } @Test diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java deleted file mode 100644 index ed4a34978..000000000 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright 2019 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backup; - -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.netflix.priam.config.IBackupRestoreConfig; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.connection.JMXNodeTool; -import com.netflix.priam.defaultimpl.IService; -import com.netflix.priam.scheduler.PriamScheduler; -import com.netflix.priam.tuner.CassandraTunerService; -import com.netflix.priam.tuner.TuneCassandra; -import com.netflix.priam.utils.BackupFileUtils; -import com.netflix.priam.utils.DateUtil; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Instant; -import java.util.Set; -import mockit.Expectations; -import mockit.Mocked; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.quartz.SchedulerException; - -/** Created by aagrawal on 3/10/19. */ -public class TestBackupService { - private final PriamScheduler scheduler; - private final CassandraTunerService cassandraTunerService; - - public TestBackupService() { - Injector injector = Guice.createInjector(new BRTestModule()); - this.scheduler = injector.getInstance(PriamScheduler.class); - this.cassandraTunerService = injector.getInstance(CassandraTunerService.class); - } - - @Before - public void cleanup() throws SchedulerException { - scheduler.getScheduler().clear(); - } - - @Test - public void testBackupDisabled( - @Mocked IConfiguration configuration, @Mocked IBackupRestoreConfig backupRestoreConfig) - throws Exception { - new Expectations() { - { - configuration.getBackupCronExpression(); - result = "-1"; - configuration.getDataFileLocation(); - result = "target/data"; - } - }; - - Path dummyDataDirectoryLocation = Paths.get(configuration.getDataFileLocation()); - Instant snapshotInstant = DateUtil.getInstant(); - - // Create one V1 snapshot. - String snapshotV1Name = DateUtil.formatInstant(DateUtil.yyyyMMdd, snapshotInstant); - BackupFileUtils.generateDummyFiles( - dummyDataDirectoryLocation, - 2, - 3, - 3, - AbstractBackup.SNAPSHOT_FOLDER, - snapshotV1Name, - true); - - String snapshotName = "meta_v2_" + snapshotV1Name; - // Create one V2 snapshot. - BackupFileUtils.generateDummyFiles( - dummyDataDirectoryLocation, - 2, - 3, - 3, - AbstractBackup.SNAPSHOT_FOLDER, - snapshotName, - false); - - IService backupService = - new BackupService( - configuration, backupRestoreConfig, scheduler, cassandraTunerService); - backupService.scheduleService(); - Assert.assertEquals(0, scheduler.getScheduler().getJobKeys(null).size()); - - // snapshot V1 name should not be there. - Set backupPaths = - AbstractBackup.getBackupDirectories(configuration, AbstractBackup.SNAPSHOT_FOLDER); - for (Path backupPath : backupPaths) { - Assert.assertTrue(Files.exists(Paths.get(backupPath.toString(), snapshotName))); - Assert.assertFalse(Files.exists(Paths.get(backupPath.toString(), snapshotV1Name))); - } - } - - @Test - public void testBackupEnabled( - @Mocked IConfiguration configuration, @Mocked IBackupRestoreConfig backupRestoreConfig) - throws Exception { - new Expectations() { - { - configuration.getBackupCronExpression(); - result = "0 0/1 * 1/1 * ? *"; - configuration.isIncrementalBackupEnabled(); - result = false; - } - }; - IService backupService = - new BackupService( - configuration, backupRestoreConfig, scheduler, cassandraTunerService); - backupService.scheduleService(); - Assert.assertEquals(2, scheduler.getScheduler().getJobKeys(null).size()); - } - - @Test - public void testBackupEnabledWithIncremental( - @Mocked IConfiguration configuration, @Mocked IBackupRestoreConfig backupRestoreConfig) - throws Exception { - new Expectations() { - { - configuration.getBackupCronExpression(); - result = "0 0/1 * 1/1 * ? *"; - configuration.isIncrementalBackupEnabled(); - result = true; - } - }; - IService backupService = - new BackupService( - configuration, backupRestoreConfig, scheduler, cassandraTunerService); - backupService.scheduleService(); - Assert.assertEquals(3, scheduler.getScheduler().getJobKeys(null).size()); - } - - @Test - public void updateService( - @Mocked IConfiguration configuration, - @Mocked IBackupRestoreConfig backupRestoreConfig, - @Mocked JMXNodeTool nodeTool, - @Mocked TuneCassandra tuneCassandra) - throws Exception { - new Expectations() { - { - configuration.getBackupCronExpression(); - result = "0 0/1 * 1/1 * ? *"; - result = "0 0/1 * 1/1 * ? *"; - result = "-1"; - result = "-1"; - configuration.isIncrementalBackupEnabled(); - result = true; - backupRestoreConfig.enableV2Backups(); - result = true; - backupRestoreConfig.getSnapshotMetaServiceCronExpression(); - result = "0 0/1 * 1/1 * ? *"; - } - }; - IService backupService = - new BackupService( - configuration, backupRestoreConfig, scheduler, cassandraTunerService); - backupService.scheduleService(); - Assert.assertEquals(3, scheduler.getScheduler().getJobKeys(null).size()); - - System.out.println("After updated"); - backupService.onChangeUpdateService(); - System.out.println(scheduler.getScheduler().getJobKeys(null)); - Assert.assertEquals(2, scheduler.getScheduler().getJobKeys(null).size()); - } -} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java index b7635a59a..3eaf55f7c 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java @@ -74,8 +74,7 @@ private void prepare() throws Exception { private BackupMetadata getBackupMetaData(Instant startTime, Status status) throws Exception { BackupMetadata backupMetadata = - new BackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, "123", new Date(startTime.toEpochMilli())); + new BackupMetadata("123", new Date(startTime.toEpochMilli())); backupMetadata.setCompleted( new Date(startTime.plus(30, ChronoUnit.MINUTES).toEpochMilli())); backupMetadata.setStatus(status); @@ -86,8 +85,7 @@ private BackupMetadata getBackupMetaData(Instant startTime, Status status) throw @Test public void testSnapshotUpdateMethod() throws Exception { Date startTime = DateUtil.getDate("198407110720"); - BackupMetadata backupMetadata = - new BackupMetadata(BackupVersion.SNAPSHOT_BACKUP, "123", startTime); + BackupMetadata backupMetadata = new BackupMetadata("123", startTime); backupStatusMgr.start(backupMetadata); Optional backupMetadata1 = backupStatusMgr.locate(startTime).stream().findFirst(); @@ -106,8 +104,7 @@ public void testSnapshotUpdateMethod() throws Exception { public void testSnapshotStatusAddFinish() throws Exception { Date startTime = DateUtil.getDate("198407110720"); - BackupMetadata backupMetadata = - new BackupMetadata(BackupVersion.SNAPSHOT_BACKUP, "123", startTime); + BackupMetadata backupMetadata = new BackupMetadata("123", startTime); backupStatusMgr.start(backupMetadata); List metadataList = backupStatusMgr.locate(startTime); Assert.assertNotNull(metadataList); @@ -130,8 +127,7 @@ public void testSnapshotStatusAddFinish() throws Exception { public void testSnapshotStatusAddFailed() throws Exception { Date startTime = DateUtil.getDate("198407120720"); - BackupMetadata backupMetadata = - new BackupMetadata(BackupVersion.SNAPSHOT_BACKUP, "123", startTime); + BackupMetadata backupMetadata = new BackupMetadata("123", startTime); backupStatusMgr.start(backupMetadata); List metadataList = backupStatusMgr.locate(startTime); Assert.assertNotNull(metadataList); @@ -158,8 +154,7 @@ public void testSnapshotStatusMultiAddFinishInADay() throws Exception { for (int i = 0; i < noOfEntries; i++) { assert startTime != null; Date time = new DateTime(startTime.getTime()).plusHours(i).toDate(); - BackupMetadata backupMetadata = - new BackupMetadata(BackupVersion.SNAPSHOT_BACKUP, "123", time); + BackupMetadata backupMetadata = new BackupMetadata("123", time); backupStatusMgr.start(backupMetadata); backupStatusMgr.finish(backupMetadata); } @@ -187,8 +182,7 @@ public void testSnapshotStatusSize() throws Exception { for (int i = 0; i < noOfEntries; i++) { assert startTime != null; Date time = new DateTime(startTime.getTime()).plusDays(i).toDate(); - BackupMetadata backupMetadata = - new BackupMetadata(BackupVersion.SNAPSHOT_BACKUP, "123", time); + BackupMetadata backupMetadata = new BackupMetadata("123", time); backupStatusMgr.start(backupMetadata); backupStatusMgr.finish(backupMetadata); } @@ -204,7 +198,6 @@ public void getLatestBackup() throws Exception { Instant start = DateUtil.parseInstant(backupDate); List list = backupStatusMgr.getLatestBackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, new DateRange( backupDate + "," @@ -221,9 +214,7 @@ public void getLatestBackup() throws Exception { public void getLatestBackupFailure() throws Exception { Optional backupMetadata = backupStatusMgr - .getLatestBackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, - new DateRange(backupDate + "," + backupDate)) + .getLatestBackupMetadata(new DateRange(backupDate + "," + backupDate)) .stream() .findFirst(); @@ -232,9 +223,7 @@ public void getLatestBackupFailure() throws Exception { backupStatusMgr.failed(getBackupMetaData(DateUtil.parseInstant(backupDate), Status.FAILED)); backupMetadata = backupStatusMgr - .getLatestBackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, - new DateRange(backupDate + "," + backupDate)) + .getLatestBackupMetadata(new DateRange(backupDate + "," + backupDate)) .stream() .findFirst(); Assert.assertFalse(backupMetadata.isPresent()); @@ -245,7 +234,6 @@ public void getLatestBackupMetadata() throws Exception { prepare(); List list = backupStatusMgr.getLatestBackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, new DateRange(backupDate + "," + "201812031000")); list.forEach(System.out::println); } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java index dd233cc12..07daf7acd 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java @@ -20,8 +20,6 @@ import com.google.inject.Guice; import com.google.inject.Injector; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.backupv2.IMetaProxy; -import com.netflix.priam.backupv2.MetaV1Proxy; import com.netflix.priam.backupv2.MetaV2Proxy; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; @@ -68,13 +66,6 @@ public TestBackupVerification() { backupStatusMgr = injector.getInstance(IBackupStatusMgr.class); } - static class MockMetaV1Proxy extends MockUp { - @Mock - public BackupVerificationResult isMetaFileValid(AbstractBackupPath metaBackupPath) { - return getBackupVerificationResult(); - } - } - static class MockMetaV2Proxy extends MockUp { @Mock public BackupVerificationResult isMetaFileValid(AbstractBackupPath metaBackupPath) { @@ -85,7 +76,6 @@ public BackupVerificationResult isMetaFileValid(AbstractBackupPath metaBackupPat @Before @After public void cleanup() { - new MockMetaV1Proxy(); new MockMetaV2Proxy(); FileUtils.deleteQuietly(new File(configuration.getBackupStatusFileLoc())); } @@ -94,16 +84,7 @@ public void cleanup() { public void noBackup() throws Exception { Optional backupVerificationResultOptinal = backupVerification.verifyLatestBackup( - BackupVersion.SNAPSHOT_BACKUP, - false, - new DateRange(Instant.now(), Instant.now())); - Assert.assertFalse(backupVerificationResultOptinal.isPresent()); - - backupVerificationResultOptinal = - backupVerification.verifyLatestBackup( - BackupVersion.SNAPSHOT_META_SERVICE, - false, - new DateRange(Instant.now(), Instant.now())); + false, new DateRange(Instant.now(), Instant.now())); Assert.assertFalse(backupVerificationResultOptinal.isPresent()); } @@ -111,12 +92,6 @@ public void noBackup() throws Exception { public void noBackupDateRange() throws Exception { List backupVerificationResults = backupVerification.verifyBackupsInRange( - BackupVersion.SNAPSHOT_BACKUP, new DateRange(Instant.now(), Instant.now())); - Assert.assertFalse(backupVerificationResults.size() > 0); - - backupVerificationResults = - backupVerification.verifyBackupsInRange( - BackupVersion.SNAPSHOT_META_SERVICE, new DateRange(Instant.now(), Instant.now())); Assert.assertFalse(backupVerificationResults.size() > 0); } @@ -125,82 +100,23 @@ private void setUp() throws Exception { Instant start = DateUtil.parseInstant(backupDate); for (int i = 0; i < numFakeBackups - 1; i++) { backupStatusMgr.finish( - getBackupMetaData( - BackupVersion.SNAPSHOT_BACKUP, - start.plus(i + 1, ChronoUnit.MINUTES), - Status.FINISHED)); + getBackupMetaData(start.plus(i + 1, ChronoUnit.MINUTES), Status.FINISHED)); } - backupStatusMgr.finish( - getBackupMetaData(BackupVersion.SNAPSHOT_BACKUP, start, Status.FINISHED)); + backupStatusMgr.finish(getBackupMetaData(start, Status.FINISHED)); backupStatusMgr.failed( - getBackupMetaData( - BackupVersion.SNAPSHOT_BACKUP, - start.plus(20, ChronoUnit.MINUTES), - Status.FAILED)); + getBackupMetaData(start.plus(20, ChronoUnit.MINUTES), Status.FAILED)); for (int i = 0; i < numFakeBackups - 1; i++) { backupStatusMgr.finish( - getBackupMetaData( - BackupVersion.SNAPSHOT_META_SERVICE, - start.plus(i + 1, ChronoUnit.MINUTES), - Status.FINISHED)); + getBackupMetaData(start.plus(i + 1, ChronoUnit.MINUTES), Status.FINISHED)); } - backupStatusMgr.finish( - getBackupMetaData(BackupVersion.SNAPSHOT_META_SERVICE, start, Status.FINISHED)); - } - - @Test - public void verifyBackupVersion1() throws Exception { - setUp(); - // Verify for backup version 1.0 - Optional backupVerificationResultOptinal = - backupVerification.verifyLatestBackup( - BackupVersion.SNAPSHOT_BACKUP, - false, - new DateRange(backupDate + "," + backupDate)); - Assert.assertTrue(backupVerificationResultOptinal.isPresent()); - Assert.assertEquals(Instant.EPOCH, backupVerificationResultOptinal.get().snapshotInstant); - Optional backupMetadata = - backupStatusMgr - .getLatestBackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, - new DateRange(backupDate + "," + backupDate)) - .stream() - .findFirst(); - Assert.assertTrue(backupMetadata.isPresent()); - Assert.assertNotNull(backupMetadata.get().getLastValidated()); - - backupMetadata = - backupStatusMgr - .getLatestBackupMetadata( - BackupVersion.SNAPSHOT_META_SERVICE, - new DateRange(backupDate + "," + backupDate)) - .stream() - .findFirst(); - Assert.assertTrue(backupMetadata.isPresent()); - Assert.assertNull(backupMetadata.get().getLastValidated()); + backupStatusMgr.finish(getBackupMetaData(start, Status.FINISHED)); } @Test public void verifyBackupVersion1DateRange() throws Exception { setUp(); - // Verify for backup version 1.0 - List backupVerificationResults = - backupVerification.verifyBackupsInRange( - BackupVersion.SNAPSHOT_BACKUP, - new DateRange(backupDate + "," + backupDateEnd)); - Assert.assertTrue(!backupVerificationResults.isEmpty()); - Assert.assertTrue(backupVerificationResults.size() == numFakeBackups); List backupMetadata = backupStatusMgr.getLatestBackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, - new DateRange(backupDate + "," + backupDateEnd)); - Assert.assertTrue(!backupMetadata.isEmpty()); - Assert.assertTrue(backupMetadata.size() == numFakeBackups); - backupMetadata.stream().forEach(b -> Assert.assertNotNull(b.getLastValidated())); - - backupMetadata = - backupStatusMgr.getLatestBackupMetadata( - BackupVersion.SNAPSHOT_META_SERVICE, new DateRange(backupDate + "," + backupDateEnd)); Assert.assertTrue(!backupMetadata.isEmpty()); Assert.assertTrue(backupMetadata.size() == numFakeBackups); @@ -213,18 +129,14 @@ public void verifyBackupVersion2() throws Exception { // Verify for backup version 2.0 Optional backupVerificationResultOptinal = backupVerification.verifyLatestBackup( - BackupVersion.SNAPSHOT_META_SERVICE, - false, - new DateRange(backupDate + "," + backupDate)); + false, new DateRange(backupDate + "," + backupDate)); Assert.assertTrue(backupVerificationResultOptinal.isPresent()); Assert.assertEquals(Instant.EPOCH, backupVerificationResultOptinal.get().snapshotInstant); Assert.assertEquals("some_random", backupVerificationResultOptinal.get().remotePath); Optional backupMetadata = backupStatusMgr - .getLatestBackupMetadata( - BackupVersion.SNAPSHOT_META_SERVICE, - new DateRange(backupDate + "," + backupDate)) + .getLatestBackupMetadata(new DateRange(backupDate + "," + backupDate)) .stream() .findFirst(); Assert.assertTrue(backupMetadata.isPresent()); @@ -233,9 +145,7 @@ public void verifyBackupVersion2() throws Exception { // Retry the verification, it should not try and re-verify backupVerificationResultOptinal = backupVerification.verifyLatestBackup( - BackupVersion.SNAPSHOT_META_SERVICE, - false, - new DateRange(backupDate + "," + backupDate)); + false, new DateRange(backupDate + "," + backupDate)); Assert.assertTrue(backupVerificationResultOptinal.isPresent()); Assert.assertEquals( DateUtil.parseInstant(backupDate), @@ -244,16 +154,6 @@ public void verifyBackupVersion2() throws Exception { Assert.assertEquals( location.subpath(1, location.getNameCount()).toString(), backupVerificationResultOptinal.get().remotePath); - - backupMetadata = - backupStatusMgr - .getLatestBackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, - new DateRange(backupDate + "," + backupDate)) - .stream() - .findFirst(); - Assert.assertTrue(backupMetadata.isPresent()); - Assert.assertNull(backupMetadata.get().getLastValidated()); } @Test @@ -262,31 +162,20 @@ public void verifyBackupVersion2DateRange() throws Exception { // Verify for backup version 2.0 List backupVerificationResults = backupVerification.verifyBackupsInRange( - BackupVersion.SNAPSHOT_META_SERVICE, new DateRange(backupDate + "," + backupDateEnd)); Assert.assertTrue(!backupVerificationResults.isEmpty()); Assert.assertTrue(backupVerificationResults.size() == numFakeBackups); List backupMetadata = backupStatusMgr.getLatestBackupMetadata( - BackupVersion.SNAPSHOT_META_SERVICE, new DateRange(backupDate + "," + backupDateEnd)); Assert.assertTrue(!backupMetadata.isEmpty()); Assert.assertTrue(backupMetadata.size() == numFakeBackups); backupMetadata.stream().forEach(b -> Assert.assertNotNull(b.getLastValidated())); - - backupMetadata = - backupStatusMgr.getLatestBackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, - new DateRange(backupDate + "," + backupDateEnd)); - Assert.assertTrue(!backupMetadata.isEmpty()); - Assert.assertTrue(backupMetadata.size() == numFakeBackups); - backupMetadata.stream().forEach(b -> Assert.assertNull(b.getLastValidated())); } - private BackupMetadata getBackupMetaData( - BackupVersion backupVersion, Instant startTime, Status status) throws Exception { + private BackupMetadata getBackupMetaData(Instant startTime, Status status) throws Exception { BackupMetadata backupMetadata = - new BackupMetadata(backupVersion, "123", new Date(startTime.toEpochMilli())); + new BackupMetadata("123", new Date(startTime.toEpochMilli())); backupMetadata.setCompleted( new Date(startTime.plus(30, ChronoUnit.MINUTES).toEpochMilli())); backupMetadata.setStatus(status); @@ -303,10 +192,4 @@ private static BackupVerificationResult getBackupVerificationResult() { result.snapshotInstant = Instant.EPOCH; return result; } - - @Test - public void testGetMetaProxy() { - IMetaProxy metaProxy = backupVerification.getMetaProxy(BackupVersion.SNAPSHOT_META_SERVICE); - Assert.assertTrue(metaProxy != null); - } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java deleted file mode 100644 index e79b45f51..000000000 --- a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java +++ /dev/null @@ -1,353 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backup; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.netflix.priam.aws.S3FileSystem; -import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.utils.DateUtil; -import java.io.IOException; -import java.util.*; -import mockit.Mock; -import mockit.MockUp; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; - -/** - * Unit test for backup file iterator - * - * @author Praveen Sadhu - */ -public class TestFileIterator { - private static Date startTime, endTime; - - private static S3FileSystem s3FileSystem; - private static String region; - private static String bucket = "TESTBUCKET"; - - @BeforeClass - public static void setup() throws InterruptedException, IOException { - AmazonS3Client s3client = new MockAmazonS3Client().getMockInstance(); - new MockObjectListing(); - - Injector injector = Guice.createInjector(new BRTestModule()); - InstanceIdentity factory = injector.getInstance(InstanceIdentity.class); - region = factory.getInstanceInfo().getRegion(); - s3FileSystem = injector.getInstance(S3FileSystem.class); - s3FileSystem.setS3Client(s3client); - - DateUtil.DateRange dateRange = new DateUtil.DateRange("201108110030,201108110530"); - startTime = new Date(dateRange.getStartTime().toEpochMilli()); - endTime = new Date(dateRange.getEndTime().toEpochMilli()); - } - - static class MockAmazonS3Client extends MockUp { - @Mock - public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) - throws AmazonClientException { - ObjectListing listing = new ObjectListing(); - listing.setBucketName(listObjectsRequest.getBucketName()); - listing.setPrefix(listObjectsRequest.getPrefix()); - return listing; - } - - @Mock - public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) - throws AmazonClientException { - ObjectListing listing = new ObjectListing(); - listing.setBucketName(previousObjectListing.getBucketName()); - listing.setPrefix(previousObjectListing.getPrefix()); - return new ObjectListing(); - } - } - - // MockObjectListing class - @Ignore - public static class MockObjectListing extends MockUp { - public static boolean truncated = true; - public static boolean firstcall = true; - public static boolean simfilter = false; // Simulate filtering - - @Mock - public List getObjectSummaries() { - if (firstcall) { - firstcall = false; - if (simfilter) return getObjectSummaryEmpty(); - return getObjectSummary(); - } else { - if (simfilter) { - simfilter = false; // reset - return getObjectSummaryEmpty(); - } else truncated = false; - return getNextObjectSummary(); - } - } - - @Mock - public boolean isTruncated() { - return truncated; - } - } - - @Test - public void testIteratorEmptySet() { - DateUtil.DateRange dateRange = new DateUtil.DateRange("201107110601,201107111101"); - Date stime = new Date(dateRange.getStartTime().toEpochMilli()); - Date etime = new Date(dateRange.getEndTime().toEpochMilli()); - - Iterator fileIterator = s3FileSystem.list(bucket, stime, etime); - Set files = new HashSet<>(); - while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); - Assert.assertEquals(0, files.size()); - } - - @Test - public void testIterator() { - MockObjectListing.truncated = false; - MockObjectListing.firstcall = true; - MockObjectListing.simfilter = false; - - Iterator fileIterator = s3FileSystem.list(bucket, startTime, endTime); - - Set files = new HashSet<>(); - while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); - Assert.assertEquals(3, files.size()); - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db")); - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db")); - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110030/META/meta.json")); - Assert.assertFalse( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db")); - } - - @Test - public void testIteratorTruncated() { - MockObjectListing.truncated = true; - MockObjectListing.firstcall = true; - MockObjectListing.simfilter = false; - - Iterator fileIterator = s3FileSystem.list(bucket, startTime, endTime); - - Set files = new HashSet<>(); - while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); - Assert.assertEquals(5, files.size()); - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db")); - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db")); - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110030/META/meta.json")); - Assert.assertFalse( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db")); - - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db")); - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db")); - Assert.assertFalse( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); - } - - @Test - public void testIteratorTruncatedOOR() { - MockObjectListing.truncated = true; - MockObjectListing.firstcall = true; - MockObjectListing.simfilter = true; - - Iterator fileIterator = s3FileSystem.list(bucket, startTime, endTime); - - Set files = new HashSet<>(); - while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); - Assert.assertEquals(2, files.size()); - Assert.assertFalse( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201107110030/SNAP/ks1/cf1/f1.db")); - Assert.assertFalse( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201107110430/SST/ks1/cf1/f2.db")); - Assert.assertFalse( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201107110030/META/meta.json")); - Assert.assertFalse( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201107110600/SST/ks1/cf1/f3.db")); - - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db")); - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db")); - Assert.assertFalse( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); - } - - @Test - public void testRestorePathIteration() { - MockObjectListing.truncated = true; - MockObjectListing.firstcall = true; - MockObjectListing.simfilter = false; - - Iterator fileIterator = - s3FileSystem.list( - "RESTOREBUCKET/test_restore_backup/fake-restore-region/fakerestorecluster", - startTime, - endTime); - - Set files = new HashSet<>(); - while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); - while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); - - Assert.assertEquals(5, files.size()); - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db")); - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db")); - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110030/META/meta.json")); - Assert.assertFalse( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db")); - - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db")); - Assert.assertTrue( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db")); - Assert.assertFalse( - files.contains( - "test_backup/" - + region - + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); - } - - private static List getObjectSummary() { - List list = new ArrayList<>(); - S3ObjectSummary summary = new S3ObjectSummary(); - summary.setKey( - "test_backup/" + region + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db"); - list.add(summary); - summary = new S3ObjectSummary(); - summary.setKey( - "test_backup/" + region + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db"); - list.add(summary); - summary = new S3ObjectSummary(); - summary.setKey( - "test_backup/" + region + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db"); - list.add(summary); - summary = new S3ObjectSummary(); - summary.setKey("test_backup/" + region + "/fakecluster/123456/201108110030/META/meta.json"); - list.add(summary); - return list; - } - - private static List getObjectSummaryEmpty() { - return new ArrayList<>(); - } - - private static List getNextObjectSummary() { - List list = new ArrayList<>(); - S3ObjectSummary summary = new S3ObjectSummary(); - summary.setKey( - "test_backup/" + region + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db"); - list.add(summary); - summary = new S3ObjectSummary(); - summary.setKey( - "test_backup/" + region + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db"); - list.add(summary); - summary = new S3ObjectSummary(); - summary.setKey( - "test_backup/" + region + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db"); - list.add(summary); - return list; - } -} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index 0a297433b..418c1ea9b 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -95,7 +95,7 @@ public void testFileUpload() throws Exception { MockS3PartUploader.setup(); AbstractFileSystem fs = injector.getInstance(NullBackupFileSystem.class); RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); - backupfile.parseLocal(localFile(), BackupFileType.SNAP); + backupfile.parseLocal(localFile(), BackupFileType.META_V2); long noOfFilesUploaded = backupMetrics.getUploadRate().count(); // temporary hack to allow tests to complete in a timely fashion // This will be removed once we stop inheriting from AbstractFileSystem @@ -125,7 +125,7 @@ public void testFileUploadFailures() throws Exception { long noOfFailures = backupMetrics.getInvalidUploads().count(); S3FileSystem fs = injector.getInstance(S3FileSystem.class); RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); - backupfile.parseLocal(localFile(), BackupFileType.SNAP); + backupfile.parseLocal(localFile(), BackupFileType.META_V2); try { // temporary hack to allow tests to complete in a timely fashion // This will be removed once we stop inheriting from AbstractFileSystem @@ -144,7 +144,7 @@ public void testFileUploadCompleteFailure() throws Exception { S3FileSystem fs = injector.getInstance(S3FileSystem.class); fs.setS3Client(new MockAmazonS3Client().getMockInstance()); RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); - backupfile.parseLocal(localFile(), BackupFileType.SNAP); + backupfile.parseLocal(localFile(), BackupFileType.META_V2); try { // temporary hack to allow tests to complete in a timely fashion // This will be removed once we stop inheriting from AbstractFileSystem diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java index a287a394b..e017ef295 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java @@ -139,8 +139,6 @@ public void testBackupEnabled( result = true; configuration.isIncrementalBackupEnabled(); result = true; - configuration.getBackupCronExpression(); - result = "-1"; } }; IService backupService = @@ -207,8 +205,6 @@ public void updateService( result = "-1"; backupRestoreConfig.getBackupTTLMonitorPeriodInSec(); result = 600; - configuration.getBackupCronExpression(); - result = "-1"; } }; IService backupService = diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java index f0e3adcd2..ba867eb50 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java @@ -76,8 +76,7 @@ public static void shouldThrow(boolean newThrowError) { } @Mock - public List verifyBackupsInRange( - BackupVersion backupVersion, DateRange dateRange) + public List verifyBackupsInRange(DateRange dateRange) throws UnsupportedTypeException, IllegalArgumentException { if (throwError) throw new IllegalArgumentException("DummyError"); return verifiedBackups; @@ -85,7 +84,7 @@ public List verifyBackupsInRange( @Mock public Optional verifyLatestBackup( - BackupVersion backupVersion, boolean force, DateRange dateRange) + boolean force, DateRange dateRange) throws UnsupportedTypeException, IllegalArgumentException { if (throwError) throw new IllegalArgumentException("DummyError"); return results.isEmpty() ? Optional.empty() : Optional.of(results.get(0)); @@ -163,7 +162,7 @@ public void testRestoreMode(@Mocked InstanceState state) throws Exception { Truth.assertThat(badVerifications.count()).isEqualTo(0); new Verifications() { { - backupVerification.verifyBackupsInRange((BackupVersion) any, (DateRange) any); + backupVerification.verifyBackupsInRange((DateRange) any); maxTimes = 0; } @@ -175,20 +174,18 @@ public void testRestoreMode(@Mocked InstanceState state) throws Exception { } private static BackupMetadata getInvalidBackupMetadata() { - return new BackupMetadata(BackupVersion.SNAPSHOT_META_SERVICE, "12345", new Date()); + return new BackupMetadata("12345", new Date()); } private static BackupMetadata getPreviouslyValidatedMetadata() { - BackupMetadata backupMetadata = - new BackupMetadata(BackupVersion.SNAPSHOT_META_SERVICE, "12345", new Date()); + BackupMetadata backupMetadata = new BackupMetadata("12345", new Date()); backupMetadata.setLastValidated( new Date(Instant.now().minus(1, ChronoUnit.HOURS).toEpochMilli())); return backupMetadata; } private static BackupMetadata getRecentlyValidatedMetadata() { - BackupMetadata backupMetadata = - new BackupMetadata(BackupVersion.SNAPSHOT_META_SERVICE, "12345", new Date()); + BackupMetadata backupMetadata = new BackupMetadata("12345", new Date()); backupMetadata.setLastValidated( new Date(Instant.now().plus(1, ChronoUnit.HOURS).toEpochMilli())); backupMetadata.setSnapshotLocation("bucket/path/to/file.db"); diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestForgottenFileManager.java b/priam/src/test/java/com/netflix/priam/backupv2/TestForgottenFileManager.java deleted file mode 100644 index 869891d99..000000000 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestForgottenFileManager.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright 2019 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backupv2; - -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.netflix.priam.backup.BRTestModule; -import com.netflix.priam.config.FakeConfiguration; -import com.netflix.priam.merics.BackupMetrics; -import com.netflix.priam.utils.DateUtil; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** Created by aagrawal on 1/1/19. */ -public class TestForgottenFileManager { - private ForgottenFilesManager forgottenFilesManager; - private TestBackupUtils testBackupUtils; - private ForgottenFilesConfiguration configuration; - private List allFiles = new ArrayList<>(); - private Instant snapshotInstant; - private Path snapshotDir; - - public TestForgottenFileManager() { - Injector injector = Guice.createInjector(new BRTestModule()); - BackupMetrics backupMetrics = injector.getInstance(BackupMetrics.class); - configuration = new ForgottenFilesConfiguration(); - forgottenFilesManager = new ForgottenFilesManager(configuration, backupMetrics); - testBackupUtils = injector.getInstance(TestBackupUtils.class); - } - - @Before - public void prep() throws Exception { - cleanup(); - Instant now = DateUtil.getInstant(); - snapshotInstant = now; - Path file1 = Paths.get(testBackupUtils.createFile("file1", now.minus(10, ChronoUnit.DAYS))); - Path file2 = Paths.get(testBackupUtils.createFile("file2", now.minus(8, ChronoUnit.DAYS))); - Path file3 = Paths.get(testBackupUtils.createFile("file3", now.minus(6, ChronoUnit.DAYS))); - Path file4 = Paths.get(testBackupUtils.createFile("file4", now.minus(4, ChronoUnit.DAYS))); - Path file5 = Paths.get(testBackupUtils.createFile("file5", now.minus(1, ChronoUnit.DAYS))); - Path file6 = - Paths.get( - testBackupUtils.createFile( - "tmplink-lb-59516-big-Index.db", now.minus(3, ChronoUnit.DAYS))); - Path file7 = - Paths.get(testBackupUtils.createFile("file7.tmp", now.minus(3, ChronoUnit.DAYS))); - - allFiles.add(file1); - allFiles.add(file2); - allFiles.add(file3); - allFiles.add(file4); - allFiles.add(file5); - allFiles.add(file6); - allFiles.add(file7); - - // Create a snapshot with file2, file3, file4. - Path columnfamilyDir = file1.getParent(); - snapshotDir = - Paths.get( - columnfamilyDir.toString(), - "snapshot", - "snap_v2_" + DateUtil.formatInstant(DateUtil.yyyyMMddHHmm, now)); - snapshotDir.toFile().mkdirs(); - Files.createLink(Paths.get(snapshotDir.toString(), file2.getFileName().toString()), file2); - Files.createLink(Paths.get(snapshotDir.toString(), file3.getFileName().toString()), file3); - Files.createLink(Paths.get(snapshotDir.toString(), file4.getFileName().toString()), file4); - } - - @After - public void cleanup() throws Exception { - String dataDir = configuration.getDataFileLocation(); - org.apache.commons.io.FileUtils.cleanDirectory(new File(dataDir)); - } - - @Test - public void testMoveForgottenFiles() throws IOException, InterruptedException { - Collection files = allFiles.stream().map(Path::toFile).collect(Collectors.toList()); - Path lostFoundDir = - Paths.get(configuration.getDataFileLocation(), forgottenFilesManager.LOST_FOUND); - - // Lets create some extra symlinks in the LOST_FOUND folder. They should not exist anymore - Path randomSymlink = Paths.get(lostFoundDir.toFile().getAbsolutePath(), "random"); - Files.createDirectory(lostFoundDir); - Files.createSymbolicLink(randomSymlink, lostFoundDir); - - forgottenFilesManager.moveForgottenFiles( - new File(configuration.getDataFileLocation()), files); - - // Extra symlinks are deleted. - Assert.assertFalse(Files.exists(randomSymlink)); - - // Symlinks are created for all the files. They are not moved yet. - Collection symlinkFiles = FileUtils.listFiles(lostFoundDir.toFile(), null, false); - Assert.assertEquals(allFiles.size(), symlinkFiles.size()); - for (Path file : allFiles) { - Path symlink = Paths.get(lostFoundDir.toString(), file.getFileName().toString()); - Assert.assertTrue(symlinkFiles.contains(symlink.toFile())); - Assert.assertTrue(Files.isSymbolicLink(symlink)); - Assert.assertTrue(Files.exists(file)); - } - - // Lets change the configuration and try again!! - configuration.setGracePeriodForgottenFileInDaysForRead(0); - forgottenFilesManager.moveForgottenFiles( - new File(configuration.getDataFileLocation()), files); - Collection movedFiles = FileUtils.listFiles(lostFoundDir.toFile(), null, false); - Assert.assertEquals(allFiles.size(), movedFiles.size()); - movedFiles - .stream() - .forEach( - file -> { - Assert.assertTrue( - Files.isRegularFile(Paths.get(file.getAbsolutePath()))); - }); - allFiles.stream() - .forEach( - file -> { - Assert.assertFalse(file.toFile().exists()); - }); - - configuration.setGracePeriodForgottenFileInDaysForRead( - ForgottenFilesConfiguration.DEFAULT_GRACE_PERIOD); - } - - @Test - public void getColumnfamilyFiles() { - - Path columnfamilyDir = allFiles.get(0).getParent(); - Collection columnfamilyFiles = - forgottenFilesManager.getColumnfamilyFiles( - snapshotInstant, columnfamilyDir.toFile()); - Assert.assertEquals(3, columnfamilyFiles.size()); - Assert.assertTrue(columnfamilyFiles.contains(allFiles.get(0).toFile())); - Assert.assertTrue(columnfamilyFiles.contains(allFiles.get(1).toFile())); - Assert.assertTrue(columnfamilyFiles.contains(allFiles.get(2).toFile())); - } - - @Test - public void findAndMoveForgottenFiles() { - Path lostFoundDir = - Paths.get(allFiles.get(0).getParent().toString(), forgottenFilesManager.LOST_FOUND); - forgottenFilesManager.findAndMoveForgottenFiles(snapshotInstant, snapshotDir.toFile()); - - // Only one potential forgotten file - file1. It will be symlink here. - Collection movedFiles = FileUtils.listFiles(lostFoundDir.toFile(), null, false); - Assert.assertEquals(1, movedFiles.size()); - Assert.assertTrue( - movedFiles - .iterator() - .next() - .getName() - .equals(allFiles.get(0).getFileName().toString())); - Assert.assertTrue( - Files.isSymbolicLink(Paths.get(movedFiles.iterator().next().getAbsolutePath()))); - - // All files still remain in columnfamily dir. - Collection cfFiles = - FileUtils.listFiles(new File(allFiles.get(0).getParent().toString()), null, false); - Assert.assertEquals(allFiles.size(), cfFiles.size()); - - // Snapshot is untouched. - Collection snapshotFiles = FileUtils.listFiles(snapshotDir.toFile(), null, false); - Assert.assertEquals(3, snapshotFiles.size()); - - // Lets change the configuration and try again!! - configuration.setGracePeriodForgottenFileInDaysForRead(0); - forgottenFilesManager.findAndMoveForgottenFiles(snapshotInstant, snapshotDir.toFile()); - configuration.setGracePeriodForgottenFileInDaysForRead( - ForgottenFilesConfiguration.DEFAULT_GRACE_PERIOD); - movedFiles = FileUtils.listFiles(lostFoundDir.toFile(), null, false); - Assert.assertEquals(1, movedFiles.size()); - Assert.assertTrue( - Files.isRegularFile(Paths.get(movedFiles.iterator().next().getAbsolutePath()))); - cfFiles = - FileUtils.listFiles(new File(allFiles.get(0).getParent().toString()), null, false); - Assert.assertEquals(6, cfFiles.size()); - int temp_file_name = 1; - for (File file : cfFiles) { - file.getName().equals(allFiles.get(temp_file_name++).getFileName().toString()); - } - - // Snapshot is untouched. - snapshotFiles = FileUtils.listFiles(snapshotDir.toFile(), null, false); - Assert.assertEquals(3, snapshotFiles.size()); - } - - private class ForgottenFilesConfiguration extends FakeConfiguration { - protected static final int DEFAULT_GRACE_PERIOD = 3; - private int gracePeriodForgottenFileInDaysForRead = DEFAULT_GRACE_PERIOD; - - @Override - public boolean isForgottenFileMoveEnabled() { - return true; - } - - @Override - public int getForgottenFileGracePeriodDaysForRead() { - return gracePeriodForgottenFileInDaysForRead; - } - - public void setGracePeriodForgottenFileInDaysForRead( - int gracePeriodForgottenFileInDaysForRead) { - this.gracePeriodForgottenFileInDaysForRead = gracePeriodForgottenFileInDaysForRead; - } - } -} diff --git a/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java b/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java index 2dbece5de..cf4f05a75 100644 --- a/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java +++ b/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java @@ -108,7 +108,7 @@ public void testNoNotificationsNonEmptyFilter( "fakeBackup", "fakeData.db"); AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); - abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST); + abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST_V2); backupNotificationMgr.notify(abstractBackupPath, UploadStatus.STARTED); new Verifications() { { @@ -149,7 +149,7 @@ public void testNotificationsEmptyFilter( "fakeBackup", "fakeData.db"); AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); - abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST); + abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST_V2); backupNotificationMgr.notify(abstractBackupPath, UploadStatus.STARTED); new Verifications() { { @@ -190,7 +190,7 @@ public void testNotificationsInvalidFilter( "fakeBackup", "fakeData.db"); AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); - abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST); + abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST_V2); backupNotificationMgr.notify(abstractBackupPath, UploadStatus.STARTED); new Verifications() { { @@ -254,7 +254,7 @@ public void testNoNotificationsPartiallyValidFilter( new Expectations() { { backupRestoreConfig.getBackupNotifyComponentIncludeList(); - result = "SOME_FAKE_FILE_TYPE_1, SOME_FAKE_FILE_TYPE_2, SST"; + result = "SOME_FAKE_FILE_TYPE_1, SOME_FAKE_FILE_TYPE_2, SST_V2"; maxTimes = 2; } }; diff --git a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java deleted file mode 100644 index ce1ae0fb5..000000000 --- a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.resources; - -import static org.junit.Assert.assertEquals; - -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.netflix.priam.backup.*; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.health.InstanceState; -import com.netflix.priam.identity.config.InstanceInfo; -import com.netflix.priam.restore.Restore; -import com.netflix.priam.utils.DateUtil; -import java.time.Instant; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import mockit.Expectations; -import mockit.Mocked; -import mockit.integration.junit4.JMockit; -import org.joda.time.DateTime; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; - -@RunWith(JMockit.class) -public class BackupServletTest { - private IConfiguration config; - private @Mocked Restore restoreObj; - private @Mocked SnapshotBackup snapshotBackup; - private BackupServlet resource; - private RestoreServlet restoreResource; - private InstanceInfo instanceInfo; - - @Before - public void setUp() { - Injector injector = Guice.createInjector(new BRTestModule()); - config = injector.getInstance(IConfiguration.class); - InstanceState instanceState = injector.getInstance(InstanceState.class); - instanceInfo = injector.getInstance(InstanceInfo.class); - resource = injector.getInstance(BackupServlet.class); - restoreResource = injector.getInstance(RestoreServlet.class); - } - - @Test - public void backup() throws Exception { - new Expectations() { - { - snapshotBackup.execute(); - } - }; - - Response response = resource.backup(); - assertEquals(200, response.getStatus()); - assertEquals("[\"ok\"]", response.getEntity()); - assertEquals( - MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); - } - - @Test - public void restore_minimal() throws Exception { - final String dateRange = null; - final String oldRegion = "us-east-1"; - new Expectations() { - { - instanceInfo.getRegion(); - result = oldRegion; - - restoreObj.restore(new DateUtil.DateRange((Instant) any, (Instant) any)); - } - }; - - expectCassandraStartup(); - - Response response = restoreResource.restore(dateRange); - assertEquals(200, response.getStatus()); - assertEquals("[\"ok\"]", response.getEntity()); - assertEquals( - MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); - } - - @Test - public void restore_withDateRange() throws Exception { - final String dateRange = "201101010000,201112312359"; - - new Expectations() { - - { - DateUtil.getDate(dateRange.split(",")[0]); - result = new DateTime(2011, 1, 1, 0, 0).toDate(); - times = 1; - DateUtil.getDate(dateRange.split(",")[1]); - result = new DateTime(2011, 12, 31, 23, 59).toDate(); - times = 1; - restoreObj.restore(new DateUtil.DateRange(dateRange)); - } - }; - - expectCassandraStartup(); - - Response response = restoreResource.restore(dateRange); - assertEquals(200, response.getStatus()); - assertEquals("[\"ok\"]", response.getEntity()); - assertEquals( - MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); - } - - // TODO: create CassandraController interface and inject, instead of static util method - private void expectCassandraStartup() { - new Expectations() { - { - config.getCassStartupScript(); - result = "/usr/bin/false"; - config.getHeapNewSize(); - result = "2G"; - config.getHeapSize(); - result = "8G"; - config.getDataFileLocation(); - result = "/var/lib/cassandra/data"; - config.getCommitLogLocation(); - result = "/var/lib/cassandra/commitlog"; - config.getBackupLocation(); - result = "backup"; - config.getCacheLocation(); - result = "/var/lib/cassandra/saved_caches"; - config.getJmxPort(); - result = 7199; - config.getMaxDirectMemory(); - result = "50G"; - } - }; - } -} diff --git a/priam/src/test/java/com/netflix/priam/resources/BackupServletV2Test.java b/priam/src/test/java/com/netflix/priam/resources/BackupServletV2Test.java index 4c3848e0f..7617535d9 100644 --- a/priam/src/test/java/com/netflix/priam/resources/BackupServletV2Test.java +++ b/priam/src/test/java/com/netflix/priam/resources/BackupServletV2Test.java @@ -158,9 +158,7 @@ public void testValidate() throws Exception { new Expectations() { { backupVerification.verifyLatestBackup( - BackupVersion.SNAPSHOT_META_SERVICE, - anyBoolean, - new DateUtil.DateRange((Instant) any, (Instant) any)); + anyBoolean, new DateUtil.DateRange((Instant) any, (Instant) any)); result = Optional.of(getBackupVerificationResult()); } }; @@ -178,9 +176,7 @@ public void testValidateNoBackups() throws Exception { new Expectations() { { backupVerification.verifyLatestBackup( - BackupVersion.SNAPSHOT_META_SERVICE, - anyBoolean, - new DateUtil.DateRange((Instant) any, (Instant) any)); + anyBoolean, new DateUtil.DateRange((Instant) any, (Instant) any)); result = Optional.empty(); } }; @@ -197,9 +193,7 @@ public void testValidateV2SnapshotByDate() throws Exception { new Expectations() { { backupVerification.verifyLatestBackup( - BackupVersion.SNAPSHOT_META_SERVICE, - anyBoolean, - new DateUtil.DateRange((Instant) any, (Instant) any)); + anyBoolean, new DateUtil.DateRange((Instant) any, (Instant) any)); result = Optional.of(getBackupVerificationResult()); } }; @@ -212,28 +206,6 @@ public void testValidateV2SnapshotByDate() throws Exception { response.getEntity().toString()); } - // @Test - // public void testListDateRange() throws Exception { - // Optional abstractBackupPath = getAbstractBackupPath(); - // String dateRange = String.format("%s,%s", - // new SimpleDateFormat("yyyymmddhhmm").format(new Date()) - // , new SimpleDateFormat("yyyymmddhhmm").format(new Date())); - // new Expectations() {{ - // backupRestoreUtil.getLatestValidMetaPath(metaV2Proxy, - // new DateUtil.DateRange((Instant) any, (Instant) any)); result = - // abstractBackupPath; - // - // backupRestoreUtil.getAllFiles( - // abstractBackupPath.get(), - // new DateUtil.DateRange((Instant) any, (Instant) any), metaV2Proxy, - // pathProvider); result = getBackupPathList(); - // }}; - // - // Response response = - // resource.list(dateRange); - // assertEquals(200, response.getStatus()); - // } - @Test public void testListDateRangeNoBackups() throws Exception { String dateRange = @@ -261,7 +233,6 @@ public void testBackUpInfo() throws Exception { new Expectations() { { backupStatusMgr.getLatestBackupMetadata( - BackupVersion.SNAPSHOT_META_SERVICE, new DateUtil.DateRange((Instant) any, (Instant) any)); result = backupMetadataList; } @@ -278,7 +249,6 @@ public void testBackUpInfoNoBackups() { new Expectations() { { backupStatusMgr.getLatestBackupMetadata( - BackupVersion.SNAPSHOT_META_SERVICE, new DateUtil.DateRange((Instant) any, (Instant) any)); result = new ArrayList<>(); } @@ -303,9 +273,7 @@ private static BackupVerificationResult getBackupVerificationResult() { private static BackupMetadata getBackupMetaData() throws Exception { BackupMetadata backupMetadata = new BackupMetadata( - BackupVersion.SNAPSHOT_META_SERVICE, - "123", - new Date(DateUtil.parseInstant(backupDate).toEpochMilli())); + "123", new Date(DateUtil.parseInstant(backupDate).toEpochMilli())); backupMetadata.setCompleted( new Date( DateUtil.parseInstant(backupDate) @@ -315,44 +283,4 @@ private static BackupMetadata getBackupMetaData() throws Exception { backupMetadata.setSnapshotLocation(location.toString()); return backupMetadata; } - - private static Optional getAbstractBackupPath() throws Exception { - Path path = - Paths.get( - configuration.getDataFileLocation(), - "keyspace1", - "columnfamily1", - "backup", - "mc-1234-Data.db"); - AbstractBackupPath abstractBackupPath = pathProvider.get(); - abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST_V2); - return Optional.of(abstractBackupPath); - } - - private static List getBackupPathList() throws Exception { - List abstractBackupPathList = new ArrayList<>(); - Path path = - Paths.get( - configuration.getDataFileLocation(), - "keyspace1", - "columnfamily1", - "backup", - "mc-1234-Data.db"); - AbstractBackupPath abstractBackupPath1 = pathProvider.get(); - abstractBackupPath1.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST_V2); - abstractBackupPathList.add(abstractBackupPath1); - - path = - Paths.get( - configuration.getDataFileLocation(), - "keyspace1", - "columnfamily1", - "backup", - "mc-1234-Data.db"); - AbstractBackupPath abstractBackupPath2 = pathProvider.get(); - abstractBackupPath2.parseLocal( - path.toFile(), AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED); - abstractBackupPathList.add(abstractBackupPath2); - return abstractBackupPathList; - } } diff --git a/priam/src/test/java/com/netflix/priam/restore/TestRestore.java b/priam/src/test/java/com/netflix/priam/restore/TestRestore.java index aeb169911..41c434229 100644 --- a/priam/src/test/java/com/netflix/priam/restore/TestRestore.java +++ b/priam/src/test/java/com/netflix/priam/restore/TestRestore.java @@ -17,6 +17,7 @@ package com.netflix.priam.restore; +import com.google.common.truth.Truth; import com.google.inject.Guice; import com.google.inject.Injector; import com.netflix.priam.backup.BRTestModule; @@ -25,7 +26,6 @@ import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.health.InstanceState; -import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.utils.DateUtil; import java.io.IOException; import java.util.ArrayList; @@ -37,7 +37,6 @@ public class TestRestore { private static FakeBackupFileSystem filesystem; private static ArrayList fileList = new ArrayList<>(); private static FakeConfiguration conf; - private static String region; private static Restore restore; private static InstanceState instanceState; @@ -46,29 +45,55 @@ public static void setup() throws InterruptedException, IOException { Injector injector = Guice.createInjector(new BRTestModule()); if (filesystem == null) filesystem = injector.getInstance(FakeBackupFileSystem.class); if (conf == null) conf = (FakeConfiguration) injector.getInstance(IConfiguration.class); - region = injector.getInstance(InstanceInfo.class).getRegion(); if (restore == null) restore = injector.getInstance(Restore.class); if (instanceState == null) instanceState = injector.getInstance(InstanceState.class); } - private static void populateBackupFileSystem(String baseDir) { + private static void populateBackupFileSystem(String cluster) { fileList.clear(); - fileList.add(baseDir + "/" + region + "/fakecluster/123456/201108110030/META/meta.json"); fileList.add( - baseDir + "/" + region + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db"); + "test_backup/" + + cluster + + "/1808575600/META_V2/1313026200000/SNAPPY/PLAINTEXT/meta_v2_201108110130.json"); fileList.add( - baseDir + "/" + region + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f2.db"); + "test_backup/" + + cluster + + "/1808575600/SST_V2/1313022601000/ks1/cf1/SNAPPY/PLAINTEXT/me-1-big-Data.db"); fileList.add( - baseDir + "/" + region + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f2.db"); - fileList.add(baseDir + "/" + region + "/fakecluster/123456/201108110530/SST/ks2/cf1/f3.db"); - fileList.add(baseDir + "/" + region + "/fakecluster/123456/201108110600/SST/ks2/cf1/f4.db"); + "test_backup/" + + cluster + + "/1808575600/SST_V2/1313022601000/ks1/cf1/SNAPPY/PLAINTEXT/me-1-big-Index.db"); + fileList.add( + "test_backup/" + + cluster + + "/1808575600/SST_V2/1313022601000/ks2/cf1/SNAPPY/PLAINTEXT/me-2-big-Data.db"); + fileList.add( + "test_backup/" + + cluster + + "/1808575600/SST_V2/1313040601000/ks2/cf1/SNAPPY/PLAINTEXT/me-2-big-Index.db"); + fileList.add( + "test_backup/" + + cluster + + "/1808575600/SST_V2/1313042400000/ks2/cf1/SNAPPY/PLAINTEXT/me-2-big-Summary.db"); + fileList.add( + "test_backup/" + + cluster + + "/1808575600/SST_V2/1313043000000/ks1/cf1/SNAPPY/PLAINTEXT/me-3-big-Data.db"); + fileList.add( + "test_backup/" + + cluster + + "/1808575600/META_V2/1313043300000/SNAPPY/PLAINTEXT/meta_v2_201108110615.json"); + fileList.add( + "test_backup/" + + cluster + + "/1808575600/META_V2/1313022540000/SNAPPY/PLAINTEXT/meta_v2_201108110029.json"); filesystem.setupTest(fileList); - conf.setRestorePrefix("RESTOREBUCKET/" + baseDir + "/" + region + "/fakecluster"); + conf.setRestorePrefix("RESTOREBUCKET/test_backup/" + cluster + ""); } @Test public void testRestore() throws Exception { - populateBackupFileSystem("test_backup"); + populateBackupFileSystem("1049_fake-app"); String dateRange = "201108110030,201108110530"; restore.restore(new DateUtil.DateRange(dateRange)); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(0))); @@ -82,8 +107,8 @@ public void testRestore() throws Exception { @Test public void testRestoreWithIncremental() throws Exception { - populateBackupFileSystem("test_backup"); - String dateRange = "201108110030,201108110730"; + populateBackupFileSystem("1049_fake-app"); + String dateRange = "201108110030,201108110601"; restore.restore(new DateUtil.DateRange(dateRange)); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(0))); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(1))); @@ -95,50 +120,53 @@ public void testRestoreWithIncremental() throws Exception { } @Test - public void testRestoreLatestWithEmptyMeta() throws Exception { - populateBackupFileSystem("test_backup"); - String metafile = - "test_backup/" + region + "/fakecluster/123456/201108110130/META/meta.json"; - filesystem.addFile(metafile); - String dateRange = "201108110030,201108110530"; + public void testRestoreWithIncrementalFromDifferentCluster() throws Exception { + populateBackupFileSystem("6089_fake-app2"); + String dateRange = "201108110030,201108110601"; restore.restore(new DateUtil.DateRange(dateRange)); - Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(0))); - Assert.assertTrue(filesystem.downloadedFiles.contains(metafile)); - Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(1))); - Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(2))); - Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(3))); - Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(4))); - Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(5))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(0))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(1))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(2))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(3))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(4))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(5))); Assert.assertEquals(Status.FINISHED, instanceState.getRestoreStatus().getStatus()); - Assert.assertEquals(metafile, instanceState.getRestoreStatus().getSnapshotMetaFile()); + } + + @Test + public void testRestoreEmptyMeta() throws Exception { + populateBackupFileSystem("1049_fake-app"); + String metafile = + "test_backup/1049_fake-app/1808575600/META_V2/1313022540000/SNAPPY/PLAINTEXT/meta_v2_201108110029.json"; + String dateRange = "201108110029,201108110030"; + restore.restore(new DateUtil.DateRange(dateRange)); + Truth.assertThat(filesystem.downloadedFiles).containsExactly(metafile); + Assert.assertEquals(Status.FAILED, instanceState.getRestoreStatus().getStatus()); + Assert.assertNull(instanceState.getRestoreStatus().getSnapshotMetaFile()); } @Test public void testRestoreLatest() throws Exception { - populateBackupFileSystem("test_backup"); + populateBackupFileSystem("1049_fake-app"); String metafile = - "test_backup/" + region + "/fakecluster/123456/201108110130/META/meta.json"; - filesystem.addFile(metafile); - String snapFile = - "test_backup/" + region + "/fakecluster/123456/201108110130/SNAP/ks1/cf1/f9.db"; - filesystem.addFile(snapFile); - String dateRange = "201108110030,201108110530"; + "test_backup/1049_fake-app/1808575600/META_V2/1313043300000/SNAPPY/PLAINTEXT/meta_v2_201108110615.json"; + String dateRange = "201108110030,201108110620"; restore.restore(new DateUtil.DateRange(dateRange)); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(0))); - Assert.assertTrue(filesystem.downloadedFiles.contains(metafile)); - Assert.assertTrue(filesystem.downloadedFiles.contains(snapFile)); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(1))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(2))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(3))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(4))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(5))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(6))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(7))); Assert.assertEquals(Status.FINISHED, instanceState.getRestoreStatus().getStatus()); Assert.assertEquals(metafile, instanceState.getRestoreStatus().getSnapshotMetaFile()); } @Test public void testNoSnapshots() throws Exception { - populateBackupFileSystem("test_backup"); + populateBackupFileSystem("1049_fake-app"); filesystem.setupTest(fileList); String dateRange = "201109110030,201109110530"; restore.restore(new DateUtil.DateRange(dateRange)); @@ -147,10 +175,9 @@ public void testNoSnapshots() throws Exception { @Test public void testRestoreFromDiffCluster() throws Exception { - populateBackupFileSystem("test_backup_new"); + populateBackupFileSystem("6089_fake-app2"); String dateRange = "201108110030,201108110530"; restore.restore(new DateUtil.DateRange(dateRange)); - System.out.println("Downloaded files: " + filesystem.downloadedFiles); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(0))); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(1))); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(2))); diff --git a/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java b/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java index c88953c2c..2d2a6f5a3 100644 --- a/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java +++ b/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java @@ -43,18 +43,16 @@ public void testAbstractPath() { Injector injector = Guice.createInjector(new BRTestModule()); IConfiguration conf = injector.getInstance(IConfiguration.class); InstanceIdentity factory = injector.getInstance(InstanceIdentity.class); - String region = factory.getInstanceInfo().getRegion(); FifoQueue queue = new FifoQueue<>(10); for (int i = 10; i < 30; i++) { RemoteBackupPath path = new RemoteBackupPath(conf, factory); path.parseRemote( "test_backup/" - + region - + "/fakecluster/123456/201108" + + "1941_fakecluster/123456/SST_V2/201108" + i + "0000" - + "/SNAP/ks1/cf2/f1" + + "/ks1/cf2/SNAPPY/PLAINTEXT/f1" + i + ".db"); queue.adjustAndAdd(path); @@ -63,12 +61,10 @@ public void testAbstractPath() { for (int i = 10; i < 30; i++) { RemoteBackupPath path = new RemoteBackupPath(conf, factory); path.parseRemote( - "test_backup/" - + region - + "/fakecluster/123456/201108" + "test_backup/1941_fakecluster/123456/SST_V2/201108" + i + "0000" - + "/SNAP/ks1/cf2/f2" + + "/ks1/cf2/SNAPPY/PLAINTEXT/f2" + i + ".db"); queue.adjustAndAdd(path); @@ -77,12 +73,10 @@ public void testAbstractPath() { for (int i = 10; i < 30; i++) { RemoteBackupPath path = new RemoteBackupPath(conf, factory); path.parseRemote( - "test_backup/" - + region - + "/fakecluster/123456/201108" + "test_backup/1941_fakecluster/123456/SST_V2/201108" + i + "0000" - + "/SNAP/ks1/cf2/f3" + + "/ks1/cf2/SNAPPY/PLAINTEXT/f3" + i + ".db"); queue.adjustAndAdd(path); @@ -90,43 +84,19 @@ public void testAbstractPath() { RemoteBackupPath path = new RemoteBackupPath(conf, factory); path.parseRemote( - "test_backup/" - + region - + "/fakecluster/123456/201108290000" - + "/SNAP/ks1/cf2/f129.db"); + "test_backup" + + "/1941_fakecluster/123456/SST_V2/201108290000" + + "/ks1/cf2/SNAPPY/PLAINTEXT/f129.db"); Assert.assertTrue(queue.contains(path)); path.parseRemote( - "test_backup/" - + region - + "/fakecluster/123456/201108290000" - + "/SNAP/ks1/cf2/f229.db"); + "test_backup" + + "/1941_fakecluster/123456/SST_V2/201108290000" + + "/ks1/cf2/SNAPPY/PLAINTEXT/f229.db"); Assert.assertTrue(queue.contains(path)); path.parseRemote( - "test_backup/" - + region - + "/fakecluster/123456/201108290000" - + "/SNAP/ks1/cf2/f329.db"); + "test_backup" + + "/1941_fakecluster/123456/SST_V2/201108290000" + + "/ks1/cf2/SNAPPY/PLAINTEXT/f329.db"); Assert.assertTrue(queue.contains(path)); - - path.parseRemote( - "test_backup/" - + region - + "/fakecluster/123456/201108260000/SNAP/ks1/cf2/f326.db To: cass/data/ks1/cf2/f326.db"); - Assert.assertEquals(path, queue.first()); - } - - @Test - public void testIgnoreIndexFiles() { - String[] testInputs = - new String[] { - "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Digest.sha1", - "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Filter.db", - "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Data.db", - "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Statistics.db", - "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Filter.db", - "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Digest.sha1", - "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Statistics.db", - "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Data.db" - }; } } diff --git a/priam/src/test/java/com/netflix/priam/utils/TestGsonJsonSerializer.java b/priam/src/test/java/com/netflix/priam/utils/TestGsonJsonSerializer.java index 08ed259e2..a4fb8a09a 100644 --- a/priam/src/test/java/com/netflix/priam/utils/TestGsonJsonSerializer.java +++ b/priam/src/test/java/com/netflix/priam/utils/TestGsonJsonSerializer.java @@ -17,7 +17,6 @@ package com.netflix.priam.utils; import com.netflix.priam.backup.BackupMetadata; -import com.netflix.priam.backup.BackupVersion; import com.netflix.priam.health.InstanceState; import java.time.LocalDateTime; import java.util.Calendar; @@ -33,9 +32,7 @@ public class TestGsonJsonSerializer { @Test public void testBackupMetaData() throws Exception { - BackupMetadata metadata = - new BackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, "123", Calendar.getInstance().getTime()); + BackupMetadata metadata = new BackupMetadata("123", Calendar.getInstance().getTime()); String json = metadata.toString(); LOG.info(json); // Deserialize it. diff --git a/priam/src/test/resources/fake-app2_meta_v2_201108110130.json b/priam/src/test/resources/fake-app2_meta_v2_201108110130.json new file mode 100644 index 000000000..c23752b94 --- /dev/null +++ b/priam/src/test/resources/fake-app2_meta_v2_201108110130.json @@ -0,0 +1,65 @@ +{ + "info": { + "version": 1, + "appName": "fake-app2", + "region": "us-east-1", + "rack": "us-east-1c", + "backupIdentifier": [ + "1808575600" + ] + }, + "data": [ + { + "keyspaceName": "ks1", + "columnfamilyName": "cf1", + "sstables": [ + { + "prefix": "me-1-big", + "sstableComponents": [ + { + "fileName": "me-1-big-Data.db", + "lastModifiedTime": 1313022601000, + "fileCreationTime": 1313022601000, + "fileSizeOnDisk": 4921307, + "compression": "SNAPPY", + "encryption": "PLAINTEXT", + "isUploaded": false, + "backupPath": "test_backup/6089_fake-app2/1808575600/SST_V2/1313022601000/ks1/cf1/SNAPPY/PLAINTEXT/me-1-big-Data.db" + }, + { + "fileName": "me-1-big-Index.db", + "lastModifiedTime": 1313022601000, + "fileCreationTime": 1313022601000, + "fileSizeOnDisk": 1828266, + "compression": "SNAPPY", + "encryption": "PLAINTEXT", + "isUploaded": false, + "backupPath": "test_backup/6089_fake-app2/1808575600/SST_V2/1313022601000/ks1/cf1/SNAPPY/PLAINTEXT/me-1-big-Index.db" + } + ] + } + ] + }, + { + "keyspaceName": "ks2", + "columnfamilyName": "cf1", + "sstables": [ + { + "prefix": "me-2-big", + "sstableComponents": [ + { + "fileName": "me-2-big-Data.db", + "lastModifiedTime": 1313022601000, + "fileCreationTime": 1313022601000, + "fileSizeOnDisk": 4921307, + "compression": "SNAPPY", + "encryption": "PLAINTEXT", + "isUploaded": false, + "backupPath": "test_backup/6089_fake-app2/1808575600/SST_V2/1313022601000/ks2/cf1/SNAPPY/PLAINTEXT/me-2-big-Data.db" + } + ] + } + ] + } + ] +} diff --git a/priam/src/test/resources/fake-app_meta_v2_201108110029.json b/priam/src/test/resources/fake-app_meta_v2_201108110029.json new file mode 100644 index 000000000..e69de29bb diff --git a/priam/src/test/resources/fake-app_meta_v2_201108110130.json b/priam/src/test/resources/fake-app_meta_v2_201108110130.json new file mode 100644 index 000000000..cd8484c5c --- /dev/null +++ b/priam/src/test/resources/fake-app_meta_v2_201108110130.json @@ -0,0 +1,65 @@ +{ + "info": { + "version": 1, + "appName": "fake-app", + "region": "us-east-1", + "rack": "us-east-1c", + "backupIdentifier": [ + "1808575600" + ] + }, + "data": [ + { + "keyspaceName": "ks1", + "columnfamilyName": "cf1", + "sstables": [ + { + "prefix": "me-1-big", + "sstableComponents": [ + { + "fileName": "me-1-big-Data.db", + "lastModifiedTime": 1313022601000, + "fileCreationTime": 1313022601000, + "fileSizeOnDisk": 4921307, + "compression": "SNAPPY", + "encryption": "PLAINTEXT", + "isUploaded": false, + "backupPath": "test_backup/1049_fake-app/1808575600/SST_V2/1313022601000/ks1/cf1/SNAPPY/PLAINTEXT/me-1-big-Data.db" + }, + { + "fileName": "me-1-big-Index.db", + "lastModifiedTime": 1313022601000, + "fileCreationTime": 1313022601000, + "fileSizeOnDisk": 1828266, + "compression": "SNAPPY", + "encryption": "PLAINTEXT", + "isUploaded": false, + "backupPath": "test_backup/1049_fake-app/1808575600/SST_V2/1313022601000/ks1/cf1/SNAPPY/PLAINTEXT/me-1-big-Index.db" + } + ] + } + ] + }, + { + "keyspaceName": "ks2", + "columnfamilyName": "cf1", + "sstables": [ + { + "prefix": "me-2-big", + "sstableComponents": [ + { + "fileName": "me-2-big-Data.db", + "lastModifiedTime": 1313022601000, + "fileCreationTime": 1313022601000, + "fileSizeOnDisk": 4921307, + "compression": "SNAPPY", + "encryption": "PLAINTEXT", + "isUploaded": false, + "backupPath": "test_backup/1049_fake-app/1808575600/SST_V2/1313022601000/ks2/cf1/SNAPPY/PLAINTEXT/me-2-big-Data.db" + } + ] + } + ] + } + ] +} diff --git a/priam/src/test/resources/fake-app_meta_v2_201108110615.json b/priam/src/test/resources/fake-app_meta_v2_201108110615.json new file mode 100644 index 000000000..65022c727 --- /dev/null +++ b/priam/src/test/resources/fake-app_meta_v2_201108110615.json @@ -0,0 +1,34 @@ +{ + "info": { + "version": 1, + "appName": "fake-app", + "region": "us-east-1", + "rack": "us-east-1c", + "backupIdentifier": [ + "1808575600" + ] + }, + "data": [ + { + "keyspaceName": "ks1", + "columnfamilyName": "cf1", + "sstables": [ + { + "prefix": "me-3-big", + "sstableComponents": [ + { + "fileName": "me-3-big-Data.db", + "lastModifiedTime": 1313043000000, + "fileCreationTime": 1313043000000, + "fileSizeOnDisk": 4921307, + "compression": "SNAPPY", + "encryption": "PLAINTEXT", + "isUploaded": false, + "backupPath": "test_backup/1049_fake-app/1808575600/SST_V2/1313043000000/ks1/cf1/SNAPPY/PLAINTEXT/me-3-big-Data.db" + } + ] + } + ] + } + ] +}