Skip to content

Commit

Permalink
Reveal hook to allow operators to restore just to the most recent sna…
Browse files Browse the repository at this point in the history 
…pshot (#1035)

* Remove unused code.

* Remove redundant comments and vertical whitespace.

* Remove debug comments and now-redundant logger, simplify if-else and tighten error message for code style.

* Use final where applicable and remove it where redundant.

* Remove redundant BackupRestoreException from getIncrementals method signature.

* Split getting incremental files and snapshot files into separate methods.

* Reveal hook to allow operators to restore to the last valid snapshot.

* Remove added non-shaded Guava dependency pursuant to review comments.
  • Loading branch information
@mattl-netflix
mattl-netflix committed Mar 29, 2023
1 parent 0fb87f9 commit 36321c4
Show file tree
Hide file tree
Showing 7 changed files with 45 additions and 83 deletions.
93 changes: 25 additions & 68 deletions priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java
View file
Expand Up @@ -30,17 +30,13 @@
import javax.inject.Provider;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** Created by aagrawal on 8/14/17. */
/** Helper methods applicable to both backup and restore */
public class BackupRestoreUtil {
private static final Logger logger = LoggerFactory.getLogger(BackupRestoreUtil.class);
private static final Pattern columnFamilyFilterPattern = Pattern.compile(".\\..");
private Map<String, List<String>> includeFilter;
private Map<String, List<String>> excludeFilter;
private final Map<String, List<String>> includeFilter;
private final Map<String, List<String>> excludeFilter;

public static final List<String> FILTER_KEYSPACE = Collections.singletonList("OpsCenter");
private static final Map<String, List<String>> FILTER_COLUMN_FAMILY =
ImmutableMap.of(
"system",
Expand All @@ -49,43 +45,26 @@ public class BackupRestoreUtil {

@Inject
public BackupRestoreUtil(String configIncludeFilter, String configExcludeFilter) {
setFilters(configIncludeFilter, configExcludeFilter);
}

public BackupRestoreUtil setFilters(String configIncludeFilter, String configExcludeFilter) {
includeFilter = getFilter(configIncludeFilter);
excludeFilter = getFilter(configExcludeFilter);
logger.info("Exclude filter set: {}", configExcludeFilter);
logger.info("Include filter set: {}", configIncludeFilter);
return this;
}

public static Optional<AbstractBackupPath> getLatestValidMetaPath(
IMetaProxy metaProxy, DateUtil.DateRange dateRange) {
// Get a list of manifest files.
List<AbstractBackupPath> metas = metaProxy.findMetaFiles(dateRange);

// Find a valid manifest file.
for (AbstractBackupPath meta : metas) {
BackupVerificationResult result = metaProxy.isMetaFileValid(meta);
if (result.valid) {
return Optional.of(meta);
}
}

return Optional.empty();
return metaProxy
.findMetaFiles(dateRange)
.stream()
.filter(meta -> metaProxy.isMetaFileValid(meta).valid)
.findFirst();
}

public static List<AbstractBackupPath> getAllFiles(
public static List<AbstractBackupPath> getMostRecentSnapshotPaths(
AbstractBackupPath latestValidMetaFile,
DateUtil.DateRange dateRange,
IMetaProxy metaProxy,
Provider<AbstractBackupPath> pathProvider)
throws Exception {
// Download the meta.json file.
Path metaFile = metaProxy.downloadMetaFile(latestValidMetaFile);
// Parse meta.json file to find the files required to download from this snapshot.
List<AbstractBackupPath> allFiles =
List<AbstractBackupPath> snapshotPaths =
metaProxy
.getSSTFilesFromMeta(metaFile)
.stream()
Expand All @@ -96,50 +75,43 @@ public static List<AbstractBackupPath> getAllFiles(
return path;
})
.collect(Collectors.toList());

FileUtils.deleteQuietly(metaFile.toFile());
return snapshotPaths;
}

// Download incremental SSTables after the snapshot meta file.
public static List<AbstractBackupPath> getIncrementalPaths(
AbstractBackupPath latestValidMetaFile,
DateUtil.DateRange dateRange,
IMetaProxy metaProxy) {
Instant snapshotTime;
if (metaProxy instanceof MetaV2Proxy) snapshotTime = latestValidMetaFile.getLastModified();
else snapshotTime = latestValidMetaFile.getTime().toInstant();

DateUtil.DateRange incrementalDateRange =
new DateUtil.DateRange(snapshotTime, dateRange.getEndTime());
Iterator<AbstractBackupPath> incremental = metaProxy.getIncrementals(incrementalDateRange);
while (incremental.hasNext()) allFiles.add(incremental.next());

return allFiles;
List<AbstractBackupPath> incrementalPaths = new ArrayList<>();
metaProxy.getIncrementals(incrementalDateRange).forEachRemaining(incrementalPaths::add);
return incrementalPaths;
}

public static final Map<String, List<String>> getFilter(String inputFilter)
public static Map<String, List<String>> getFilter(String inputFilter)
throws IllegalArgumentException {
if (StringUtils.isEmpty(inputFilter)) return null;

final Map<String, List<String>> columnFamilyFilter =
new HashMap<>(); // key: keyspace, value: a list of CFs within the keyspace

final Map<String, List<String>> columnFamilyFilter = new HashMap<>();
String[] filters = inputFilter.split(",");
for (String cfFilter :
filters) { // process filter of form keyspace.* or keyspace.columnfamily
for (String cfFilter : filters) {
if (columnFamilyFilterPattern.matcher(cfFilter).find()) {

String[] filter = cfFilter.split("\\.");
String keyspaceName = filter[0];
String columnFamilyName = filter[1];

if (columnFamilyName.contains("-"))
columnFamilyName = columnFamilyName.substring(0, columnFamilyName.indexOf("-"));

List<String> existingCfs =
columnFamilyFilter.getOrDefault(keyspaceName, new ArrayList<>());
if (!columnFamilyName.equalsIgnoreCase("*")) existingCfs.add(columnFamilyName);
columnFamilyFilter.put(keyspaceName, existingCfs);

} else {
throw new IllegalArgumentException(
"Column family filter format is not valid. Format needs to be \"keyspace.columnfamily\". Invalid input: "
+ cfFilter);
"Invalid format: " + cfFilter + ". \"keyspace.columnfamily\" is required.");
}
}
return columnFamilyFilter;
Expand All @@ -154,34 +126,19 @@ public static final Map<String, List<String>> getFilter(String inputFilter)
*/
public final boolean isFiltered(String keyspace, String columnFamilyDir) {
if (StringUtils.isEmpty(keyspace) || StringUtils.isEmpty(columnFamilyDir)) return false;

String columnFamilyName = columnFamilyDir.split("-")[0];
// column family is in list of global CF filter
if (FILTER_COLUMN_FAMILY.containsKey(keyspace)
&& FILTER_COLUMN_FAMILY.get(keyspace).contains(columnFamilyName)) return true;

if (excludeFilter != null)
if (excludeFilter.containsKey(keyspace)
&& (excludeFilter.get(keyspace).isEmpty()
|| excludeFilter.get(keyspace).contains(columnFamilyName))) {
logger.debug(
"Skipping: keyspace: {}, CF: {} is part of exclude list.",
keyspace,
columnFamilyName);
return true;
}

if (includeFilter != null)
if (!(includeFilter.containsKey(keyspace)
return !(includeFilter.containsKey(keyspace)
&& (includeFilter.get(keyspace).isEmpty()
|| includeFilter.get(keyspace).contains(columnFamilyName)))) {
logger.debug(
"Skipping: keyspace: {}, CF: {} is not part of include list.",
keyspace,
columnFamilyName);
return true;
}

|| includeFilter.get(keyspace).contains(columnFamilyName)));
return false;
}
}
4 changes: 1 addition & 3 deletions priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java
View file
Expand Up @@ -77,10 +77,8 @@ public interface IMetaProxy {
*
* @param dateRange the time period to scan in the remote file system for incremental files.
* @return iterator containing the list of path on the remote file system satisfying criteria.
* @throws BackupRestoreException if there is an issue contacting remote file system.
*/
Iterator<AbstractBackupPath> getIncrementals(DateUtil.DateRange dateRange)
throws BackupRestoreException;
Iterator<AbstractBackupPath> getIncrementals(DateUtil.DateRange dateRange);

/**
* Validate that all the files mentioned in the meta file actually exists on remote file system.
Expand Down
3 changes: 1 addition & 2 deletions priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java
View file
Expand Up @@ -169,8 +169,7 @@ public List<String> getSSTFilesFromMeta(Path localMetaPath) throws Exception {
}

@Override
public Iterator<AbstractBackupPath> getIncrementals(DateUtil.DateRange dateRange)
throws BackupRestoreException {
public Iterator<AbstractBackupPath> getIncrementals(DateUtil.DateRange dateRange) {
String prefix = fs.getPrefix().toString();
Iterator<AbstractBackupPath> iterator =
fs.list(
Expand Down
3 changes: 1 addition & 2 deletions priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java
View file
Expand Up @@ -79,8 +79,7 @@ private String getMatch(
}

@Override
public Iterator<AbstractBackupPath> getIncrementals(DateUtil.DateRange dateRange)
throws BackupRestoreException {
public Iterator<AbstractBackupPath> getIncrementals(DateUtil.DateRange dateRange) {
String incrementalPrefix = getMatch(dateRange, AbstractBackupPath.BackupFileType.SST_V2);
String marker =
getMatch(
Expand Down
5 changes: 5 additions & 0 deletions priam/src/main/java/com/netflix/priam/config/IConfiguration.java
View file
Expand Up @@ -1178,6 +1178,11 @@ default long getCompressionTransitionEpochMillis() {
/** @return whether to enable auto_snapshot */
boolean getAutoSnapshot();

/** @return whether incremental backups should be skipped in a restore */
default boolean skipIncrementalRestore() {
return false;
}

/**
* Escape hatch for getting any arbitrary property by key This is useful so we don't have to
* keep adding methods to this interface for every single configuration option ever. Also
Expand Down
7 changes: 5 additions & 2 deletions priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java
View file
Expand Up @@ -147,8 +147,11 @@ public Response list(@PathParam("daterange") String daterange) throws Exception
return Response.ok("No valid meta found!").build();
}
List<AbstractBackupPath> allFiles =
BackupRestoreUtil.getAllFiles(
latestValidMetaFile.get(), dateRange, metaProxy, pathProvider);
BackupRestoreUtil.getMostRecentSnapshotPaths(
latestValidMetaFile.get(), metaProxy, pathProvider);
allFiles.addAll(
BackupRestoreUtil.getIncrementalPaths(
latestValidMetaFile.get(), dateRange, metaProxy));

return Response.ok(
GsonJsonSerializer.getGson()
Expand Down
13 changes: 7 additions & 6 deletions priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java
View file
Expand Up @@ -114,10 +114,6 @@ public static final boolean isRestoreEnabled(IConfiguration conf, InstanceInfo i
return (isRestoreMode && isBackedupRac);
}

public void setRestoreConfiguration(String restoreIncludeCFList, String restoreExcludeCFList) {
backupRestoreUtil.setFilters(restoreIncludeCFList, restoreExcludeCFList);
}

private List<Future<Path>> download(
Iterator<AbstractBackupPath> fsIterator, boolean waitForCompletion) throws Exception {
List<Future<Path>> futureList = new ArrayList<>();
Expand Down Expand Up @@ -247,8 +243,13 @@ public void restore(DateUtil.DateRange dateRange) throws Exception {
.setSnapshotMetaFile(latestValidMetaFile.get().getRemotePath());

List<AbstractBackupPath> allFiles =
BackupRestoreUtil.getAllFiles(
latestValidMetaFile.get(), dateRange, metaProxy, pathProvider);
BackupRestoreUtil.getMostRecentSnapshotPaths(
latestValidMetaFile.get(), metaProxy, pathProvider);
if (!config.skipIncrementalRestore()) {
allFiles.addAll(
BackupRestoreUtil.getIncrementalPaths(
latestValidMetaFile.get(), dateRange, metaProxy));
}

// Download snapshot which is listed in the meta file.
List<Future<Path>> futureList = new ArrayList<>();
Expand Down

0 comments on commit 36321c4

Please sign in to comment.