Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Return missing files from snapshot verification continuing to marginalize BackupVerificationResult. #1039

Open
wants to merge 1 commit into
base: 3.11
Choose a base branch
from
+151 −178
Open

Return missing files from snapshot verification continuing to marginalize BackupVerificationResult. #1039

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
42 changes: 13 additions & 29 deletions priam/src/main/java/com/netflix/priam/backup/BackupVerification.java
View file
Open in desktop
Expand Up @@ -13,6 +13,8 @@
*/
package com.netflix.priam.backup;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.netflix.priam.backupv2.IMetaProxy;
import com.netflix.priam.utils.DateUtil;
import com.netflix.priam.utils.DateUtil.DateRange;
Expand Down Expand Up @@ -68,46 +70,29 @@ public IMetaProxy getMetaProxy(BackupVersion backupVersion) {
return null;
}

public Optional<BackupVerificationResult> verifyLatestBackup(
public ImmutableMap<BackupMetadata, ImmutableSet<String>> findMissingBackupFilesInRange(
BackupVersion backupVersion, boolean force, DateRange dateRange)
throws IllegalArgumentException {
IMetaProxy metaProxy = getMetaProxy(backupVersion);
ImmutableMap.Builder<BackupMetadata, ImmutableSet<String>> mapBuilder =
ImmutableMap.builder();
for (BackupMetadata backupMetadata :
backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange)) {
if (backupMetadata.getLastValidated() == null || force) {
Optional<BackupVerificationResult> result = verifyBackup(metaProxy, backupMetadata);
if (result.isPresent()) {
return result;
}
} else {
updateLatestResult(backupMetadata);
return Optional.of(latestResult);
}
List<String> missingFiles =
backupMetadata.getLastValidated() == null || force
? verifyBackup(metaProxy, backupMetadata).filesInMetaOnly
: new ArrayList<>();
mapBuilder.put(backupMetadata, ImmutableSet.copyOf(missingFiles));
}
latestResult = null;
return Optional.empty();
}

public List<BackupMetadata> verifyBackupsInRange(
BackupVersion backupVersion, DateRange dateRange) throws IllegalArgumentException {
IMetaProxy metaProxy = getMetaProxy(backupVersion);
List<BackupMetadata> results = new ArrayList<>();
for (BackupMetadata backupMetadata :
backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange)) {
if (backupMetadata.getLastValidated() != null
|| verifyBackup(metaProxy, backupMetadata).isPresent()) {
results.add(backupMetadata);
}
}
return results;
return mapBuilder.build();
}

/** returns the latest valid backup verification result if we have found one within the SLO * */
public Optional<Instant> getLatestVerfifiedBackupTime() {
return latestResult == null ? Optional.empty() : Optional.of(latestResult.snapshotInstant);
}

private Optional<BackupVerificationResult> verifyBackup(
private BackupVerificationResult verifyBackup(
IMetaProxy metaProxy, BackupMetadata latestBackupMetaData) {
Path metadataLocation = Paths.get(latestBackupMetaData.getSnapshotLocation());
metadataLocation = metadataLocation.subpath(1, metadataLocation.getNameCount());
Expand All @@ -119,9 +104,8 @@ private Optional<BackupVerificationResult> verifyBackup(
Date now = new Date(DateUtil.getInstant().toEpochMilli());
latestBackupMetaData.setLastValidated(now);
backupStatusMgr.update(latestBackupMetaData);
return Optional.of(result);
}
return Optional.empty();
return result;
}

private void updateLatestResult(BackupMetadata backupMetadata) {
Expand Down
25 changes: 17 additions & 8 deletions priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java
View file
Open in desktop
Expand Up @@ -30,7 +30,9 @@
import com.netflix.priam.utils.DateUtil.DateRange;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.slf4j.Logger;
Expand Down Expand Up @@ -84,21 +86,28 @@ public void execute() throws Exception {
Instant slo =
now.minus(backupRestoreConfig.getBackupVerificationSLOInHours(), ChronoUnit.HOURS);
DateRange dateRange = new DateRange(slo, now);
List<BackupMetadata> verifiedBackups =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_META_SERVICE, dateRange);
Set<BackupMetadata> verifiedBackups =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE, false /* force */, dateRange)
.entrySet()
.stream()
.filter(entry -> entry.getValue().isEmpty())
.map(Map.Entry::getKey)
.collect(Collectors.toSet());

verifiedBackups
.stream()
.filter(result -> result.getLastValidated().toInstant().isAfter(now))
.filter(metadata -> metadata.getLastValidated().toInstant().isAfter(now))
.forEach(
result -> {
metadata -> {
logger.info(
"Sending {} message for backup: {}",
AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED,
result.getSnapshotLocation());
metadata.getSnapshotLocation());
backupNotificationMgr.notify(
result.getSnapshotLocation(), result.getStart().toInstant());
metadata.getSnapshotLocation(),
metadata.getStart().toInstant());
});

if (verifiedBackups.isEmpty()) {
Expand Down
18 changes: 7 additions & 11 deletions priam/src/main/java/com/netflix/priam/resources/BackupServlet.java
View file
Open in desktop
Expand Up @@ -16,13 +16,16 @@
*/
package com.netflix.priam.resources;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.netflix.priam.backup.*;
import com.netflix.priam.backup.AbstractBackupPath.BackupFileType;
import com.netflix.priam.config.IBackupRestoreConfig;
import com.netflix.priam.config.IConfiguration;
import com.netflix.priam.scheduler.PriamScheduler;
import com.netflix.priam.utils.DateUtil;
import com.netflix.priam.utils.DateUtil.DateRange;
import com.netflix.priam.utils.GsonJsonSerializer;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.*;
Expand Down Expand Up @@ -215,19 +218,12 @@ public Response snapshotsByDate(@PathParam("date") String date) throws Exception
@Produces(MediaType.APPLICATION_JSON)
public Response validateSnapshotByDate(
@PathParam("daterange") String daterange,
@DefaultValue("false") @QueryParam("force") boolean force)
throws Exception {
@DefaultValue("false") @QueryParam("force") boolean force) {
DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange);
Optional<BackupVerificationResult> result =
backupVerification.verifyLatestBackup(
ImmutableMap<BackupMetadata, ImmutableSet<String>> result =
backupVerification.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_BACKUP, force, dateRange);
if (!result.isPresent()) {
return Response.noContent()
.entity("No valid meta found for provided time range")
.build();
}

return Response.ok(result.get().toString()).build();
return Response.ok(GsonJsonSerializer.getGson().toJson(result)).build();
}

/*
Expand Down
17 changes: 6 additions & 11 deletions priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java
View file
Open in desktop
Expand Up @@ -17,6 +17,8 @@

package com.netflix.priam.resources;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.netflix.priam.backup.*;
import com.netflix.priam.backupv2.BackupTTLTask;
import com.netflix.priam.backupv2.BackupV2Service;
Expand Down Expand Up @@ -121,19 +123,12 @@ public Response info(@PathParam("date") String date) {
@Path("/validate/{daterange}")
public Response validateV2SnapshotByDate(
@PathParam("daterange") String daterange,
@DefaultValue("false") @QueryParam("force") boolean force)
throws Exception {
@DefaultValue("false") @QueryParam("force") boolean force) {
DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange);
Optional<BackupVerificationResult> result =
backupVerification.verifyLatestBackup(
ImmutableMap<BackupMetadata, ImmutableSet<String>> result =
backupVerification.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE, force, dateRange);
if (!result.isPresent()) {
return Response.noContent()
.entity("No valid meta found for provided time range")
.build();
}

return Response.ok(result.get().toString()).build();
return Response.ok(GsonJsonSerializer.getGson().toJson(result)).build();
}

@GET
Expand Down
112 changes: 57 additions & 55 deletions priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java
View file
Open in desktop
Expand Up @@ -17,6 +17,9 @@

package com.netflix.priam.backup;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.truth.Truth;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.netflix.priam.backup.AbstractBackupPath.BackupFileType;
Expand All @@ -31,9 +34,7 @@
import java.nio.file.Paths;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.*;
import mockit.Mock;
import mockit.MockUp;
import org.apache.commons.io.FileUtils;
Expand Down Expand Up @@ -91,34 +92,43 @@ public void cleanup() {
}

@Test
public void noBackup() throws Exception {
Optional<BackupVerificationResult> backupVerificationResultOptinal =
backupVerification.verifyLatestBackup(
public void noBackup() {
ImmutableMap<BackupMetadata, ImmutableSet<String>> missingFiles =
backupVerification.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_BACKUP,
false,
new DateRange(Instant.now(), Instant.now()));
Assert.assertFalse(backupVerificationResultOptinal.isPresent());
Truth.assertThat(missingFiles).isEmpty();

backupVerificationResultOptinal =
backupVerification.verifyLatestBackup(
missingFiles =
backupVerification.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
false,
new DateRange(Instant.now(), Instant.now()));
Assert.assertFalse(backupVerificationResultOptinal.isPresent());
Truth.assertThat(missingFiles).isEmpty();
}

@Test
public void noBackupDateRange() throws Exception {
List<BackupMetadata> backupVerificationResults =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_BACKUP, new DateRange(Instant.now(), Instant.now()));
Assert.assertFalse(backupVerificationResults.size() > 0);
long foundBackups =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_BACKUP,
false /* force */,
new DateRange(Instant.now(), Instant.now()))
.entrySet()
.size();
Truth.assertThat(foundBackups).isEqualTo(0L);

backupVerificationResults =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
new DateRange(Instant.now(), Instant.now()));
Assert.assertFalse(backupVerificationResults.size() > 0);
foundBackups =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
false /* force */,
new DateRange(Instant.now(), Instant.now()))
.entrySet()
.size();
Truth.assertThat(foundBackups).isEqualTo(0L);
}

private void setUp() throws Exception {
Expand Down Expand Up @@ -152,13 +162,12 @@ private void setUp() throws Exception {
public void verifyBackupVersion1() throws Exception {
setUp();
// Verify for backup version 1.0
Optional<BackupVerificationResult> backupVerificationResultOptinal =
backupVerification.verifyLatestBackup(
ImmutableMap<BackupMetadata, ImmutableSet<String>> missingFiles =
backupVerification.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_BACKUP,
false,
new DateRange(backupDate + "," + backupDate));
Assert.assertTrue(backupVerificationResultOptinal.isPresent());
Assert.assertEquals(Instant.EPOCH, backupVerificationResultOptinal.get().snapshotInstant);
Truth.assertThat(missingFiles.values().stream().allMatch(Set::isEmpty)).isTrue();
Optional<BackupMetadata> backupMetadata =
backupStatusMgr
.getLatestBackupMetadata(
Expand All @@ -184,12 +193,17 @@ public void verifyBackupVersion1() throws Exception {
public void verifyBackupVersion1DateRange() throws Exception {
setUp();
// Verify for backup version 1.0
List<BackupMetadata> backupVerificationResults =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_BACKUP,
new DateRange(backupDate + "," + backupDateEnd));
Assert.assertTrue(!backupVerificationResults.isEmpty());
Assert.assertTrue(backupVerificationResults.size() == numFakeBackups);
long missingFilesCount =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_BACKUP,
false /* force */,
new DateRange(backupDate + "," + backupDateEnd))
.values()
.stream()
.filter(AbstractCollection::isEmpty)
.count();
Truth.assertThat(missingFilesCount).isEqualTo(numFakeBackups);
List<BackupMetadata> backupMetadata =
backupStatusMgr.getLatestBackupMetadata(
BackupVersion.SNAPSHOT_BACKUP,
Expand All @@ -211,14 +225,12 @@ public void verifyBackupVersion1DateRange() throws Exception {
public void verifyBackupVersion2() throws Exception {
setUp();
// Verify for backup version 2.0
Optional<BackupVerificationResult> backupVerificationResultOptinal =
backupVerification.verifyLatestBackup(
ImmutableMap<BackupMetadata, ImmutableSet<String>> missingFiles =
backupVerification.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
false,
new DateRange(backupDate + "," + backupDate));
Assert.assertTrue(backupVerificationResultOptinal.isPresent());
Assert.assertEquals(Instant.EPOCH, backupVerificationResultOptinal.get().snapshotInstant);
Assert.assertEquals("some_random", backupVerificationResultOptinal.get().remotePath);
Truth.assertThat(missingFiles.values().stream().allMatch(Set::isEmpty)).isTrue();

Optional<BackupMetadata> backupMetadata =
backupStatusMgr
Expand All @@ -230,21 +242,6 @@ public void verifyBackupVersion2() throws Exception {
Assert.assertTrue(backupMetadata.isPresent());
Assert.assertNotNull(backupMetadata.get().getLastValidated());

// Retry the verification, it should not try and re-verify
backupVerificationResultOptinal =
backupVerification.verifyLatestBackup(
BackupVersion.SNAPSHOT_META_SERVICE,
false,
new DateRange(backupDate + "," + backupDate));
Assert.assertTrue(backupVerificationResultOptinal.isPresent());
Assert.assertEquals(
DateUtil.parseInstant(backupDate),
backupVerificationResultOptinal.get().snapshotInstant);
Assert.assertNotEquals("some_random", backupVerificationResultOptinal.get().remotePath);
Assert.assertEquals(
location.subpath(1, location.getNameCount()).toString(),
backupVerificationResultOptinal.get().remotePath);

backupMetadata =
backupStatusMgr
.getLatestBackupMetadata(
Expand All @@ -260,12 +257,17 @@ public void verifyBackupVersion2() throws Exception {
public void verifyBackupVersion2DateRange() throws Exception {
setUp();
// Verify for backup version 2.0
List<BackupMetadata> backupVerificationResults =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
new DateRange(backupDate + "," + backupDateEnd));
Assert.assertTrue(!backupVerificationResults.isEmpty());
Assert.assertTrue(backupVerificationResults.size() == numFakeBackups);
long missingFilesCount =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
false /* force */,
new DateRange(backupDate + "," + backupDateEnd))
.values()
.stream()
.filter(AbstractCollection::isEmpty)
.count();
Truth.assertThat(missingFilesCount).isEqualTo(numFakeBackups);
List<BackupMetadata> backupMetadata =
backupStatusMgr.getLatestBackupMetadata(
BackupVersion.SNAPSHOT_META_SERVICE,
Expand Down