Add metrics for back TTL jobs

Netflix · May 20, 2023 · 1715925 · 1715925
1 parent 882dbc9
commit 1715925
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 10 deletions.
diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java
@@ -22,6 +22,7 @@
 import com.netflix.priam.config.IConfiguration;
 import com.netflix.priam.health.InstanceState;
 import com.netflix.priam.identity.token.TokenRetriever;
+import com.netflix.priam.merics.BackupMetrics;
 import com.netflix.priam.scheduler.SimpleTimer;
 import com.netflix.priam.scheduler.Task;
 import com.netflix.priam.scheduler.TaskTimer;
@@ -56,18 +57,19 @@
 @Singleton
 public class BackupTTLTask extends Task {
     private static final Logger logger = LoggerFactory.getLogger(BackupTTLTask.class);
-    private IBackupRestoreConfig backupRestoreConfig;
-    private IMetaProxy metaProxy;
-    private IBackupFileSystem fileSystem;
-    private Provider<AbstractBackupPath> abstractBackupPathProvider;
-    private InstanceState instanceState;
+    private final IBackupRestoreConfig backupRestoreConfig;
+    private final IMetaProxy metaProxy;
+    private final IBackupFileSystem fileSystem;
+    private final Provider<AbstractBackupPath> abstractBackupPathProvider;
+    private final InstanceState instanceState;
     public static final String JOBNAME = "BackupTTLService";
-    private Map<String, Boolean> filesInMeta = new HashMap<>();
-    private List<Path> filesToDelete = new ArrayList<>();
+    private final Map<String, Boolean> filesInMeta = new HashMap<>();
+    private final List<Path> filesToDelete = new ArrayList<>();
     private static final Lock lock = new ReentrantLock();
-    private final int BATCH_SIZE = 1000;
+    private static final int BATCH_SIZE = 1000;
     private final Instant start_of_feature = DateUtil.parseInstant("201801010000");
     private final int maxWaitMillis;
+    private final BackupMetrics backupMetrics;
 
     @Inject
     public BackupTTLTask(
@@ -77,7 +79,8 @@ public BackupTTLTask(
             IFileSystemContext backupFileSystemCtx,
             Provider<AbstractBackupPath> abstractBackupPathProvider,
             InstanceState instanceState,
-            TokenRetriever tokenRetriever)
+            TokenRetriever tokenRetriever,
+            BackupMetrics backupMetrics)
             throws Exception {
         super(configuration);
         this.backupRestoreConfig = backupRestoreConfig;
@@ -89,6 +92,7 @@ public BackupTTLTask(
                 1_000
                         * backupRestoreConfig.getBackupTTLMonitorPeriodInSec()
                         / tokenRetriever.getRingPosition().getDenominator();
+        this.backupMetrics = backupMetrics;
     }
 
     @Override
@@ -221,6 +225,8 @@ private void deleteFile(AbstractBackupPath path, boolean forceClear)
 
         if (forceClear || filesToDelete.size() >= BATCH_SIZE) {
             fileSystem.deleteRemoteFiles(filesToDelete);
+            backupMetrics.incrementBackupTtlFiles(filesToDelete.size());
+            logger.info("Deleted(TTL-ed) {} META backup files", filesToDelete.size());
             filesToDelete.clear();
         }
     }

diff --git a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java
@@ -36,7 +36,8 @@ public class BackupMetrics {
             snsNotificationSuccess,
             snsNotificationFailure,
             forgottenFiles,
-            backupVerificationFailure;
+            backupVerificationFailure,
+            backupTTLFiles;
     public static final String uploadQueueSize = Metrics.METRIC_PREFIX + "upload.queue.size";
     public static final String downloadQueueSize = Metrics.METRIC_PREFIX + "download.queue.size";
 
@@ -56,6 +57,7 @@ public BackupMetrics(Registry registry) {
         forgottenFiles = registry.counter(Metrics.METRIC_PREFIX + "forgotten.files");
         backupVerificationFailure =
                 registry.counter(Metrics.METRIC_PREFIX + "backup.verification.failure");
+        backupTTLFiles = registry.counter(Metrics.METRIC_PREFIX + "backup.ttl.files");
     }
 
     public DistributionSummary getUploadRate() {
@@ -130,6 +132,10 @@ public void incrementValidDownloads() {
         this.validDownloads.increment();
     }
 
+    public void incrementBackupTtlFiles(int amount){
+        this.backupTTLFiles.increment(amount);
+    }
+
     public Registry getRegistry() {
         return registry;
     }