OAK-10281 : draft of handling recoveryDelayMillis non static

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentStore.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+public abstract class BaseDocumentStore implements DocumentStore {
+
+    long recoveryDelayMillis = 0;
+
+    @Override
+    public long getRecoveryDelayMillis() {
+        return recoveryDelayMillis;
+    }
+
+    public void setRecoveryDelayMillis(long recoveryDelayMillis) {
+        this.recoveryDelayMillis = recoveryDelayMillis;
+    }
+}

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java

@@ -240,13 +240,6 @@ static RecoverLockState fromString(String state) {
     /** OAK-10281 : default millis to delay a recovery after a lease timeout */
     static final long DEFAULT_RECOVERY_DELAY_MILLIS = 0;
 
-    /**
-     * Actual millis to delay a recovery after a lease timeout.
-     * <p>
-     * Initialized by DocumentNodeStore constructor.
-     */
-    static long recoveryDelayMillis = DEFAULT_RECOVERY_DELAY_MILLIS;
-
     /**
      * The Oak version.
      */
@@ -1235,19 +1228,6 @@ static void resetClockToDefault() {
         clock = Clock.SIMPLE;
     }
 
-    static long getRecoveryDelayMillis() {
-        return recoveryDelayMillis;
-    }
-
-    static void setRecoveryDelayMillis(long recoveryDelayMillis) {
-        ClusterNodeInfo.recoveryDelayMillis = recoveryDelayMillis;
-    }
-
-    /** <b>only used for testing</b> **/
-    static void resetRecoveryDelayMillisToDefault() {
-        recoveryDelayMillis = DEFAULT_RECOVERY_DELAY_MILLIS;
-    }
-
     private static long getProcessId() {
         try {
             String name = ManagementFactory.getRuntimeMXBean().getName();

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java

@@ -52,6 +52,12 @@ public class ClusterNodeInfoDocument extends Document {
      */
     private final long created = Revision.getCurrentTimestamp();
 
+    private final long recoveryDelayMillis;
+
+    public ClusterNodeInfoDocument(DocumentStore store) {
+        this.recoveryDelayMillis = store.getRecoveryDelayMillis();
+    }
+
     /**
      * @return the timestamp when this document was created.
      */
@@ -108,7 +114,7 @@ public boolean isBeingRecovered(){
      */
     public boolean isRecoveryNeeded(long currentTimeMillis) {
         return isActive() &&
-                (currentTimeMillis - getLeaseEndTime() > ClusterNodeInfo.getRecoveryDelayMillis() ||
+                (currentTimeMillis - getLeaseEndTime() > recoveryDelayMillis ||
                         isBeingRecovered());
     }
 

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Collection.java

@@ -54,7 +54,7 @@ public NodeDocument newDocument(DocumentStore store) {
                 @Override
                 @NotNull
                 public ClusterNodeInfoDocument newDocument(DocumentStore store) {
-                    return new ClusterNodeInfoDocument();
+                    return new ClusterNodeInfoDocument(store);
                 }
             };
 

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java

@@ -576,6 +576,10 @@ public DocumentNodeStore(DocumentNodeStoreBuilder<?> builder) {
             this.simpleRevisionCounter = new AtomicInteger(0);
         }
         DocumentStore s = builder.getDocumentStore();
+        if (s instanceof BaseDocumentStore) {
+            BaseDocumentStore bds = (BaseDocumentStore) s;
+            bds.setRecoveryDelayMillis(builder.getRecoveryDelayMillis());
+        }
         checkServerTimeDifference(s);
         if (builder.getTiming()) {
             s = new TimingDocumentStoreWrapper(s);

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java

@@ -524,9 +524,6 @@ public void handleLeaseFailure() {
         if (isThrottlingEnabled(builder)) {
             builder.setThrottlingStatsCollector(new ThrottlingStatsCollectorImpl(statisticsProvider));
         }
-
-        // initialize the (global) recoveryDelayMillis
-        ClusterNodeInfo.setRecoveryDelayMillis(builder.getRecoveryDelayMillis());
     }
 
     private boolean isWrappingCustomBlobStore() {

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java

@@ -571,4 +571,6 @@ default <T extends Document> List<T> findAndUpdate(final @NotNull Collection<T>
                                                        final @NotNull List<UpdateOp> updateOps) throws DocumentStoreException {
         return updateOps.stream().map(op -> findAndUpdate(collection, op)).collect(toList());
     }
+
+    long getRecoveryDelayMillis();
 }

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java

@@ -30,6 +30,7 @@
 import org.apache.jackrabbit.guava.common.collect.ImmutableMap;
 import org.apache.jackrabbit.guava.common.collect.Maps;
 import org.apache.jackrabbit.oak.cache.CacheStats;
+import org.apache.jackrabbit.oak.plugins.document.BaseDocumentStore;
 import org.apache.jackrabbit.oak.plugins.document.Collection;
 import org.apache.jackrabbit.oak.plugins.document.Document;
 import org.apache.jackrabbit.oak.plugins.document.DocumentStore;
@@ -58,7 +59,7 @@
  * Emulates a MongoDB store (possibly consisting of multiple shards and
  * replicas).
  */
-public class MemoryDocumentStore implements DocumentStore {
+public class MemoryDocumentStore extends BaseDocumentStore {
 
     /**
      * The 'nodes' collection.

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java

@@ -60,6 +60,7 @@
 import org.apache.jackrabbit.guava.common.util.concurrent.UncheckedExecutionException;
 import org.apache.jackrabbit.oak.cache.CacheStats;
 import org.apache.jackrabbit.oak.cache.CacheValue;
+import org.apache.jackrabbit.oak.plugins.document.BaseDocumentStore;
 import org.apache.jackrabbit.oak.plugins.document.Collection;
 import org.apache.jackrabbit.oak.plugins.document.Document;
 import org.apache.jackrabbit.oak.plugins.document.DocumentStore;
@@ -144,7 +145,7 @@
 /**
  * A document store that uses MongoDB as the backend.
  */
-public class MongoDocumentStore implements DocumentStore {
+public class MongoDocumentStore extends BaseDocumentStore {
 
     private static final Logger LOG = LoggerFactory.getLogger(MongoDocumentStore.class);
     private static final PerfLogger PERFLOG = new PerfLogger(

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java

@@ -64,6 +64,7 @@
 import org.apache.jackrabbit.oak.cache.CacheStats;
 import org.apache.jackrabbit.oak.cache.CacheValue;
 import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier;
+import org.apache.jackrabbit.oak.plugins.document.BaseDocumentStore;
 import org.apache.jackrabbit.oak.plugins.document.Collection;
 import org.apache.jackrabbit.oak.plugins.document.Document;
 import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder;
@@ -248,7 +249,7 @@
  * "deletedOnce", and "_modified". Attempts to use a different indexed property
  * will cause a {@link DocumentStoreException}.
  */
-public class RDBDocumentStore implements DocumentStore {
+public class RDBDocumentStore extends BaseDocumentStore {
 
     /**
      * Creates a {@linkplain RDBDocumentStore} instance using the provided

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LeaseCheckDocumentStoreWrapper.java

@@ -52,6 +52,11 @@ public LeaseCheckDocumentStoreWrapper(final DocumentStore delegate, final Cluste
         this.clusterNodeInfo = clusterNodeInfo;
     }
 
+    @Override
+    public long getRecoveryDelayMillis() {
+        return delegate.getRecoveryDelayMillis();
+    }
+
     private final void performLeaseCheck() {
         if (clusterNodeInfo != null) {
             clusterNodeInfo.performLeaseCheck();

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java

@@ -66,6 +66,11 @@ public LoggingDocumentStoreWrapper withPrefix(String prefix) {
         return this;
     }
 
+    @Override
+    public long getRecoveryDelayMillis() {
+        return store.getRecoveryDelayMillis();
+    }
+
     @Override
     public <T extends Document> T find(final Collection<T> collection,
                                        final String key) {

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java

@@ -41,6 +41,11 @@ public SynchronizingDocumentStoreWrapper(DocumentStore store) {
         this.store = store;
     }
 
+    @Override
+    public long getRecoveryDelayMillis() {
+        return store.getRecoveryDelayMillis();
+    }
+
     @Override
     public synchronized <T extends Document> T find(final Collection<T> collection, final String key) {
         return store.find(collection, key);

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/ThrottlingDocumentStoreWrapper.java

@@ -59,6 +59,11 @@ public ThrottlingDocumentStoreWrapper(final @NotNull DocumentStore store,
         this.throttlingStatsCollector = throttlingStatsCollector;
     }
 
+    @Override
+    public long getRecoveryDelayMillis() {
+        return store.getRecoveryDelayMillis();
+    }
+
     @Override
     public <T extends Document> T find(final Collection<T> collection, final String key) {
         return store.find(collection, key);

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java

@@ -110,6 +110,11 @@ private boolean logCommonCall() {
         return callCount % 10 == 0;
     }
 
+    @Override
+    public long getRecoveryDelayMillis() {
+        return base.getRecoveryDelayMillis();
+    }
+
     @Override
     @Nullable
     public <T extends Document> T find(Collection<T> collection, String key) {

oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java

@@ -79,7 +79,6 @@ public void before() throws Exception {
     @After
     public void after() {
         ClusterNodeInfo.resetClockToDefault();
-        ClusterNodeInfo.resetRecoveryDelayMillisToDefault();
     }
 
     @Test
@@ -745,7 +744,7 @@ public void recoveryNeededNoDelay() throws Exception {
 
     @Test
     public void recoveryNeededWithDelay() throws Exception {
-        ClusterNodeInfo.setRecoveryDelayMillis(60000);
+        setRecoveryDelayMillis(store, 60000);
         ClusterNodeInfo info = newClusterNodeInfo(1);
         String key = String.valueOf(info.getId());
         ClusterNodeInfoDocument doc = store.find(Collection.CLUSTER_NODES, key);
@@ -758,6 +757,12 @@ public void recoveryNeededWithDelay() throws Exception {
         assertTrue(doc.isRecoveryNeeded(clock.getTime()));
     }
 
+    private void setRecoveryDelayMillis(DocumentStore s, int rdm) {
+        assertTrue(s instanceof BaseDocumentStore);
+        BaseDocumentStore bds = (BaseDocumentStore) s;
+        bds.setRecoveryDelayMillis(rdm);
+    }
+
     private void assertLeaseFailure() throws Exception {
         for (int i = 0; i < 100; i++) {
             if (handler.isLeaseFailure()) {
@@ -852,7 +857,7 @@ public <T extends Document> T findAndUpdate(Collection<T> collection,
             T doc = super.findAndUpdate(collection, update);
             maybeThrow(failAfterUpdate, "update failed after");
             if (getFindAndUpdateShouldAlterReturnDocument()) {
-                ClusterNodeInfoDocument cdoc = new ClusterNodeInfoDocument();
+                ClusterNodeInfoDocument cdoc = new ClusterNodeInfoDocument(this);
                 cdoc.data.putAll(getMapAlterReturnDocument());
                 cdoc.seal();
                 return (T)cdoc;
@@ -868,7 +873,7 @@ public <T extends Document> T find(Collection<T> collection,
             maybeThrow(failFind, "find failed");
             T doc = super.find(collection, key);
             if (getFindShouldAlterReturnDocument()) {
-                ClusterNodeInfoDocument cdoc = new ClusterNodeInfoDocument();
+                ClusterNodeInfoDocument cdoc = new ClusterNodeInfoDocument(this);
                 doc.deepCopy(cdoc);
                 cdoc.data.putAll(getMapAlterReturnDocument());
                 cdoc.seal();

oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/CountingDocumentStore.java

@@ -77,6 +77,11 @@ private Stats getStats(Collection collection) {
         }
     }
 
+    @Override
+    public long getRecoveryDelayMillis() {
+        return delegate.getRecoveryDelayMillis();
+    }
+
     @Override
     public <T extends Document> T find(Collection<T> collection, String key) {
         getStats(collection).numFindCalls++;

oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceTest.java

@@ -77,7 +77,6 @@ public void setUp() throws  Exception {
     public void tearDown() throws Exception {
         MockOsgi.deactivate(service, context.bundleContext());
         MongoUtils.dropCollections(MongoUtils.DB);
-        ClusterNodeInfo.resetRecoveryDelayMillisToDefault();
     }
 
     @Test
@@ -363,7 +362,7 @@ private void doRecoveryDelayMillis(long recoveryDelayMillis) {
         MockOsgi.activate(service, context.bundleContext());
 
         DocumentNodeStore dns = context.getService(DocumentNodeStore.class);
-        assertEquals(recoveryDelayMillis, ClusterNodeInfo.getRecoveryDelayMillis());
+        assertEquals(recoveryDelayMillis, dns.getDocumentStore().getRecoveryDelayMillis());
     }
 
     @NotNull

oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentStoreWrapper.java

@@ -35,6 +35,11 @@ public DocumentStoreWrapper(DocumentStore store) {
         this.store = store;
     }
 
+    @Override
+    public long getRecoveryDelayMillis() {
+        return store.getRecoveryDelayMillis();
+    }
+
     @Override
     public <T extends Document> T find(Collection<T> collection, String key) {
         return store.find(collection, key);