Bug 36591594 - [32909258->22.06.9] CQC and NearCache results in disco…

…nnected state after snapshot recovery (merge 14.1.1.2206 -> ce/22.06.9 @ 108876)

[git-p4: depot-paths = "//dev/coherence-ce/release/coherence-ce-v22.06/": change = 108884]

prj/coherence-core-components/src/main/java/com/tangosol/coherence/component/util/SafeService.java

@@ -1,6 +1,6 @@
 
 /*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates.
  *
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * https://oss.oracle.com/licenses/upl.
@@ -948,6 +948,12 @@ public void memberLeft(com.tangosol.net.MemberEvent evt)
         translateEvent(evt);
         }
 
+    @Override
+    public void memberRecovered(com.tangosol.net.MemberEvent evt)
+        {
+        translateEvent(evt);
+        }
+
     // Declared at the super level
     /**
      * The "component has been initialized" method-notification called out of

prj/coherence-core-components/src/main/java/com/tangosol/coherence/component/util/daemon/queueProcessor/service/Grid.java

@@ -1,6 +1,6 @@
 
 /*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates.
  *
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * https://oss.oracle.com/licenses/upl.
@@ -150,6 +150,7 @@
  * 12   * RESERVED for join protocol compatibility *
  * 13   * RESERVED for join protocol compatibility *
  * 17   * RESERVED for join protocol compatibility *
+ * 18    MemberRecovered
  */
 @SuppressWarnings({"deprecation", "rawtypes", "unused", "unchecked", "ConstantConditions", "DuplicatedCode", "ForLoopReplaceableByForEach", "IfCanBeSwitch", "RedundantArrayCreation", "RedundantSuppression", "SameParameterValue", "TryFinallyCanBeTryWithResources", "TryWithIdenticalCatches", "UnnecessaryBoxing", "UnnecessaryUnboxing", "UnusedAssignment", "JavadocBlankLines", "JavadocDeclaration", "EnhancedSwitchMigration"})
 public abstract class Grid
@@ -553,6 +554,7 @@ private static void __initStatic()
         __mapChildren.put("DispatchNotification", Grid.DispatchNotification.get_CLASS());
         __mapChildren.put("MemberConfigUpdate", Grid.MemberConfigUpdate.get_CLASS());
         __mapChildren.put("MemberJoined", Grid.MemberJoined.get_CLASS());
+        __mapChildren.put("MemberRecovered", Grid.MemberRecovered.get_CLASS());
         __mapChildren.put("MemberWelcome", Grid.MemberWelcome.get_CLASS());
         __mapChildren.put("MemberWelcomeRequest", Grid.MemberWelcomeRequest.get_CLASS());
         __mapChildren.put("MemberWelcomeRequestTask", Grid.MemberWelcomeRequestTask.get_CLASS());
@@ -8720,6 +8722,170 @@ public void write(com.tangosol.io.WriteBuffer.BufferOutput output)
             }
         }
 
+    /**
+     * This Message is used to signal that recovery has completed either as 
+     * part of active persistence or snapshot recovery.
+     */
+    @SuppressWarnings({"deprecation", "rawtypes", "unused", "unchecked", "ConstantConditions", "DuplicatedCode", "ForLoopReplaceableByForEach", "IfCanBeSwitch", "RedundantArrayCreation", "RedundantSuppression", "SameParameterValue", "TryFinallyCanBeTryWithResources", "TryWithIdenticalCatches", "UnnecessaryBoxing", "UnnecessaryUnboxing", "UnusedAssignment"})
+    public static class MemberRecovered
+            extends    com.tangosol.coherence.component.net.Message
+        {
+        // ---- Fields declarations ----
+
+        /**
+         * Property MemberId
+         *
+         * The ID of the member who has recovered.
+         */
+        private int __m_MemberId;
+
+        // Default constructor
+        public MemberRecovered()
+            {
+            this(null, null, true);
+            }
+
+        // Initializing constructor
+        public MemberRecovered(String sName, com.tangosol.coherence.Component compParent, boolean fInit)
+            {
+            super(sName, compParent, false);
+
+            if (fInit)
+                {
+                __init();
+                }
+            }
+
+        // Main initializer
+        public void __init()
+            {
+            // private initialization
+            __initPrivate();
+
+            // state initialization: public and protected properties
+            try
+                {
+                setMessageType(18);
+                }
+            catch (java.lang.Exception e)
+                {
+                // re-throw as a runtime exception
+                throw new com.tangosol.util.WrapperException(e);
+                }
+
+            // signal the end of the initialization
+            set_Constructed(true);
+            }
+
+        // Private initializer
+        protected void __initPrivate()
+            {
+
+            super.__initPrivate();
+            }
+
+        //++ getter for static property _Instance
+        /**
+         * Getter for property _Instance.<p>
+         * Auto generated
+         */
+        public static com.tangosol.coherence.Component get_Instance()
+            {
+            return new com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.MemberRecovered();
+            }
+
+        //++ getter for static property _CLASS
+        /**
+         * Getter for property _CLASS.<p>
+         * Property with auto-generated accessor that returns the Class object
+         * for a given component.
+         */
+        public static Class get_CLASS()
+            {
+            Class clz;
+            try
+                {
+                clz = Class.forName("com.tangosol.coherence/component/util/daemon/queueProcessor/service/Grid$MemberRecovered".replace('/', '.'));
+                }
+            catch (ClassNotFoundException e)
+                {
+                throw new NoClassDefFoundError(e.getMessage());
+                }
+            return clz;
+            }
+
+        //++ getter for autogen property _Module
+        /**
+         * This is an auto-generated method that returns the global [design
+         * time] parent component.
+         * 
+         * Note: the class generator will ignore any custom implementation for
+         * this behavior.
+         */
+        private com.tangosol.coherence.Component get_Module()
+            {
+            return this.get_Parent();
+            }
+
+        // Declared at the super level
+        /**
+         * Getter for property Description.<p>
+         * Used for debugging purposes (from toString). Create a human-readable
+         * description of the specific Message data.
+         */
+        public String getDescription()
+            {
+            return "MemberRecovered=" + getFromMember();
+            }
+
+        // Accessor for the property "MemberId"
+        /**
+         * Getter for property MemberId.<p>
+         * The ID of the member who has recovered.
+         */
+        public int getMemberId()
+            {
+            return __m_MemberId;
+            }
+
+        // Declared at the super level
+        public void onReceived()
+            {
+            super.onReceived();
+            Grid   service = (Grid) get_Module();
+            Member member  = service.getServiceMemberSet().getMember(getMemberId());
+            service.dispatchMemberEvent(member, MemberEvent.MEMBER_RECOVERED);
+            }
+
+        // Declared at the super level
+        public void read(com.tangosol.io.ReadBuffer.BufferInput input)
+                throws java.io.IOException
+            {
+            super.read(input);
+
+            setMemberId(input.readInt());
+            }
+
+        // Accessor for the property "MemberId"
+        /**
+         * Setter for property MemberId.<p>
+         * The ID of the member who has recovered.
+         */
+        public void setMemberId(int nId)
+            {
+            __m_MemberId = nId;
+            }
+
+        // Declared at the super level
+        public void write(com.tangosol.io.WriteBuffer.BufferOutput output)
+                throws java.io.IOException
+            {
+            super.write(output);
+
+            output.writeInt(getMemberId());
+            }
+        }
+
     // ---- class: com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$MemberWelcome
 
     /**

prj/coherence-core-components/src/main/java/com/tangosol/coherence/component/util/daemon/queueProcessor/service/grid/PartitionedService.java

@@ -21,6 +21,7 @@
 import com.tangosol.coherence.component.net.message.requestMessage.chainedRequest.BackupRequest;
 import com.tangosol.coherence.component.util.DistributionStrategy;
 import com.tangosol.coherence.component.util.PartialJob;
+import com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid;
 import com.oracle.coherence.common.base.Blocking;
 import com.oracle.coherence.common.base.Continuation;
 import com.oracle.coherence.common.base.MutableLong;
@@ -666,6 +667,7 @@ private static void __initStatic()
         __mapChildren.put("DistributionRequest", PartitionedService.DistributionRequest.get_CLASS());
         __mapChildren.put("MemberConfigUpdate", PartitionedService.MemberConfigUpdate.get_CLASS());
         __mapChildren.put("MemberJoined", com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.MemberJoined.get_CLASS());
+        __mapChildren.put("MemberRecovered", com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.MemberRecovered.get_CLASS());
         __mapChildren.put("MemberWelcome", PartitionedService.MemberWelcome.get_CLASS());
         __mapChildren.put("MemberWelcomeRequest", PartitionedService.MemberWelcomeRequest.get_CLASS());
         __mapChildren.put("MemberWelcomeRequestTask", com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.MemberWelcomeRequestTask.get_CLASS());
@@ -28788,6 +28790,7 @@ public void onRecoveryCompleted(String sSnapshot, com.tangosol.net.partition.Par
                 addNotification(PersistenceManagerMBean.RECOVER_SNAPSHOT_END, sMessage, sUserData);
 
                 reset();
+                sendMemberRecovered();
                 }
 
             /**
@@ -29159,6 +29162,19 @@ public void retrieveArchivedSnapshot(String sSnapshot)
                 addNotification(PersistenceManagerMBean.RETRIEVE_ARCHIVED_SNAPSHOT_BEGIN, getOperationStatus(), "");
                 }
 
+            /**
+             * Send a Member RECOVERED message to signal that this member's recovery has completed.
+             */
+            public void sendMemberRecovered()
+                {
+                PartitionedService   service = (PartitionedService) get_Module();
+                Grid.MemberRecovered msg     = (MemberRecovered) service.instantiateMessage("MemberRecovered");
+                Member               member  = service.getThisMember();
+                msg.setMemberId(member.getId());
+                msg.setToMemberSet(service.getServiceMemberSet());
+                service.post(msg);
+                }
+
             // Accessor for the property "ResumeOnCompletion"
             /**
              * Setter for property ResumeOnCompletion.<p>

prj/coherence-core-components/src/main/java/com/tangosol/coherence/component/util/daemon/queueProcessor/service/grid/partitionedService/PartitionedCache.java

@@ -645,6 +645,7 @@ private static void __initStatic()
         __mapChildren.put("MapEvent", PartitionedCache.MapEvent.get_CLASS());
         __mapChildren.put("MemberConfigUpdate", com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.PartitionedService.MemberConfigUpdate.get_CLASS());
         __mapChildren.put("MemberJoined", com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.MemberJoined.get_CLASS());
+        __mapChildren.put("MemberRecovered", com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.MemberRecovered.get_CLASS());
         __mapChildren.put("MemberWelcome", PartitionedCache.MemberWelcome.get_CLASS());
         __mapChildren.put("MemberWelcomeRequest", PartitionedCache.MemberWelcomeRequest.get_CLASS());
         __mapChildren.put("MemberWelcomeRequestTask", com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.MemberWelcomeRequestTask.get_CLASS());

prj/coherence-core/src/main/java/com/tangosol/net/MemberEvent.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2022, Oracle and/or its affiliates.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates.
  *
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * https://oss.oracle.com/licenses/upl.
@@ -121,6 +121,10 @@ public void dispatch(EventListener[] aListeners)
                 case MEMBER_LEFT:
                     target.memberLeft(this);
                     break;
+
+                case MEMBER_RECOVERED:
+                    target.memberRecovered(this);
+                    break;
                  }
             }
         }
@@ -191,10 +195,15 @@ public String toString()
     */
     public static final int MEMBER_LEFT = 3;
 
+    /**
+     * This event indicates that a Member has performed persistence recovery.
+     */
+    public static final int MEMBER_RECOVERED = 4;
+
     /**
     * Descriptions of the various event IDs.
     */
-    private static final String[] DESCRIPTIONS = {"<unknown>", "JOINED", "LEAVING", "LEFT"};
+    private static final String[] DESCRIPTIONS = {"<unknown>", "JOINED", "LEAVING", "LEFT", "RECOVERED"};
 
 
     // ----- data members ---------------------------------------------------

prj/coherence-core/src/main/java/com/tangosol/net/MemberListener.java

@@ -1,8 +1,8 @@
 /*
- * Copyright (c) 2000, 2020, Oracle and/or its affiliates.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates.
  *
  * Licensed under the Universal Permissive License v 1.0 as shown at
- * http://oss.oracle.com/licenses/upl.
+ * https://oss.oracle.com/licenses/upl.
  */
 
 package com.tangosol.net;
@@ -60,4 +60,13 @@ public interface MemberListener
     * @param evt  the MemberEvent.MEMBER_LEFT event
     */
     public void memberLeft(MemberEvent evt);
+
+    /**
+     * Invoked when a Member has recovered from persistence.
+     *
+     * @param evt  the MemberEvent.MEMBER_RECOVERED event
+     */
+    default public void memberRecovered(MemberEvent evt)
+        {
+        }
     }

prj/coherence-core/src/main/java/com/tangosol/net/cache/ContinuousQueryCache.java

@@ -2208,6 +2208,12 @@ public void memberLeft(MemberEvent evt)
                 }
             }
 
+        @Override
+        public void memberRecovered(MemberEvent evt)
+            {
+            changeState(STATE_DISCONNECTED);
+            }
+
         /**
          * Produce a human-readable description of this object.
          *

prj/coherence-core/src/main/java/com/tangosol/net/cache/NearCache.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates.
  *
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * https://oss.oracle.com/licenses/upl.
@@ -721,6 +721,14 @@ public void memberLeft(MemberEvent evt)
                     }
                 }
             }
+
+        /**
+         * Invoked when a Member has recovered.
+         */
+        public void memberRecovered(MemberEvent evt)
+            {
+            resetFrontMap();
+            }
         }