chore: Integration test for old pod migration (#190)

Makefile

@@ -64,7 +64,7 @@ test-cov:
 
 .PHONY: test-int
 test-int:
-	poetry run pytest  -m "integration" -s
+	poetry run pytest  -m "integration" -s -v
 
 .PHONY: report-to-coveralls
 report-to-coveralls:

app/handlers/handlers_connectors.py

@@ -153,7 +153,8 @@ def twingate_connector_update(body, memo, logger, new, diff, status, namespace,
     client = TwingateAPIClient(settings)
 
     crd = TwingateConnectorCRD(**body)
-    if len(diff) == 1 and diff[0][:3] == ("add", ("id"), None):
+    # (('add', ('id',), None, 'Q29ubmVjdG9yOjUwNjE3NQ=='),)
+    if len(diff) == 1 and diff[0][:3] == ("add", ("id",), None):
         return success(twingate_id=crd.spec.id, message="No update required")
 
     if not crd.spec.id:

app/handlers/tests/test_handlers_connector.py

@@ -170,7 +170,7 @@ def test_twingate_connector_update_only_id_does_nothing(
         crd,
         MagicMock(),
         new={},
-        diff=(("add", ("id"), None, "123"),),
+        diff=(("add", ("id",), None, "123"),),
     )
     assert run.result == {
         "success": True,

tests_integration/test_connector_flows.py

@@ -1,30 +1,43 @@
-import os
 import time
+from contextlib import contextmanager
 from subprocess import CalledProcessError
-from unittest.mock import ANY, patch
+from unittest.mock import ANY
 
 import pytest
 from kopf.testing import KopfRunner
 
 from tests_integration.utils import (
+    kubectl,
     kubectl_create,
     kubectl_delete,
     kubectl_get,
     kubectl_patch,
 )
 
 
-@pytest.fixture(autouse=True)
-def _connector_reconciler():
-    with patch.dict(
-        os.environ,
-        {"CONNECTOR_RECONCILER_INTERVAL": "1", "CONNECTOR_RECONCILER_INIT_DELAY": "1"},
-    ):
-        yield
+@pytest.fixture(scope="session")
+def run_kopf(kopf_runner_args, kopf_settings):
+    @contextmanager
+    def inner():
+        with KopfRunner(
+            kopf_runner_args,
+            settings=kopf_settings,
+            env={
+                "CONNECTOR_RECONCILER_INTERVAL": "1",
+                "CONNECTOR_RECONCILER_INIT_DELAY": "1",
+            },
+        ) as runner:
+            time.sleep(5)
+            yield
 
+        assert runner.exception is None
+        assert runner.exit_code == 0
 
-def test_connector_flows(kopf_settings, kopf_runner_args, ci_run_number):
-    connector_name = f"test-connector-{ci_run_number}"
+    return inner
+
+
+def test_connector_flows(run_kopf, ci_run_number):
+    connector_name = f"test-{ci_run_number}"
     OBJ = f"""
         apiVersion: twingate.com/v1beta
         kind: TwingateConnector
@@ -41,8 +54,7 @@ def test_connector_flows(kopf_settings, kopf_runner_args, ci_run_number):
                 version: "^1.0.0"
     """
 
-    with KopfRunner(kopf_runner_args, settings=kopf_settings) as runner:
-        time.sleep(5)
+    with run_kopf():
         kubectl_create(OBJ)
         time.sleep(10)
 
@@ -88,12 +100,9 @@ def test_connector_flows(kopf_settings, kopf_runner_args, ci_run_number):
         with pytest.raises(CalledProcessError):
             kubectl_get("pod", connector_name)
 
-    assert runner.exception is None
-    assert runner.exit_code == 0
-
 
-def test_connector_flows_image_change(kopf_settings, kopf_runner_args, ci_run_number):
-    connector_name = f"test-connector-image-{ci_run_number}"
+def test_connector_flows_image_change(run_kopf, ci_run_number):
+    connector_name = f"test-image-{ci_run_number}"
     OBJ = f"""
         apiVersion: twingate.com/v1beta
         kind: TwingateConnector
@@ -106,8 +115,7 @@ def test_connector_flows_image_change(kopf_settings, kopf_runner_args, ci_run_nu
                 tag: "1.62.0"
     """
 
-    with KopfRunner(kopf_runner_args, settings=kopf_settings) as runner:
-        time.sleep(5)
+    with run_kopf():
         kubectl_create(OBJ)
         time.sleep(5)
 
@@ -133,13 +141,13 @@ def test_connector_flows_image_change(kopf_settings, kopf_runner_args, ci_run_nu
         # Change image tag
         # kubectl patch tc/test-connector-image-local -p '{"spec": {"image": {"tag": "1.63.0"}}}' --type=merge
         kubectl_patch(f"tc/{connector_name}", {"spec": {"image": {"tag": "1.63.0"}}})
-        time.sleep(5)
+        time.sleep(10)
         pod = kubectl_get("pod", connector_name)
         assert pod["status"]["phase"] == "Running"
         assert pod["spec"]["containers"][0]["image"] == "twingate/connector:1.63.0"
 
         kubectl_delete(f"tc/{connector_name}")
-        time.sleep(5)
+        time.sleep(10)
 
         # secret & pod are deleted
         with pytest.raises(CalledProcessError):
@@ -148,14 +156,9 @@ def test_connector_flows_image_change(kopf_settings, kopf_runner_args, ci_run_nu
         with pytest.raises(CalledProcessError):
             kubectl_get("pod", connector_name)
 
-    assert runner.exception is None
-    assert runner.exit_code == 0
 
-
-def test_connector_flows_pod_gone_while_operator_down(
-    kopf_settings, kopf_runner_args, ci_run_number
-):
-    connector_name = f"test-connector-gone-{ci_run_number}"
+def test_connector_flows_pod_gone_while_operator_down(run_kopf, ci_run_number):
+    connector_name = f"test-gone-{ci_run_number}"
     OBJ = f"""
         apiVersion: twingate.com/v1beta
         kind: TwingateConnector
@@ -168,8 +171,7 @@ def test_connector_flows_pod_gone_while_operator_down(
                 tag: "1.63.0"
     """
 
-    with KopfRunner(kopf_runner_args, settings=kopf_settings) as _runner:
-        time.sleep(5)
+    with run_kopf():
         kubectl_create(OBJ)
         time.sleep(10)
 
@@ -188,8 +190,8 @@ def test_connector_flows_pod_gone_while_operator_down(
     kubectl_delete(f"pod/{connector_name}")
 
     # run operator again
-    with KopfRunner(kopf_runner_args, settings=kopf_settings) as _runner:
-        time.sleep(10)
+    with run_kopf():
+        time.sleep(5)
 
         # pod was recreated
         pod = kubectl_get("pod", connector_name)
@@ -198,3 +200,66 @@ def test_connector_flows_pod_gone_while_operator_down(
         # Test done, delete connector
         kubectl_delete(f"tc/{connector_name}")
         time.sleep(5)
+
+
+def test_connector_flows_pod_migration_from_older_pod_with_finalizers(
+    run_kopf, ci_run_number
+):
+    connector_name = f"test-migration-{ci_run_number}"
+    OBJ = f"""
+        apiVersion: twingate.com/v1beta
+        kind: TwingateConnector
+        metadata:
+            name: {connector_name}
+        spec:
+            name: {connector_name}
+            hasStatusNotificationsEnabled: false
+            image:
+                tag: "1.63.0"
+    """
+
+    with run_kopf():
+        kubectl_create(OBJ)
+        time.sleep(10)
+
+        connector = kubectl_get("tc", connector_name)
+        kubectl_get("secret", connector_name)
+        pod = kubectl_get("pod", connector_name)
+
+        while pod["status"]["phase"] == "Pending":
+            time.sleep(1)
+            pod = kubectl_get("pod", connector_name)
+
+        # connector was properly provisioned
+        expected_status = {"success": True, "ts": ANY, "twingate_id": ANY}
+        assert connector["status"]["twingate_connector_create"] == expected_status
+
+        # Patch pod to add finalizers and make operator think its old
+        kubectl_patch(
+            f"pod/{connector_name}",
+            {
+                "metadata": {
+                    "finalizers": ["twingate.com/finalizer"],
+                }
+            },
+        )
+        kubectl(
+            f"annotate pod/{connector_name} --overwrite twingate.com/connector-podspec-version=not_what_op_expects"
+        )
+
+        # wait for timer to run
+        time.sleep(10)
+
+        # check pod was recreated
+        pod = kubectl_get("pod", connector_name)
+        if pod["status"]["phase"] == "Pending":
+            time.sleep(5)
+            pod = kubectl_get("pod", connector_name)
+
+        assert pod["metadata"]["annotations"]["twingate.com/connector-podspec-version"] == "v1"  # fmt: skip
+        # assert pod["status"]["phase"] in ["Running", "Pending"]
+        assert pod["status"]["phase"] == "Running"
+
+        # Test done, delete connector
+        kubectl_delete(f"tc/{connector_name}")
+        time.sleep(5)