feat: Update naas.secret with new method (#438)

* first commit

* fix: Few updates to make it more like the legacy

* feat: create remote bulk secrets list

* move old func at the end

* check for naas api

* fix: better error message

* fix: lint secret.py

* fix: Bump naas-python

* fix: get secret return None

* fix: Bump ci python version to 3.9

---------

Co-authored-by: Maxime Jublou <jubloum@gmail.com>
Co-authored-by: Florent <48032461+FlorentLvr@users.noreply.github.com>
Co-authored-by: Maxime Jublou <maxime@jublou.fr>

.github/workflows/pull_request.yml

@@ -12,7 +12,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v1
       with:
-        python-version: '3.8'
+        python-version: '3.9'
     - uses: actions/cache@v2
       with:
         path: ~/.cache/pip

docker-compose.yaml

@@ -13,6 +13,7 @@ services:
     ports:
       - 8888:8888
       - 5000:5000
+      - 38745:38745
     volumes:
       - ~/.ssh:/home/ftp/.ssh
       - .:/home/ftp/naas

naas/secret.py

@@ -2,10 +2,140 @@
 from .runner.env_var import n_env
 import pandas as pd
 import requests
+import naas_python
 
 
 class Secret:
-    def list(self, raw=False):
+
+    def __get_remote_secret(self, name: str):
+        try:
+            if self.__check_api():
+                return naas_python.secret.get(name)
+            else:
+                print(
+                    "Naas.api service request failed. Please retry again in few seconds."
+                )
+                return None
+        except naas_python.domains.secret.SecretSchema.SecretNotFound:
+            print("Secret not found.")
+            return None
+        except:
+            print("Secret get failed. Please retry again in few seconds.")
+            return None
+
+    def __create_remote_secret(self, name: str, value: str):
+        try:
+            if self.__check_api():
+                naas_python.secret.create(name, value)
+                return True
+                # print("\nYour Secret has been moved to naas.ai 👌\n")
+            else:
+                print(
+                    "Naas.api service request failed. Please retry again in few seconds."
+                )
+                return False
+        except:
+            print("Secret creation failed. Please retry again in few seconds.")
+            return False
+
+    def __create_remote_bulk_secret(self, secrets: str):
+        try:
+            if self.__check_api():
+                naas_python.secret.bulk_create(secrets)
+                return True
+                # print("\nYour Secret has been moved to naas.ai 👌\n")
+            else:
+                print(
+                    "Naas.api service request failed. Please retry again in few seconds."
+                )
+                return False
+        except:
+            print("Secret creation failed. Please retry again in few seconds.")
+            return False
+
+    def __delete_remote_secret(self, name: str):
+        try:
+            if self.__check_api():
+                naas_python.secret.delete(name=name)
+                return True
+            else:
+                print(
+                    "Naas.api service request failed. Please retry again in few seconds."
+                )
+                return False
+        except:
+            # print("Secret not found")
+            return False
+
+    def __list_remote_secret(self):
+        # try:
+        if self.__check_api():
+            return naas_python.secret.list()
+        else:
+            print("Naas.api service request failed. Please retry again in few seconds.")
+            return False
+        # except:
+        #     print("Secret list failed")
+        #     return False
+
+    def __check_api(self, url="https://api.naas.ai/"):
+        response = requests.get(url)
+        if response.status_code == 200:
+            data = response.json()
+            if "status" in data and data["status"] == "ok":
+                return True
+        else:
+            return False
+
+    def list(self):
+        local_secret = None
+        remote_secret_list = self.__list_remote_secret()
+
+        # convert remote secret to a dataframe
+        remote_secret = pd.DataFrame(columns=["name", "secret"])
+        for secret in remote_secret_list:
+            new_row = pd.DataFrame({"name": [secret.name], "secret": [secret.value]})
+            remote_secret = remote_secret.append(new_row, ignore_index=True)
+
+        # local secret dataframe
+        local_secret = self.__old_list()
+
+        if local_secret.empty:
+            return remote_secret
+
+        local_secret = local_secret.drop("id", axis=1)
+        local_secret = local_secret.drop("lastUpdate", axis=1)
+
+        merged_secrets_df = pd.merge(
+            remote_secret, local_secret, how="outer", indicator=True
+        )
+
+        # If the secret exists locally AND in api.naas.ai, I remove the local version of the secret.
+        selected_secrets = merged_secrets_df[merged_secrets_df["_merge"] == "both"]
+        for index, row in selected_secrets.iterrows():
+            self.__old_delete(row["name"])
+
+        # If the secret exists locally and does not exists in api.naas.ai, I create it in api.naas.ai and I delete the local version.
+        secrets_list = []
+        selected_secrets = merged_secrets_df[
+            merged_secrets_df["_merge"] == "right_only"
+        ]
+        for index, row in selected_secrets.iterrows():
+            # self.__create_remote_secret(name=row['name'], value=row['secret'])
+            name = row["name"]
+            value = row["secret"]
+            new_secret = {"name": name, "value": value}
+            secrets_list.append(new_secret)
+
+            new_row = pd.DataFrame({"name": [row["name"]], "secret": [row["secret"]]})
+            remote_secret = remote_secret.append(new_row, ignore_index=True)
+
+            self.__old_delete(row["name"])
+        self.__create_remote_bulk_secret({"secrets": secrets_list})
+
+        return remote_secret
+
+    def __old_list(self, raw=False):
         try:
             r = requests.get(f"{n_env.api}/{t_secret}")
             r.raise_for_status()
@@ -22,21 +152,55 @@ def list(self, raw=False):
             raise
 
     def add(self, name=None, secret=None):
-        obj = {"name": name, "secret": secret, "status": t_add}
-        try:
-            r = requests.post(f"{n_env.api}/{t_secret}", json=obj)
-            r.raise_for_status()
-            print("👌 Well done! Your Secret has been sent to production. \n")
-            print('PS: to remove the "Secret" feature, just replace .add by .delete')
-        except requests.exceptions.ConnectionError as err:
-            print(error_busy, err)
-            raise
-        except requests.exceptions.HTTPError as err:
-            print(error_reject, err)
-            raise
+        if name is None or secret is None:
+            return "Incomplete secret"
+        self.__create_remote_secret(name=name, value=secret)
+        self.__old_delete(name)
 
     def get(self, name=None, default_value=None):
-        all_secret = self.list(True)
+        all_secret = self.__old_list(True)
+        local_secret = None
+        remote_secret = None
+
+        # Find local_secret
+        for item in all_secret:
+            if name == item["name"]:
+                local_secret = item
+                break
+
+        # try:
+        remote_secret = self.__get_remote_secret(name=name)
+
+        # except:
+        #     print("Try Again")
+        #     return None
+
+        # if the secret exists on api.naas.ai
+        # AND exists locally, then I remove the local version.
+        if remote_secret is not None and local_secret is not None:
+            if remote_secret.name == local_secret["name"]:
+                self.__old_delete(name=local_secret["name"])
+                return remote_secret.value
+
+        # If the secret does not exists on api.naas.ai
+        # If the secret exists locally
+        # I take the local secret and create it on api.naas.ai.
+        # I delete the local version of the secret.
+        elif remote_secret is None and local_secret is not None:
+            self.__create_remote_secret(local_secret["name"], local_secret["secret"])
+            secret = self.__get_remote_secret(local_secret["name"])
+            self.__old_delete(name=local_secret["name"])
+            return secret.value
+
+        else:  # local_secret is None
+            # If the secret exists on api.naas.ai
+            # I use that value
+            if remote_secret is not None:
+                return remote_secret.value
+        return default_value
+
+    def __old_get(self, name=None, default_value=None):
+        all_secret = self.__old_list(True)
         secret_item = None
         for item in all_secret:
             if name == item["name"]:
@@ -47,11 +211,31 @@ def get(self, name=None, default_value=None):
         return default_value
 
     def delete(self, name=None):
+        if name is None:
+            return "Incomplete name"
+        self.__delete_remote_secret(name=name)
+        self.__old_delete(name)
+
+    def __old_delete(self, name=None):
         obj = {"name": name, "secret": "", "status": t_delete}
         try:
             r = requests.post(f"{n_env.api}/{t_secret}", json=obj)
             r.raise_for_status()
-            print("👌 Well done! Your Secret has been remove in production. \n")
+            # print("👌 Well done! Your Secret has been remove in production. \n")
+        except requests.exceptions.ConnectionError as err:
+            print(error_busy, err)
+            raise
+        except requests.exceptions.HTTPError as err:
+            print(error_reject, err)
+            raise
+
+    def __old_add(self, name=None, secret=None):
+        obj = {"name": name, "secret": secret, "status": t_add}
+        try:
+            r = requests.post(f"{n_env.api}/{t_secret}", json=obj)
+            r.raise_for_status()
+            print("👌 Well done! Your Secret has been sent to production. \n")
+            print('PS: to remove the "Secret" feature, just replace .add by .delete')
         except requests.exceptions.ConnectionError as err:
             print(error_busy, err)
             raise

setup.py

@@ -74,10 +74,11 @@
         "pymongo[srv]==3.11.3",
         "psycopg2-binary==2.9.1",
         "mprop==0.16.0",
-        "pydash==5.1.0",
+        "pydash==7.0.7",
         "pyvis==0.3.0",
         "rich",
-        "tzlocal==2.1"
+        "tzlocal==2.1",
+        "naas-python==1.3.0"
     ],
     classifiers=[
         "Programming Language :: Python :: 3.9",