Merge pull request #46 from GoogleCloudPlatform/dev

Merging polong's hot fix from dev into main
GoogleCloudPlatform · Feb 24, 2023 · d8be4a5 · d8be4a5
2 parents 4c1114c + a840c21
commit d8be4a5
Show file tree

Hide file tree

Showing 9 changed files with 112 additions and 5,585 deletions.
diff --git a/00_environment_setup.ipynb b/00_environment_setup.ipynb
@@ -9,7 +9,7 @@
    },
    "outputs": [],
    "source": [
-    "# Copyright 2022 Google LLC\n",
+    "# Copyright 2023 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
@@ -30,7 +30,7 @@
     "id": "fsv4jGuU89rX"
    },
    "source": [
-    "# Fraudfinder - Environment Setup\n",
+    "# FraudFinder - Environment Setup\n",
     "\n",
     "<table align=\"left\">\n",
     "  <td>\n",
@@ -60,7 +60,7 @@
    "source": [
     "## Overview\n",
     "\n",
-    "[Fraudfinder](https://github.com/googlecloudplatform/fraudfinder) is a series of labs on how to build a real-time fraud detection system on Google Cloud. Throughout the Fraudfinder labs, you will learn how to read historical bank transaction data stored in data warehouse, read from a live stream of new transactions, perform exploratory data analysis (EDA), do feature engineering, ingest features into a feature store, train a model using feature store, register your model in a model registry, evaluate your model, deploy your model to an endpoint, do real-time inference on your model with feature store, and monitor your model."
+    "[FraudFinder](https://github.com/googlecloudplatform/fraudfinder) is a series of labs on how to build a real-time fraud detection system on Google Cloud. Throughout the FraudFinder labs, you will learn how to read historical bank transaction data stored in data warehouse, read from a live stream of new transactions, perform exploratory data analysis (EDA), do feature engineering, ingest features into a feature store, train a model using feature store, register your model in a model registry, evaluate your model, deploy your model to an endpoint, do real-time inference on your model with feature store, and monitor your model."
    ]
   },
   {

diff --git a/01_exploratory_data_analysis.ipynb b/01_exploratory_data_analysis.ipynb
@@ -9,7 +9,7 @@
    },
    "outputs": [],
    "source": [
-    "# Copyright 2022 Google LLC\n",
+    "# Copyright 2023 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
@@ -535,6 +535,13 @@
     "\n",
     "Now you can go to the next notebook `02_feature_engineering_batch.ipynb`"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -545,9 +552,9 @@
   },
   "environment": {
    "kernel": "python3",
-   "name": "common-cpu.m102",
+   "name": "common-cpu.m103",
    "type": "gcloud",
-   "uri": "gcr.io/deeplearning-platform-release/base-cpu:m102"
+   "uri": "gcr.io/deeplearning-platform-release/base-cpu:m103"
   },
   "kernelspec": {
    "display_name": "Python 3",

diff --git a/02_feature_engineering_batch.ipynb b/02_feature_engineering_batch.ipynb
@@ -8,7 +8,7 @@
    },
    "outputs": [],
    "source": [
-    "# Copyright 2022 Google LLC\n",
+    "# Copyright 2023 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
@@ -29,7 +29,7 @@
     "id": "JAPoU8Sm5E6e"
    },
    "source": [
-    "# Fraudfinder - Feature Engineering (batch)\n",
+    "# FraudFinder - Feature Engineering (batch)\n",
     "\n",
     "<table align=\"left\">\n",
     "  <td>\n",
@@ -59,7 +59,7 @@
    "source": [
     "## Overview\n",
     "\n",
-    "[Fraudfinder](https://github.com/googlecloudplatform/fraudfinder) is a series of labs on how to build a real-time fraud detection system on Google Cloud. Throughout the Fraudfinder labs, you will learn how to read historical bank transaction data stored in data warehouse, read from a live stream of new transactions, perform exploratory data analysis (EDA), do feature engineering, ingest features into a feature store, train a model using feature store, register your model in a model registry, evaluate your model, deploy your model to an endpoint, do real-time inference on your model with feature store, and monitor your model."
+    "[FraudFinder](https://github.com/googlecloudplatform/fraudfinder) is a series of labs on how to build a real-time fraud detection system on Google Cloud. Throughout the FraudFinder labs, you will learn how to read historical bank transaction data stored in data warehouse, read from a live stream of new transactions, perform exploratory data analysis (EDA), do feature engineering, ingest features into a feature store, train a model using feature store, register your model in a model registry, evaluate your model, deploy your model to an endpoint, do real-time inference on your model with feature store, and monitor your model."
    ]
   },
   {
@@ -1364,7 +1364,7 @@
     "id": "4c5b719f3dcd"
    },
    "source": [
-    "## (DO NOT RUN) Clean up"
+    "## Clean up"
    ]
   },
   {

diff --git a/03_feature_engineering_streaming.ipynb b/03_feature_engineering_streaming.ipynb
@@ -8,7 +8,7 @@
    },
    "outputs": [],
    "source": [
-    "# Copyright 2022 Google LLC\n",
+    "# Copyright 2023 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
@@ -29,7 +29,7 @@
     "id": "JAPoU8Sm5E6e"
    },
    "source": [
-    "# Fraudfinder - Feature Engineering (streaming)\n",
+    "# FraudFinder - Feature Engineering (streaming)\n",
     "\n",
     "<table align=\"left\">\n",
     "  <td>\n",
@@ -59,7 +59,7 @@
    "source": [
     "## Overview\n",
     "\n",
-    "[Fraudfinder](https://github.com/googlecloudplatform/fraudfinder) is a series of labs on how to build a real-time fraud detection system on Google Cloud. Throughout the Fraudfinder labs, you will learn how to read historical bank transaction data stored in data warehouse, read from a live stream of new transactions, perform exploratory data analysis (EDA), do feature engineering, ingest features into a feature store, train a model using feature store, register your model in a model registry, evaluate your model, deploy your model to an endpoint, do real-time inference on your model with feature store, and monitor your model."
+    "[FraudFinder](https://github.com/googlecloudplatform/fraudfinder) is a series of labs on how to build a real-time fraud detection system on Google Cloud. Throughout the FraudFinder labs, you will learn how to read historical bank transaction data stored in data warehouse, read from a live stream of new transactions, perform exploratory data analysis (EDA), do feature engineering, ingest features into a feature store, train a model using feature store, register your model in a model registry, evaluate your model, deploy your model to an endpoint, do real-time inference on your model with feature store, and monitor your model."
    ]
   },
   {
@@ -282,7 +282,7 @@
     "%%writefile -a {PYTHON_SCRIPT}\n",
     "\n",
     "# Pub/Sub variables\n",
-    "SUBSCRIPTION_NAME = \"ff-tx-sub\"\n",
+    "SUBSCRIPTION_NAME = \"ff-tx-for-feat-eng-sub\"\n",
     "SUBSCRIPTION_PATH = f\"projects/{PROJECT_ID}/subscriptions/{SUBSCRIPTION_NAME}\"\n",
     "\n",
     "# Dataflow variables\n",
@@ -348,7 +348,6 @@
     "            'CUSTOMER_ID': element['CUSTOMER_ID'],\n",
     "            'TERMINAL_ID': element['TERMINAL_ID'],\n",
     "            'TX_AMOUNT': element['TX_AMOUNT'],\n",
-    "            'TX_FRAUD': element['TX_FRAUD'],\n",
     "            'CUSTOMER_ID_COMPOSITE_KEY': f\"{element['CUSTOMER_ID']}_{window_end_dt}\",\n",
     "            'TERMINAL_ID_COMPOSITE_KEY': f\"{element['TERMINAL_ID']}_{window_end_dt}\",\n",
     "            'TS_DIFF': window.end - timestamp\n",
@@ -738,13 +737,6 @@
     "\n",
     "Or else, if you want to explore Vertex ML pipeline, go here: `vertex_ai/04_experimentation.ipynb`"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

diff --git a/README.md b/README.md
@@ -57,6 +57,7 @@ Please make sure that you have selected a Google Cloud project as shown below:
   gcloud services enable iam.googleapis.com
 
   gcloud pubsub subscriptions create "ff-tx-sub" --topic="ff-tx" --topic-project="cymbal-fraudfinder"
+  gcloud pubsub subscriptions create "ff-tx-for-feat-eng-sub" --topic="ff-tx" --topic-project="cymbal-fraudfinder"
   gcloud pubsub subscriptions create "ff-txlabels-sub" --topic="ff-txlabels" --topic-project="cymbal-fraudfinder"
 
   # Run the following command to grant the Compute Engine default service account access to read and write pipeline artifacts in Google Cloud Storage.
@@ -66,14 +67,20 @@ Please make sure that you have selected a Google Cloud project as shown below:
         --member="serviceAccount:${PROJECT_NUM}-compute@developer.gserviceaccount.com"\
         --role='roles/storage.admin'
   gcloud projects add-iam-policy-binding $PROJECT_ID \
-      --member="serviceAccount:${PROJECT_NUM}@cloudbuild.gserviceaccount.com"\
-      --role='roles/aiplatform.admin'
+        --member="serviceAccount:${PROJECT_NUM}@cloudbuild.gserviceaccount.com"\
+        --role='roles/aiplatform.admin'
   gcloud projects add-iam-policy-binding $PROJECT_ID \
-    --member="serviceAccount:service-${PROJECT_NUM}@gcp-sa-aiplatform-cc.iam.gserviceaccount.com"\
-    --role='roles/artifactregistry.writer'
+        --member="serviceAccount:$PROJECT_NUM-compute@developer.gserviceaccount.com"\
+        --role='roles/run.admin'
   gcloud projects add-iam-policy-binding $PROJECT_ID \
-    --member="serviceAccount:service-${PROJECT_NUM}@gcp-sa-aiplatform-cc.iam.gserviceaccount.com"\
-    --role='roles/storage.objectAdmin'   
+        --member="serviceAccount:$PROJECT_NUM-compute@developer.gserviceaccount.com"\
+        --role='roles/resourcemanager.projectIamAdmin'
+  gcloud projects add-iam-policy-binding $PROJECT_ID \
+        --member="serviceAccount:service-${PROJECT_NUM}@gcp-sa-aiplatform-cc.iam.gserviceaccount.com"\
+        --role='roles/artifactregistry.writer'
+  gcloud projects add-iam-policy-binding $PROJECT_ID \
+        --member="serviceAccount:service-${PROJECT_NUM}@gcp-sa-aiplatform-cc.iam.gserviceaccount.com"\
+        --role='roles/storage.objectAdmin'   
   ```
 
 #### Step 2: Create a User-Managed Notebook instance on Vertex AI Workbench