[Notebooks] Update hunting notebooks (#235)

Azure · Sep 21, 2023 · 7873abb · 7873abb
1 parent fe2e1ed
commit 7873abb
Show file tree

Hide file tree

Showing 2 changed files with 92 additions and 63 deletions.
diff --git a/...rio-notebooks/Hunting-Notebooks/Hunting-AutomatedDataQueryAndIngestionToCustomTable.ipynb b/...rio-notebooks/Hunting-Notebooks/Hunting-AutomatedDataQueryAndIngestionToCustomTable.ipynb
@@ -66,14 +66,20 @@
         "resource_group_name_source = \"\"\r\n",
         "workspace_name_source = \"\"\r\n",
         "workspace_id_source = \"\"\r\n",
-        "workspace_resource_id_source = \"/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{2}\".format(subscription_id_source, resource_group_name_source, workspace_name_source)"
+        "workspace_resource_id_source = \"/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{2}\".format(subscription_id_source, resource_group_name_source, workspace_name_source)\r\n"
       ],
       "outputs": [],
       "execution_count": null,
       "metadata": {
-        "tags": [
-          "parameters"
-        ]
+        "jupyter": {
+          "source_hidden": false,
+          "outputs_hidden": false
+        },
+        "nteract": {
+          "transient": {
+            "deleting": false
+          }
+        }
       }
     },
     {
@@ -87,7 +93,6 @@
         "location = \"\"\r\n",
         "workspace_name = ''\r\n",
         "workspace_resource_id = \"/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{2}\".format(subscription_id, resource_group_name, workspace_name)\r\n",
-        "\r\n",
         "data_collection_endpoint_name = \"\"\r\n",
         "data_collection_rule_name = \"\"\r\n",
         "custom_table_name = \"\"\r\n",
@@ -245,7 +250,7 @@
     {
       "cell_type": "code",
       "source": [
-        "# Use Dror's test LA table\r\n",
+        "# Use test LA table, set 24 hours looking back\r\n",
         "query_template = \"let t1 = SecurityAlert | extend ent = parse_json(Entities)| extend ip = tostring(ent[0]['Address']) | project-keep TimeGenerated, ip; let t2 = CommonSecurityLog | where TimeGenerated > ago({0}{1}) and TimeGenerated <= ago({2}{1}) | project ip = DestinationIP; t1 | join kind=innerunique t2 on ip\"\r\n",
         "lookback_start = '24'\r\n",
         "\r\n",
@@ -266,25 +271,6 @@
         }
       }
     },
-    {
-      "cell_type": "code",
-      "source": [
-        "df_final"
-      ],
-      "outputs": [],
-      "execution_count": null,
-      "metadata": {
-        "jupyter": {
-          "source_hidden": false,
-          "outputs_hidden": false
-        },
-        "nteract": {
-          "transient": {
-            "deleting": false
-          }
-        }
-      }
-    },
     {
       "cell_type": "markdown",
       "source": [
@@ -301,7 +287,7 @@
     {
       "cell_type": "code",
       "source": [
-        "# Calling Microsoft MDTI API for List, the same template can be used for calling other Azure REST APIs with different parameters.\r\n",
+        "# Call Microsoft MDTI API for List, the same template can be used for calling other Azure REST APIs with different parameters.\r\n",
         "# For different environments, such as national clouds, you may need to use different root_url, please contact with your admins.\r\n",
         "# It can be ---.azure.us, ---.azure.microsoft.scloud, ---.azure.eaglex.ic.gov, etc.\r\n",
         "def call_mdti_api_for_read(token, resource):\r\n",
@@ -347,9 +333,9 @@
     {
       "cell_type": "code",
       "source": [
-        "# Calling MDTI API, hosts as example\r\n",
-        "#header_token_value = \"Bearer {}\".format(get_token_for_graph())\r\n",
-        "#response_mdti_host = call_mdti_api_for_read(header_token_value, \"hosts('www.microsoft.com')\")"
+        "# Call MDTI API, hosts as example\r\n",
+        "header_token_value = \"Bearer {}\".format(get_token_for_graph())\r\n",
+        "response_mdti_host = call_mdti_api_for_read(header_token_value, \"hosts('www.microsoft.com')\")"
       ],
       "outputs": [],
       "execution_count": null,
@@ -368,8 +354,9 @@
     {
       "cell_type": "code",
       "source": [
-        "#sample_data = pd.read_json('abfs://modsynapsefiles/synapse/workspaces/modsynapse/mdti_host.json', typ='series').to_dict()\r\n",
-        "#df_host = pd.DataFrame(sample_data, index=[0])"
+        "# Merge data\r\n",
+        "df_final.loc[df_final['ip'].str.startswith('23.'), 'Fact'] = response_mdti_host.json()[\"registrar\"]\r\n",
+        "df_merged = df_final.rename(columns = {'TimeGenerated': 'TimeGenerated', 'ip': 'IP', 'Fact': 'Fact'})[['TimeGenerated', 'IP', 'Fact']]"
       ],
       "outputs": [],
       "execution_count": null,
@@ -388,29 +375,7 @@
     {
       "cell_type": "code",
       "source": [
-        "# Data process\r\n",
-        "#df_merged = pd.merge(df_final, df_host[['id','firstSeenDateTime','registrar']], left_on='ip', right_on='id', how=\"outer\")\r\n",
-        "#df_merged = df_merged.rename(columns = {'TimeGenerated': 'TimeGenerated', 'ip': 'Url', 'registrar': 'Fact'})[['TimeGenerated', 'Url', 'Fact']]\r\n",
-        "#df_merged = df_merged.fillna(numpy.nan).replace([numpy.nan], [None])"
-      ],
-      "outputs": [],
-      "execution_count": null,
-      "metadata": {
-        "jupyter": {
-          "source_hidden": false,
-          "outputs_hidden": false
-        },
-        "nteract": {
-          "transient": {
-            "deleting": false
-          }
-        }
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "#df_merged = df_final"
+        "#df_merged"
       ],
       "outputs": [],
       "execution_count": null,
@@ -465,7 +430,9 @@
         "        index_block = len(df) // partition_number\r\n",
         "\r\n",
         "        list_df = [df[i:i+index_block] for i in range(0,df.shape[0],index_block)]\r\n",
-        "        return list_df"
+        "        return list_df\r\n",
+        "    else:\r\n",
+        "        return [df]"
       ],
       "outputs": [],
       "execution_count": null,

diff --git a/...g-Notebooks/Scheduled_Hunting-AutomatedDataQueryAndMDTIAPIAndIngestionToCustomTable.ipynb b/...g-Notebooks/Scheduled_Hunting-AutomatedDataQueryAndMDTIAPIAndIngestionToCustomTable.ipynb
@@ -64,7 +64,7 @@
         "resource_group_name_source = \"\"\r\n",
         "workspace_name_source = \"\"\r\n",
         "workspace_id_source = \"\"\r\n",
-        "workspace_resource_id_source = \"/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{2}\".format(subscription_id_source, resource_group_name_source, workspace_name_source)"
+        "workspace_resource_id_source = \"/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{2}\".format(subscription_id_source, resource_group_name_source, workspace_name_source)\r\n"
       ],
       "outputs": [],
       "execution_count": null,
@@ -84,14 +84,13 @@
       "cell_type": "code",
       "source": [
         "# User input for Log Analytics workspace for data ingestion\r\n",
+        "resource_group_name = \"\"\r\n",
+        "location = \"\"\r\n",
+        "workspace_name = \"\"\r\n",
         "tenant_id = \"\"\r\n",
         "subscription_id = \"\"\r\n",
         "workspace_id = \"\"\r\n",
-        "resource_group_name = \"\"\r\n",
-        "location = \"\"\r\n",
-        "workspace_name = ''\r\n",
         "workspace_resource_id = \"/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{2}\".format(subscription_id, resource_group_name, workspace_name)\r\n",
-        "\r\n",
         "data_collection_endpoint_name = \"\"\r\n",
         "data_collection_rule_name = \"\"\r\n",
         "custom_table_name = \"\"\r\n",
@@ -249,7 +248,7 @@
     {
       "cell_type": "code",
       "source": [
-        "# Use Dror's test LA table\r\n",
+        "# Use test LA table\r\n",
         "query_template = \"let t1 = SecurityAlert | extend ent = parse_json(Entities)| extend ip = tostring(ent[0]['Address']) | project-keep TimeGenerated, ip; let t2 = CommonSecurityLog | where TimeGenerated > ago({0}{1}) and TimeGenerated <= ago({2}{1}) | project ip = DestinationIP; t1 | join kind=innerunique t2 on ip\"\r\n",
         "lookback_start = '24'\r\n",
         "\r\n",
@@ -286,7 +285,7 @@
     {
       "cell_type": "code",
       "source": [
-        "# Calling Microsoft MDTI API for List, the same template can be used for calling other Azure REST APIs with different parameters.\r\n",
+        "# Call Microsoft MDTI API for List, the same template can be used for calling other Azure REST APIs with different parameters.\r\n",
         "# For different environments, such as national clouds, you may need to use different root_url, please contact with your admins.\r\n",
         "# It can be ---.azure.us, ---.azure.microsoft.scloud, ---.azure.eaglex.ic.gov, etc.\r\n",
         "def call_mdti_api_for_read(token, resource):\r\n",
@@ -325,6 +324,67 @@
         }
       }
     },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Call MDTI API, hosts as example\r\n",
+        "header_token_value = \"Bearer {}\".format(get_token_for_graph())\r\n",
+        "response_mdti_host = call_mdti_api_for_read(header_token_value, \"hosts('www.microsoft.com')\")"
+      ],
+      "outputs": [],
+      "execution_count": null,
+      "metadata": {
+        "jupyter": {
+          "source_hidden": false,
+          "outputs_hidden": false
+        },
+        "nteract": {
+          "transient": {
+            "deleting": false
+          }
+        }
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Merge data\r\n",
+        "df_final.loc[df_final['ip'].str.startswith('23.'), 'Fact'] = response_mdti_host.json()[\"registrar\"]\r\n",
+        "df_merged = df_final.rename(columns = {'TimeGenerated': 'TimeGenerated', 'ip': 'IP', 'Fact': 'Fact'})[['TimeGenerated', 'IP', 'Fact']]"
+      ],
+      "outputs": [],
+      "execution_count": null,
+      "metadata": {
+        "jupyter": {
+          "source_hidden": false,
+          "outputs_hidden": false
+        },
+        "nteract": {
+          "transient": {
+            "deleting": false
+          }
+        }
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#df_merged"
+      ],
+      "outputs": [],
+      "execution_count": null,
+      "metadata": {
+        "jupyter": {
+          "source_hidden": false,
+          "outputs_hidden": false
+        },
+        "nteract": {
+          "transient": {
+            "deleting": false
+          }
+        }
+      }
+    },
     {
       "cell_type": "markdown",
       "source": [
@@ -364,7 +424,9 @@
         "        index_block = len(df) // partition_number\r\n",
         "\r\n",
         "        list_df = [df[i:i+index_block] for i in range(0,df.shape[0],index_block)]\r\n",
-        "        return list_df"
+        "        return list_df\r\n",
+        "    else:\r\n",
+        "        return [df]"
       ],
       "outputs": [],
       "execution_count": null,