Merge pull request #80 from Azure/ianhelle/msticpy_1_0_changes-2021-0…

…3-16 Ianhelle/msticpy 1 0 changes 2021 03 16
Azure · Apr 15, 2021 · 7bad46a · 7bad46a
2 parents 4c400ca + 43a17bf
commit 7bad46a
Show file tree

Hide file tree

Showing 35 changed files with 14,794 additions and 10,215 deletions.
diff --git a/.gitignore b/.gitignore
@@ -332,3 +332,4 @@ ASALocalRun/
 **/Kqlmagic_temp_files/**
 **/.mypy_cache/**
 **/kqlmagic/**
+/.vscode/**
diff --git a/A Getting Started Guide For Azure Sentinel ML Notebooks.ipynb b/A Getting Started Guide For Azure Sentinel ML Notebooks.ipynb
diff --git a/A Getting Started Guide For Azure Sentinel Notebooks.ipynb b/A Getting Started Guide For Azure Sentinel Notebooks.ipynb
@@ -878,9 +878,9 @@
  "metadata": {
   "hide_input": false,
   "kernelspec": {
-   "display_name": "Python 3.6",
+   "display_name": "Python 3.8 - AzureML",
    "language": "python",
-   "name": "python36"
+   "name": "python38-azureml"
   },
   "language_info": {
    "codemirror_mode": {

diff --git a/ConfiguringNotebookEnvironment.ipynb b/ConfiguringNotebookEnvironment.ipynb
diff --git a/Entity Explorer - Account.ipynb b/Entity Explorer - Account.ipynb
diff --git a/Entity Explorer - Domain and URL.ipynb b/Entity Explorer - Domain and URL.ipynb
diff --git a/Entity Explorer - IP Address.ipynb b/Entity Explorer - IP Address.ipynb
diff --git a/Entity Explorer - Linux Host.ipynb b/Entity Explorer - Linux Host.ipynb
diff --git a/Entity Explorer - Windows Host.ipynb b/Entity Explorer - Windows Host.ipynb
diff --git a/Guided Hunting - Anomalous Office365 Exchange Sessions.ipynb b/Guided Hunting - Anomalous Office365 Exchange Sessions.ipynb
diff --git a/Guided Hunting - Base64-Encoded Linux Commands.ipynb b/Guided Hunting - Base64-Encoded Linux Commands.ipynb
@@ -16,7 +16,7 @@
     "\n",
     " **Notebook Version:** 1.0<br>\n",
     " **Python Version:** Python 3.6 (including Python 3.6 - AzureML)<br>\n",
-    " **Required Packages**: kqlmagic, msticpy, pandas, pandas_bokeh, numpy, matplotlib, networkx, seaborn, datetime, ipywidgets, ipython, dnspython, ipwhois, folium, maxminddb_geolite2, BeautifulSoup<br>\n",
+    " **Required Packages**: kqlmagic, msticpy, pandas, numpy, matplotlib, networkx, seaborn, datetime, ipywidgets, ipython, dnspython, folium, maxminddb_geolite2, BeautifulSoup<br>\n",
     " **Platforms Supported**:\n",
     " - Azure Notebooks Free Compute\n",
     " - Azure Notebooks DSVM\n",
@@ -192,65 +192,27 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617824890131
+    },
     "scrolled": true
    },
    "outputs": [],
    "source": [
     "from pathlib import Path\n",
-    "import os\n",
-    "import sys\n",
-    "import warnings\n",
-    "from IPython.display import display, HTML, Markdown\n",
+    "from pathlib import Path\n",
+    "from IPython.display import display, HTML\n",
     "\n",
-    "REQ_PYTHON_VER=(3, 6)\n",
-    "REQ_MSTICPY_VER=(0, 5, 0)\n",
+    "REQ_PYTHON_VER = \"3.6\"\n",
+    "REQ_MSTICPY_VER = \"1.0.0\"\n",
     "\n",
     "display(HTML(\"<h3>Starting Notebook setup...</h3>\"))\n",
     "if Path(\"./utils/nb_check.py\").is_file():\n",
-    "    from utils.nb_check import check_python_ver, check_mp_ver\n",
-    "\n",
-    "    check_python_ver(min_py_ver=REQ_PYTHON_VER)\n",
-    "    try:\n",
-    "        check_mp_ver(min_msticpy_ver=REQ_MSTICPY_VER)\n",
-    "    except ImportError:\n",
-    "        !pip install --user --upgrade msticpy\n",
-    "        if \"msticpy\" in sys.modules:\n",
-    "            importlib.reload(msticpy)\n",
-    "        else:\n",
-    "            import msticpy\n",
-    "        check_mp_ver(REQ_PYTHON_VER)\n",
+    "    from utils.nb_check import check_versions\n",
+    "    check_versions(REQ_PYTHON_VER, REQ_MSTICPY_VER)\n",
     "            \n",
     "from msticpy.nbtools import nbinit\n",
-    "extra_imports = [\n",
-    "    \"msticpy.nbtools, observationlist\",\n",
-    "    \"msticpy.nbtools.foliummap, get_map_center\",\n",
-    "    \"pyvis.network, Network\",\n",
-    "    \"re\",\n",
-    "    \"ipwhois, IPWhois\",\n",
-    "    \"pandas_bokeh\",\n",
-    "    \"bokeh.palettes, viridis\",\n",
-    "    \"dns, reversename\",\n",
-    "    \"dns, resolver\"\n",
-    "]\n",
-    "additional_packages = [\n",
-    "    \"oauthlib\", \"pyvis\", \"python-whois\", \"pandas_bokeh\"\n",
-    "]\n",
-    "nbinit.init_notebook(\n",
-    "    namespace=globals(),\n",
-    "    additional_packages=additional_packages,\n",
-    "    extra_imports=extra_imports,\n",
-    ");\n",
-    "\n",
-    "WIDGET_DEFAULTS = {\n",
-    "    \"layout\": widgets.Layout(width=\"95%\"),\n",
-    "    \"style\": {\"description_width\": \"initial\"},\n",
-    "}\n",
-    "\n",
-    "from msticpy.sectools import auditdextract\n",
-    "from msticpy.sectools.cmd_line import *\n",
-    "from msticpy.sectools.ip_utils import convert_to_ip_entities\n",
-    "from msticpy.sectools.syslog_utils import *\n",
-    "from msticpy.sectools.syslog_utils import create_host_record, cluster_syslog_logons_df, risky_sudo_sessions\n"
+    "nbinit.init_notebook(namespace=globals());\n"
    ]
   },
   {
@@ -273,43 +235,37 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617824899543
+    },
     "scrolled": true
    },
    "outputs": [],
    "source": [
-    "#See if we have an Azure Sentinel Workspace defined in our config file, if not let the user specify Workspace and Tenant IDs\n",
-    "from msticpy.nbtools.wsconfig import WorkspaceConfig\n",
+    "# See if we have an Azure Sentinel Workspace defined in our config file.\n",
+    "# If not, let the user specify Workspace and Tenant IDs\n",
+    "\n",
     "ws_config = WorkspaceConfig()\n",
-    "print(ws_config)\n",
-    "try:\n",
-    "    ws_id = ws_config['workspace_id']\n",
-    "    ten_id = ws_config['tenant_id']\n",
-    "    display(HTML(\"Workspace details collected from config file\"))\n",
-    "    config = True\n",
-    "except:\n",
-    "    display(HTML('Please go to your Log Analytics workspace, copy the workspace ID'\n",
-    "                 ' and/or tenant Id and paste here to enable connection to the workspace and querying of it..<br> '))\n",
-    "    ws_id = nbwidgets.GetEnvironmentKey(env_var='WORKSPACE_ID',\n",
-    "                                        prompt='Please enter your Log Analytics Workspace Id:', auto_display=True)\n",
-    "    ten_id = nbwidgets.GetEnvironmentKey(env_var='TENANT_ID',\n",
-    "                                         prompt='Please enter your Log Analytics Tenant Id:', auto_display=True)\n",
-    "    config = False\n"
+    "if not ws_config.config_loaded:\n",
+    "    ws_config.prompt_for_ws()\n",
+    "    \n",
+    "qry_prov = QueryProvider(data_environment=\"AzureSentinel\")\n",
+    "print(\"done\")\n"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617824912258
+    },
     "scrolled": true
    },
    "outputs": [],
    "source": [
-    "# Establish a query provider for Azure Sentinel and connect to it\n",
-    "if config is False:\n",
-    "    ws_id = ws_id.value\n",
-    "    ten_id = ten_id.value\n",
-    "qry_prov = QueryProvider('LogAnalytics')\n",
-    "qry_prov.connect(connection_str=ws_config.code_connect_str)\n"
+    "# Authenticate to Azure Sentinel workspace\n",
+    "qry_prov.connect(ws_config)"
    ]
   },
   {
@@ -337,7 +293,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "scrolled": false
+    "gather": {
+     "logged": 1617824918576
+    }
    },
    "outputs": [],
    "source": [
@@ -366,7 +324,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "scrolled": false
+    "gather": {
+     "logged": 1617824925882
+    }
    },
    "outputs": [],
    "source": [
@@ -407,6 +367,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617824930857
+    },
     "scrolled": true
    },
    "outputs": [],
@@ -460,6 +423,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617824937794
+    },
     "scrolled": true
    },
    "outputs": [],
@@ -480,7 +446,7 @@
     "\n",
     "# Print function headings and descriptions\n",
     "for fn in range(len(function_descriptions)):\n",
-    "    display(HTML(f\"<b>{function_names[fn].text}</b>:&nbsp;{function_descriptions[fn].text}\")) "
+    "    display(HTML(f\"<b>{function_names[fn].text}</b>:&nbsp;{function_descriptions[fn].text}<br>\")) "
    ]
   },
   {
@@ -494,6 +460,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617824990809
+    },
     "scrolled": true
    },
    "outputs": [],
@@ -584,6 +553,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617824991098
+    },
     "scrolled": true
    },
    "outputs": [],
@@ -651,6 +623,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617825049100
+    },
     "scrolled": true
    },
    "outputs": [],
@@ -694,14 +669,18 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617825052658
+    },
     "scrolled": true
    },
    "outputs": [],
    "source": [
     "ti_lookup = TILookup()\n",
     "\n",
     "ti_lookup.reload_providers()\n",
-    "ti_lookup.provider_status"
+    "if not ti_lookup.provider_status:\n",
+    "    md_warn(\"You have no TI providers configured, please see the documentation link above.\")\n"
    ]
   },
   {
@@ -797,6 +776,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617825071437
+    },
     "scrolled": true
    },
    "outputs": [],
@@ -880,6 +862,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617825078094
+    },
     "scrolled": true
    },
    "outputs": [],
@@ -905,6 +890,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617825099112
+    },
     "scrolled": true
    },
    "outputs": [],
@@ -985,6 +973,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617825102642
+    },
     "scrolled": true
    },
    "outputs": [],
@@ -1024,6 +1015,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
+    "gather": {
+     "logged": 1617825107478
+    },
     "scrolled": true
    },
    "outputs": [],
@@ -1121,20 +1115,16 @@
    "source": [
     "nbdisplay.display_timeline(data=b64_df, source_columns=['DecodedCommand', 'Categories'])"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
+  "kernel_info": {
+   "name": "python38-azureml"
+  },
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3.8 - AzureML",
    "language": "python",
-   "name": "python3"
+   "name": "python38-azureml"
   },
   "language_info": {
    "codemirror_mode": {
@@ -1146,7 +1136,17 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.6"
+   "version": "3.8.1"
+  },
+  "nteract": {
+   "version": "nteract-front-end@1.0.0"
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {},
+    "version_major": 2,
+    "version_minor": 0
+   }
   }
  },
  "nbformat": 4,