droid.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import bagit\n",
    "import ntpath\n",
    "from collections import Counter\n",
    "from ipywidgets import interact"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook is going to give us some rough whole of collection statistics about metrics like format distribution, duplication and modified dates. It does this by looping through a directory, hoovering up droid csv files (either created using Droid or Siegfried with the -droid switch). \n",
    "\n",
    "Assumptions:\n",
    "Each package is in a bagit structure.\n",
    "Each package has a droid csv report,with a name ending with 'droid.csv'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def droid_sniff(file):\n",
    "    'this will help us determine whether a given file is a droid csv'\n",
    "    if file.endswith('csv'):\n",
    "        with open(file) as f:\n",
    "            sample = f.read(16)\n",
    "            if 'PARENT_ID' in sample:\n",
    "                return True\n",
    "            else:\n",
    "                return False\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This will read each droid csv report into a Pandas dataframe. It also appends a checksum column from the checksums in the bag manifest, as we don't use the checksum option in Droid usually."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "def norm_path(path):\n",
    "    try:\n",
    "        path = ntpath.normpath('data'+path.split('data')[1])\n",
    "        return path\n",
    "    except:\n",
    "        return None\n",
    "\n",
    "directory = 'corpus'\n",
    "frames = []\n",
    "bag_names = []\n",
    "for root, _, files in os.walk(directory):\n",
    "    for file in files:\n",
    "        f_path = os.path.join(root, file)\n",
    "        if droid_sniff(f_path):\n",
    "            bag = bagit.Bag(root.split('data')[0])\n",
    "            droid = pd.read_csv(f_path, parse_dates=['LAST_MODIFIED'], index_col=False)\n",
    "            #I had some empty droid csvs in the corpus for some reason, so:\n",
    "            if len(droid) > 0:\n",
    "                #Here, we're going to append the hashes from the bag to the dataframe. \n",
    "                #Unnecessary if you use the hash function in droid\n",
    "                droid['NORM_PATH'] = droid['FILE_PATH'].apply(norm_path)\n",
    "                hashes = pd.Series(\n",
    "                    {ntpath.normpath(k) : v.get('sha256') for k, v in bag.payload_entries().items()}, name='SHA256')\n",
    "                frame = droid.join(hashes, on='NORM_PATH')\n",
    "                frames.append(frame)\n",
    "                bag_names.append(os.path.split(bag.path)[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We then concatenate the dataframes. We'll also create a subset for unique files by dropping duplicates on the checksum column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10f16cc458694a8eb3a719eea21ad18a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='item', options=('sip_2014.0037.00599', 'sip_2014.0037.00600', 'sip…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<function __main__.f(item)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def fstring(x):\n",
    "    s = '{0} {1} {2}'.format(*x)\n",
    "    if s == 'nan nan nan':\n",
    "        s = 'Unidentified'\n",
    "    else:\n",
    "        s = s.replace('nan', '').strip()\n",
    "    return s\n",
    "    \n",
    "\n",
    "result = pd.concat(frames, keys=bag_names, names=['bag', 'index'], sort=False)\n",
    "#We're going to concatenate a couple of columns to create a distinct string for each format and version\n",
    "result['FORMAT_STRING'] = result[['PUID', 'FORMAT_NAME', 'FORMAT_VERSION']].apply(fstring, axis=1)\n",
    "\n",
    "#where droid isn't sure about a format, it creates lines for each possible match\n",
    "#so we're just going to set dupe formats as multiple matches and drop the dupes\n",
    "#also concatanate the PUIDs. If there's a less verbose way of doing it, I'd like to know.\n",
    "\n",
    "uri = None\n",
    "row = None\n",
    "ind = None\n",
    "for i, row in result.loc[result.duplicated(subset='URI', keep=False), ['URI', 'PUID', 'FORMAT_NAME', 'FORMAT_STRING']].iterrows():\n",
    "    if uri != row['URI']:\n",
    "        if uri is not None:\n",
    "            result.at[ind, 'PUID'] = '; '.join(puid)\n",
    "        uri = row['URI']\n",
    "        puid = [row['PUID']]\n",
    "        ind = i\n",
    "    else:\n",
    "        puid.append(row['PUID'])\n",
    "result.loc[result.duplicated(subset='URI', keep=False), ['FORMAT_NAME', 'FORMAT_STRING', 'FORMAT_VERSION', 'MIME_TYPE']] = 'Multiple matches'\n",
    "result.drop_duplicates(subset='URI', keep='first', inplace=True)\n",
    "\n",
    "files = result[result['TYPE'] == 'File']\n",
    "unique_files = files.drop_duplicates(subset='SHA256')\n",
    "\n",
    "\n",
    "def f(item):\n",
    "    return files.xs(item, level=\"bag\")\n",
    "\n",
    "itemlist = sorted(set([x[0] for x in files.index.tolist()]))\n",
    "interact(f, item=itemlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2befea3f8af54f7d883e8013f2aebb2e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='format', options=('Multiple matches', 'Unidentified', 'fmt/1070 Pr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<function __main__.format_filter(format)>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def format_filter(format):\n",
    "    return files[files['FORMAT_STRING'] == format]\n",
    "interact(format_filter, format=files.FORMAT_STRING.drop_duplicates().sort_values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we're going to graph some data. Our semi-obvious first step is to make some pie charts of the format distribution. We'll do this by number of files and total file sizes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "        <script type=\"text/javascript\">\n",
       "        window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
       "        if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
       "        if (typeof require !== 'undefined') {\n",
       "        require.undef(\"plotly\");\n",
       "        requirejs.config({\n",
       "            paths: {\n",
       "                'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
       "            }\n",
       "        });\n",
       "        require(['plotly'], function(Plotly) {\n",
       "            window._Plotly = Plotly;\n",
       "        });\n",
       "        }\n",
       "        </script>\n",
       "        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "linkText": "Export to plot.ly",
        "plotlyServerURL": "https://plot.ly",
        "showLink": false
       },
       "data": [
        {
         "domain": {
          "column": 1
         },
         "hole": 0.4,
         "hoverinfo": "label+percent",
         "labels": [
          "fmt/126 Microsoft Powerpoint Presentation 97-2003",
          "x-fmt/391 Exchangeable Image File Format (Compressed) 2.2",
          "fmt/40 Microsoft Word Document 97-2003",
          "x-fmt/92 Adobe Photoshop",
          "fmt/124 Encapsulated PostScript File Format 3",
          "x-fmt/390 Exchangeable Image File Format (Compressed) 2.1",
          "fmt/17 Acrobat PDF 1.3 - Portable Document Format 1.3",
          "fmt/950 MIME Email 1.0",
          "Unidentified",
          "x-fmt/65 Microsoft Word for Macintosh Document 5.0",
          "x-fmt/64 Microsoft Word for Macintosh Document 4.0",
          "fmt/61 Microsoft Excel 97 Workbook (xls) 8",
          "fmt/341 Macintosh PICT Image 2.0",
          "fmt/43 JPEG File Interchange Format 1.01",
          "fmt/44 JPEG File Interchange Format 1.02",
          "fmt/18 Acrobat PDF 1.4 - Portable Document Format 1.4",
          "fmt/1070 Preferred Executable Format",
          "fmt/20 Acrobat PDF 1.6 - Portable Document Format 1.6",
          "fmt/353 Tagged Image File Format",
          "fmt/116 Windows Bitmap 3.0",
          "fmt/412 Microsoft Word for Windows 2007 onwards",
          "fmt/61 Microsoft Excel 97 Workbook (xls) 8.0",
          "fmt/41 Raw JPEG Stream",
          "fmt/19 Acrobat PDF 1.5 - Portable Document Format 1.5",
          "fmt/39 Microsoft Word Document 6.0/95",
          "fmt/500 Internet Explorer for Mac cache file",
          "Multiple matches",
          "fmt/38 Microsoft Word for Windows Document 2.0",
          "x-fmt/410 Windows New Executable",
          "fmt/42 JPEG File Interchange Format 1.0",
          "fmt/50 Rich Text Format 1.5-1.6",
          "fmt/4 Graphics Interchange Format 89a",
          "x-fmt/275 Microsoft Word for MS-DOS Document 5.0",
          "fmt/16 Acrobat PDF 1.2 - Portable Document Format 1.2",
          "x-fmt/236 WordStar for MS-DOS Document 5.5",
          "fmt/96 Hypertext Markup Language",
          "fmt/899 Windows Portable Executable 32 bit",
          "x-fmt/45 Microsoft Word Document Template 97-2003",
          "fmt/111 OLE2 Compound Document Format",
          "fmt/45 Rich Text Format 1.0-1.4",
          "x-fmt/205 WordStar for MS-DOS Document 5.0",
          "fmt/214 Microsoft Excel for Windows 2007 onwards",
          "x-fmt/1 Microsoft Word for Macintosh Document 3.0",
          "fmt/609 Microsoft Word (Generic) 6.0-2003",
          "x-fmt/18 Comma Separated Values",
          "x-fmt/354 SAP Document",
          "x-fmt/229 Intergraph Raster Image",
          "fmt/90 PCX 5",
          "fmt/879 Fortran",
          "fmt/210 Statistica Report File",
          "x-fmt/44 WordPerfect for MS-DOS/Windows Document 6.0",
          "x-fmt/89 Freelance File 1.0-2.1",
          "x-fmt/20 Adobe Illustrator 1.0 / 1.1",
          "fmt/37 Microsoft Word for Windows Document 1.0",
          "x-fmt/425 Generic Library File",
          "x-fmt/151 Micrografx Designer",
          "x-fmt/381 Dia Graphics Format",
          "x-fmt/156 Ventura Publisher",
          "fmt/212 Information or Setup File",
          "fmt/503 AppleDouble Resource Fork 2",
          "x-fmt/111 Plain Text File"
         ],
         "name": "Formats by size",
         "showlegend": false,
         "textinfo": "value",
         "type": "pie",
         "uid": "4db3fc8d-295b-41bd-b20b-0e8383cb0748",
         "values": [
          599290880,
          573947302,
          513213440,
          386050196,
          174197015,
          151462522,
          135529096,
          134161982,
          97959168,
          90845696,
          42260768,
          17823744,
          10907484,
          9968880,
          8305659,
          6232480,
          5507561,
          4977903,
          3403520,
          3129344,
          2867726,
          2419712,
          2375079,
          1847733,
          1722368,
          1578220,
          1107093,
          1093120,
          1057273,
          996495,
          705822,
          601875,
          451584,
          386182,
          321152,
          302804,
          294912,
          251904,
          228376,
          197847,
          180352,
          70295,
          64000,
          33280,
          32825,
          24625,
          19750,
          15656,
          13908,
          10034,
          6819,
          6402,
          4637,
          2560,
          1475,
          1162,
          716,
          644,
          405,
          82,
          8
         ]
        },
        {
         "domain": {
          "column": 0
         },
         "hole": 0.4,
         "hoverinfo": "label+percent",
         "labels": [
          "x-fmt/65 Microsoft Word for Macintosh Document 5.0",
          "Unidentified",
          "fmt/40 Microsoft Word Document 97-2003",
          "x-fmt/64 Microsoft Word for Macintosh Document 4.0",
          "fmt/950 MIME Email 1.0",
          "Multiple matches",
          "x-fmt/391 Exchangeable Image File Format (Compressed) 2.2",
          "fmt/61 Microsoft Excel 97 Workbook (xls) 8",
          "fmt/96 Hypertext Markup Language",
          "x-fmt/390 Exchangeable Image File Format (Compressed) 2.1",
          "fmt/61 Microsoft Excel 97 Workbook (xls) 8.0",
          "fmt/38 Microsoft Word for Windows Document 2.0",
          "fmt/43 JPEG File Interchange Format 1.01",
          "fmt/44 JPEG File Interchange Format 1.02",
          "x-fmt/275 Microsoft Word for MS-DOS Document 5.0",
          "x-fmt/92 Adobe Photoshop",
          "fmt/39 Microsoft Word Document 6.0/95",
          "fmt/412 Microsoft Word for Windows 2007 onwards",
          "fmt/126 Microsoft Powerpoint Presentation 97-2003",
          "fmt/18 Acrobat PDF 1.4 - Portable Document Format 1.4",
          "fmt/500 Internet Explorer for Mac cache file",
          "x-fmt/229 Intergraph Raster Image",
          "fmt/879 Fortran",
          "fmt/116 Windows Bitmap 3.0",
          "fmt/50 Rich Text Format 1.5-1.6",
          "fmt/17 Acrobat PDF 1.3 - Portable Document Format 1.3",
          "fmt/111 OLE2 Compound Document Format",
          "x-fmt/45 Microsoft Word Document Template 97-2003",
          "x-fmt/236 WordStar for MS-DOS Document 5.5",
          "x-fmt/354 SAP Document",
          "fmt/124 Encapsulated PostScript File Format 3",
          "x-fmt/89 Freelance File 1.0-2.1",
          "x-fmt/205 WordStar for MS-DOS Document 5.0",
          "fmt/214 Microsoft Excel for Windows 2007 onwards",
          "fmt/210 Statistica Report File",
          "fmt/19 Acrobat PDF 1.5 - Portable Document Format 1.5",
          "x-fmt/381 Dia Graphics Format",
          "x-fmt/1 Microsoft Word for Macintosh Document 3.0",
          "fmt/45 Rich Text Format 1.0-1.4",
          "fmt/41 Raw JPEG Stream",
          "fmt/4 Graphics Interchange Format 89a",
          "fmt/20 Acrobat PDF 1.6 - Portable Document Format 1.6",
          "fmt/1070 Preferred Executable Format",
          "fmt/212 Information or Setup File",
          "x-fmt/151 Micrografx Designer",
          "fmt/37 Microsoft Word for Windows Document 1.0",
          "x-fmt/18 Comma Separated Values",
          "fmt/16 Acrobat PDF 1.2 - Portable Document Format 1.2",
          "x-fmt/20 Adobe Illustrator 1.0 / 1.1",
          "fmt/341 Macintosh PICT Image 2.0",
          "x-fmt/425 Generic Library File",
          "fmt/90 PCX 5",
          "fmt/353 Tagged Image File Format",
          "fmt/899 Windows Portable Executable 32 bit",
          "x-fmt/44 WordPerfect for MS-DOS/Windows Document 6.0",
          "fmt/609 Microsoft Word (Generic) 6.0-2003",
          "x-fmt/111 Plain Text File",
          "fmt/42 JPEG File Interchange Format 1.0",
          "x-fmt/156 Ventura Publisher",
          "fmt/503 AppleDouble Resource Fork 2",
          "x-fmt/410 Windows New Executable"
         ],
         "name": "Formats by size",
         "showlegend": false,
         "textinfo": "value",
         "type": "pie",
         "uid": "b0492510-0fe4-437a-978a-10191a98de00",
         "values": [
          9377,
          6027,
          5276,
          3539,
          2468,
          462,
          358,
          260,
          227,
          112,
          66,
          44,
          37,
          29,
          23,
          22,
          21,
          20,
          19,
          19,
          18,
          12,
          9,
          8,
          7,
          7,
          6,
          5,
          5,
          4,
          4,
          4,
          3,
          2,
          2,
          2,
          2,
          2,
          2,
          2,
          2,
          2,
          2,
          1,
          1,
          1,
          1,
          1,
          1,
          1,
          1,
          1,
          1,
          1,
          1,
          1,
          1,
          1,
          1,
          1,
          1
         ]
        }
       ],
       "layout": {
        "annotations": [
         {
          "font": {
           "size": 20
          },
          "showarrow": false,
          "text": "By count",
          "x": 0.18,
          "y": 0.5
         },
         {
          "font": {
           "size": 20
          },
          "showarrow": false,
          "text": "By size",
          "x": 0.81,
          "y": 0.5
         }
        ],
        "grid": {
         "columns": 2,
         "rows": 1
        },
        "legend": {
         "orientation": "h"
        },
        "title": {
         "text": "File formats"
        }
       }
      },
      "text/html": [
       "<div>\n",
       "        \n",
       "        \n",
       "            <div id=\"a74f5a32-3f21-49eb-9825-222beb114d68\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                require([\"plotly\"], function(Plotly) {\n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    window.PLOTLYENV.BASE_URL='https://plot.ly';\n",
       "                    \n",
       "                if (document.getElementById(\"a74f5a32-3f21-49eb-9825-222beb114d68\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        'a74f5a32-3f21-49eb-9825-222beb114d68',\n",
       "                        [{\"domain\": {\"column\": 1}, \"hole\": 0.4, \"hoverinfo\": \"label+percent\", \"labels\": [\"fmt/126 Microsoft Powerpoint Presentation 97-2003\", \"x-fmt/391 Exchangeable Image File Format (Compressed) 2.2\", \"fmt/40 Microsoft Word Document 97-2003\", \"x-fmt/92 Adobe Photoshop\", \"fmt/124 Encapsulated PostScript File Format 3\", \"x-fmt/390 Exchangeable Image File Format (Compressed) 2.1\", \"fmt/17 Acrobat PDF 1.3 - Portable Document Format 1.3\", \"fmt/950 MIME Email 1.0\", \"Unidentified\", \"x-fmt/65 Microsoft Word for Macintosh Document 5.0\", \"x-fmt/64 Microsoft Word for Macintosh Document 4.0\", \"fmt/61 Microsoft Excel 97 Workbook (xls) 8\", \"fmt/341 Macintosh PICT Image 2.0\", \"fmt/43 JPEG File Interchange Format 1.01\", \"fmt/44 JPEG File Interchange Format 1.02\", \"fmt/18 Acrobat PDF 1.4 - Portable Document Format 1.4\", \"fmt/1070 Preferred Executable Format\", \"fmt/20 Acrobat PDF 1.6 - Portable Document Format 1.6\", \"fmt/353 Tagged Image File Format\", \"fmt/116 Windows Bitmap 3.0\", \"fmt/412 Microsoft Word for Windows 2007 onwards\", \"fmt/61 Microsoft Excel 97 Workbook (xls) 8.0\", \"fmt/41 Raw JPEG Stream\", \"fmt/19 Acrobat PDF 1.5 - Portable Document Format 1.5\", \"fmt/39 Microsoft Word Document 6.0/95\", \"fmt/500 Internet Explorer for Mac cache file\", \"Multiple matches\", \"fmt/38 Microsoft Word for Windows Document 2.0\", \"x-fmt/410 Windows New Executable\", \"fmt/42 JPEG File Interchange Format 1.0\", \"fmt/50 Rich Text Format 1.5-1.6\", \"fmt/4 Graphics Interchange Format 89a\", \"x-fmt/275 Microsoft Word for MS-DOS Document 5.0\", \"fmt/16 Acrobat PDF 1.2 - Portable Document Format 1.2\", \"x-fmt/236 WordStar for MS-DOS Document 5.5\", \"fmt/96 Hypertext Markup Language\", \"fmt/899 Windows Portable Executable 32 bit\", \"x-fmt/45 Microsoft Word Document Template 97-2003\", \"fmt/111 OLE2 Compound Document Format\", \"fmt/45 Rich Text Format 1.0-1.4\", \"x-fmt/205 WordStar for MS-DOS Document 5.0\", \"fmt/214 Microsoft Excel for Windows 2007 onwards\", \"x-fmt/1 Microsoft Word for Macintosh Document 3.0\", \"fmt/609 Microsoft Word (Generic) 6.0-2003\", \"x-fmt/18 Comma Separated Values\", \"x-fmt/354 SAP Document\", \"x-fmt/229 Intergraph Raster Image\", \"fmt/90 PCX 5\", \"fmt/879 Fortran\", \"fmt/210 Statistica Report File\", \"x-fmt/44 WordPerfect for MS-DOS/Windows Document 6.0\", \"x-fmt/89 Freelance File 1.0-2.1\", \"x-fmt/20 Adobe Illustrator 1.0 / 1.1\", \"fmt/37 Microsoft Word for Windows Document 1.0\", \"x-fmt/425 Generic Library File\", \"x-fmt/151 Micrografx Designer\", \"x-fmt/381 Dia Graphics Format\", \"x-fmt/156 Ventura Publisher\", \"fmt/212 Information or Setup File\", \"fmt/503 AppleDouble Resource Fork 2\", \"x-fmt/111 Plain Text File\"], \"name\": \"Formats by size\", \"showlegend\": false, \"textinfo\": \"value\", \"type\": \"pie\", \"uid\": \"4db3fc8d-295b-41bd-b20b-0e8383cb0748\", \"values\": [599290880.0, 573947302.0, 513213440.0, 386050196.0, 174197015.0, 151462522.0, 135529096.0, 134161982.0, 97959168.0, 90845696.0, 42260768.0, 17823744.0, 10907484.0, 9968880.0, 8305659.0, 6232480.0, 5507561.0, 4977903.0, 3403520.0, 3129344.0, 2867726.0, 2419712.0, 2375079.0, 1847733.0, 1722368.0, 1578220.0, 1107093.0, 1093120.0, 1057273.0, 996495.0, 705822.0, 601875.0, 451584.0, 386182.0, 321152.0, 302804.0, 294912.0, 251904.0, 228376.0, 197847.0, 180352.0, 70295.0, 64000.0, 33280.0, 32825.0, 24625.0, 19750.0, 15656.0, 13908.0, 10034.0, 6819.0, 6402.0, 4637.0, 2560.0, 1475.0, 1162.0, 716.0, 644.0, 405.0, 82.0, 8.0]}, {\"domain\": {\"column\": 0}, \"hole\": 0.4, \"hoverinfo\": \"label+percent\", \"labels\": [\"x-fmt/65 Microsoft Word for Macintosh Document 5.0\", \"Unidentified\", \"fmt/40 Microsoft Word Document 97-2003\", \"x-fmt/64 Microsoft Word for Macintosh Document 4.0\", \"fmt/950 MIME Email 1.0\", \"Multiple matches\", \"x-fmt/391 Exchangeable Image File Format (Compressed) 2.2\", \"fmt/61 Microsoft Excel 97 Workbook (xls) 8\", \"fmt/96 Hypertext Markup Language\", \"x-fmt/390 Exchangeable Image File Format (Compressed) 2.1\", \"fmt/61 Microsoft Excel 97 Workbook (xls) 8.0\", \"fmt/38 Microsoft Word for Windows Document 2.0\", \"fmt/43 JPEG File Interchange Format 1.01\", \"fmt/44 JPEG File Interchange Format 1.02\", \"x-fmt/275 Microsoft Word for MS-DOS Document 5.0\", \"x-fmt/92 Adobe Photoshop\", \"fmt/39 Microsoft Word Document 6.0/95\", \"fmt/412 Microsoft Word for Windows 2007 onwards\", \"fmt/126 Microsoft Powerpoint Presentation 97-2003\", \"fmt/18 Acrobat PDF 1.4 - Portable Document Format 1.4\", \"fmt/500 Internet Explorer for Mac cache file\", \"x-fmt/229 Intergraph Raster Image\", \"fmt/879 Fortran\", \"fmt/116 Windows Bitmap 3.0\", \"fmt/50 Rich Text Format 1.5-1.6\", \"fmt/17 Acrobat PDF 1.3 - Portable Document Format 1.3\", \"fmt/111 OLE2 Compound Document Format\", \"x-fmt/45 Microsoft Word Document Template 97-2003\", \"x-fmt/236 WordStar for MS-DOS Document 5.5\", \"x-fmt/354 SAP Document\", \"fmt/124 Encapsulated PostScript File Format 3\", \"x-fmt/89 Freelance File 1.0-2.1\", \"x-fmt/205 WordStar for MS-DOS Document 5.0\", \"fmt/214 Microsoft Excel for Windows 2007 onwards\", \"fmt/210 Statistica Report File\", \"fmt/19 Acrobat PDF 1.5 - Portable Document Format 1.5\", \"x-fmt/381 Dia Graphics Format\", \"x-fmt/1 Microsoft Word for Macintosh Document 3.0\", \"fmt/45 Rich Text Format 1.0-1.4\", \"fmt/41 Raw JPEG Stream\", \"fmt/4 Graphics Interchange Format 89a\", \"fmt/20 Acrobat PDF 1.6 - Portable Document Format 1.6\", \"fmt/1070 Preferred Executable Format\", \"fmt/212 Information or Setup File\", \"x-fmt/151 Micrografx Designer\", \"fmt/37 Microsoft Word for Windows Document 1.0\", \"x-fmt/18 Comma Separated Values\", \"fmt/16 Acrobat PDF 1.2 - Portable Document Format 1.2\", \"x-fmt/20 Adobe Illustrator 1.0 / 1.1\", \"fmt/341 Macintosh PICT Image 2.0\", \"x-fmt/425 Generic Library File\", \"fmt/90 PCX 5\", \"fmt/353 Tagged Image File Format\", \"fmt/899 Windows Portable Executable 32 bit\", \"x-fmt/44 WordPerfect for MS-DOS/Windows Document 6.0\", \"fmt/609 Microsoft Word (Generic) 6.0-2003\", \"x-fmt/111 Plain Text File\", \"fmt/42 JPEG File Interchange Format 1.0\", \"x-fmt/156 Ventura Publisher\", \"fmt/503 AppleDouble Resource Fork 2\", \"x-fmt/410 Windows New Executable\"], \"name\": \"Formats by size\", \"showlegend\": false, \"textinfo\": \"value\", \"type\": \"pie\", \"uid\": \"b0492510-0fe4-437a-978a-10191a98de00\", \"values\": [9377, 6027, 5276, 3539, 2468, 462, 358, 260, 227, 112, 66, 44, 37, 29, 23, 22, 21, 20, 19, 19, 18, 12, 9, 8, 7, 7, 6, 5, 5, 4, 4, 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}],\n",
       "                        {\"annotations\": [{\"font\": {\"size\": 20}, \"showarrow\": false, \"text\": \"By count\", \"x\": 0.18, \"y\": 0.5}, {\"font\": {\"size\": 20}, \"showarrow\": false, \"text\": \"By size\", \"x\": 0.81, \"y\": 0.5}], \"grid\": {\"columns\": 2, \"rows\": 1}, \"legend\": {\"orientation\": \"h\"}, \"title\": {\"text\": \"File formats\"}},\n",
       "                        {\"showLink\": false, \"linkText\": \"Export to plot.ly\", \"plotlyServerURL\": \"https://plot.ly\", \"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('a74f5a32-3f21-49eb-9825-222beb114d68');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                });\n",
       "            </script>\n",
       "        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import plotly.graph_objs as go\n",
    "from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\n",
    "init_notebook_mode(connected=True)\n",
    "\n",
    "f_numbers  = unique_files['FORMAT_STRING'].value_counts(dropna=False)\n",
    "sizes = unique_files.fillna('unidentified').groupby('FORMAT_STRING')['SIZE'].sum().sort_values(ascending=False)\n",
    "sizes = sizes.reset_index()\n",
    "f_numbers = f_numbers.reset_index()\n",
    "\n",
    "fig = {\n",
    "    \"data\": [{\"values\": sizes['SIZE'], \"labels\": sizes['FORMAT_STRING'], \"hoverinfo\": 'label+percent', \"showlegend\": False,\n",
    "              \"domain\": {\"column\": 1}, \"name\": \"Formats by size\", \"textinfo\": 'value', \"hole\": .4, \"type\": \"pie\"},\n",
    "            {\"values\": f_numbers['FORMAT_STRING'], \"labels\": f_numbers['index'], \"hoverinfo\": 'label+percent', \"showlegend\": False,\n",
    "             \"textinfo\": 'value', \"domain\": {\"column\": 0}, \"name\": \"Formats by size\", \n",
    "             \"textinfo\": 'value', \"hole\": .4, \"type\": \"pie\"}], \n",
    "    \"layout\": {\"legend\": {\"orientation\": \"h\"},\n",
    "               \"title\": \"File formats\",\n",
    "               \"grid\": {\"rows\": 1, \"columns\": 2},\n",
    "               \"annotations\": [\n",
    "                   {\"text\": \"By count\", \"x\": 0.18, \"y\": 0.5, \"font_size\": 20, \"showarrow\": False},\n",
    "                   {\"text\": \"By size\", \"x\": 0.81, \"y\": 0.5, \"font_size\": 20, \"showarrow\": False}]}}\n",
    "\n",
    "iplot(fig, filename=\"formats\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "linkText": "Export to plot.ly",
        "plotlyServerURL": "https://plot.ly",
        "showLink": false
       },
       "data": [
        {
         "mode": "markers",
         "name": "Microsoft Word Document",
         "type": "scatter",
         "uid": "16f75349-2c0c-4dd4-97e6-75c3ec5ac12c",
         "x": [
          "1996-10",
          "1998-10",
          "1998-12",
          "1999-01",
          "1999-04",
          "1999-05",
          "1999-11",
          "2000-03",
          "2000-04",
          "2000-05",
          "2000-06",
          "2000-07",
          "2000-08",
          "2000-09",
          "2000-10",
          "2000-11",
          "2000-12",
          "2001-01",
          "2001-02",
          "2001-03",
          "2001-04",
          "2001-05",
          "2001-06",
          "2001-07",
          "2001-08",
          "2001-09",
          "2001-10",
          "2001-11",
          "2001-12",
          "2002-01",
          "2002-02",
          "2002-03",
          "2002-04",
          "2002-05",
          "2002-06",
          "2002-07",
          "2002-08",
          "2002-09",
          "2002-10",
          "2002-11",
          "2002-12",
          "2003-01",
          "2003-02",
          "2003-03",
          "2003-04",
          "2003-05",
          "2003-06",
          "2003-07",
          "2003-08",
          "2003-09",
          "2003-10",
          "2003-11",
          "2003-12",
          "2004-01",
          "2004-02",
          "2004-03",
          "2004-04",
          "2004-05",
          "2004-06",
          "2004-07",
          "2004-08",
          "2004-09",
          "2004-10",
          "2004-11",
          "2004-12",
          "2005-01",
          "2005-02",
          "2005-03",
          "2005-04",
          "2005-05",
          "2005-06",
          "2005-07",
          "2005-08",
          "2005-09",
          "2005-10",
          "2005-11",
          "2005-12",
          "2006-01",
          "2006-02",
          "2006-03",
          "2006-04",
          "2006-05",
          "2006-06",
          "2006-07",
          "2006-08",
          "2006-09",
          "2006-12",
          "2007-01",
          "2007-04",
          "2007-05",
          "2007-10",
          "2007-11",
          "2007-12",
          "2008-02",
          "2009-02",
          "2009-03",
          "2009-04",
          "2010-01",
          "2010-07",
          "2010-08",
          "2010-09",
          "2010-10",
          "2010-11",
          "2010-12",
          "2011-01"
         ],
         "y": [
          1,
          2,
          2,
          3,
          1,
          1,
          1,
          4,
          12,
          7,
          23,
          23,
          107,
          132,
          134,
          26,
          28,
          156,
          202,
          250,
          123,
          58,
          76,
          110,
          98,
          63,
          102,
          82,
          70,
          78,
          63,
          71,
          52,
          95,
          70,
          88,
          61,
          65,
          79,
          78,
          140,
          57,
          60,
          86,
          58,
          75,
          70,
          73,
          77,
          75,
          122,
          99,
          72,
          100,
          65,
          69,
          98,
          80,
          142,
          60,
          72,
          45,
          33,
          72,
          45,
          76,
          42,
          59,
          30,
          37,
          38,
          16,
          16,
          27,
          45,
          74,
          12,
          3,
          4,
          3,
          4,
          4,
          3,
          7,
          2,
          7,
          6,
          1,
          2,
          3,
          2,
          8,
          3,
          51,
          20,
          8,
          2,
          2,
          6,
          1,
          1,
          1,
          3,
          2,
          12
         ]
        },
        {
         "mode": "markers",
         "name": "WordPerfect for MS-DOS/Windows Document",
         "type": "scatter",
         "uid": "93567289-8ea3-4047-89c7-fc0b5e01906f",
         "x": [
          "2000-12"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Microsoft Word for Macintosh Document",
         "type": "scatter",
         "uid": "fa20f247-7024-4063-9d2b-a97f1687800c",
         "x": [
          "1989-10",
          "1989-11",
          "1989-12",
          "1990-01",
          "1990-02",
          "1990-03",
          "1990-04",
          "1990-05",
          "1990-06",
          "1990-07",
          "1990-08",
          "1990-09",
          "1990-10",
          "1990-11",
          "1990-12",
          "1991-01",
          "1991-02",
          "1991-03",
          "1991-04",
          "1991-05",
          "1991-06",
          "1991-07",
          "1991-08",
          "1991-09",
          "1991-10",
          "1991-11",
          "1991-12",
          "1992-01",
          "1992-02",
          "1992-03",
          "1992-04",
          "1992-05",
          "1992-06",
          "1992-07",
          "1992-08",
          "1992-09",
          "1992-10",
          "1992-11",
          "1992-12",
          "1993-01",
          "1993-02",
          "1993-03",
          "1993-04",
          "1993-05",
          "1993-06",
          "1993-07",
          "1993-08",
          "1993-09",
          "1993-10",
          "1993-11",
          "1993-12",
          "1994-01",
          "1994-02",
          "1994-03",
          "1994-04",
          "1994-05",
          "1994-06",
          "1994-07",
          "1994-08",
          "1994-09",
          "1994-10",
          "1994-11",
          "1994-12",
          "1995-01",
          "1995-02",
          "1995-03",
          "1995-04",
          "1995-05",
          "1995-06",
          "1995-07",
          "1995-08",
          "1995-09",
          "1995-10",
          "1995-11",
          "1995-12",
          "1996-01",
          "1996-02",
          "1996-03",
          "1996-04",
          "1996-05",
          "1996-06",
          "1996-07",
          "1996-08",
          "1996-09",
          "1996-10",
          "1996-11",
          "1996-12",
          "1997-01",
          "1997-02",
          "1997-03",
          "1997-04",
          "1997-05",
          "1997-06",
          "1997-07",
          "1997-08",
          "1997-09",
          "1997-10",
          "1997-11",
          "1997-12",
          "1998-01",
          "1998-02",
          "1998-03",
          "1998-04",
          "1998-05",
          "1998-06",
          "1998-07",
          "1998-08",
          "1998-09",
          "1998-10",
          "1998-11",
          "1998-12",
          "1999-01",
          "1999-02",
          "1999-03",
          "1999-04",
          "1999-05",
          "1999-06",
          "1999-07",
          "1999-08",
          "1999-09",
          "1999-10",
          "1999-11",
          "1999-12",
          "2000-01",
          "2000-02",
          "2000-03",
          "2000-04",
          "2000-05",
          "2000-06",
          "2000-07",
          "2000-08",
          "2000-09",
          "2000-10",
          "2000-11",
          "2000-12",
          "2001-01",
          "2001-02",
          "2001-03",
          "2001-04",
          "2001-05",
          "2001-06",
          "2001-08",
          "2001-09",
          "2001-11",
          "2001-12",
          "2002-10",
          "2003-05",
          "2004-05",
          "2004-08",
          "2005-10",
          "2009-04"
         ],
         "y": [
          45,
          23,
          23,
          22,
          19,
          38,
          23,
          18,
          25,
          19,
          20,
          27,
          24,
          17,
          4,
          16,
          12,
          40,
          17,
          28,
          90,
          49,
          49,
          71,
          78,
          140,
          241,
          105,
          80,
          104,
          90,
          109,
          106,
          97,
          116,
          91,
          163,
          102,
          84,
          89,
          105,
          114,
          93,
          80,
          94,
          147,
          137,
          160,
          119,
          108,
          101,
          94,
          158,
          125,
          98,
          148,
          94,
          142,
          87,
          91,
          103,
          106,
          73,
          50,
          89,
          72,
          96,
          97,
          99,
          115,
          100,
          77,
          124,
          128,
          59,
          154,
          84,
          89,
          86,
          121,
          117,
          131,
          91,
          133,
          75,
          95,
          110,
          156,
          167,
          109,
          97,
          138,
          161,
          84,
          135,
          130,
          100,
          102,
          48,
          51,
          166,
          182,
          194,
          80,
          73,
          82,
          83,
          121,
          80,
          76,
          51,
          137,
          188,
          120,
          158,
          123,
          95,
          109,
          147,
          69,
          53,
          50,
          125,
          112,
          94,
          67,
          44,
          77,
          115,
          250,
          86,
          70,
          65,
          77,
          25,
          4,
          14,
          8,
          19,
          11,
          2,
          3,
          3,
          2,
          4,
          2,
          1,
          1,
          41,
          37,
          21
         ]
        },
        {
         "mode": "markers",
         "name": "nan",
         "type": "scatter",
         "uid": "ea26a771-1ff7-4fe6-b2c3-174d7b331be1",
         "x": [],
         "y": []
        },
        {
         "mode": "markers",
         "name": "Microsoft Word for Windows Document",
         "type": "scatter",
         "uid": "34f8fece-74ae-4c55-b4bf-58e404ec20e3",
         "x": [
          "1998-04",
          "1998-07",
          "1998-12",
          "1999-01"
         ],
         "y": [
          39,
          1,
          3,
          2
         ]
        },
        {
         "mode": "markers",
         "name": "Exchangeable Image File Format (Compressed)",
         "type": "scatter",
         "uid": "c5acf427-31d0-4db7-85d2-0f6c18dec179",
         "x": [
          "2001-08",
          "2003-01",
          "2003-09",
          "2003-10",
          "2003-11",
          "2003-12",
          "2004-01",
          "2004-02",
          "2004-05",
          "2004-06",
          "2005-03",
          "2005-07",
          "2005-10",
          "2005-11",
          "2006-02",
          "2006-03",
          "2006-04"
         ],
         "y": [
          2,
          1,
          1,
          8,
          28,
          1,
          54,
          23,
          32,
          1,
          140,
          37,
          37,
          18,
          54,
          7,
          26
         ]
        },
        {
         "mode": "markers",
         "name": "Microsoft Excel 97 Workbook (xls)",
         "type": "scatter",
         "uid": "9f042ee6-f3ba-43bd-9f52-9e4c63e957b1",
         "x": [
          "1999-11",
          "1999-12",
          "2000-01",
          "2000-02",
          "2000-03",
          "2000-04",
          "2000-05",
          "2000-06",
          "2000-07",
          "2000-08",
          "2000-09",
          "2000-10",
          "2000-11",
          "2000-12",
          "2001-01",
          "2001-02",
          "2001-03",
          "2001-04",
          "2001-05",
          "2001-07",
          "2001-08",
          "2001-11",
          "2001-12",
          "2002-01",
          "2002-02",
          "2002-04",
          "2002-05",
          "2002-06",
          "2002-08",
          "2002-11",
          "2002-12",
          "2003-01",
          "2003-02",
          "2003-03",
          "2003-05",
          "2003-08",
          "2003-09",
          "2003-10",
          "2003-11",
          "2003-12",
          "2004-01",
          "2004-02",
          "2004-03",
          "2004-04",
          "2004-05",
          "2004-06",
          "2004-08",
          "2004-10",
          "2004-11",
          "2005-02",
          "2005-07",
          "2005-08",
          "2005-09",
          "2005-10",
          "2005-11"
         ],
         "y": [
          6,
          5,
          2,
          8,
          7,
          1,
          9,
          8,
          4,
          2,
          1,
          3,
          6,
          2,
          2,
          4,
          4,
          1,
          6,
          1,
          4,
          8,
          4,
          2,
          6,
          2,
          11,
          1,
          9,
          14,
          1,
          2,
          12,
          3,
          11,
          28,
          5,
          4,
          13,
          8,
          2,
          17,
          5,
          1,
          14,
          1,
          6,
          2,
          8,
          26,
          1,
          1,
          1,
          7,
          4
         ]
        },
        {
         "mode": "markers",
         "name": "JPEG File Interchange Format",
         "type": "scatter",
         "uid": "11bf0e93-b895-4e84-847a-e69bc963475f",
         "x": [
          "2003-01",
          "2003-02",
          "2003-08",
          "2003-09",
          "2003-10",
          "2003-11",
          "2004-01",
          "2004-03",
          "2004-04",
          "2004-05",
          "2004-10",
          "2005-01"
         ],
         "y": [
          8,
          5,
          6,
          3,
          1,
          5,
          1,
          5,
          11,
          1,
          1,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Tagged Image File Format",
         "type": "scatter",
         "uid": "9fded2f4-9e9c-465e-b60b-d19245793837",
         "x": [
          "2003-03"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Raw JPEG Stream",
         "type": "scatter",
         "uid": "9e6e3a1a-9217-4ebd-b3d3-a4d29cd4b142",
         "x": [
          "2002-12",
          "2003-11"
         ],
         "y": [
          1,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Plain Text File",
         "type": "scatter",
         "uid": "59617e6b-8f4f-407a-8035-e734aa1235cd",
         "x": [
          "2005-03"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Microsoft Powerpoint Presentation",
         "type": "scatter",
         "uid": "a345fbb7-4970-41e6-98b5-e16abee82601",
         "x": [
          "2003-10",
          "2003-11",
          "2004-04",
          "2004-06",
          "2005-10",
          "2006-05",
          "2006-06"
         ],
         "y": [
          6,
          2,
          3,
          2,
          2,
          2,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Macintosh PICT Image",
         "type": "scatter",
         "uid": "7abb923a-3b0c-4b33-8d9e-41fe8b84c50f",
         "x": [],
         "y": []
        },
        {
         "mode": "markers",
         "name": "MIME Email",
         "type": "scatter",
         "uid": "ce9756ff-7132-4c7e-b36f-ab1d9801aceb",
         "x": [
          "2000-01",
          "2001-02",
          "2001-03",
          "2001-04",
          "2001-06",
          "2001-08",
          "2001-09",
          "2001-11",
          "2001-12",
          "2002-10",
          "2002-11",
          "2002-12",
          "2003-01",
          "2003-02",
          "2003-05",
          "2003-08",
          "2003-10",
          "2003-11",
          "2005-01"
         ],
         "y": [
          1,
          1,
          2,
          5,
          1,
          1,
          2,
          16,
          7,
          108,
          329,
          222,
          78,
          116,
          5,
          1,
          559,
          5,
          1009
         ]
        },
        {
         "mode": "markers",
         "name": "Internet Explorer for Mac cache file",
         "type": "scatter",
         "uid": "ba8a6b5c-052c-433b-aaab-cbf6061e81a5",
         "x": [
          "2001-03",
          "2003-04",
          "2003-08"
         ],
         "y": [
          15,
          1,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Adobe Photoshop",
         "type": "scatter",
         "uid": "c6cb526f-d02a-4b40-a73a-29dfab72331e",
         "x": [
          "2003-01",
          "2003-10",
          "2004-01"
         ],
         "y": [
          9,
          10,
          3
         ]
        },
        {
         "mode": "markers",
         "name": "Acrobat PDF 1.3 - Portable Document Format",
         "type": "scatter",
         "uid": "5dc1dc30-2b3e-44a1-9179-6635b25d2b5f",
         "x": [
          "2003-08",
          "2003-11",
          "2010-07"
         ],
         "y": [
          1,
          4,
          2
         ]
        },
        {
         "mode": "markers",
         "name": "Rich Text Format",
         "type": "scatter",
         "uid": "44836816-5c6b-4982-aaa0-38e1613872d8",
         "x": [
          "1998-12",
          "2000-06",
          "2001-05",
          "2001-08",
          "2001-09",
          "2001-10",
          "2004-01"
         ],
         "y": [
          1,
          1,
          2,
          2,
          1,
          1,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Fortran",
         "type": "scatter",
         "uid": "4290745f-4514-4fa4-a40a-c97b5370ff7c",
         "x": [
          "2001-07",
          "2001-08",
          "2001-10",
          "2001-11"
         ],
         "y": [
          3,
          4,
          1,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Statistica Report File",
         "type": "scatter",
         "uid": "4f887426-2b3f-4a12-bc3c-6edd4d9a9619",
         "x": [
          "2001-08"
         ],
         "y": [
          2
         ]
        },
        {
         "mode": "markers",
         "name": "Intergraph Raster Image",
         "type": "scatter",
         "uid": "adb2c281-f82a-4fe9-840c-cf759fd16277",
         "x": [
          "2001-07",
          "2001-08",
          "2001-10",
          "2001-11"
         ],
         "y": [
          3,
          4,
          2,
          3
         ]
        },
        {
         "mode": "markers",
         "name": "Dia Graphics Format",
         "type": "scatter",
         "uid": "00954df9-7a62-484c-b38d-208df2f4ebea",
         "x": [
          "2001-08",
          "2001-10"
         ],
         "y": [
          1,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Micrografx Designer",
         "type": "scatter",
         "uid": "e6a1901e-60c1-4f8e-818f-10469348d3c8",
         "x": [
          "2001-08"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Multiple matches",
         "type": "scatter",
         "uid": "066cb8a6-8613-41e8-87df-45ba690406be",
         "x": [
          "1980-01",
          "1985-01",
          "1985-06",
          "1985-08",
          "1985-09",
          "1985-10",
          "1985-11",
          "1985-12",
          "1986-01",
          "1986-06",
          "1986-08",
          "1986-09",
          "1986-10",
          "1987-05",
          "1987-06",
          "1987-07",
          "1987-08",
          "1987-11",
          "1987-12",
          "1988-01",
          "1988-02",
          "1988-03",
          "1988-06",
          "1988-08",
          "1989-01",
          "1989-05",
          "1989-09",
          "1990-04",
          "1991-12",
          "1995-10",
          "1999-05",
          "1999-09",
          "1999-10",
          "2000-02",
          "2000-03",
          "2000-05",
          "2000-06",
          "2000-08",
          "2000-10",
          "2000-11",
          "2001-04",
          "2001-07",
          "2001-08",
          "2002-12",
          "2003-02",
          "2003-08",
          "2005-01",
          "2005-10"
         ],
         "y": [
          34,
          10,
          2,
          5,
          3,
          11,
          9,
          23,
          10,
          11,
          5,
          27,
          3,
          17,
          10,
          13,
          66,
          26,
          4,
          6,
          13,
          2,
          35,
          43,
          1,
          24,
          1,
          1,
          2,
          1,
          2,
          1,
          2,
          1,
          1,
          1,
          1,
          3,
          1,
          2,
          1,
          1,
          1,
          2,
          3,
          19,
          1,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Freelance File",
         "type": "scatter",
         "uid": "50ab983e-cc79-4c2a-a004-ab0a1b21254a",
         "x": [
          "1989-05",
          "2001-08"
         ],
         "y": [
          1,
          3
         ]
        },
        {
         "mode": "markers",
         "name": "Generic Library File",
         "type": "scatter",
         "uid": "6e028f73-3c89-450b-b9d5-156e72316825",
         "x": [
          "2001-08"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "SAP Document",
         "type": "scatter",
         "uid": "cc2936c8-66d7-4e70-b99f-ed7b9194b70c",
         "x": [
          "2001-07",
          "2001-08",
          "2001-10"
         ],
         "y": [
          1,
          2,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Information or Setup File",
         "type": "scatter",
         "uid": "64e7695e-d41a-42d5-b512-87bd7d13e810",
         "x": [
          "2001-10"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Ventura Publisher",
         "type": "scatter",
         "uid": "2d1e9b32-1cfa-4ef1-a540-a67a2d896cc6",
         "x": [
          "2001-10"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Hypertext Markup Language",
         "type": "scatter",
         "uid": "a1e69abb-c37a-4942-b071-935c6e9120c4",
         "x": [
          "2002-10",
          "2002-11",
          "2002-12",
          "2003-01",
          "2003-02",
          "2003-10",
          "2005-01"
         ],
         "y": [
          11,
          46,
          16,
          8,
          9,
          47,
          90
         ]
        },
        {
         "mode": "markers",
         "name": "Windows New Executable",
         "type": "scatter",
         "uid": "2b62edb3-e502-4925-9842-801c347d7a79",
         "x": [
          "1997-01"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Acrobat PDF 1.4 - Portable Document Format",
         "type": "scatter",
         "uid": "3bc2668b-bdf2-4ec8-9750-6430ca73fa14",
         "x": [
          "2004-01",
          "2004-08",
          "2010-07"
         ],
         "y": [
          11,
          1,
          7
         ]
        },
        {
         "mode": "markers",
         "name": "Preferred Executable Format",
         "type": "scatter",
         "uid": "0a6ebea0-f128-4429-b673-b934fa6ba86e",
         "x": [
          "1999-08",
          "2001-04"
         ],
         "y": [
          1,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "PCX",
         "type": "scatter",
         "uid": "070cb9ff-022a-44a2-a3ab-42a4e25e161b",
         "x": [
          "2004-01"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "AppleDouble Resource Fork",
         "type": "scatter",
         "uid": "234a5588-3d45-4f30-b6f6-8778bd3de485",
         "x": [
          "2009-02"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Graphics Interchange Format",
         "type": "scatter",
         "uid": "99df9237-d502-4b41-a393-2a5ad0947248",
         "x": [
          "2003-01",
          "2003-11"
         ],
         "y": [
          1,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Microsoft Word for Windows",
         "type": "scatter",
         "uid": "3eb7f652-5584-4ac5-9f17-953088ecf809",
         "x": [
          "2010-07",
          "2010-08",
          "2010-09",
          "2010-12",
          "2011-01",
          "2011-05",
          "2011-06",
          "2011-09",
          "2013-06"
         ],
         "y": [
          2,
          3,
          1,
          2,
          1,
          1,
          1,
          1,
          8
         ]
        },
        {
         "mode": "markers",
         "name": "Comma Separated Values",
         "type": "scatter",
         "uid": "9f62d183-ddce-4a5f-8e41-cda907ffee62",
         "x": [
          "2003-12"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "OLE2 Compound Document Format",
         "type": "scatter",
         "uid": "93b38ade-56d0-4cde-8826-d98f18bab748",
         "x": [
          "2003-08",
          "2005-07"
         ],
         "y": [
          4,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Encapsulated PostScript File Format",
         "type": "scatter",
         "uid": "f30db5f2-3398-4bb6-b96d-3c401d9a3d69",
         "x": [
          "2003-10"
         ],
         "y": [
          4
         ]
        },
        {
         "mode": "markers",
         "name": "Microsoft Word for MS-DOS Document",
         "type": "scatter",
         "uid": "03650ac5-a9ce-4300-8110-379cf88bc253",
         "x": [
          "1989-09",
          "1989-10",
          "1990-01",
          "1990-04",
          "1990-05"
         ],
         "y": [
          4,
          4,
          5,
          9,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "WordStar for MS-DOS Document",
         "type": "scatter",
         "uid": "4cc86d02-468f-49bd-9554-f817cd0b3297",
         "x": [
          "1992-09",
          "1992-10"
         ],
         "y": [
          1,
          7
         ]
        },
        {
         "mode": "markers",
         "name": "Adobe Illustrator",
         "type": "scatter",
         "uid": "7a6f120c-d65e-4207-86ee-6646dd51068f",
         "x": [
          "1988-04"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Microsoft Word Document Template",
         "type": "scatter",
         "uid": "82a289dc-981b-464b-8da9-9e532c063afb",
         "x": [
          "2000-04",
          "2003-08"
         ],
         "y": [
          1,
          4
         ]
        },
        {
         "mode": "markers",
         "name": "Microsoft Word (Generic)",
         "type": "scatter",
         "uid": "83f9e99e-6e87-411f-8a45-240f7865f1d0",
         "x": [
          "2001-01"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Windows Bitmap",
         "type": "scatter",
         "uid": "5bde9731-71ae-4d9e-8ee3-bc8b71cfd8c5",
         "x": [
          "2003-08"
         ],
         "y": [
          8
         ]
        },
        {
         "mode": "markers",
         "name": "Microsoft Excel for Windows",
         "type": "scatter",
         "uid": "9fcb3eb7-de1d-4caf-842d-0a675b94a6e5",
         "x": [
          "2010-07"
         ],
         "y": [
          2
         ]
        },
        {
         "mode": "markers",
         "name": "Windows Portable Executable",
         "type": "scatter",
         "uid": "f5fbd3d4-b87a-4b06-a565-d33e7a73f1d5",
         "x": [
          "2005-05"
         ],
         "y": [
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Acrobat PDF 1.5 - Portable Document Format",
         "type": "scatter",
         "uid": "fa6dba19-18b8-4262-8fa6-c1c61c7865e9",
         "x": [
          "2006-06",
          "2010-07"
         ],
         "y": [
          1,
          1
         ]
        },
        {
         "mode": "markers",
         "name": "Acrobat PDF 1.6 - Portable Document Format",
         "type": "scatter",
         "uid": "e804baf2-b946-475d-bdf6-9fe6f6952abe",
         "x": [
          "2010-07"
         ],
         "y": [
          2
         ]
        },
        {
         "mode": "markers",
         "name": "Acrobat PDF 1.2 - Portable Document Format",
         "type": "scatter",
         "uid": "b246ae5d-5368-4b15-b4d5-9a3f15c68ec5",
         "x": [
          "2010-07"
         ],
         "y": [
          1
         ]
        }
       ],
       "layout": {
        "legend": {
         "orientation": "v"
        },
        "title": {
         "text": "Format counts by modification date"
        }
       }
      },
      "text/html": [
       "<div>\n",
       "        \n",
       "        \n",
       "            <div id=\"1af1b17b-0b8a-4b2d-8bff-161c7c54d4e4\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                require([\"plotly\"], function(Plotly) {\n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    window.PLOTLYENV.BASE_URL='https://plot.ly';\n",
       "                    \n",
       "                if (document.getElementById(\"1af1b17b-0b8a-4b2d-8bff-161c7c54d4e4\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '1af1b17b-0b8a-4b2d-8bff-161c7c54d4e4',\n",
       "                        [{\"mode\": \"markers\", \"name\": \"Microsoft Word Document\", \"type\": \"scatter\", \"uid\": \"16f75349-2c0c-4dd4-97e6-75c3ec5ac12c\", \"x\": [\"1996-10\", \"1998-10\", \"1998-12\", \"1999-01\", \"1999-04\", \"1999-05\", \"1999-11\", \"2000-03\", \"2000-04\", \"2000-05\", \"2000-06\", \"2000-07\", \"2000-08\", \"2000-09\", \"2000-10\", \"2000-11\", \"2000-12\", \"2001-01\", \"2001-02\", \"2001-03\", \"2001-04\", \"2001-05\", \"2001-06\", \"2001-07\", \"2001-08\", \"2001-09\", \"2001-10\", \"2001-11\", \"2001-12\", \"2002-01\", \"2002-02\", \"2002-03\", \"2002-04\", \"2002-05\", \"2002-06\", \"2002-07\", \"2002-08\", \"2002-09\", \"2002-10\", \"2002-11\", \"2002-12\", \"2003-01\", \"2003-02\", \"2003-03\", \"2003-04\", \"2003-05\", \"2003-06\", \"2003-07\", \"2003-08\", \"2003-09\", \"2003-10\", \"2003-11\", \"2003-12\", \"2004-01\", \"2004-02\", \"2004-03\", \"2004-04\", \"2004-05\", \"2004-06\", \"2004-07\", \"2004-08\", \"2004-09\", \"2004-10\", \"2004-11\", \"2004-12\", \"2005-01\", \"2005-02\", \"2005-03\", \"2005-04\", \"2005-05\", \"2005-06\", \"2005-07\", \"2005-08\", \"2005-09\", \"2005-10\", \"2005-11\", \"2005-12\", \"2006-01\", \"2006-02\", \"2006-03\", \"2006-04\", \"2006-05\", \"2006-06\", \"2006-07\", \"2006-08\", \"2006-09\", \"2006-12\", \"2007-01\", \"2007-04\", \"2007-05\", \"2007-10\", \"2007-11\", \"2007-12\", \"2008-02\", \"2009-02\", \"2009-03\", \"2009-04\", \"2010-01\", \"2010-07\", \"2010-08\", \"2010-09\", \"2010-10\", \"2010-11\", \"2010-12\", \"2011-01\"], \"y\": [1, 2, 2, 3, 1, 1, 1, 4, 12, 7, 23, 23, 107, 132, 134, 26, 28, 156, 202, 250, 123, 58, 76, 110, 98, 63, 102, 82, 70, 78, 63, 71, 52, 95, 70, 88, 61, 65, 79, 78, 140, 57, 60, 86, 58, 75, 70, 73, 77, 75, 122, 99, 72, 100, 65, 69, 98, 80, 142, 60, 72, 45, 33, 72, 45, 76, 42, 59, 30, 37, 38, 16, 16, 27, 45, 74, 12, 3, 4, 3, 4, 4, 3, 7, 2, 7, 6, 1, 2, 3, 2, 8, 3, 51, 20, 8, 2, 2, 6, 1, 1, 1, 3, 2, 12]}, {\"mode\": \"markers\", \"name\": \"WordPerfect for MS-DOS/Windows Document\", \"type\": \"scatter\", \"uid\": \"93567289-8ea3-4047-89c7-fc0b5e01906f\", \"x\": [\"2000-12\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"Microsoft Word for Macintosh Document\", \"type\": \"scatter\", \"uid\": \"fa20f247-7024-4063-9d2b-a97f1687800c\", \"x\": [\"1989-10\", \"1989-11\", \"1989-12\", \"1990-01\", \"1990-02\", \"1990-03\", \"1990-04\", \"1990-05\", \"1990-06\", \"1990-07\", \"1990-08\", \"1990-09\", \"1990-10\", \"1990-11\", \"1990-12\", \"1991-01\", \"1991-02\", \"1991-03\", \"1991-04\", \"1991-05\", \"1991-06\", \"1991-07\", \"1991-08\", \"1991-09\", \"1991-10\", \"1991-11\", \"1991-12\", \"1992-01\", \"1992-02\", \"1992-03\", \"1992-04\", \"1992-05\", \"1992-06\", \"1992-07\", \"1992-08\", \"1992-09\", \"1992-10\", \"1992-11\", \"1992-12\", \"1993-01\", \"1993-02\", \"1993-03\", \"1993-04\", \"1993-05\", \"1993-06\", \"1993-07\", \"1993-08\", \"1993-09\", \"1993-10\", \"1993-11\", \"1993-12\", \"1994-01\", \"1994-02\", \"1994-03\", \"1994-04\", \"1994-05\", \"1994-06\", \"1994-07\", \"1994-08\", \"1994-09\", \"1994-10\", \"1994-11\", \"1994-12\", \"1995-01\", \"1995-02\", \"1995-03\", \"1995-04\", \"1995-05\", \"1995-06\", \"1995-07\", \"1995-08\", \"1995-09\", \"1995-10\", \"1995-11\", \"1995-12\", \"1996-01\", \"1996-02\", \"1996-03\", \"1996-04\", \"1996-05\", \"1996-06\", \"1996-07\", \"1996-08\", \"1996-09\", \"1996-10\", \"1996-11\", \"1996-12\", \"1997-01\", \"1997-02\", \"1997-03\", \"1997-04\", \"1997-05\", \"1997-06\", \"1997-07\", \"1997-08\", \"1997-09\", \"1997-10\", \"1997-11\", \"1997-12\", \"1998-01\", \"1998-02\", \"1998-03\", \"1998-04\", \"1998-05\", \"1998-06\", \"1998-07\", \"1998-08\", \"1998-09\", \"1998-10\", \"1998-11\", \"1998-12\", \"1999-01\", \"1999-02\", \"1999-03\", \"1999-04\", \"1999-05\", \"1999-06\", \"1999-07\", \"1999-08\", \"1999-09\", \"1999-10\", \"1999-11\", \"1999-12\", \"2000-01\", \"2000-02\", \"2000-03\", \"2000-04\", \"2000-05\", \"2000-06\", \"2000-07\", \"2000-08\", \"2000-09\", \"2000-10\", \"2000-11\", \"2000-12\", \"2001-01\", \"2001-02\", \"2001-03\", \"2001-04\", \"2001-05\", \"2001-06\", \"2001-08\", \"2001-09\", \"2001-11\", \"2001-12\", \"2002-10\", \"2003-05\", \"2004-05\", \"2004-08\", \"2005-10\", \"2009-04\"], \"y\": [45, 23, 23, 22, 19, 38, 23, 18, 25, 19, 20, 27, 24, 17, 4, 16, 12, 40, 17, 28, 90, 49, 49, 71, 78, 140, 241, 105, 80, 104, 90, 109, 106, 97, 116, 91, 163, 102, 84, 89, 105, 114, 93, 80, 94, 147, 137, 160, 119, 108, 101, 94, 158, 125, 98, 148, 94, 142, 87, 91, 103, 106, 73, 50, 89, 72, 96, 97, 99, 115, 100, 77, 124, 128, 59, 154, 84, 89, 86, 121, 117, 131, 91, 133, 75, 95, 110, 156, 167, 109, 97, 138, 161, 84, 135, 130, 100, 102, 48, 51, 166, 182, 194, 80, 73, 82, 83, 121, 80, 76, 51, 137, 188, 120, 158, 123, 95, 109, 147, 69, 53, 50, 125, 112, 94, 67, 44, 77, 115, 250, 86, 70, 65, 77, 25, 4, 14, 8, 19, 11, 2, 3, 3, 2, 4, 2, 1, 1, 41, 37, 21]}, {\"mode\": \"markers\", \"name\": \"nan\", \"type\": \"scatter\", \"uid\": \"ea26a771-1ff7-4fe6-b2c3-174d7b331be1\", \"x\": [], \"y\": []}, {\"mode\": \"markers\", \"name\": \"Microsoft Word for Windows Document\", \"type\": \"scatter\", \"uid\": \"34f8fece-74ae-4c55-b4bf-58e404ec20e3\", \"x\": [\"1998-04\", \"1998-07\", \"1998-12\", \"1999-01\"], \"y\": [39, 1, 3, 2]}, {\"mode\": \"markers\", \"name\": \"Exchangeable Image File Format (Compressed)\", \"type\": \"scatter\", \"uid\": \"c5acf427-31d0-4db7-85d2-0f6c18dec179\", \"x\": [\"2001-08\", \"2003-01\", \"2003-09\", \"2003-10\", \"2003-11\", \"2003-12\", \"2004-01\", \"2004-02\", \"2004-05\", \"2004-06\", \"2005-03\", \"2005-07\", \"2005-10\", \"2005-11\", \"2006-02\", \"2006-03\", \"2006-04\"], \"y\": [2, 1, 1, 8, 28, 1, 54, 23, 32, 1, 140, 37, 37, 18, 54, 7, 26]}, {\"mode\": \"markers\", \"name\": \"Microsoft Excel 97 Workbook (xls)\", \"type\": \"scatter\", \"uid\": \"9f042ee6-f3ba-43bd-9f52-9e4c63e957b1\", \"x\": [\"1999-11\", \"1999-12\", \"2000-01\", \"2000-02\", \"2000-03\", \"2000-04\", \"2000-05\", \"2000-06\", \"2000-07\", \"2000-08\", \"2000-09\", \"2000-10\", \"2000-11\", \"2000-12\", \"2001-01\", \"2001-02\", \"2001-03\", \"2001-04\", \"2001-05\", \"2001-07\", \"2001-08\", \"2001-11\", \"2001-12\", \"2002-01\", \"2002-02\", \"2002-04\", \"2002-05\", \"2002-06\", \"2002-08\", \"2002-11\", \"2002-12\", \"2003-01\", \"2003-02\", \"2003-03\", \"2003-05\", \"2003-08\", \"2003-09\", \"2003-10\", \"2003-11\", \"2003-12\", \"2004-01\", \"2004-02\", \"2004-03\", \"2004-04\", \"2004-05\", \"2004-06\", \"2004-08\", \"2004-10\", \"2004-11\", \"2005-02\", \"2005-07\", \"2005-08\", \"2005-09\", \"2005-10\", \"2005-11\"], \"y\": [6, 5, 2, 8, 7, 1, 9, 8, 4, 2, 1, 3, 6, 2, 2, 4, 4, 1, 6, 1, 4, 8, 4, 2, 6, 2, 11, 1, 9, 14, 1, 2, 12, 3, 11, 28, 5, 4, 13, 8, 2, 17, 5, 1, 14, 1, 6, 2, 8, 26, 1, 1, 1, 7, 4]}, {\"mode\": \"markers\", \"name\": \"JPEG File Interchange Format\", \"type\": \"scatter\", \"uid\": \"11bf0e93-b895-4e84-847a-e69bc963475f\", \"x\": [\"2003-01\", \"2003-02\", \"2003-08\", \"2003-09\", \"2003-10\", \"2003-11\", \"2004-01\", \"2004-03\", \"2004-04\", \"2004-05\", \"2004-10\", \"2005-01\"], \"y\": [8, 5, 6, 3, 1, 5, 1, 5, 11, 1, 1, 1]}, {\"mode\": \"markers\", \"name\": \"Tagged Image File Format\", \"type\": \"scatter\", \"uid\": \"9fded2f4-9e9c-465e-b60b-d19245793837\", \"x\": [\"2003-03\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"Raw JPEG Stream\", \"type\": \"scatter\", \"uid\": \"9e6e3a1a-9217-4ebd-b3d3-a4d29cd4b142\", \"x\": [\"2002-12\", \"2003-11\"], \"y\": [1, 1]}, {\"mode\": \"markers\", \"name\": \"Plain Text File\", \"type\": \"scatter\", \"uid\": \"59617e6b-8f4f-407a-8035-e734aa1235cd\", \"x\": [\"2005-03\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"Microsoft Powerpoint Presentation\", \"type\": \"scatter\", \"uid\": \"a345fbb7-4970-41e6-98b5-e16abee82601\", \"x\": [\"2003-10\", \"2003-11\", \"2004-04\", \"2004-06\", \"2005-10\", \"2006-05\", \"2006-06\"], \"y\": [6, 2, 3, 2, 2, 2, 1]}, {\"mode\": \"markers\", \"name\": \"Macintosh PICT Image\", \"type\": \"scatter\", \"uid\": \"7abb923a-3b0c-4b33-8d9e-41fe8b84c50f\", \"x\": [], \"y\": []}, {\"mode\": \"markers\", \"name\": \"MIME Email\", \"type\": \"scatter\", \"uid\": \"ce9756ff-7132-4c7e-b36f-ab1d9801aceb\", \"x\": [\"2000-01\", \"2001-02\", \"2001-03\", \"2001-04\", \"2001-06\", \"2001-08\", \"2001-09\", \"2001-11\", \"2001-12\", \"2002-10\", \"2002-11\", \"2002-12\", \"2003-01\", \"2003-02\", \"2003-05\", \"2003-08\", \"2003-10\", \"2003-11\", \"2005-01\"], \"y\": [1, 1, 2, 5, 1, 1, 2, 16, 7, 108, 329, 222, 78, 116, 5, 1, 559, 5, 1009]}, {\"mode\": \"markers\", \"name\": \"Internet Explorer for Mac cache file\", \"type\": \"scatter\", \"uid\": \"ba8a6b5c-052c-433b-aaab-cbf6061e81a5\", \"x\": [\"2001-03\", \"2003-04\", \"2003-08\"], \"y\": [15, 1, 1]}, {\"mode\": \"markers\", \"name\": \"Adobe Photoshop\", \"type\": \"scatter\", \"uid\": \"c6cb526f-d02a-4b40-a73a-29dfab72331e\", \"x\": [\"2003-01\", \"2003-10\", \"2004-01\"], \"y\": [9, 10, 3]}, {\"mode\": \"markers\", \"name\": \"Acrobat PDF 1.3 - Portable Document Format\", \"type\": \"scatter\", \"uid\": \"5dc1dc30-2b3e-44a1-9179-6635b25d2b5f\", \"x\": [\"2003-08\", \"2003-11\", \"2010-07\"], \"y\": [1, 4, 2]}, {\"mode\": \"markers\", \"name\": \"Rich Text Format\", \"type\": \"scatter\", \"uid\": \"44836816-5c6b-4982-aaa0-38e1613872d8\", \"x\": [\"1998-12\", \"2000-06\", \"2001-05\", \"2001-08\", \"2001-09\", \"2001-10\", \"2004-01\"], \"y\": [1, 1, 2, 2, 1, 1, 1]}, {\"mode\": \"markers\", \"name\": \"Fortran\", \"type\": \"scatter\", \"uid\": \"4290745f-4514-4fa4-a40a-c97b5370ff7c\", \"x\": [\"2001-07\", \"2001-08\", \"2001-10\", \"2001-11\"], \"y\": [3, 4, 1, 1]}, {\"mode\": \"markers\", \"name\": \"Statistica Report File\", \"type\": \"scatter\", \"uid\": \"4f887426-2b3f-4a12-bc3c-6edd4d9a9619\", \"x\": [\"2001-08\"], \"y\": [2]}, {\"mode\": \"markers\", \"name\": \"Intergraph Raster Image\", \"type\": \"scatter\", \"uid\": \"adb2c281-f82a-4fe9-840c-cf759fd16277\", \"x\": [\"2001-07\", \"2001-08\", \"2001-10\", \"2001-11\"], \"y\": [3, 4, 2, 3]}, {\"mode\": \"markers\", \"name\": \"Dia Graphics Format\", \"type\": \"scatter\", \"uid\": \"00954df9-7a62-484c-b38d-208df2f4ebea\", \"x\": [\"2001-08\", \"2001-10\"], \"y\": [1, 1]}, {\"mode\": \"markers\", \"name\": \"Micrografx Designer\", \"type\": \"scatter\", \"uid\": \"e6a1901e-60c1-4f8e-818f-10469348d3c8\", \"x\": [\"2001-08\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"Multiple matches\", \"type\": \"scatter\", \"uid\": \"066cb8a6-8613-41e8-87df-45ba690406be\", \"x\": [\"1980-01\", \"1985-01\", \"1985-06\", \"1985-08\", \"1985-09\", \"1985-10\", \"1985-11\", \"1985-12\", \"1986-01\", \"1986-06\", \"1986-08\", \"1986-09\", \"1986-10\", \"1987-05\", \"1987-06\", \"1987-07\", \"1987-08\", \"1987-11\", \"1987-12\", \"1988-01\", \"1988-02\", \"1988-03\", \"1988-06\", \"1988-08\", \"1989-01\", \"1989-05\", \"1989-09\", \"1990-04\", \"1991-12\", \"1995-10\", \"1999-05\", \"1999-09\", \"1999-10\", \"2000-02\", \"2000-03\", \"2000-05\", \"2000-06\", \"2000-08\", \"2000-10\", \"2000-11\", \"2001-04\", \"2001-07\", \"2001-08\", \"2002-12\", \"2003-02\", \"2003-08\", \"2005-01\", \"2005-10\"], \"y\": [34, 10, 2, 5, 3, 11, 9, 23, 10, 11, 5, 27, 3, 17, 10, 13, 66, 26, 4, 6, 13, 2, 35, 43, 1, 24, 1, 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 3, 1, 2, 1, 1, 1, 2, 3, 19, 1, 1]}, {\"mode\": \"markers\", \"name\": \"Freelance File\", \"type\": \"scatter\", \"uid\": \"50ab983e-cc79-4c2a-a004-ab0a1b21254a\", \"x\": [\"1989-05\", \"2001-08\"], \"y\": [1, 3]}, {\"mode\": \"markers\", \"name\": \"Generic Library File\", \"type\": \"scatter\", \"uid\": \"6e028f73-3c89-450b-b9d5-156e72316825\", \"x\": [\"2001-08\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"SAP Document\", \"type\": \"scatter\", \"uid\": \"cc2936c8-66d7-4e70-b99f-ed7b9194b70c\", \"x\": [\"2001-07\", \"2001-08\", \"2001-10\"], \"y\": [1, 2, 1]}, {\"mode\": \"markers\", \"name\": \"Information or Setup File\", \"type\": \"scatter\", \"uid\": \"64e7695e-d41a-42d5-b512-87bd7d13e810\", \"x\": [\"2001-10\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"Ventura Publisher\", \"type\": \"scatter\", \"uid\": \"2d1e9b32-1cfa-4ef1-a540-a67a2d896cc6\", \"x\": [\"2001-10\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"Hypertext Markup Language\", \"type\": \"scatter\", \"uid\": \"a1e69abb-c37a-4942-b071-935c6e9120c4\", \"x\": [\"2002-10\", \"2002-11\", \"2002-12\", \"2003-01\", \"2003-02\", \"2003-10\", \"2005-01\"], \"y\": [11, 46, 16, 8, 9, 47, 90]}, {\"mode\": \"markers\", \"name\": \"Windows New Executable\", \"type\": \"scatter\", \"uid\": \"2b62edb3-e502-4925-9842-801c347d7a79\", \"x\": [\"1997-01\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"Acrobat PDF 1.4 - Portable Document Format\", \"type\": \"scatter\", \"uid\": \"3bc2668b-bdf2-4ec8-9750-6430ca73fa14\", \"x\": [\"2004-01\", \"2004-08\", \"2010-07\"], \"y\": [11, 1, 7]}, {\"mode\": \"markers\", \"name\": \"Preferred Executable Format\", \"type\": \"scatter\", \"uid\": \"0a6ebea0-f128-4429-b673-b934fa6ba86e\", \"x\": [\"1999-08\", \"2001-04\"], \"y\": [1, 1]}, {\"mode\": \"markers\", \"name\": \"PCX\", \"type\": \"scatter\", \"uid\": \"070cb9ff-022a-44a2-a3ab-42a4e25e161b\", \"x\": [\"2004-01\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"AppleDouble Resource Fork\", \"type\": \"scatter\", \"uid\": \"234a5588-3d45-4f30-b6f6-8778bd3de485\", \"x\": [\"2009-02\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"Graphics Interchange Format\", \"type\": \"scatter\", \"uid\": \"99df9237-d502-4b41-a393-2a5ad0947248\", \"x\": [\"2003-01\", \"2003-11\"], \"y\": [1, 1]}, {\"mode\": \"markers\", \"name\": \"Microsoft Word for Windows\", \"type\": \"scatter\", \"uid\": \"3eb7f652-5584-4ac5-9f17-953088ecf809\", \"x\": [\"2010-07\", \"2010-08\", \"2010-09\", \"2010-12\", \"2011-01\", \"2011-05\", \"2011-06\", \"2011-09\", \"2013-06\"], \"y\": [2, 3, 1, 2, 1, 1, 1, 1, 8]}, {\"mode\": \"markers\", \"name\": \"Comma Separated Values\", \"type\": \"scatter\", \"uid\": \"9f62d183-ddce-4a5f-8e41-cda907ffee62\", \"x\": [\"2003-12\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"OLE2 Compound Document Format\", \"type\": \"scatter\", \"uid\": \"93b38ade-56d0-4cde-8826-d98f18bab748\", \"x\": [\"2003-08\", \"2005-07\"], \"y\": [4, 1]}, {\"mode\": \"markers\", \"name\": \"Encapsulated PostScript File Format\", \"type\": \"scatter\", \"uid\": \"f30db5f2-3398-4bb6-b96d-3c401d9a3d69\", \"x\": [\"2003-10\"], \"y\": [4]}, {\"mode\": \"markers\", \"name\": \"Microsoft Word for MS-DOS Document\", \"type\": \"scatter\", \"uid\": \"03650ac5-a9ce-4300-8110-379cf88bc253\", \"x\": [\"1989-09\", \"1989-10\", \"1990-01\", \"1990-04\", \"1990-05\"], \"y\": [4, 4, 5, 9, 1]}, {\"mode\": \"markers\", \"name\": \"WordStar for MS-DOS Document\", \"type\": \"scatter\", \"uid\": \"4cc86d02-468f-49bd-9554-f817cd0b3297\", \"x\": [\"1992-09\", \"1992-10\"], \"y\": [1, 7]}, {\"mode\": \"markers\", \"name\": \"Adobe Illustrator\", \"type\": \"scatter\", \"uid\": \"7a6f120c-d65e-4207-86ee-6646dd51068f\", \"x\": [\"1988-04\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"Microsoft Word Document Template\", \"type\": \"scatter\", \"uid\": \"82a289dc-981b-464b-8da9-9e532c063afb\", \"x\": [\"2000-04\", \"2003-08\"], \"y\": [1, 4]}, {\"mode\": \"markers\", \"name\": \"Microsoft Word (Generic)\", \"type\": \"scatter\", \"uid\": \"83f9e99e-6e87-411f-8a45-240f7865f1d0\", \"x\": [\"2001-01\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"Windows Bitmap\", \"type\": \"scatter\", \"uid\": \"5bde9731-71ae-4d9e-8ee3-bc8b71cfd8c5\", \"x\": [\"2003-08\"], \"y\": [8]}, {\"mode\": \"markers\", \"name\": \"Microsoft Excel for Windows\", \"type\": \"scatter\", \"uid\": \"9fcb3eb7-de1d-4caf-842d-0a675b94a6e5\", \"x\": [\"2010-07\"], \"y\": [2]}, {\"mode\": \"markers\", \"name\": \"Windows Portable Executable\", \"type\": \"scatter\", \"uid\": \"f5fbd3d4-b87a-4b06-a565-d33e7a73f1d5\", \"x\": [\"2005-05\"], \"y\": [1]}, {\"mode\": \"markers\", \"name\": \"Acrobat PDF 1.5 - Portable Document Format\", \"type\": \"scatter\", \"uid\": \"fa6dba19-18b8-4262-8fa6-c1c61c7865e9\", \"x\": [\"2006-06\", \"2010-07\"], \"y\": [1, 1]}, {\"mode\": \"markers\", \"name\": \"Acrobat PDF 1.6 - Portable Document Format\", \"type\": \"scatter\", \"uid\": \"e804baf2-b946-475d-bdf6-9fe6f6952abe\", \"x\": [\"2010-07\"], \"y\": [2]}, {\"mode\": \"markers\", \"name\": \"Acrobat PDF 1.2 - Portable Document Format\", \"type\": \"scatter\", \"uid\": \"b246ae5d-5368-4b15-b4d5-9a3f15c68ec5\", \"x\": [\"2010-07\"], \"y\": [1]}],\n",
       "                        {\"legend\": {\"orientation\": \"v\"}, \"title\": {\"text\": \"Format counts by modification date\"}},\n",
       "                        {\"showLink\": false, \"linkText\": \"Export to plot.ly\", \"plotlyServerURL\": \"https://plot.ly\", \"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('1af1b17b-0b8a-4b2d-8bff-161c7c54d4e4');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                });\n",
       "            </script>\n",
       "        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = []\n",
    "f = 'FORMAT_NAME'\n",
    "for puid in files[f].unique():\n",
    "    filtered = unique_files[unique_files[f] == puid]\n",
    "    filtered = filtered['LAST_MODIFIED'].dt.strftime('%Y-%m').value_counts().sort_index()\n",
    "    filtered = filtered.reset_index()\n",
    "    filtered = filtered[filtered['index'] < '2014-01']\n",
    "    data.append(go.Scatter(x=filtered['index'], y=filtered['LAST_MODIFIED'], name=puid, mode='markers'))\n",
    "layout = go.Layout(\n",
    "    legend={'orientation': 'v'}, title=\"Format counts by modification date\")\n",
    "fig = go.Figure(data=data, layout=layout)\n",
    "iplot(fig, filename='pandas-time-series')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "linkText": "Export to plot.ly",
        "plotlyServerURL": "https://plot.ly",
        "showLink": false
       },
       "data": [
        {
         "name": "Unique files",
         "type": "scatter",
         "uid": "566f8a8e-07e7-4894-a5af-687f9405da8e",
         "x": [
          "1980",
          "1984",
          "1985",
          "1986",
          "1987",
          "1988",
          "1989",
          "1990",
          "1991",
          "1992",
          "1993",
          "1994",
          "1995",
          "1996",
          "1997",
          "1998",
          "1999",
          "2000",
          "2001",
          "2002",
          "2003",
          "2004",
          "2005",
          "2006",
          "2007"
         ],
         "y": [
          76,
          1,
          151,
          168,
          313,
          266,
          397,
          327,
          834,
          1310,
          1372,
          1320,
          1108,
          1287,
          1431,
          1292,
          1408,
          1653,
          2963,
          2645,
          3059,
          1135,
          3520,
          135,
          19
         ]
        },
        {
         "name": "Total files",
         "type": "scatter",
         "uid": "5f9dadf7-06d6-49b2-bb35-9546c4aac916",
         "x": [
          "1980",
          "1984",
          "1985",
          "1986",
          "1987",
          "1988",
          "1989",
          "1990",
          "1991",
          "1992",
          "1993",
          "1994",
          "1995",
          "1996",
          "1997",
          "1998",
          "1999",
          "2000",
          "2001",
          "2002",
          "2003",
          "2004",
          "2005",
          "2006",
          "2007"
         ],
         "y": [
          107,
          1,
          186,
          249,
          360,
          451,
          915,
          593,
          1976,
          4905,
          5389,
          4871,
          4456,
          4742,
          4627,
          4211,
          5387,
          5668,
          10823,
          8596,
          9380,
          2321,
          9268,
          137,
          19
         ]
        }
       ],
       "layout": {
        "title": {
         "text": "File counts by modification date (year)"
        }
       }
      },
      "text/html": [
       "<div>\n",
       "        \n",
       "        \n",
       "            <div id=\"7628c9ac-3b26-4fc3-a66d-0087515318e4\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                require([\"plotly\"], function(Plotly) {\n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    window.PLOTLYENV.BASE_URL='https://plot.ly';\n",
       "                    \n",
       "                if (document.getElementById(\"7628c9ac-3b26-4fc3-a66d-0087515318e4\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '7628c9ac-3b26-4fc3-a66d-0087515318e4',\n",
       "                        [{\"name\": \"Unique files\", \"type\": \"scatter\", \"uid\": \"566f8a8e-07e7-4894-a5af-687f9405da8e\", \"x\": [\"1980\", \"1984\", \"1985\", \"1986\", \"1987\", \"1988\", \"1989\", \"1990\", \"1991\", \"1992\", \"1993\", \"1994\", \"1995\", \"1996\", \"1997\", \"1998\", \"1999\", \"2000\", \"2001\", \"2002\", \"2003\", \"2004\", \"2005\", \"2006\", \"2007\"], \"y\": [76, 1, 151, 168, 313, 266, 397, 327, 834, 1310, 1372, 1320, 1108, 1287, 1431, 1292, 1408, 1653, 2963, 2645, 3059, 1135, 3520, 135, 19]}, {\"name\": \"Total files\", \"type\": \"scatter\", \"uid\": \"5f9dadf7-06d6-49b2-bb35-9546c4aac916\", \"x\": [\"1980\", \"1984\", \"1985\", \"1986\", \"1987\", \"1988\", \"1989\", \"1990\", \"1991\", \"1992\", \"1993\", \"1994\", \"1995\", \"1996\", \"1997\", \"1998\", \"1999\", \"2000\", \"2001\", \"2002\", \"2003\", \"2004\", \"2005\", \"2006\", \"2007\"], \"y\": [107, 1, 186, 249, 360, 451, 915, 593, 1976, 4905, 5389, 4871, 4456, 4742, 4627, 4211, 5387, 5668, 10823, 8596, 9380, 2321, 9268, 137, 19]}],\n",
       "                        {\"title\": {\"text\": \"File counts by modification date (year)\"}},\n",
       "                        {\"showLink\": false, \"linkText\": \"Export to plot.ly\", \"plotlyServerURL\": \"https://plot.ly\", \"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('7628c9ac-3b26-4fc3-a66d-0087515318e4');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                });\n",
       "            </script>\n",
       "        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "years = files['LAST_MODIFIED'].dt.strftime('%Y').value_counts().sort_index()[:25]\n",
    "uyears = unique_files['LAST_MODIFIED'].dt.strftime('%Y').value_counts().sort_index()[:25]\n",
    "uyears = uyears.reset_index()\n",
    "years = years.reset_index()\n",
    "year_fig = go.Figure(data=[\n",
    "    go.Scatter(x=uyears['index'], y=uyears['LAST_MODIFIED'], name='Unique files'),\n",
    "    go.Scatter(x=years['index'], y=years['LAST_MODIFIED'], name='Total files')],\n",
    "    layout_title_text=\"File counts by modification date (year)\")\n",
    "iplot(year_fig, filename='pandas-time-series')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "linkText": "Export to plot.ly",
        "plotlyServerURL": "https://plot.ly",
        "showLink": false
       },
       "data": [
        {
         "name": "Unique files",
         "type": "scatter",
         "uid": "fbe44191-7a7a-47d2-8d47-eba224cf5128",
         "x": [
          "1980-01",
          "1984-01",
          "1985-01",
          "1985-05",
          "1985-06",
          "1985-07",
          "1985-08",
          "1985-09",
          "1985-10",
          "1985-11",
          "1985-12",
          "1986-01",
          "1986-04",
          "1986-06",
          "1986-08",
          "1986-09",
          "1986-10",
          "1986-11",
          "1986-12",
          "1987-01",
          "1987-02",
          "1987-03",
          "1987-04",
          "1987-05",
          "1987-06",
          "1987-07",
          "1987-08",
          "1987-09",
          "1987-11",
          "1987-12",
          "1988-01",
          "1988-02",
          "1988-03",
          "1988-04",
          "1988-05",
          "1988-06",
          "1988-07",
          "1988-08",
          "1988-09",
          "1988-10",
          "1988-11",
          "1988-12",
          "1989-01",
          "1989-02",
          "1989-03",
          "1989-04",
          "1989-05",
          "1989-06",
          "1989-07",
          "1989-08",
          "1989-09",
          "1989-10",
          "1989-11",
          "1989-12",
          "1990-01",
          "1990-02",
          "1990-03",
          "1990-04",
          "1990-05",
          "1990-06",
          "1990-07",
          "1990-08",
          "1990-09",
          "1990-10",
          "1990-11",
          "1990-12",
          "1991-01",
          "1991-02",
          "1991-03",
          "1991-04",
          "1991-05",
          "1991-06",
          "1991-07",
          "1991-08",
          "1991-09",
          "1991-10",
          "1991-11",
          "1991-12",
          "1992-01",
          "1992-02",
          "1992-03",
          "1992-04",
          "1992-05",
          "1992-06",
          "1992-07",
          "1992-08",
          "1992-09",
          "1992-10",
          "1992-11",
          "1992-12",
          "1993-01",
          "1993-02",
          "1993-03",
          "1993-04",
          "1993-05",
          "1993-06",
          "1993-07",
          "1993-08",
          "1993-09",
          "1993-10",
          "1993-11",
          "1993-12",
          "1994-01",
          "1994-02",
          "1994-03",
          "1994-04",
          "1994-05",
          "1994-06",
          "1994-07",
          "1994-08",
          "1994-09",
          "1994-10",
          "1994-11",
          "1994-12",
          "1995-01",
          "1995-02",
          "1995-03",
          "1995-04",
          "1995-05",
          "1995-06",
          "1995-07",
          "1995-08",
          "1995-09",
          "1995-10",
          "1995-11",
          "1995-12",
          "1996-01",
          "1996-02",
          "1996-03",
          "1996-04",
          "1996-05",
          "1996-06",
          "1996-07",
          "1996-08",
          "1996-09",
          "1996-10",
          "1996-11",
          "1996-12",
          "1997-01",
          "1997-02",
          "1997-03",
          "1997-04",
          "1997-05",
          "1997-06",
          "1997-07",
          "1997-08",
          "1997-09",
          "1997-10",
          "1997-11",
          "1997-12",
          "1998-01",
          "1998-02",
          "1998-03",
          "1998-04",
          "1998-05",
          "1998-06",
          "1998-07",
          "1998-08",
          "1998-09",
          "1998-10",
          "1998-11",
          "1998-12",
          "1999-01",
          "1999-02",
          "1999-03",
          "1999-04",
          "1999-05",
          "1999-06",
          "1999-07",
          "1999-08",
          "1999-09",
          "1999-10",
          "1999-11",
          "1999-12",
          "2000-01",
          "2000-02",
          "2000-03",
          "2000-04",
          "2000-05",
          "2000-06",
          "2000-07",
          "2000-08",
          "2000-09",
          "2000-10",
          "2000-11",
          "2000-12",
          "2001-01",
          "2001-02",
          "2001-03",
          "2001-04",
          "2001-05",
          "2001-06",
          "2001-07",
          "2001-08",
          "2001-09",
          "2001-10",
          "2001-11",
          "2001-12",
          "2002-01",
          "2002-02",
          "2002-03",
          "2002-04",
          "2002-05",
          "2002-06",
          "2002-07",
          "2002-08",
          "2002-09",
          "2002-10",
          "2002-11",
          "2002-12",
          "2003-01",
          "2003-02",
          "2003-03",
          "2003-04",
          "2003-05",
          "2003-06",
          "2003-07",
          "2003-08",
          "2003-09",
          "2003-10",
          "2003-11",
          "2003-12",
          "2004-01",
          "2004-02",
          "2004-03",
          "2004-04",
          "2004-05",
          "2004-06",
          "2004-07",
          "2004-08",
          "2004-09",
          "2004-10",
          "2004-11",
          "2004-12",
          "2005-01",
          "2005-02",
          "2005-03",
          "2005-04",
          "2005-05",
          "2005-06",
          "2005-07",
          "2005-08",
          "2005-09",
          "2005-10",
          "2005-11",
          "2005-12",
          "2006-01",
          "2006-02",
          "2006-03",
          "2006-04",
          "2006-05",
          "2006-06",
          "2006-07",
          "2006-08",
          "2006-09",
          "2006-12",
          "2007-01",
          "2007-04",
          "2007-05",
          "2007-10",
          "2007-11",
          "2007-12",
          "2008-02",
          "2009-02",
          "2009-03",
          "2009-04",
          "2010-01",
          "2010-07",
          "2010-08",
          "2010-09",
          "2010-10",
          "2010-11",
          "2010-12",
          "2011-01",
          "2011-05",
          "2011-06",
          "2011-09",
          "2013-06"
         ],
         "y": [
          76,
          1,
          27,
          1,
          4,
          1,
          16,
          8,
          23,
          21,
          50,
          31,
          1,
          22,
          47,
          51,
          9,
          6,
          1,
          1,
          4,
          8,
          10,
          21,
          15,
          21,
          168,
          14,
          41,
          10,
          15,
          27,
          9,
          15,
          24,
          65,
          4,
          70,
          6,
          15,
          11,
          5,
          14,
          6,
          9,
          24,
          52,
          34,
          30,
          38,
          56,
          81,
          26,
          27,
          34,
          27,
          41,
          35,
          20,
          25,
          48,
          25,
          27,
          24,
          17,
          4,
          16,
          12,
          40,
          17,
          28,
          91,
          49,
          49,
          71,
          78,
          140,
          243,
          105,
          80,
          104,
          144,
          109,
          106,
          97,
          117,
          92,
          170,
          102,
          84,
          89,
          106,
          115,
          94,
          82,
          101,
          154,
          141,
          160,
          119,
          108,
          103,
          94,
          158,
          125,
          98,
          149,
          94,
          142,
          87,
          91,
          103,
          106,
          73,
          50,
          89,
          72,
          96,
          97,
          99,
          116,
          100,
          77,
          125,
          128,
          59,
          154,
          84,
          89,
          86,
          121,
          117,
          131,
          91,
          133,
          76,
          95,
          110,
          157,
          167,
          109,
          97,
          139,
          162,
          84,
          136,
          130,
          100,
          102,
          48,
          51,
          168,
          182,
          235,
          80,
          73,
          83,
          83,
          121,
          83,
          76,
          57,
          142,
          188,
          120,
          159,
          127,
          98,
          110,
          148,
          70,
          58,
          58,
          130,
          115,
          104,
          80,
          59,
          94,
          150,
          277,
          199,
          203,
          204,
          112,
          56,
          165,
          222,
          282,
          150,
          77,
          80,
          351,
          760,
          81,
          421,
          269,
          105,
          82,
          69,
          71,
          54,
          106,
          71,
          88,
          70,
          65,
          324,
          979,
          666,
          274,
          353,
          97,
          59,
          92,
          70,
          73,
          152,
          84,
          1565,
          158,
          82,
          175,
          105,
          79,
          114,
          128,
          148,
          60,
          120,
          45,
          36,
          80,
          45,
          2801,
          68,
          200,
          30,
          38,
          38,
          56,
          17,
          28,
          135,
          97,
          12,
          3,
          58,
          10,
          30,
          7,
          5,
          7,
          2,
          7,
          6,
          1,
          2,
          3,
          2,
          8,
          3,
          51,
          21,
          8,
          23,
          2,
          23,
          4,
          2,
          1,
          3,
          4,
          13,
          1,
          1,
          1,
          8
         ]
        },
        {
         "name": "Total files",
         "type": "scatter",
         "uid": "9142194d-139d-4fd4-812b-3213875239eb",
         "x": [
          "1980-01",
          "1984-01",
          "1985-01",
          "1985-05",
          "1985-06",
          "1985-07",
          "1985-08",
          "1985-09",
          "1985-10",
          "1985-11",
          "1985-12",
          "1986-01",
          "1986-04",
          "1986-06",
          "1986-08",
          "1986-09",
          "1986-10",
          "1986-11",
          "1986-12",
          "1987-01",
          "1987-02",
          "1987-03",
          "1987-04",
          "1987-05",
          "1987-06",
          "1987-07",
          "1987-08",
          "1987-09",
          "1987-11",
          "1987-12",
          "1988-01",
          "1988-02",
          "1988-03",
          "1988-04",
          "1988-05",
          "1988-06",
          "1988-07",
          "1988-08",
          "1988-09",
          "1988-10",
          "1988-11",
          "1988-12",
          "1989-01",
          "1989-02",
          "1989-03",
          "1989-04",
          "1989-05",
          "1989-06",
          "1989-07",
          "1989-08",
          "1989-09",
          "1989-10",
          "1989-11",
          "1989-12",
          "1990-01",
          "1990-02",
          "1990-03",
          "1990-04",
          "1990-05",
          "1990-06",
          "1990-07",
          "1990-08",
          "1990-09",
          "1990-10",
          "1990-11",
          "1990-12",
          "1991-01",
          "1991-02",
          "1991-03",
          "1991-04",
          "1991-05",
          "1991-06",
          "1991-07",
          "1991-08",
          "1991-09",
          "1991-10",
          "1991-11",
          "1991-12",
          "1992-01",
          "1992-02",
          "1992-03",
          "1992-04",
          "1992-05",
          "1992-06",
          "1992-07",
          "1992-08",
          "1992-09",
          "1992-10",
          "1992-11",
          "1992-12",
          "1993-01",
          "1993-02",
          "1993-03",
          "1993-04",
          "1993-05",
          "1993-06",
          "1993-07",
          "1993-08",
          "1993-09",
          "1993-10",
          "1993-11",
          "1993-12",
          "1994-01",
          "1994-02",
          "1994-03",
          "1994-04",
          "1994-05",
          "1994-06",
          "1994-07",
          "1994-08",
          "1994-09",
          "1994-10",
          "1994-11",
          "1994-12",
          "1995-01",
          "1995-02",
          "1995-03",
          "1995-04",
          "1995-05",
          "1995-06",
          "1995-07",
          "1995-08",
          "1995-09",
          "1995-10",
          "1995-11",
          "1995-12",
          "1996-01",
          "1996-02",
          "1996-03",
          "1996-04",
          "1996-05",
          "1996-06",
          "1996-07",
          "1996-08",
          "1996-09",
          "1996-10",
          "1996-11",
          "1996-12",
          "1997-01",
          "1997-02",
          "1997-03",
          "1997-04",
          "1997-05",
          "1997-06",
          "1997-07",
          "1997-08",
          "1997-09",
          "1997-10",
          "1997-11",
          "1997-12",
          "1998-01",
          "1998-02",
          "1998-03",
          "1998-04",
          "1998-05",
          "1998-06",
          "1998-07",
          "1998-08",
          "1998-09",
          "1998-10",
          "1998-11",
          "1998-12",
          "1999-01",
          "1999-02",
          "1999-03",
          "1999-04",
          "1999-05",
          "1999-06",
          "1999-07",
          "1999-08",
          "1999-09",
          "1999-10",
          "1999-11",
          "1999-12",
          "2000-01",
          "2000-02",
          "2000-03",
          "2000-04",
          "2000-05",
          "2000-06",
          "2000-07",
          "2000-08",
          "2000-09",
          "2000-10",
          "2000-11",
          "2000-12",
          "2001-01",
          "2001-02",
          "2001-03",
          "2001-04",
          "2001-05",
          "2001-06",
          "2001-07",
          "2001-08",
          "2001-09",
          "2001-10",
          "2001-11",
          "2001-12",
          "2002-01",
          "2002-02",
          "2002-03",
          "2002-04",
          "2002-05",
          "2002-06",
          "2002-07",
          "2002-08",
          "2002-09",
          "2002-10",
          "2002-11",
          "2002-12",
          "2003-01",
          "2003-02",
          "2003-03",
          "2003-04",
          "2003-05",
          "2003-06",
          "2003-07",
          "2003-08",
          "2003-09",
          "2003-10",
          "2003-11",
          "2003-12",
          "2004-01",
          "2004-02",
          "2004-03",
          "2004-04",
          "2004-05",
          "2004-06",
          "2004-07",
          "2004-08",
          "2004-09",
          "2004-10",
          "2004-11",
          "2004-12",
          "2005-01",
          "2005-02",
          "2005-03",
          "2005-04",
          "2005-05",
          "2005-06",
          "2005-07",
          "2005-08",
          "2005-09",
          "2005-10",
          "2005-11",
          "2005-12",
          "2006-01",
          "2006-02",
          "2006-03",
          "2006-04",
          "2006-05",
          "2006-06",
          "2006-07",
          "2006-08",
          "2006-09",
          "2006-12",
          "2007-01",
          "2007-04",
          "2007-05",
          "2007-10",
          "2007-11",
          "2007-12",
          "2008-02",
          "2008-03",
          "2009-02",
          "2009-03",
          "2009-04",
          "2010-01",
          "2010-07",
          "2010-08",
          "2010-09",
          "2010-10",
          "2010-11",
          "2010-12",
          "2011-01",
          "2011-05",
          "2011-06",
          "2011-09",
          "2013-06"
         ],
         "y": [
          107,
          1,
          38,
          1,
          4,
          1,
          16,
          8,
          24,
          23,
          71,
          57,
          1,
          36,
          60,
          78,
          10,
          6,
          1,
          2,
          4,
          8,
          10,
          23,
          15,
          41,
          173,
          20,
          44,
          20,
          25,
          76,
          10,
          20,
          38,
          106,
          4,
          114,
          6,
          35,
          12,
          5,
          15,
          6,
          15,
          27,
          94,
          120,
          64,
          86,
          197,
          160,
          61,
          70,
          65,
          65,
          66,
          66,
          29,
          38,
          66,
          36,
          48,
          54,
          43,
          17,
          30,
          26,
          61,
          41,
          36,
          169,
          137,
          122,
          195,
          287,
          383,
          489,
          379,
          279,
          382,
          390,
          420,
          397,
          387,
          365,
          425,
          696,
          414,
          371,
          407,
          504,
          466,
          401,
          340,
          434,
          510,
          431,
          743,
          480,
          344,
          329,
          350,
          684,
          566,
          391,
          558,
          328,
          410,
          304,
          312,
          369,
          347,
          252,
          158,
          371,
          320,
          314,
          440,
          426,
          373,
          452,
          279,
          553,
          470,
          300,
          502,
          304,
          359,
          302,
          441,
          391,
          611,
          375,
          499,
          283,
          378,
          297,
          409,
          629,
          269,
          278,
          420,
          539,
          260,
          376,
          470,
          348,
          426,
          203,
          193,
          654,
          560,
          508,
          347,
          295,
          333,
          251,
          397,
          227,
          249,
          197,
          689,
          882,
          366,
          531,
          384,
          317,
          370,
          614,
          230,
          238,
          204,
          562,
          416,
          370,
          225,
          210,
          305,
          514,
          1204,
          936,
          572,
          450,
          301,
          165,
          510,
          1651,
          1429,
          573,
          220,
          253,
          1096,
          2319,
          258,
          1333,
          819,
          362,
          263,
          256,
          268,
          181,
          367,
          266,
          445,
          267,
          202,
          1020,
          2983,
          2078,
          781,
          1670,
          279,
          170,
          281,
          207,
          226,
          349,
          224,
          3222,
          1767,
          204,
          309,
          202,
          157,
          227,
          265,
          321,
          126,
          265,
          99,
          86,
          155,
          109,
          8309,
          116,
          236,
          41,
          50,
          43,
          56,
          17,
          31,
          244,
          113,
          12,
          3,
          58,
          10,
          32,
          7,
          5,
          7,
          2,
          7,
          6,
          1,
          2,
          3,
          2,
          8,
          3,
          54,
          1,
          21,
          8,
          44,
          2,
          23,
          4,
          2,
          1,
          3,
          4,
          13,
          1,
          1,
          1,
          8
         ]
        }
       ],
       "layout": {
        "title": {
         "text": "File counts by modification date (month)"
        }
       }
      },
      "text/html": [
       "<div>\n",
       "        \n",
       "        \n",
       "            <div id=\"f9968c6a-5905-4144-b093-24956281b7d5\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                require([\"plotly\"], function(Plotly) {\n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    window.PLOTLYENV.BASE_URL='https://plot.ly';\n",
       "                    \n",
       "                if (document.getElementById(\"f9968c6a-5905-4144-b093-24956281b7d5\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        'f9968c6a-5905-4144-b093-24956281b7d5',\n",
       "                        [{\"name\": \"Unique files\", \"type\": \"scatter\", \"uid\": \"fbe44191-7a7a-47d2-8d47-eba224cf5128\", \"x\": [\"1980-01\", \"1984-01\", \"1985-01\", \"1985-05\", \"1985-06\", \"1985-07\", \"1985-08\", \"1985-09\", \"1985-10\", \"1985-11\", \"1985-12\", \"1986-01\", \"1986-04\", \"1986-06\", \"1986-08\", \"1986-09\", \"1986-10\", \"1986-11\", \"1986-12\", \"1987-01\", \"1987-02\", \"1987-03\", \"1987-04\", \"1987-05\", \"1987-06\", \"1987-07\", \"1987-08\", \"1987-09\", \"1987-11\", \"1987-12\", \"1988-01\", \"1988-02\", \"1988-03\", \"1988-04\", \"1988-05\", \"1988-06\", \"1988-07\", \"1988-08\", \"1988-09\", \"1988-10\", \"1988-11\", \"1988-12\", \"1989-01\", \"1989-02\", \"1989-03\", \"1989-04\", \"1989-05\", \"1989-06\", \"1989-07\", \"1989-08\", \"1989-09\", \"1989-10\", \"1989-11\", \"1989-12\", \"1990-01\", \"1990-02\", \"1990-03\", \"1990-04\", \"1990-05\", \"1990-06\", \"1990-07\", \"1990-08\", \"1990-09\", \"1990-10\", \"1990-11\", \"1990-12\", \"1991-01\", \"1991-02\", \"1991-03\", \"1991-04\", \"1991-05\", \"1991-06\", \"1991-07\", \"1991-08\", \"1991-09\", \"1991-10\", \"1991-11\", \"1991-12\", \"1992-01\", \"1992-02\", \"1992-03\", \"1992-04\", \"1992-05\", \"1992-06\", \"1992-07\", \"1992-08\", \"1992-09\", \"1992-10\", \"1992-11\", \"1992-12\", \"1993-01\", \"1993-02\", \"1993-03\", \"1993-04\", \"1993-05\", \"1993-06\", \"1993-07\", \"1993-08\", \"1993-09\", \"1993-10\", \"1993-11\", \"1993-12\", \"1994-01\", \"1994-02\", \"1994-03\", \"1994-04\", \"1994-05\", \"1994-06\", \"1994-07\", \"1994-08\", \"1994-09\", \"1994-10\", \"1994-11\", \"1994-12\", \"1995-01\", \"1995-02\", \"1995-03\", \"1995-04\", \"1995-05\", \"1995-06\", \"1995-07\", \"1995-08\", \"1995-09\", \"1995-10\", \"1995-11\", \"1995-12\", \"1996-01\", \"1996-02\", \"1996-03\", \"1996-04\", \"1996-05\", \"1996-06\", \"1996-07\", \"1996-08\", \"1996-09\", \"1996-10\", \"1996-11\", \"1996-12\", \"1997-01\", \"1997-02\", \"1997-03\", \"1997-04\", \"1997-05\", \"1997-06\", \"1997-07\", \"1997-08\", \"1997-09\", \"1997-10\", \"1997-11\", \"1997-12\", \"1998-01\", \"1998-02\", \"1998-03\", \"1998-04\", \"1998-05\", \"1998-06\", \"1998-07\", \"1998-08\", \"1998-09\", \"1998-10\", \"1998-11\", \"1998-12\", \"1999-01\", \"1999-02\", \"1999-03\", \"1999-04\", \"1999-05\", \"1999-06\", \"1999-07\", \"1999-08\", \"1999-09\", \"1999-10\", \"1999-11\", \"1999-12\", \"2000-01\", \"2000-02\", \"2000-03\", \"2000-04\", \"2000-05\", \"2000-06\", \"2000-07\", \"2000-08\", \"2000-09\", \"2000-10\", \"2000-11\", \"2000-12\", \"2001-01\", \"2001-02\", \"2001-03\", \"2001-04\", \"2001-05\", \"2001-06\", \"2001-07\", \"2001-08\", \"2001-09\", \"2001-10\", \"2001-11\", \"2001-12\", \"2002-01\", \"2002-02\", \"2002-03\", \"2002-04\", \"2002-05\", \"2002-06\", \"2002-07\", \"2002-08\", \"2002-09\", \"2002-10\", \"2002-11\", \"2002-12\", \"2003-01\", \"2003-02\", \"2003-03\", \"2003-04\", \"2003-05\", \"2003-06\", \"2003-07\", \"2003-08\", \"2003-09\", \"2003-10\", \"2003-11\", \"2003-12\", \"2004-01\", \"2004-02\", \"2004-03\", \"2004-04\", \"2004-05\", \"2004-06\", \"2004-07\", \"2004-08\", \"2004-09\", \"2004-10\", \"2004-11\", \"2004-12\", \"2005-01\", \"2005-02\", \"2005-03\", \"2005-04\", \"2005-05\", \"2005-06\", \"2005-07\", \"2005-08\", \"2005-09\", \"2005-10\", \"2005-11\", \"2005-12\", \"2006-01\", \"2006-02\", \"2006-03\", \"2006-04\", \"2006-05\", \"2006-06\", \"2006-07\", \"2006-08\", \"2006-09\", \"2006-12\", \"2007-01\", \"2007-04\", \"2007-05\", \"2007-10\", \"2007-11\", \"2007-12\", \"2008-02\", \"2009-02\", \"2009-03\", \"2009-04\", \"2010-01\", \"2010-07\", \"2010-08\", \"2010-09\", \"2010-10\", \"2010-11\", \"2010-12\", \"2011-01\", \"2011-05\", \"2011-06\", \"2011-09\", \"2013-06\"], \"y\": [76, 1, 27, 1, 4, 1, 16, 8, 23, 21, 50, 31, 1, 22, 47, 51, 9, 6, 1, 1, 4, 8, 10, 21, 15, 21, 168, 14, 41, 10, 15, 27, 9, 15, 24, 65, 4, 70, 6, 15, 11, 5, 14, 6, 9, 24, 52, 34, 30, 38, 56, 81, 26, 27, 34, 27, 41, 35, 20, 25, 48, 25, 27, 24, 17, 4, 16, 12, 40, 17, 28, 91, 49, 49, 71, 78, 140, 243, 105, 80, 104, 144, 109, 106, 97, 117, 92, 170, 102, 84, 89, 106, 115, 94, 82, 101, 154, 141, 160, 119, 108, 103, 94, 158, 125, 98, 149, 94, 142, 87, 91, 103, 106, 73, 50, 89, 72, 96, 97, 99, 116, 100, 77, 125, 128, 59, 154, 84, 89, 86, 121, 117, 131, 91, 133, 76, 95, 110, 157, 167, 109, 97, 139, 162, 84, 136, 130, 100, 102, 48, 51, 168, 182, 235, 80, 73, 83, 83, 121, 83, 76, 57, 142, 188, 120, 159, 127, 98, 110, 148, 70, 58, 58, 130, 115, 104, 80, 59, 94, 150, 277, 199, 203, 204, 112, 56, 165, 222, 282, 150, 77, 80, 351, 760, 81, 421, 269, 105, 82, 69, 71, 54, 106, 71, 88, 70, 65, 324, 979, 666, 274, 353, 97, 59, 92, 70, 73, 152, 84, 1565, 158, 82, 175, 105, 79, 114, 128, 148, 60, 120, 45, 36, 80, 45, 2801, 68, 200, 30, 38, 38, 56, 17, 28, 135, 97, 12, 3, 58, 10, 30, 7, 5, 7, 2, 7, 6, 1, 2, 3, 2, 8, 3, 51, 21, 8, 23, 2, 23, 4, 2, 1, 3, 4, 13, 1, 1, 1, 8]}, {\"name\": \"Total files\", \"type\": \"scatter\", \"uid\": \"9142194d-139d-4fd4-812b-3213875239eb\", \"x\": [\"1980-01\", \"1984-01\", \"1985-01\", \"1985-05\", \"1985-06\", \"1985-07\", \"1985-08\", \"1985-09\", \"1985-10\", \"1985-11\", \"1985-12\", \"1986-01\", \"1986-04\", \"1986-06\", \"1986-08\", \"1986-09\", \"1986-10\", \"1986-11\", \"1986-12\", \"1987-01\", \"1987-02\", \"1987-03\", \"1987-04\", \"1987-05\", \"1987-06\", \"1987-07\", \"1987-08\", \"1987-09\", \"1987-11\", \"1987-12\", \"1988-01\", \"1988-02\", \"1988-03\", \"1988-04\", \"1988-05\", \"1988-06\", \"1988-07\", \"1988-08\", \"1988-09\", \"1988-10\", \"1988-11\", \"1988-12\", \"1989-01\", \"1989-02\", \"1989-03\", \"1989-04\", \"1989-05\", \"1989-06\", \"1989-07\", \"1989-08\", \"1989-09\", \"1989-10\", \"1989-11\", \"1989-12\", \"1990-01\", \"1990-02\", \"1990-03\", \"1990-04\", \"1990-05\", \"1990-06\", \"1990-07\", \"1990-08\", \"1990-09\", \"1990-10\", \"1990-11\", \"1990-12\", \"1991-01\", \"1991-02\", \"1991-03\", \"1991-04\", \"1991-05\", \"1991-06\", \"1991-07\", \"1991-08\", \"1991-09\", \"1991-10\", \"1991-11\", \"1991-12\", \"1992-01\", \"1992-02\", \"1992-03\", \"1992-04\", \"1992-05\", \"1992-06\", \"1992-07\", \"1992-08\", \"1992-09\", \"1992-10\", \"1992-11\", \"1992-12\", \"1993-01\", \"1993-02\", \"1993-03\", \"1993-04\", \"1993-05\", \"1993-06\", \"1993-07\", \"1993-08\", \"1993-09\", \"1993-10\", \"1993-11\", \"1993-12\", \"1994-01\", \"1994-02\", \"1994-03\", \"1994-04\", \"1994-05\", \"1994-06\", \"1994-07\", \"1994-08\", \"1994-09\", \"1994-10\", \"1994-11\", \"1994-12\", \"1995-01\", \"1995-02\", \"1995-03\", \"1995-04\", \"1995-05\", \"1995-06\", \"1995-07\", \"1995-08\", \"1995-09\", \"1995-10\", \"1995-11\", \"1995-12\", \"1996-01\", \"1996-02\", \"1996-03\", \"1996-04\", \"1996-05\", \"1996-06\", \"1996-07\", \"1996-08\", \"1996-09\", \"1996-10\", \"1996-11\", \"1996-12\", \"1997-01\", \"1997-02\", \"1997-03\", \"1997-04\", \"1997-05\", \"1997-06\", \"1997-07\", \"1997-08\", \"1997-09\", \"1997-10\", \"1997-11\", \"1997-12\", \"1998-01\", \"1998-02\", \"1998-03\", \"1998-04\", \"1998-05\", \"1998-06\", \"1998-07\", \"1998-08\", \"1998-09\", \"1998-10\", \"1998-11\", \"1998-12\", \"1999-01\", \"1999-02\", \"1999-03\", \"1999-04\", \"1999-05\", \"1999-06\", \"1999-07\", \"1999-08\", \"1999-09\", \"1999-10\", \"1999-11\", \"1999-12\", \"2000-01\", \"2000-02\", \"2000-03\", \"2000-04\", \"2000-05\", \"2000-06\", \"2000-07\", \"2000-08\", \"2000-09\", \"2000-10\", \"2000-11\", \"2000-12\", \"2001-01\", \"2001-02\", \"2001-03\", \"2001-04\", \"2001-05\", \"2001-06\", \"2001-07\", \"2001-08\", \"2001-09\", \"2001-10\", \"2001-11\", \"2001-12\", \"2002-01\", \"2002-02\", \"2002-03\", \"2002-04\", \"2002-05\", \"2002-06\", \"2002-07\", \"2002-08\", \"2002-09\", \"2002-10\", \"2002-11\", \"2002-12\", \"2003-01\", \"2003-02\", \"2003-03\", \"2003-04\", \"2003-05\", \"2003-06\", \"2003-07\", \"2003-08\", \"2003-09\", \"2003-10\", \"2003-11\", \"2003-12\", \"2004-01\", \"2004-02\", \"2004-03\", \"2004-04\", \"2004-05\", \"2004-06\", \"2004-07\", \"2004-08\", \"2004-09\", \"2004-10\", \"2004-11\", \"2004-12\", \"2005-01\", \"2005-02\", \"2005-03\", \"2005-04\", \"2005-05\", \"2005-06\", \"2005-07\", \"2005-08\", \"2005-09\", \"2005-10\", \"2005-11\", \"2005-12\", \"2006-01\", \"2006-02\", \"2006-03\", \"2006-04\", \"2006-05\", \"2006-06\", \"2006-07\", \"2006-08\", \"2006-09\", \"2006-12\", \"2007-01\", \"2007-04\", \"2007-05\", \"2007-10\", \"2007-11\", \"2007-12\", \"2008-02\", \"2008-03\", \"2009-02\", \"2009-03\", \"2009-04\", \"2010-01\", \"2010-07\", \"2010-08\", \"2010-09\", \"2010-10\", \"2010-11\", \"2010-12\", \"2011-01\", \"2011-05\", \"2011-06\", \"2011-09\", \"2013-06\"], \"y\": [107, 1, 38, 1, 4, 1, 16, 8, 24, 23, 71, 57, 1, 36, 60, 78, 10, 6, 1, 2, 4, 8, 10, 23, 15, 41, 173, 20, 44, 20, 25, 76, 10, 20, 38, 106, 4, 114, 6, 35, 12, 5, 15, 6, 15, 27, 94, 120, 64, 86, 197, 160, 61, 70, 65, 65, 66, 66, 29, 38, 66, 36, 48, 54, 43, 17, 30, 26, 61, 41, 36, 169, 137, 122, 195, 287, 383, 489, 379, 279, 382, 390, 420, 397, 387, 365, 425, 696, 414, 371, 407, 504, 466, 401, 340, 434, 510, 431, 743, 480, 344, 329, 350, 684, 566, 391, 558, 328, 410, 304, 312, 369, 347, 252, 158, 371, 320, 314, 440, 426, 373, 452, 279, 553, 470, 300, 502, 304, 359, 302, 441, 391, 611, 375, 499, 283, 378, 297, 409, 629, 269, 278, 420, 539, 260, 376, 470, 348, 426, 203, 193, 654, 560, 508, 347, 295, 333, 251, 397, 227, 249, 197, 689, 882, 366, 531, 384, 317, 370, 614, 230, 238, 204, 562, 416, 370, 225, 210, 305, 514, 1204, 936, 572, 450, 301, 165, 510, 1651, 1429, 573, 220, 253, 1096, 2319, 258, 1333, 819, 362, 263, 256, 268, 181, 367, 266, 445, 267, 202, 1020, 2983, 2078, 781, 1670, 279, 170, 281, 207, 226, 349, 224, 3222, 1767, 204, 309, 202, 157, 227, 265, 321, 126, 265, 99, 86, 155, 109, 8309, 116, 236, 41, 50, 43, 56, 17, 31, 244, 113, 12, 3, 58, 10, 32, 7, 5, 7, 2, 7, 6, 1, 2, 3, 2, 8, 3, 54, 1, 21, 8, 44, 2, 23, 4, 2, 1, 3, 4, 13, 1, 1, 1, 8]}],\n",
       "                        {\"title\": {\"text\": \"File counts by modification date (month)\"}},\n",
       "                        {\"showLink\": false, \"linkText\": \"Export to plot.ly\", \"plotlyServerURL\": \"https://plot.ly\", \"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('f9968c6a-5905-4144-b093-24956281b7d5');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                });\n",
       "            </script>\n",
       "        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "months = files['LAST_MODIFIED'].dt.strftime('%Y-%m').value_counts().sort_index()\n",
    "umonths = unique_files['LAST_MODIFIED'].dt.strftime('%Y-%m').value_counts().sort_index()\n",
    "umonths = umonths.reset_index()\n",
    "months = months.reset_index()\n",
    "umonths = umonths[umonths['index'] < '2014-01']\n",
    "months = months[months['index'] < '2014-01']\n",
    "month_fig = go.Figure(data=[\n",
    "    go.Scatter(x=umonths['index'], y=umonths['LAST_MODIFIED'], name='Unique files'),\n",
    "    go.Scatter(x=months['index'], y=months['LAST_MODIFIED'], name='Total files')],\n",
    "    layout_title_text=\"File counts by modification date (month)\")\n",
    "\n",
    "iplot(month_fig, filename='pandas-time-series')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}