SOCRATA_GET_KEYWORDS_ONE_SITE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#import the required libraries\n",
    "import csv\n",
    "import requests\n",
    "import simplejson as json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "targeturl='https://data.lacity.org/' #change this to the SOCRATA portal you want to target, don't forget ending /\n",
    "descriptor='LOS ANGELES'   #change this to a recognizable descriptor for yourself"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "r=requests.get(targeturl+\"api/dcat.json\") #build string according to SOCRATA's convention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nSOCRATA has a limit to how many requests can be made every hour from a public pool without an application token.\\nThis can especially be a problem if your portal has over 100 datasets. Every time this program is run, you are\\nmaking (x+1) request \"pings\" against SOCRATA servers, where x is the number of datasets on the target portal\\n\\nIf you are running into this, you will need to register an account with SOCRATA and append the following code\\nbehind your API calls:\\n\\n?$$app_token=INSERT-YOUR-APP-TOKEN-HERE\\n'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "SOCRATA has a limit to how many requests can be made every hour from a public pool without an application token.\n",
    "This can especially be a problem if your portal has over 100 datasets. Every time this program is run, you are\n",
    "making (x+1) request \"pings\" against SOCRATA servers, where x is the number of datasets on the target portal\n",
    "\n",
    "If you are running into this, you will need to register an account with SOCRATA and append the following code\n",
    "behind your API calls:\n",
    "\n",
    "?$$app_token=INSERT-YOUR-APP-TOKEN-HERE\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "j=r.json() #parse the json into a dictionary named j, coincidentally j's KVPs are also dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "successfully fetched json data, http return code 200\n"
     ]
    }
   ],
   "source": [
    "#if it fetched the data successfully, continue; otherwise stop\n",
    "#this could probably be implemented more pythonically.. but it works for now\n",
    "if r.status_code==200:\n",
    "    print \"\\nsuccessfully fetched json data, http return code 200\"\n",
    "else:\n",
    "    sys.exit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#this cell retrieves the list of keywords from all datasets and loads them into one list named masterlist\n",
    "\n",
    "masterlist=[] #dim masterlist as a empty list\n",
    "\n",
    "for i in j:\n",
    "    if len(i['identifier'])==9:\n",
    "        unified=i['keyword'].replace(';',',')\n",
    "        strlist=unified.split(',')\n",
    "        for x in strlist:\n",
    "            masterlist.append(x.lstrip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "master keyword list built: 1517 elements\n"
     ]
    }
   ],
   "source": [
    "masterlist.sort() #sort masterlist\n",
    "print \"master keyword list built:\", len(masterlist),\"elements\" #print how many elements are in masterlist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "master keyword list file opened, starting to write rows\n"
     ]
    }
   ],
   "source": [
    "keywords=open(descriptor+' - KEYWORDS.csv', 'wb') #open the csv file for writing\n",
    "print \"master keyword list file opened, starting to write rows\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in masterlist:\n",
    "    csv.writer(keywords).writerow([i.encode(\"utf-8\")])\n",
    "#this may need to be tweaked to optimize encoding to handle errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "master keyword list file closed, all rows written \n",
      "\n"
     ]
    }
   ],
   "source": [
    "keywords.close() #close csv writing, release all locks\n",
    "print \"master keyword list file closed, all rows written \\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#the below dumps out identifiers, views, titles and descriptions, created, modified and publisher\n",
    "#this can be modified to produce specific metadata elements YOU want, examine /api/dcat.json as needed\n",
    "\n",
    "metadata=open(descriptor+' - METADATA.csv', 'wb')\n",
    "csv.writer(metadata).writerow(['identifier','views','title','description','created','modified'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "error, continuing\n",
      "10 of 386 rows written, 376 remaining\n",
      "error, continuing\n",
      "20 of 386 rows written, 366 remaining\n",
      "30 of 386 rows written, 356 remaining\n",
      "40 of 386 rows written, 346 remaining\n",
      "50 of 386 rows written, 336 remaining\n",
      "60 of 386 rows written, 326 remaining\n",
      "70 of 386 rows written, 316 remaining\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "80 of 386 rows written, 306 remaining\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "90 of 386 rows written, 296 remaining\n",
      "100 of 386 rows written, 286 remaining\n",
      "110 of 386 rows written, 276 remaining\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "120 of 386 rows written, 266 remaining\n",
      "error, continuing\n",
      "130 of 386 rows written, 256 remaining\n",
      "error, continuing\n",
      "error, continuing\n",
      "140 of 386 rows written, 246 remaining\n",
      "150 of 386 rows written, 236 remaining\n",
      "error, continuing\n",
      "error, continuing\n",
      "160 of 386 rows written, 226 remaining\n",
      "error, continuing\n",
      "170 of 386 rows written, 216 remaining\n",
      "error, continuing\n",
      "error, continuing\n",
      "180 of 386 rows written, 206 remaining\n",
      "error, continuing\n",
      "error, continuing\n",
      "190 of 386 rows written, 196 remaining\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "200 of 386 rows written, 186 remaining\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "210 of 386 rows written, 176 remaining\n",
      "error, continuing\n",
      "error, continuing\n",
      "error, continuing\n",
      "220 of 386 rows written, 166 remaining\n",
      "error, continuing\n",
      "error, continuing\n",
      "230 of 386 rows written, 156 remaining\n",
      "240 of 386 rows written, 146 remaining\n",
      "250 of 386 rows written, 136 remaining\n",
      "error, continuing\n",
      "error, continuing\n",
      "260 of 386 rows written, 126 remaining\n",
      "error, continuing\n",
      "270 of 386 rows written, 116 remaining\n",
      "280 of 386 rows written, 106 remaining\n",
      "290 of 386 rows written, 96 remaining\n",
      "error, continuing\n",
      "300 of 386 rows written, 86 remaining\n",
      "310 of 386 rows written, 76 remaining\n",
      "320 of 386 rows written, 66 remaining\n",
      "330 of 386 rows written, 56 remaining\n",
      "340 of 386 rows written, 46 remaining\n",
      "error, continuing\n",
      "350 of 386 rows written, 36 remaining\n",
      "360 of 386 rows written, 26 remaining\n",
      "370 of 386 rows written, 16 remaining\n",
      "380 of 386 rows written, 6 remaining\n"
     ]
    }
   ],
   "source": [
    "counter=0\n",
    "for i in j:\n",
    "    if len(i['identifier']) == 9:\n",
    "        counter=counter+1\n",
    "        try:\n",
    "            if counter%10==0: #modify the modulus to change the frequency of printouts\n",
    "                print counter,\"of\",len(j)-1,\"rows written,\",(len(j)-1)-counter,\"remaining\"\n",
    "            metastring=targeturl+\"api/views/\"+i['identifier']+\".json\"\n",
    "            x=requests.request('get',metastring).json()\n",
    "            csv.writer(metadata).writerow([i['identifier'].encode(\"utf-8\"),x['viewCount'], i['title'].encode(\"utf-8\"), i['description'].encode(\"utf-8\"),i['created'],i['modified']]) #write one line to csv file, list of elements only!\n",
    "        except:\n",
    "            print \"error, continuing\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "386 of 386 rows written, 0 remaining\n"
     ]
    }
   ],
   "source": [
    "metadata.close() #Close the output file, release all locks\n",
    "print len(j)-1,\"of\",len(j)-1,\"rows written, 0 remaining\" #print final completion notice"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}