SOCRATA_GET_KEYWORDS_ONE_SITE_CHHS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#import the required libraries\n",
    "import csv\n",
    "import requests\n",
    "import simplejson as json\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "targeturl='http://chhs.data.ca.gov/' #change this to the SOCRATA portal you want to target, don't forget ending /\n",
    "descriptor='CHHS OPEN DATA PORTAL'   #change this to a recognizable descriptor for yourself"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "r=requests.get(targeturl+\"api/dcat.json\") #build string according to SOCRATA's convention, this is in json format\n",
    "\n",
    "'''\n",
    "SOCRATA has a limit to how many requests can be made every hour from a public pool without an application token.\n",
    "This can especially be a problem if your portal has over 100 datasets. Every time this program is run, you are\n",
    "making (x+1) request \"pings\" against SOCRATA servers, where x is the number of datasets on the target portal\n",
    "\n",
    "If you are running into this, you will need to register an account with SOCRATA and append the following code\n",
    "behind your API calls:\n",
    "\n",
    "?$$app_token=INSERT-YOUR-APP-TOKEN-HERE\n",
    "'''\n",
    "\n",
    "j=r.json() #parse the json into a dictionary named j, coincidentally j's KVPs are also dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://chhs.data.ca.gov/api/dcat.json\n",
      "2015-06-27\n"
     ]
    }
   ],
   "source": [
    "print r.url\n",
    "today=datetime.date.today()\n",
    "print str(today)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#\"dcat-files/\"+url_clean+\".json\", 'w'\n",
    "\n",
    "with open(\"chhs-json-repo/chhs-dcat-\"+str(today)+\".json\", 'w') as outfile:\n",
    "    outfile.write(r.text.encode(\"utf-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "successfully fetched json data, http return code 200\n"
     ]
    }
   ],
   "source": [
    "#if it fetched the data successfully, continue; otherwise stop\n",
    "#this could probably be implemented more pythonically.. but it works for now\n",
    "if r.status_code==200:\n",
    "    print \"\\nsuccessfully fetched json data, http return code 200\"\n",
    "else:\n",
    "    sys.exit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#this cell retrieves the list of keywords from all datasets and loads them into one list named masterlist\n",
    "\n",
    "masterlist=[] #dim masterlist as a empty list\n",
    "\n",
    "for i in j:\n",
    "    if len(i['identifier'])==9:\n",
    "        unified=i['keyword'].replace(';',',')\n",
    "        strlist=unified.split(',')\n",
    "        for x in strlist:\n",
    "            masterlist.append(x.lstrip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "master keyword list built: 990 elements\n"
     ]
    }
   ],
   "source": [
    "masterlist.sort() #sort masterlist\n",
    "print \"master keyword list built:\", len(masterlist),\"elements\" #print how many elements are in masterlist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "master keyword list file opened, starting to write rows\n"
     ]
    }
   ],
   "source": [
    "keywords=open(descriptor+' - KEYWORDS.csv', 'wb') #open the csv file for writing\n",
    "print \"master keyword list file opened, starting to write rows\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in masterlist:\n",
    "    csv.writer(keywords).writerow([i.encode(\"utf-8\")])\n",
    "#this may need to be tweaked to optimize encoding to handle errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "master keyword list file closed, all rows written \n",
      "\n"
     ]
    }
   ],
   "source": [
    "keywords.close() #close csv writing, release all locks\n",
    "print \"master keyword list file closed, all rows written \\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#the below dumps out identifiers, views, titles and descriptions, created, modified and publisher\n",
    "#this can be modified to produce specific metadata elements YOU want, examine /api/dcat.json as needed\n",
    "\n",
    "metadata=open(descriptor+' - METADATA.csv', 'wb')\n",
    "csv.writer(metadata).writerow(['identifier','views','title','description','created','modified','publisher'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 of 87 rows written, 77 remaining\n",
      "20 of 87 rows written, 67 remaining\n",
      "30 of 87 rows written, 57 remaining\n",
      "40 of 87 rows written, 47 remaining\n",
      "50 of 87 rows written, 37 remaining\n",
      "60 of 87 rows written, 27 remaining\n",
      "70 of 87 rows written, 17 remaining\n",
      "80 of 87 rows written, 7 remaining\n"
     ]
    }
   ],
   "source": [
    "counter=0\n",
    "for i in j:\n",
    "    if len(i['identifier']) == 9:\n",
    "        counter=counter+1\n",
    "        if counter%10==0: #modify the modulus to change the frequency of printouts\n",
    "            print counter,\"of\",len(j)-1,\"rows written,\",(len(j)-1)-counter,\"remaining\"\n",
    "        metastring=targeturl+\"api/views/\"+i['identifier']+\".json\"\n",
    "        x=requests.request('get',metastring).json()\n",
    "        publisher=x['metadata']['custom_fields']['Dataset Summary']['Publisher']\n",
    "        csv.writer(metadata).writerow([i['identifier'].encode(\"utf-8\"),x['viewCount'], i['title'].encode(\"utf-8\"), i['description'].encode(\"utf-8\"),i['created'],i['modified'],publisher]) #write one line to csv file, list of elements only!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87 of 87 rows written, 0 remaining\n"
     ]
    }
   ],
   "source": [
    "metadata.close() #Close the output file, release all locks\n",
    "print len(j)-1,\"of\",len(j)-1,\"rows written, 0 remaining\" #print final completion notice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}