affiliations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d29ec16b-d5b3-4f6c-bddb-9be8968a16d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (3 of 3) |##########################| Elapsed Time: 0:00:14 Time:  0:00:14\n",
      "100% (2228 of 2228) |####################| Elapsed Time: 0:00:23 Time:  0:00:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No publication date set - something is odd with the record  {'_form_uuid': 'beae3039-29ed-4e20-bd21-6ed6e994afa5', '_oai': {'id': 'oai:data.caltech.edu:recid/20197', 'updated': '2022-06-14T19:02:07Z'}, 'access_right': 'open', 'authors': [{'authorAffiliation': 'Caltech', 'authorName': 'Kay, Justin'}, {'authorAffiliation': 'Cornell', 'authorName': 'Van Horn, Grant'}, {'authorAffiliation': 'Caltech', 'authorName': 'Stathatos, Suzanne'}, {'authorAffiliation': 'Caltech', 'authorName': 'Kulits, Peter'}, {'authorAffiliation': 'Caltech', 'authorName': 'Beery, Sara'}, {'authorAffiliation': 'Amazon', 'authorName': 'Deng, Tiffany'}, {'authorAffiliation': 'Caltech', 'authorName': 'Perona, Pietro'}], 'brief_authors': \"<a href='/search?q=Kay, Justin'>Kay, Justin</a>; <a href='/search?q=Van Horn, Grant'>Van Horn, Grant</a>; <a href='/search?q=Stathatos, Suzanne'>Stathatos, Suzanne</a>; (4 more)\", 'brief_information_bar': [{'icon': 'tag-1', 'text': 'Dataset'}], 'brief_subtitle': '', 'brief_summary': '', 'brief_title': 'Caltech Fish Counting Dataset 2022', 'control_number': '20197', 'descriptions': [{'descriptionType': 'Other', 'descriptionValue': 'A full instructional guide is provided here: https://github.com/visipedia/caltech-fish-counting\\n\\nThe Caltech Fish Counting Dataset is a large-scale dataset for detecting, tracking, and counting fish in sonar videos. '}], 'language': 'eng', 'owners': [445], 'pid_value': '20197', 'resourceType': {'resourceTypeGeneral': 'Dataset'}, 'resource_type': {'color': '#2980BA', 'icon': 'dataset', 'title': 'Dataset'}, 'rightsList': {'rights': 'mit', 'rightsURI': 'https://opensource.org/licenses/MIT'}, 'subjects': [{'subject': 'Detection, Tracking, Counting, Generalization, Benchmark, Video Dataset, Sonar Dataset, Fisheries'}], 'version': '1.0', 'titles': [{'title': 'Caltech Fish Counting Dataset 2022'}]}\n"
     ]
    }
   ],
   "source": [
    "#Get metadata from CaltechDATA\n",
    "\n",
    "import os\n",
    "from ames.harvesters import get_caltechdata\n",
    "\n",
    "if os.path.isdir('data') == False:\n",
    "    os.mkdir('data')\n",
    "os.chdir('data')\n",
    "\n",
    "production = True\n",
    "collection = 'caltechdata.ds'\n",
    "\n",
    "get_caltechdata(collection,production)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d0fa8940-d625-4f09-8744-2aa80caaf7be",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ames.harvesters import get_records\n",
    "from py_dataset import dataset\n",
    "import csv\n",
    "\n",
    "dot_paths = ['._Key','.contributors','.creators']\n",
    "keys = dataset.keys(collection)\n",
    "metadata = get_records(dot_paths,'dois',collection,keys)\n",
    "\n",
    "all_aff= set()\n",
    "\n",
    "for record in metadata:\n",
    "    if 'creators' in record:\n",
    "        creators = record['creators']\n",
    "        for cre in creators:\n",
    "            if 'affiliations' in cre:\n",
    "                affiliations = cre['affiliations']\n",
    "                all_aff.update(affiliations)\n",
    "    if 'contributors' in record:\n",
    "        contributors = record['contributors']\n",
    "        for con in contributors:\n",
    "            if 'affiliations' in con:\n",
    "                affiliations = con['affiliations']\n",
    "                all_aff.update(affiliations)\n",
    "                \n",
    "with open(\"../affiliations.csv\", \"w\", newline=\"\", encoding=\"utf-8-sig\") as fout:\n",
    "    file_out = csv.writer(fout)\n",
    "    for aff in all_aff:\n",
    "        file_out.writerow([aff])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fbfdc68c-2560-42bc-824f-21511fb1ede5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ames.harvesters import get_records\n",
    "from py_dataset import dataset\n",
    "import csv\n",
    "\n",
    "dot_paths = ['._Key','.fundingReferences']\n",
    "keys = dataset.keys(collection)\n",
    "metadata = get_records(dot_paths,'dois',collection,keys)\n",
    "\n",
    "all_funders= set()\n",
    "\n",
    "for record in metadata:\n",
    "    if 'fundingReferences' in record:\n",
    "        fund = record['fundingReferences']\n",
    "        for f in fund:\n",
    "            if 'funderName' in f:\n",
    "                funder = (f['funderName']).strip('\"')\n",
    "                if 'funderIdentifier' not in f:\n",
    "                    all_funders.update([funder])\n",
    "                \n",
    "with open(\"../funders.csv\", \"w\", newline=\"\", encoding=\"utf-8-sig\") as fout:\n",
    "    file_out = csv.writer(fout)\n",
    "    for fun in all_funders:\n",
    "        file_out.writerow([fun])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e6ab1ed-65fb-4098-9e13-6794fcf9a99c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Then  python RORRetriever.py -af funders.csv"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}