Migrations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[website.addons.osfstorage.settings]  WARNING: No local.py settings file found\n",
      "[website.addons.wiki.settings]  WARNING: No local.py settings file found\n",
      "[root]  DEBUG: Setting storage backends\n",
      "[root]  INFO: Sentry disabled; Flask's debug mode enabled\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Flask 'framework.flask'>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from website.app import init_app\n",
    "\n",
    "init_app()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# LOAD BLACKLIST GUIDS\n",
    "from django.db import transaction\n",
    "from framework.guid.model import BlacklistGuid\n",
    "from osf_models.models import BlackListGuid\n",
    "import gevent\n",
    "\n",
    "odm_blacklist = BlacklistGuid.find()\n",
    "total = len(odm_blacklist)\n",
    "count = 0\n",
    "page_size = 1000\n",
    "\n",
    "\n",
    "def migrate_blacklist_item(guid):\n",
    "    pg_guid = BlackListGuid.objects.create(guid=guid._id)\n",
    "\n",
    "while count < total:\n",
    "    with transaction.atomic():\n",
    "        page = odm_blacklist[count:count+page_size]\n",
    "        threads = []\n",
    "        for guid in page:\n",
    "            threads.append(gevent.spawn(migrate_blacklist_item, guid))\n",
    "            count += 1\n",
    "        gevent.joinall(threads)\n",
    "        print 'Committing {} through {}'.format(count-page_size, count)\n",
    "    \n",
    "print count\n",
    "print total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# LOAD GUIDS\n",
    "from django.db import transaction\n",
    "from framework.guid.model import Guid as MODMGuid\n",
    "from osf_models.models import Guid\n",
    "import gevent\n",
    "\n",
    "\n",
    "guids = MODMGuid.find()\n",
    "total = len(guids)\n",
    "count = 0\n",
    "page_size = 30000\n",
    "\n",
    "def migrate_guid(guid):\n",
    "    return Guid.objects.create(guid=guid)\n",
    "\n",
    "while count < total:\n",
    "    with transaction.atomic():\n",
    "        threads = []\n",
    "        for guid in guids[count:count+page_size].get_keys():\n",
    "            threads.append(gevent.spawn(migrate_guid, guid))\n",
    "            count += 1\n",
    "            if count % page_size == 0:\n",
    "                print count\n",
    "        gevent.joinall(threads)\n",
    "        print 'Committing {} through {}'.format(count-page_size, count)\n",
    "        \n",
    "print total\n",
    "print count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# VERIFY GUIDS\n",
    "from framework.guid.model import Guid as MODMGuid\n",
    "from osf_models.models import Guid\n",
    "import gevent\n",
    "import copy\n",
    "\n",
    "modm_guids = MODMGuid.find()\n",
    "print 'MODM Guids: {}'.format(len(modm_guids))\n",
    "\n",
    "guids = Guid.objects.filter(guid__in=modm_guids.get_keys())\n",
    "filtered_count = len(guids)\n",
    "total_count = Guid.objects.count()\n",
    "\n",
    "if len(modm_guids) == filtered_count == total_count:\n",
    "    print 'WINNING'\n",
    "else:\n",
    "    print 'LOSING'\n",
    "\n",
    "print 'Postgres Guids: {}'.format(Guid.objects.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "1000\n",
      "Nodes: 2095\n",
      "Users: 3862\n",
      "Tags: 0\n",
      "MODM: 1000\n",
      "PG: 1000\n"
     ]
    }
   ],
   "source": [
    "# MIGRATE NODES\n",
    "\n",
    "from website.models import Node as MODMNode\n",
    "from osf_models.models import Node\n",
    "from modularodm import Q as MODMQ\n",
    "from django.core.serializers.json import DjangoJSONEncoder\n",
    "import json\n",
    "from django.db import IntegrityError\n",
    "import pytz\n",
    "from datetime import datetime\n",
    "\n",
    "modm_nodes = MODMNode.find()[1000:2000]\n",
    "\n",
    "total = len(modm_nodes)\n",
    "count = 0\n",
    "print total\n",
    "\n",
    "fk_node_fields = [\n",
    "    'forked_from',\n",
    "    'registered_from',\n",
    "    'root',\n",
    "    'parent_node',\n",
    "    'template_node'\n",
    "]\n",
    "m2m_node_fields = [\n",
    "    'nodes',\n",
    "]\n",
    "fk_user_fields = [\n",
    "    'registered_user',\n",
    "    'creator',\n",
    "    'merged_by'\n",
    "]\n",
    "m2m_user_fields = [\n",
    "    'permissions',\n",
    "    'recently_added',\n",
    "    'users_watching_node',\n",
    "    'contributors'\n",
    "]\n",
    "m2m_tag_fields = [\n",
    "    'tags',\n",
    "    'system_tags'\n",
    "]\n",
    "\n",
    "node_cache = {\n",
    "    # 'xyz12': {\n",
    "    #     'modm': modm_object,\n",
    "    #     'django': django_object,\n",
    "    # }\n",
    "}\n",
    "user_cache = []\n",
    "tag_cache  = []\n",
    "\n",
    "node_key_blacklist = [\n",
    "    '__backrefs', \n",
    "    '_version',\n",
    "    'expanded',\n",
    "    # foreign keys not yet implemented\n",
    "    'logs', \n",
    "    'primary_institution', \n",
    "    'registration_approval',\n",
    "    'alternative_citations',\n",
    "    'registered_schema',\n",
    "    'affiliated_institutions',\n",
    "    'retraction',\n",
    "    'embargo',\n",
    "    'node_license',\n",
    "] + m2m_node_fields + m2m_user_fields + m2m_tag_fields\n",
    "user_key_blacklist = ['__backrefs', '_version', 'affiliated_institutions', 'watched', 'external_accounts',] + m2m_node_fields + m2m_user_fields + m2m_tag_fields\n",
    "\n",
    "def process_node_fk_fields(modm_object):\n",
    "    fk_nodes = {}\n",
    "    for fk_node_field in fk_node_fields:\n",
    "        value = getattr(modm_object, fk_node_field, None)\n",
    "        if value is not None:\n",
    "            if fk_node_field != 'root' and value != modm_object:\n",
    "                node = get_or_create_node(value)\n",
    "                if node is not None:\n",
    "                    fk_nodes[fk_node_field] = node\n",
    "            else:\n",
    "                fk_nodes[fk_node_field] = None\n",
    "    return fk_nodes\n",
    "\n",
    "def process_node_m2m_fields(modm_object):\n",
    "    m2m_nodes = {}\n",
    "    for m2m_node_field in m2m_node_fields:\n",
    "        value = getattr(modm_object, m2m_node_field, None)\n",
    "        if value is not None:\n",
    "            if isinstance(value, list):\n",
    "                for nv in value:\n",
    "                    if m2m_node_field in m2m_nodes:\n",
    "                        node = get_or_create_node(nv)\n",
    "                        if node is not None:\n",
    "                            m2m_nodes[m2m_node_field].append(node)\n",
    "                        else:\n",
    "                            m2m_nodes[m2m_node_field] = [node, ]\n",
    "            else:\n",
    "                if m2m_node_field in m2m_nodes:\n",
    "                    node = get_or_create_node(value)\n",
    "                    if node is not None:\n",
    "                        m2m_nodes[m2m_node_field].append(node)\n",
    "                    else:\n",
    "                        m2m_nodes[m2m_node_field] = [node, ]\n",
    "    return m2m_nodes\n",
    "\n",
    "def process_user_fk_fields(modm_object):\n",
    "    fk_users = {}\n",
    "    for fk_user_field in fk_user_fields:\n",
    "        modm_user = getattr(modm_object, fk_user_field, None)\n",
    "        if modm_user is not None:\n",
    "            user = get_or_create_user(modm_user, fk_user_field, modm_object)\n",
    "            if not user is None:\n",
    "                fk_users[fk_user_field] = user\n",
    "    return fk_users\n",
    "\n",
    "def process_user_m2m_fields(modm_object):\n",
    "    m2m_users = {}\n",
    "    for m2m_user_field in m2m_user_fields:\n",
    "        value = getattr(modm_object, m2m_user_field, None)\n",
    "        if isinstance(value, list):\n",
    "            for uv in value:\n",
    "                if m2m_user_field in m2m_users:\n",
    "                    user = get_or_create_user(uv)\n",
    "                    if user is not None:\n",
    "                        m2m_users[m2m_user_field].append(user)\n",
    "                    else:\n",
    "                         m2m_users[m2m_user_field] = [user, ]\n",
    "                            \n",
    "    return m2m_users\n",
    "\n",
    "def process_tag_m2m_fields(modm_object):\n",
    "    m2m_tags = {}\n",
    "    for m2m_tag_field in m2m_tag_fields:\n",
    "        value = getattr(modm_object, m2m_tag_field, None)\n",
    "        if isinstance(value, list):\n",
    "            for tv in value:\n",
    "                if m2m_tag_field in m2m_tags:\n",
    "                    tag = get_or_create_tag(tv)\n",
    "                    if tag is not None:\n",
    "                        m2m_tags[m2m_tag_field].append(user)\n",
    "                    else:\n",
    "                        m2m_tags[m2m_tag_field] = [tag, ]\n",
    "                        \n",
    "    return m2m_tags\n",
    "\n",
    "def set_m2m_fields(object, fields):\n",
    "    for key, value in fields.iteritems():\n",
    "        attr = getattr(object, key)\n",
    "        attr.add(*value)\n",
    "    object.save()\n",
    "\n",
    "def get_or_create_user(modm_user, field_name, modm_node=None):\n",
    "    try:\n",
    "        user = User.objects.get(guid__guid=modm_user._id)\n",
    "    except User.DoesNotExist:\n",
    "        user_fk_nodes = process_node_fk_fields(modm_user)\n",
    "        user_m2m_nodes = process_node_m2m_fields(modm_user)\n",
    "        user_fk_users = process_user_fk_fields(modm_user)\n",
    "        user_m2m_users = process_user_m2m_fields(modm_user)\n",
    "        user_m2m_tags = process_tag_m2m_fields(modm_user)\n",
    "        user_fields = {}\n",
    "        user_fields.update(modm_user.to_storage())\n",
    "        user_fields.update(user_fk_nodes)\n",
    "        user_fields.update(user_fk_users)\n",
    "        user_fields = {k:v for k, v in user_fields.iteritems() if v is not None}\n",
    "        for k, v in user_fields.iteritems():\n",
    "            if isinstance(v, datetime):\n",
    "                user_fields[k] = pytz.utc.localize(v)\n",
    "        user = User.objects.create(**{key: user_fields[key] for key in user_fields if key not in user_key_blacklist})\n",
    "        set_m2m_fields(user, user_m2m_nodes)\n",
    "        set_m2m_fields(user, user_m2m_users)\n",
    "        set_m2m_fields(user, user_m2m_tags)\n",
    "    user_cache.append(user)\n",
    "    return user\n",
    "        \n",
    "\n",
    "def get_or_create_tag(modm_tag, field_name, modm_node=None):\n",
    "    try:\n",
    "        tag = Tag.objects.get(_id=modm_tag._id)\n",
    "    except Tag.DoesNotExist:\n",
    "        tag = Tag.objects.create(**modm_tag.to_storage())\n",
    "    tag_cache.append(tag)\n",
    "    return tag\n",
    "\n",
    "def get_or_create_node(modm_node):\n",
    "    try:\n",
    "        # try and get the node\n",
    "        node = Node.objects.get(guid__guid=modm_node._id)\n",
    "    except Node.DoesNotExist:  \n",
    "        # if it doesn't exist, check to see if the guid does\n",
    "        try:\n",
    "            guid = Guid.objects.get(guid=modm_node._id)\n",
    "        except Guid.DoesNotExist:\n",
    "            # fail if the guid doesn't exist\n",
    "            print 'GUID {} DoesNotExist'.format(modm_guid)\n",
    "        else:\n",
    "            fk_nodes = process_node_fk_fields(modm_node)\n",
    "            \n",
    "            m2m_nodes = process_node_m2m_fields(modm_node)\n",
    "            \n",
    "            fk_users = process_user_fk_fields(modm_node)\n",
    "            \n",
    "            m2m_users = process_user_m2m_fields(modm_node)\n",
    "                    \n",
    "            m2m_tags = process_tag_m2m_fields(modm_node)\n",
    "                                    \n",
    "            node_fields = {}\n",
    "            node_fields.update(modm_node.to_storage())\n",
    "            node_fields.update(fk_nodes)\n",
    "            node_fields.update(fk_users)\n",
    "            cleaned_node = {key: node_fields[key] for key in node_fields if key not in node_key_blacklist}\n",
    "            for k, v in cleaned_node.iteritems():\n",
    "                if isinstance(v, datetime):\n",
    "                    cleaned_node[k] = pytz.utc.localize(v)\n",
    "            # this shouldn't need to be here, not sure why it has to be\n",
    "            cleaned_node['is_collection'] = cleaned_node.pop('is_folder')\n",
    "            cleaned_node['is_bookmark_collection'] = cleaned_node.pop('is_dashboard')\n",
    "            cleaned_node = {k:v for k, v in cleaned_node.iteritems() if v is not None}\n",
    "            node = Node.objects.create(**cleaned_node)\n",
    "            set_m2m_fields(node, m2m_nodes)\n",
    "            set_m2m_fields(node, m2m_users)\n",
    "            set_m2m_fields(node, m2m_tags)\n",
    "    if modm_node._id not in node_cache:\n",
    "        node_cache[modm_node._id] = dict()\n",
    "    node_cache[modm_node._id]['django'] = node\n",
    "    return node                \n",
    "\n",
    "for modm_node in modm_nodes:\n",
    "    node_cache[modm_node._id] = {'modm': modm_node}\n",
    "    noooood = get_or_create_node(modm_node)\n",
    "    count +=1\n",
    "    if count % 1000 == 0:\n",
    "        print count\n",
    "\n",
    "print 'Nodes: {}'.format(len(node_cache))\n",
    "print 'Users: {}'.format(len(user_cache))\n",
    "print 'Tags: {}'.format(len(tag_cache))\n",
    "            \n",
    "print 'MODM: {}'.format(total)\n",
    "print 'PG: {}'.format(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# MIGRATE REGISTRATIONS\n",
    "\n",
    "from website.models import Node as MODMNode\n",
    "from osf_models.models import Registration\n",
    "from modularodm import Q as MODMQ\n",
    "from django.core.serializers.json import DjangoJSONEncoder\n",
    "import json\n",
    "from django.db import IntegrityError\n",
    "\n",
    "modm_nodes = MODMNode.find(\n",
    "    MODMQ('is_registration', 'eq', True) &\n",
    "    MODMQ('is_folder', 'ne', True)\n",
    ")\n",
    "\n",
    "total = len(modm_nodes)\n",
    "count = 0\n",
    "print total\n",
    "\n",
    "for modm_node in modm_nodes:\n",
    "    try:\n",
    "        guid = Guid.objects.get(guid=modm_node._id)\n",
    "    except Guid.DoesNotExist:\n",
    "        print 'Guid {} DoesNotExist'.format(modm_guid)\n",
    "    else:\n",
    "        try:\n",
    "            node = Registration.objects.create(guid=guid, data=modm_node.to_storage())\n",
    "        except IntegrityError as ex:\n",
    "            print 'Registration existed: {}'.format(ex)\n",
    "        count +=1\n",
    "        if count % 100 == 0:\n",
    "            print count\n",
    "            \n",
    "print 'MODM: {}'.format(total)\n",
    "print 'PG: {}'.format(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# MIGRATE COLLECTIONS\n",
    "\n",
    "from website.models import Node as MODMNode\n",
    "from osf_models.models import Collection\n",
    "from modularodm import Q as MODMQ\n",
    "from django.core.serializers.json import DjangoJSONEncoder\n",
    "import json\n",
    "from django.db import IntegrityError\n",
    "\n",
    "modm_nodes = MODMNode.find(\n",
    "    MODMQ('is_folder', 'eq', True)\n",
    ")\n",
    "\n",
    "total = len(modm_nodes)\n",
    "count = 0\n",
    "print total\n",
    "\n",
    "for modm_node in modm_nodes:\n",
    "    try:\n",
    "        guid = Guid.objects.get(guid=modm_node._id)\n",
    "    except Guid.DoesNotExist:\n",
    "        print 'Guid {} DoesNotExist'.format(modm_guid)\n",
    "    else:\n",
    "        try:\n",
    "            node = Collection.objects.create(guid=guid, data=modm_node.to_storage())\n",
    "        except IntegrityError as ex:\n",
    "            print 'Collection existed: {}'.format(ex)\n",
    "        count +=1\n",
    "        if count % 100 == 0:\n",
    "            print count\n",
    "            \n",
    "print 'MODM: {}'.format(total)\n",
    "print 'PG: {}'.format(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# BENCHMARK /users/me/nodes/\n",
    "\n",
    "import timeit\n",
    "\n",
    "modm = \"\"\"\n",
    "from website.models import Node\n",
    "from modularodm import Q\n",
    "nodes = Node.find(Q('contributors','eq','cdi38') & Q('is_deleted','eq', False) & Q('is_folder','eq', False))[:60]\n",
    "for node in nodes:\n",
    "    id = node._id\n",
    "\"\"\"\n",
    "django = \"\"\"\n",
    "from osf_models.models import Node\n",
    "nodes = Node.objects.filter(data__contributors__contains='\"cdi38\"', data__contains='{\"is_deleted\": false, \"is_folder\": false}')[:60]\n",
    "for node in nodes:\n",
    "    id = node.guid_id\n",
    "    id = node.guid.id\n",
    "\"\"\"\n",
    "print \"django, mongo\"\n",
    "for i in range(0,20):\n",
    "    d = timeit.timeit(stmt=django, number=50)\n",
    "    m = timeit.timeit(stmt=modm, number=50)\n",
    "    print '{} {}'.format(d, m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from osf_models.models import Node\n",
    "nodes = Node.objects.filter(data__contributors__contains='\"cdi38\"')\n",
    "print nodes.query\n",
    "print nodes.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mia_guids = Guid.objects.filter(referent_node=None, referent_registration=None, referent_collection=None).count()\n",
    "accounted_for_guids = Guid.objects.exclude(referent_node=None, referent_registration=None, referent_collection=None).count()\n",
    "total_guids = Guid.objects.all().count()\n",
    "total_guids == accounted_for_guids + mia_guids"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Django Shell-Plus",
   "language": "python",
   "name": "django_extensions"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}