time_conversion.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Determining the time of diagnosis\n",
    "\n",
    "this notebook contains the code necessary to calculate the intervals in which the diagnosis most likely took place (as referenced in Section 1 of the Supplementary Information. \n",
    "\n",
    "First, we load the required modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dateutil.relativedelta import relativedelta\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since several of the conversions are similarly referenced by users, we defined several categories of phrases as part of our mapping. These categories are defined below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "_YEARS_AGO = {'19 years ago': 19, '36 years ago': 36, '2 years ago': 2, '8 years ago': 8,\n",
    "              '5 years ago': 5, '3 years ago': 3, 'About 5 years ago': 5, 'Six years ago': 6,\n",
    "              '20 years ago': 20, '4 years ago': 4, '13 years ago': 13, '7 years ago': 7,\n",
    "              '14 years ago': 14, 'Two years ago': 2, 'One year ago': 1, 'A year ago': 1,\n",
    "              '25 years ago': 25, '6 years ago': 6, 'About 4 years ago': 4, 'About 15 years ago': 15,\n",
    "              '11 years ago': 11, '15 years ago': 15, 'About 3 years ago': 3, 'About 2 years ago': 2,\n",
    "              'Four years ago': 4, '29 years ago': 29, '23 years ago': 23, '12 years ago': 12,\n",
    "              'Eight years ago': 8, '10 years ago': 10, '9 years ago': 9, 'Five year ago': 5,\n",
    "              'In the last year': 1, 'In the past year': 1, 'For 9 years': 9, 'Nearly 5 years ago': 5,\n",
    "              'For 6 years': 6, 'Last year': 1, 'last year': 1}\n",
    "_MONTHS_AGO = {'5 months ago': 5, '18 months ago': 18, 'Two years and one month ago': 25,\n",
    "               '4 months ago': 4, '16 months ago': 16, 'A month ago': 1, 'Half a year ago': 6,\n",
    "               '6 months ago': 6, 'One year and one month ago': 18, '7 months ago': 7,\n",
    "               'A year and a half ago': 18, '2 and a half years ago': 30, '8 months since': 8,\n",
    "               'Two months since': 2, 'Two years this month since': 24}\n",
    "_DAYS_AGO = {'Three days ago': 3, '5 days ago': 5}\n",
    "_WEEKS_AGO = {'2 weeks ago': 2, 'Two weeks ago': 2, '3 weeks ago': 3}\n",
    "_FEW_YEARS = ['A couple of years ago', 'A few years ago', 'Few years ago', '2-3 years ago',\n",
    "              'Some years ago', 'Several year ago']\n",
    "_FEW_MONTHS = ['A few months ago', 'Few months back', 'A couple of months ago']\n",
    "_YEARS_AGO_DATE = {'A year ago today': 1, '5 years ago today': 5, 'Four years ago today': 4,\n",
    "                   '10 years ago today': 10, 'One year ago today': 1, '6 years ago today': 6,\n",
    "                   '4 years ago today': 4, '3 years ago today': 3, '2 years ago today': 2,\n",
    "                   '2 year ago today': 2, 'Today three years ago': 3, '1 year ago today': 1,\n",
    "                   '7 years since': 7, '1 year since': 1, 'A year since': 1, 'Exactly a year ago': 1,\n",
    "                   '4 years since': 4, '2 years since': 2, '5 years since': 5}\n",
    "_ALMOST_YEAR_AGO = {'Nearly four years since': 4, 'Almost 25 years ago': 25, 'Almost a year since': 1,\n",
    "                    'Almost two years ago': 2, 'Under 2 years ago': 2, 'Just under 2 years ago': 2,\n",
    "                    'Almost 6 years ago': 6, 'For almost 5 years': 5, 'Been almost 10 year': 10,\n",
    "                    'Almost 3 years ago': 3, 'Almost 9 years since': 9, 'Almost a year ago': 1}\n",
    "_OVER_YEAR_AGO = {'For over a year': 1, 'Over a year ago': 1, 'Over 5 years ago': 5,\n",
    "                  'A little over 9 years ago': 9}\n",
    "_IN_MONTH = {'In january': 1, 'In February': 2, 'Back in august': 8, 'In december': 12,\n",
    "             'In october': 10, 'In April': 4, 'In november': 11, 'In august': 8,\n",
    "             'In October': 10, 'In June': 6, 'In July': 7}\n",
    "_IN_YEAR = {\"In 1979\": 1979, \"In 1993\": 1993, \"In 1994\": 1994, \"Since 1996\": 1996,\n",
    "            \"In 1997\": 1997, \"In 2001\": 2001, \"In 2003\": 2003, \"In 2005\": 2005,\n",
    "            \"In 2006\": 2006, \"In 2007\": 2007, \"In 2008\": 2008, \"Since 2008\": 2008,\n",
    "            \"In 2009\": 2009, \"In 2010\": 2010, \"In 2011\": 2011, \"Since 2011\": 2011,\n",
    "            \"In 2012\": 2012, \"In 2013\": 2013, \"In 2014\": 2014, \"In 2015\": 2015,\n",
    "            \"In 2016\": 2016, \"In 2017\": 2017}\n",
    "_IN_MONTH_YEAR = {'May 2015': 'May 2015', 'In september 2015': 'september 2015',\n",
    "                  'In may 2015': 'may 2015', 'In October 2014': 'October 2014',\n",
    "                  'May of 2012': 'May 2012', 'In January 2018': 'January 2018',\n",
    "                  'In november 2016': 'november 2016', 'Around October 2013': 'October 2013',\n",
    "                  'In march 2017': 'march 2017', 'In december 2014': 'december 2014',\n",
    "                  'In februari 2017': 'february 2017', 'In februari 2008': 'february 2008'}\n",
    "_IN_PART_OF_YEAR = {'Late 2015': (\"October 2015\", \"December 2015\"),\n",
    "                    'End of 2015': (\"October 2015\", \"December 2015\"),\n",
    "                    'Beginning of 2016': (\"January 2016\", \"March 2016\"),\n",
    "                    'Early 2015': (\"January 2015\", \"March 2015\"),\n",
    "                    'Early 2017': (\"January 2017\", \"March 2017\"),\n",
    "                    'Late last year': (\"October 2017\", \"December 2017\"),\n",
    "                    'Over the summer': (\"June 2017\", \"August 2017\"),\n",
    "                    'Last summer': (\"June 2017\", \"August 2017\")}\n",
    "_IN_INTERVAL = {\n",
    "    '2013 - today': pd.Timestamp(\"2013-1-1\", tz=\"UTC\"),\n",
    "    'Somewhere between 2009 and today': pd.Timestamp(\"2009-1-1\", tz=\"UTC\"),\n",
    "    'In the last six months': relativedelta(months=6),\n",
    "    'Within the last two years': relativedelta(years=2),\n",
    "}\n",
    "_ON_WEEKDAY = {\"On tuesday\": 1, \"On friday\": 4, \"Last friday\": 4}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For some tweets, we found a reference to an exact date. If this date did not contain a year, inferred the year manually based on the time at which the tweet was posted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ON_DATE = {\n",
    "    'Last year march 27': pd.Timestamp(\"2017 march 27\", tz=\"UTC\"),\n",
    "    'On February 10 2010': pd.Timestamp(\"February 10 2010\", tz=\"UTC\"),\n",
    "    'On friday 28 October 2016': pd.Timestamp(\"friday 28 October 2016\", tz=\"UTC\"),\n",
    "    'On Wednesday October 11': pd.Timestamp(\"Wednesday October 11 2017\", tz=\"UTC\"),\n",
    "    'On 9/3/14': pd.Timestamp(\"9/3/14\", tz=\"UTC\"),\n",
    "    'On June 27 2017': pd.Timestamp(\"June 27 2017\", tz=\"UTC\"),\n",
    "    'On feb 15': pd.Timestamp(\"2016 feb 15\", tz=\"UTC\"),\n",
    "    'On 6 feb 2018': pd.Timestamp(\"6 feb 2018\", tz=\"UTC\"),\n",
    "    'Recently on august 17': pd.Timestamp(\"august 17 2017\", tz=\"UTC\")\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we define the conversion function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert(ts, freq):\n",
    "    return ts.to_period(freq).to_timestamp().date()\n",
    "\n",
    "def convert_timestamp(row, which=\"start\"):\n",
    "    if which not in [\"start\", \"end\"]:\n",
    "        raise(NotImplementedError)\n",
    "\n",
    "    if pd.isna(row.time_indication):\n",
    "        return pd.NaT\n",
    "    elif row.time_indication in [\"today\", \"Just got\", \"Just been\"]:\n",
    "        return row.created_at.date()\n",
    "    elif row.time_indication == \"yesterday\":\n",
    "        return row.created_at.date() - pd.Timedelta(\"1 day\")\n",
    "    elif row.time_indication in [\"recently\", \"Recently\"]:\n",
    "        if which == \"start\":\n",
    "            return row.created_at.date() - relativedelta(months=3)\n",
    "        else:\n",
    "            return row.created_at.date()\n",
    "    elif row.time_indication in _YEARS_AGO.keys():\n",
    "        diff = _YEARS_AGO[row.time_indication]\n",
    "        to_year = convert(row.created_at, \"Y\")\n",
    "        if which == \"start\":\n",
    "            return to_year - relativedelta(years=diff)\n",
    "        else:\n",
    "            return to_year - relativedelta(years=diff - 1, days=1)\n",
    "    elif row.time_indication in _FEW_YEARS:\n",
    "        to_year = convert(row.created_at, \"Y\")\n",
    "        if which == \"start\":\n",
    "            return to_year - relativedelta(years=3)\n",
    "        else:\n",
    "            return to_year - relativedelta(years=1, days=1)\n",
    "    elif row.time_indication == \"This time last year\":\n",
    "        to_month = convert(row.created_at, \"M\")\n",
    "        if which == \"start\":\n",
    "            return to_month - relativedelta(months=12)\n",
    "        else:\n",
    "            return to_month - relativedelta(months=11, days=1)\n",
    "    elif row.time_indication == \"Earlier this year\":\n",
    "        to_year = convert(row.created_at, \"Y\")\n",
    "        if which == \"start\":\n",
    "            to_month = convert(row.created_at, \"M\")\n",
    "            if to_year == to_month:  # If posted in Jan, assume previous year\n",
    "                return to_year - relativedelta(years=1)\n",
    "            else:\n",
    "                return to_year\n",
    "        else:\n",
    "            return row.created_at.date()\n",
    "    elif row.time_indication == \"Last week\":\n",
    "        if which == \"start\":\n",
    "            return row.created_at.date() - relativedelta(days=row.created_at.weekday() + 7)\n",
    "        else:\n",
    "            return row.created_at.date() - relativedelta(days=row.created_at.weekday() + 1)\n",
    "    elif row.time_indication in _FEW_MONTHS:\n",
    "        to_month = convert(row.created_at, \"M\")\n",
    "        if which == \"start\":\n",
    "            return to_month - relativedelta(months=3)\n",
    "        else:\n",
    "            return to_month - relativedelta(months=1, days=1)\n",
    "    elif row.time_indication == \"This year\":\n",
    "        if which == \"start\":\n",
    "            return convert(row.created_at, \"Y\")\n",
    "        else:\n",
    "            return row.created_at.date()\n",
    "    elif row.time_indication in _YEARS_AGO_DATE.keys():\n",
    "        diff = _YEARS_AGO_DATE[row.time_indication]\n",
    "        return row.created_at.date() - relativedelta(years=diff)\n",
    "    elif row.time_indication in _MONTHS_AGO.keys():\n",
    "        diff = _MONTHS_AGO[row.time_indication]\n",
    "        to_month = convert(row.created_at, \"M\")\n",
    "        if which == \"start\":\n",
    "            return to_month - relativedelta(months=diff)\n",
    "        else:\n",
    "            return to_month - relativedelta(months=diff - 1, days=1)\n",
    "    elif row.time_indication == \"Within the last year\":\n",
    "        if which == \"start\":\n",
    "            return row.created_at.date() - relativedelta(years=1)\n",
    "        else:\n",
    "            return row.created_at.date()\n",
    "    elif row.time_indication in _DAYS_AGO.keys():\n",
    "        return row.created_at.date() - relativedelta(days=_DAYS_AGO[row.time_indication])\n",
    "    elif row.time_indication in _WEEKS_AGO.keys():\n",
    "        diff = _WEEKS_AGO[row.time_indication]\n",
    "        to_week = convert(row.created_at, \"W\")\n",
    "        if which == \"start\":\n",
    "            return to_week - relativedelta(weeks=diff)\n",
    "        else:\n",
    "            return to_week - relativedelta(weeks=diff - 1, days=1)\n",
    "    elif row.time_indication in ['Few weeks ago', 'A couple of weeks ago']:\n",
    "        to_week = convert(row.created_at, \"W\")\n",
    "        if which == \"start\":\n",
    "            return to_week - relativedelta(weeks=3)\n",
    "        else:\n",
    "            return to_week - relativedelta(weeks=1, days=1)\n",
    "    elif row.time_indication in _IN_MONTH:\n",
    "        month_num = _IN_MONTH[row.time_indication]\n",
    "        year = row.created_at.year - 1 if row.created_at.month < month_num else row.created_at.year\n",
    "        dt = pd.Timestamp(\"{y}-{m}-1\".format(y=year, m=month_num), tz=\"UTC\").date()\n",
    "        if which == \"start\":\n",
    "            return dt\n",
    "        else:\n",
    "            return dt + relativedelta(months=1, days=-1)\n",
    "    elif row.time_indication == \"In late 90’s\":\n",
    "        if which == \"start\":\n",
    "            return pd.Timestamp(\"1997-1-1\", tz=\"UTC\").date()\n",
    "        else:\n",
    "            return pd.Timestamp(\"1999-12-31\", tz=\"UTC\").date()\n",
    "    elif row.time_indication == \"About a year ago\":\n",
    "        to_month = convert(row.created_at, \"M\")\n",
    "        if which == \"start\":\n",
    "            return to_month - relativedelta(months=13)\n",
    "        else:\n",
    "            return to_month - relativedelta(months=10, days=1)\n",
    "    elif row.time_indication in _ON_DATE:\n",
    "        return _ON_DATE[row.time_indication].date()\n",
    "    elif row.time_indication == \"About 3-4 years ago\":\n",
    "        to_year = convert(row.created_at, \"Y\")\n",
    "        if which == \"start\":\n",
    "            return to_year - relativedelta(years=4)\n",
    "        else:\n",
    "            return to_year - relativedelta(years=2, days=1)\n",
    "    elif row.time_indication in _IN_MONTH_YEAR:\n",
    "        dt = pd.Timestamp(_IN_MONTH_YEAR[row.time_indication], tz=\"UTC\")\n",
    "        if which == \"start\":\n",
    "            return dt.date()\n",
    "        else:\n",
    "            return dt.date() + relativedelta(months=1, days=-1)\n",
    "    elif row.time_indication == \"Over ten years ago\":\n",
    "        to_year = convert(row.created_at, \"Y\")\n",
    "        if which == \"start\":\n",
    "            return to_year - relativedelta(years=15)\n",
    "        else:\n",
    "            return to_year - relativedelta(years=9, days=1)\n",
    "    elif row.time_indication in _ON_WEEKDAY:\n",
    "        reference = _ON_WEEKDAY[row.time_indication]\n",
    "        dt = row.created_at.date()\n",
    "        wkd = row.created_at.weekday()\n",
    "        if wkd == reference:\n",
    "            return dt\n",
    "        elif wkd > reference:\n",
    "            return dt - relativedelta(days=wkd - reference)\n",
    "        else:\n",
    "            return dt - relativedelta(days=(7 - (reference - wkd)))\n",
    "    elif row.time_indication in _IN_PART_OF_YEAR:\n",
    "        dt_strs = _IN_PART_OF_YEAR[row.time_indication]\n",
    "        if which == \"start\":\n",
    "            return pd.Timestamp(dt_strs[0], tz=\"UTC\").date()\n",
    "        else:\n",
    "            return pd.Timestamp(dt_strs[1], tz=\"UTC\").date() + relativedelta(months=1, days=-1)\n",
    "    elif row.time_indication in _IN_YEAR:\n",
    "        year = _IN_YEAR[row.time_indication]\n",
    "        if which == \"start\":\n",
    "            return pd.Timestamp(\"{}-1-1\".format(year), tz=\"UTC\").date()\n",
    "        else:\n",
    "            #  in case someone references the current year\n",
    "            return min(row.created_at.date(), pd.Timestamp(\"{}-12-31\".format(year), tz=\"UTC\").date())\n",
    "    elif row.time_indication == \"In 2010-2011\":\n",
    "        if which == \"start\":\n",
    "            return pd.Timestamp(\"2010-1-1\", tz=\"UTC\").date()\n",
    "        else:\n",
    "            return pd.Timestamp(\"2011-12-31\", tz=\"UTC\").date()\n",
    "    elif row.time_indication == \"In 2012-2013\":\n",
    "        if which == \"start\":\n",
    "            return pd.Timestamp(\"2012-1-1\", tz=\"UTC\").date()\n",
    "        else:\n",
    "            return pd.Timestamp(\"2013-12-31\", tz=\"UTC\").date()\n",
    "    elif row.time_indication in _IN_INTERVAL:\n",
    "        conversion = _IN_INTERVAL[row.time_indication]\n",
    "        if which == \"start\":\n",
    "            if isinstance(conversion, relativedelta):\n",
    "                return row.created_at.date() - conversion\n",
    "            else:\n",
    "                return conversion\n",
    "        else:\n",
    "            return row.created_at.date()\n",
    "    elif row.time_indication == \"Within 10 days ago to yesterday\":\n",
    "        dt = row.created_at.date()\n",
    "        if which == \"start\":\n",
    "            return dt - relativedelta(days=10)\n",
    "        else:\n",
    "            return dt - relativedelta(days=1)\n",
    "    elif row.time_indication == \"A year ago tomorrow\":\n",
    "        return row.created_at.date() - relativedelta(years=1, days=-1)\n",
    "    elif row.time_indication in _ALMOST_YEAR_AGO:\n",
    "        diff = _ALMOST_YEAR_AGO[row.time_indication]\n",
    "        dt = row.created_at - relativedelta(years=diff)\n",
    "        if which == \"start\":\n",
    "            return dt.date()\n",
    "        else:\n",
    "            return convert(dt, \"M\") + relativedelta(months=3, days=-1)\n",
    "    elif row.time_indication in _OVER_YEAR_AGO:\n",
    "        diff = _OVER_YEAR_AGO[row.time_indication]\n",
    "        dt = row.created_at - relativedelta(years=diff)\n",
    "        if which == \"start\":\n",
    "            return convert(dt, \"M\") - relativedelta(months=3)\n",
    "        else:\n",
    "            return dt.date()\n",
    "    elif row.time_indication == \"A year or more ago\":\n",
    "        if which == \"start\":\n",
    "            return convert(row.created_at, \"Y\") - relativedelta(years=1)\n",
    "        else:\n",
    "            return row.created_at.date() - relativedelta(years=1)\n",
    "    elif row.time_indication == \"6 years ago in the weeks after this date\":\n",
    "        dt = row.created_at - relativedelta(years=6)\n",
    "        if which == \"start\":\n",
    "            return convert(dt, \"W\")\n",
    "        else:\n",
    "            return convert(dt, \"W\") + relativedelta(weeks=4)\n",
    "    elif row.time_indication == \"Over a half year or so\":\n",
    "        dt = row.created_at.date() - relativedelta(months=6)\n",
    "        if which == \"start\":\n",
    "            return dt\n",
    "        else:\n",
    "            return dt - relativedelta(months=2)\n",
    "    elif row.time_indication == \"Over 2 hand a half years ago\":\n",
    "        dt = row.created_at.date() - relativedelta(months=30)\n",
    "        if which == \"start\":\n",
    "            return dt\n",
    "        else:\n",
    "            return dt - relativedelta(months=6)\n",
    "\n",
    "    elif row.time_indication == \"A year or two ago\":\n",
    "        to_year = convert(row.created_at, \"Y\")\n",
    "        if which == \"start\":\n",
    "            return to_year - relativedelta(years=2)\n",
    "        else:\n",
    "            return to_year\n",
    "    else:\n",
    "        return pd.NaT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then, we load the file that contains the time indications."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_fn = \"data/diagnosis_labelling_depression_final.tsv\"\n",
    "labels = pd.read_csv(label_fn, sep=\"\\t\", index_col=\"tweet_id\")\n",
    "labels[\"created_at\"] = pd.to_datetime(labels[\"created_at\"], utc=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we construct the intervals based on the time indications in the diagnosis tweets. Note that these conversions are already contained in the file named `diagnosis_labelling_depression_final.tsv`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>time_indication</th>\n",
       "      <th>diag_start_interval</th>\n",
       "      <th>diag_end_interval</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tweet_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>td5725616</th>\n",
       "      <td>2018-05-01 21:39:49+00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>td3894375</th>\n",
       "      <td>2018-01-03 00:58:18+00:00</td>\n",
       "      <td>In 2012</td>\n",
       "      <td>2012-01-01</td>\n",
       "      <td>2012-12-31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>td5637801</th>\n",
       "      <td>2018-04-27 10:59:42+00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>td5731074</th>\n",
       "      <td>2018-05-02 03:29:41+00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>td5747722</th>\n",
       "      <td>2018-05-02 22:06:57+00:00</td>\n",
       "      <td>In 2015</td>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>2015-12-31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         created_at time_indication diag_start_interval  \\\n",
       "tweet_id                                                                  \n",
       "td5725616 2018-05-01 21:39:49+00:00             NaN                 NaT   \n",
       "td3894375 2018-01-03 00:58:18+00:00         In 2012          2012-01-01   \n",
       "td5637801 2018-04-27 10:59:42+00:00             NaN                 NaT   \n",
       "td5731074 2018-05-02 03:29:41+00:00             NaN                 NaT   \n",
       "td5747722 2018-05-02 22:06:57+00:00         In 2015          2015-01-01   \n",
       "\n",
       "          diag_end_interval  \n",
       "tweet_id                     \n",
       "td5725616               NaT  \n",
       "td3894375        2012-12-31  \n",
       "td5637801               NaT  \n",
       "td5731074               NaT  \n",
       "td5747722        2015-12-31  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conversions = labels[[\"created_at\", \"time_indication\"]].copy()\n",
    "conversions[\"diag_start_interval\"] = conversions.apply(convert_timestamp, axis=1)\n",
    "conversions[\"diag_end_interval\"] = conversions.apply(convert_timestamp, which=\"end\", axis=1)\n",
    "conversions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}