mt-scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "antonio-feregrino-bolanos\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "import slugify\n",
    "import os\n",
    "from urllib.parse import urljoin\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "print(slugify.slugify('Antonio Feregrino Bolaños'))\n",
    "\n",
    "if not os.path.exists(\"mt-scraper\"):\n",
    "    os.makedirs(\"mt-scraper/defensiva\")\n",
    "    os.makedirs(\"mt-scraper/ofensiva\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get base page\n",
    "base_url = \"http://www.mediotiempo.com/liga/futbol/ligamx/tabla-general/\"\n",
    "base_page = requests.get(base_url).text\n",
    "base_soup = BeautifulSoup(base_page, \"lxml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tournament_container = base_soup.find(\"div\", { \"class\" : \"dropdown-container\" })\n",
    "ul = tournament_container.find('ul')\n",
    "tournaments = []\n",
    "for li in ul.findAll('li'):\n",
    "    tournaments.append(li.get('value'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get tables\n",
    "c = { 'Team': 0, 'PTS':1, 'JJ':2, 'DG':3, 'JG':4,'JE':5, 'JP':6, 'GF': 7, 'GC': 8 }\n",
    "print(\"Tournaments\", len(tournaments))\n",
    "scraped = {}\n",
    "for tournament in tournaments:\n",
    "    results = []\n",
    "    url = urljoin(base_url, tournament)\n",
    "    tournament_page = requests.get(url).text\n",
    "    tournament_soup = BeautifulSoup(tournament_page, \"lxml\")\n",
    "    tables = tournament_soup.findAll(\"div\", { \"class\" :'table-positions' })\n",
    "    for table in tables:\n",
    "        # need to find the table inside div.scroll:\n",
    "        table = table.find('div', {'class':'scroll'}).find('table', {'class':'mt-table'})\n",
    "        rows = table.tbody.findAll('tr')\n",
    "        for row in rows:\n",
    "            tds = row.findAll('td')\n",
    "            team = tds[c['Team']].text.strip()\n",
    "            pts =  tds[c['PTS']].text.strip()\n",
    "            jj =  tds[c['JJ']].text.strip()\n",
    "            dg =  tds[c['DG']].text.strip()\n",
    "            jg =  tds[c['JG']].text.strip()\n",
    "            je =  tds[c['JE']].text.strip()\n",
    "            jp =  tds[c['JP']].text.strip()\n",
    "            gf =  tds[c['GF']].text.strip()\n",
    "            gc =  tds[c['GC']].text.strip()\n",
    "            team_stat = {\n",
    "                'team':team,\n",
    "                'pts':pts,\n",
    "                'jj':jj,\n",
    "                'dg':dg,\n",
    "                'jg':jg,\n",
    "                'je':je,\n",
    "                'jp':jp,\n",
    "                'gf':gf,\n",
    "                'gc':gc\n",
    "            }\n",
    "            results.append(team_stat)\n",
    "    scraped[tournament] = results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torneo_largo = re.compile('(\\w+)-([0-9]{4})-+([0-9]{4})')\n",
    "torneo_corto = re.compile('(\\w+)-([0-9]{4})')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "dos = set(['invierno', 'apertura'])\n",
    "uno = set(['verano', 'clausura', 'bicentenario'])\n",
    "intermediate = []\n",
    "for torneo in scraped:\n",
    "    match_torneo_largo = torneo_largo.search(torneo)\n",
    "    match_torneo_corto = torneo_corto.search(torneo)\n",
    "    name = ''\n",
    "    if match_torneo_largo:\n",
    "        t = match_torneo_largo.group(1)\n",
    "        inicio = int(match_torneo_largo.group(2).upper())\n",
    "        fin = int(match_torneo_largo.group(3).upper())\n",
    "        if t == \"temporada\":\n",
    "            name = \"Temporada de \" + str(inicio) + \" a \"+ str(fin)\n",
    "        if t == \"liguilla\":\n",
    "            name = \"Liguilla de \" + str(inicio) + \" a \"+ str(fin)\n",
    "    elif match_torneo_corto:\n",
    "        t = match_torneo_corto.group(1)\n",
    "        c = int(match_torneo_corto.group(2).upper())\n",
    "        if t in dos:\n",
    "            name = \"Torneo corto \" + str(c)  + \"-2\"\n",
    "        if t in uno:\n",
    "            name = \"Torneo corto \" + str(c)  + \"-1\"\n",
    "    for result in scraped[torneo]:\n",
    "        intermediate.append([\n",
    "            torneo,\n",
    "            result['team'],\n",
    "            result['pts'],\n",
    "            result['jj'],\n",
    "            result['dg'],\n",
    "            result['jg'],\n",
    "            result['je'],\n",
    "            result['jp'],\n",
    "            result['gf'],\n",
    "            result['gc']\n",
    "        ])\n",
    "tournament_df = pd.DataFrame(intermediate)\n",
    "tournament_df.columns = ['tournament', 'team', 'pts', 'jj', 'dg', 'jg','je', 'jp', 'gf', 'gc']\n",
    "tournament_df.set_index(['tournament', 'team'], inplace=True)\n",
    "print(tournament_df.tail())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tournament_df.to_csv('mt-scraper/tournaments.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tournament_df = pd.read_csv('mt-scraper/tournaments.csv', index_col=[0, 1])\n",
    "print(tournament_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_tournaments = tournament_df.index.get_level_values(0).unique().values\n",
    "url = \"http://www.mediotiempo.com/liga/futbol/ligamx/calendario/\"\n",
    "seasons = []\n",
    "seasonRounds = []\n",
    "rounds = []\n",
    "for season in all_tournaments:\n",
    "    #print(\"Scraping season\", season)\n",
    "    season_url = url + tournament + \"/\"\n",
    "    season_page = requests.get(season_url).text\n",
    "    season_soup = BeautifulSoup(season_page, \"lxml\")\n",
    "    seasonRound_ul = season_soup.find('ul', { \"name\": \"seasonRound\"})\n",
    "    if seasonRound_ul is None: \n",
    "        continue\n",
    "    for li in seasonRound_ul.findAll('li'):\n",
    "        seasonRound = li.get('value')\n",
    "        seasonRound_url = season_url + seasonRound + \"/\"\n",
    "        seasonRound_page = requests.get(seasonRound_url).text\n",
    "        seasonRound_soup = BeautifulSoup(seasonRound_page, \"lxml\")\n",
    "        round_ul = seasonRound_soup.find('ul', { \"name\": \"round\"})\n",
    "        if round_ul is None: \n",
    "            continue\n",
    "        for li in round_ul.findAll('li'):\n",
    "            _round = li.get('value')\n",
    "            seasons.append(season)\n",
    "            seasonRounds.append(seasonRound)\n",
    "            rounds.append(_round)\n",
    "        \n",
    "print(\"Found\",len(seasons), len(seasonRounds), len(rounds), \"rounds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "url = \"http://www.mediotiempo.com/liga/futbol/ligamx/calendario/%s/%s/%s\"\n",
    "\n",
    "matches_seasons = []\n",
    "matches_seasonRounds = []\n",
    "matches_rounds = []\n",
    "matches_date = []\n",
    "matches_time = []\n",
    "matches_home_team = []\n",
    "matches_result = []\n",
    "matches_away_team = []\n",
    "matches_venue = []\n",
    "\n",
    "for season,seasonRound,_round in zip(seasons,seasonRounds,rounds):\n",
    "    query_url = url % (season,seasonRound,_round)\n",
    "#    print(query_url)\n",
    "    scrape = requests.get(query_url).text\n",
    "    scrape_soup = BeautifulSoup(scrape, \"lxml\")\n",
    "    calendar_groups = scrape_soup.findAll('div', {\"class\":\"mt-calendar-group\"})\n",
    "    for calendar_group in calendar_groups:\n",
    "        date = calendar_group.find('div', {\"class\":\"calendar-date-wrapper\"}).text.strip()\n",
    "        match_wrappers = calendar_group.findAll('div', {\"class\":\"mt-calendar-match\"},recursive=False)\n",
    "        for match_wrapper in match_wrappers:\n",
    "            try:\n",
    "                divs = match_wrapper.findAll('div')\n",
    "                time = divs[0].text.strip()\n",
    "                _as = divs[1].findAll('a')\n",
    "                home_team = _as[0].text.strip()\n",
    "                result = _as[1].text.replace(\"\\n\", \" \").strip()\n",
    "                away_team = _as[2].text.strip()\n",
    "                venue_div =  divs[1].find('div', {'class':'venue-wrapper'})\n",
    "                if venue_div is None:\n",
    "                    venue = ''\n",
    "                else:\n",
    "                    venue = venue_div.text.strip()\n",
    "\n",
    "                matches_seasons.append(season)\n",
    "                matches_seasonRounds.append(seasonRound)\n",
    "                matches_rounds.append(_round)\n",
    "                matches_date.append(date)\n",
    "                matches_time.append(time)\n",
    "                matches_home_team.append(home_team)\n",
    "                matches_result.append(result)\n",
    "                matches_away_team.append(away_team)\n",
    "                matches_venue.append(venue)\n",
    "            except:\n",
    "                print(\"Error\", query_url)\n",
    "print(\"Done scraping\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "matches_df = pd.DataFrame({\n",
    "    'season' : matches_seasons,\n",
    "    'season_round': matches_seasonRounds,\n",
    "    'round': matches_rounds,\n",
    "    'date': matches_date,\n",
    "    'time': matches_time,\n",
    "    'home_team': matches_home_team,\n",
    "    'result': matches_result,\n",
    "    'away_team': matches_away_team,\n",
    "    'venue': matches_venue\n",
    "})\n",
    "\n",
    "print(matches_df.info())\n",
    "matches_df.to_csv('mt-scraper/matches_raw.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "matches_df = pd.read_csv('mt-scraper/matches_raw.csv', index_col=0)\n",
    "\n",
    "month_dict = {\n",
    "    'enero': 1,\n",
    "    'febrero': 2,\n",
    "    'marzo': 3,\n",
    "    'abril':4,\n",
    "    'mayo': 5,\n",
    "    'junio': 6,\n",
    "    'julio': 7,\n",
    "    'agosto': 8,\n",
    "    'septiembre': 9,\n",
    "    'octubre': 10,\n",
    "    'noviembre': 11,\n",
    "    'diciembre': 12\n",
    "}\n",
    "\n",
    "mt_date_re = re.compile('(\\w{3})\\s([0-9]+)\\sde\\s(\\w+),\\s([0-9]{4})\\s([0-9]{2}):([0-9]{2})')\n",
    "def parse_dates(text_date):\n",
    "    match = mt_date_re.search(text_date)\n",
    "    if match:\n",
    "        day = int(match.group(2))\n",
    "        month = month_dict[match.group(3)] \n",
    "        year = int(match.group(4))\n",
    "        hour =int(match.group(5))\n",
    "        minute =int(match.group(6))\n",
    "        dt_str = \"%04d-%02d-%02d %02d:%02d\" % (year,month,day,hour,minute)\n",
    "        try:\n",
    "            return pd.to_datetime(dt_str)\n",
    "        except:\n",
    "            print(dt_str)\n",
    "\n",
    "mt_score = re.compile('([0-9]+)\\s*-\\s*([0-9]+)')\n",
    "def get_scores(raw_score):\n",
    "    match = mt_score.search(raw_score)\n",
    "    if match:\n",
    "        return int(match.group(1)), int(match.group(2))\n",
    "    return np.nan, np.nan\n",
    "    \n",
    "    \n",
    "date_time = matches_df['date'] + \" \" + matches_df[\"time\"]\n",
    "\n",
    "#matches_df['match_datetime']\n",
    "matches_df['match_datetime'] = date_time.apply(parse_dates)\n",
    "matches_df['home_score'],  matches_df['away_score'] = zip(*matches_df['result'].apply(get_scores))\n",
    "matches_df.tail()\n",
    "\n",
    "#del matches_df['date'], matches_df['time'], matches_df['result']\n",
    "\n",
    "#matches_df.info()\n",
    "matches_df.to_csv('mt-scraper/matches_processed.csv')\n",
    "no_date = matches_df[matches_df['match_datetime'].isnull()]\n",
    "no_date.head(11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "matches_processed_df = pd.read_csv('mt-scraper/matches_processed.csv', index_col=0, parse_dates=['match_datetime'])\n",
    "matches_processed_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "no_date = matches_processed_df[matches_processed_df['match_datetime'].isnull()]\n",
    "no_date.head(11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get details about each match (insane):\n",
    "_ = '''\n",
    "url = \"http://www.mediotiempo.com/partido/futbol/ligamx/%s/%s/ficha\"\n",
    "url_alt = \"http://www.mediotiempo.com/partido/futbol/liga-mx/%s/%s/ficha\"\n",
    "a = matches_processed_df[['home_team','away_team','match_datetime']].values\n",
    "for r in a[4300:4305]:\n",
    "    s = r[0] + \" vs \" + r[1]\n",
    "    _url = url % (slugify.slugify(s), pd.to_datetime(r[2]).strftime(\"%Y/%m/%d\"))\n",
    "    rq = requests.get(_url)\n",
    "    if rq.status_code != 200:\n",
    "        _url = url_alt % (slugify.slugify(s), pd.to_datetime(r[2]).strftime(\"%Y/%m/%d\"))\n",
    "        rq = requests.get(_url)\n",
    "        if rq.status_code != 200:\n",
    "            rq = None\n",
    "            _url = None\n",
    "    \n",
    "    if rq is not None:\n",
    "        print(_url)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_tournaments = tournament_df.index.get_level_values(0).unique().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Ofensiva\n",
    "of_url = \"http://www.mediotiempo.com/liga/futbol/ligamx/estadisticas/equipos/%s/ofensiva?tabla=mas-goleadores\"\n",
    "ofensiva_tables = {}\n",
    "for season in all_tournaments:\n",
    "    url = of_url % season\n",
    "    r = requests.get(url)\n",
    "    scrape_soup = BeautifulSoup(r.text, \"lxml\")\n",
    "    table = scrape_soup.find('div', {'class': 'table-containers'})\n",
    "    if table is not None:\n",
    "        table_body = table.find('div', {'class': 'scroll'}).find('tbody', {'class':'mt-table-body'})\n",
    "        rows = table_body.findAll('tr')\n",
    "        if len(rows) == 0:\n",
    "            continue\n",
    "        \n",
    "        gf = []\n",
    "        equipo = []\n",
    "        tt = []\n",
    "        tg = []\n",
    "        prec = []\n",
    "        g_c = []\n",
    "        ll = []\n",
    "        lla = []\n",
    "        fdl = []\n",
    "        \n",
    "        for row in rows:\n",
    "            all_tds = row.findAll('td')\n",
    "            equipo.append(all_tds[0].text.strip())\n",
    "            if len(all_tds) == 2:\n",
    "                gf.append(int(all_tds[1].text.strip()))\n",
    "                tt.append(np.nan)\n",
    "                tg.append(np.nan)\n",
    "                prec.append(np.nan)\n",
    "                g_c.append(np.nan)\n",
    "                ll.append(np.nan)\n",
    "                lla.append(np.nan)\n",
    "                fdl.append(np.nan)\n",
    "            else:\n",
    "                gf.append(int(all_tds[3].text.strip()))\n",
    "                tt.append(int(all_tds[1].text.strip()))\n",
    "                tg.append(int(all_tds[2].text.strip()))\n",
    "                prec.append(all_tds[4].text.strip())\n",
    "                g_c.append(float(all_tds[5].text.strip()))\n",
    "                ll.append(int(all_tds[6].text.strip()))\n",
    "                lla.append(int(all_tds[7].text.strip()))\n",
    "                fdl.append(int(all_tds[8].text.strip()))\n",
    "        ofensiva_tables[season] = pd.DataFrame({'Equipo': equipo, \n",
    "                                                'GF': gf,\n",
    "                                                'TT': tt,\n",
    "                                                'TG': tg,\n",
    "                                                'PREC': prec,\n",
    "                                                'G_C': g_c,\n",
    "                                                'LL': ll,\n",
    "                                                'LLA': lla,\n",
    "                                                'FDL': fdl})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Defensiva\n",
    "of_url = \"http://www.mediotiempo.com/liga/futbol/ligamx/estadisticas/equipos/%s/defensiva?tabla=menos-goleados\"\n",
    "defensiva_tables = {}\n",
    "for season in all_tournaments:\n",
    "    url = of_url % season\n",
    "    r = requests.get(url)\n",
    "    scrape_soup = BeautifulSoup(r.text, \"lxml\")\n",
    "    table = scrape_soup.find('div', {'class': 'table-containers'})\n",
    "    if table is not None:\n",
    "        table_body = table.find('div', {'class': 'scroll'}).find('tbody', {'class':'mt-table-body'})\n",
    "        rows = table_body.findAll('tr')\n",
    "        if len(rows) == 0:\n",
    "            continue\n",
    "        \n",
    "        gc = []\n",
    "        equipo = []\n",
    "        ttp = []\n",
    "        tgp = []\n",
    "        blq = []\n",
    "        tblq = []\n",
    "        cblq = []\n",
    "        pblq = []\n",
    "        _int = []\n",
    "        \n",
    "        for row in rows:\n",
    "            all_tds = row.findAll('td')\n",
    "            equipo.append(all_tds[0].text.strip())\n",
    "            if len(all_tds) == 2:\n",
    "                gc.append(int(all_tds[1].text.strip()))\n",
    "                ttp.append(np.nan)\n",
    "                tgp.append(np.nan)\n",
    "                blq.append(np.nan)\n",
    "                tblq.append(np.nan)\n",
    "                cblq.append(np.nan)\n",
    "                pblq.append(np.nan)\n",
    "                _int.append(np.nan)\n",
    "            else:\n",
    "                gc.append(int(all_tds[3].text.strip()))\n",
    "                ttp.append(int(all_tds[1].text.strip()))\n",
    "                tgp.append(int(all_tds[2].text.strip()))\n",
    "                blq.append(int(all_tds[4].text.strip()))\n",
    "                tblq.append(int(all_tds[5].text.strip()))\n",
    "                cblq.append(int(all_tds[6].text.strip()))\n",
    "                pblq.append(int(all_tds[7].text.strip()))\n",
    "                _int.append(int(all_tds[8].text.strip()))\n",
    "        defensiva_tables[season] = pd.DataFrame({'Equipo': equipo, \n",
    "                                                 'TTP': ttp,\n",
    "                                                 'TGP': tgp,\n",
    "                                                 'GC': gc,\n",
    "                                                 'BLQ': blq,\n",
    "                                                 'TBLQ': tblq,\n",
    "                                                 'CBLQ': cblq,\n",
    "                                                 'PBLQ': pblq,\n",
    "                                                 'INT': _int})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for season in all_tournaments:\n",
    "    if season in defensiva_tables and season in ofensiva_tables:\n",
    "        defensiva_tables[season].to_csv('mt-scraper/defensiva/' + season +'.csv')\n",
    "        ofensiva_tables[season].to_csv('mt-scraper/ofensiva/' + season +'.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}