Scrape_Genius.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Scrape Genius.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNjrvD7SqaJsZyRks+HoaeZ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/amfakh/eapanicobacoba/blob/master/Scrape_Genius.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dU4aiyBw-smJ",
        "colab_type": "text"
      },
      "source": [
        "Sauce:\n",
        "\n",
        "https://medium.com/analytics-vidhya/how-to-scrape-song-lyrics-a-gentle-python-tutorial-5b1d4ab351d2\n",
        "\n",
        "https://www.promptcloud.com/blog/scraping-song-lyrics-using-python-from-genius/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Ta0QGUq1byw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import urllib.request\n",
        "import urllib.parse\n",
        "import urllib.error\n",
        "import ssl\n",
        "import os\n",
        "import re\n",
        "import requests\n",
        "\n",
        "from urllib.request import Request, urlopen\n",
        "from bs4 import BeautifulSoup"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P414fELH1zUA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "GENIUS_API_TOKEN='PXpvxuPl_XyMM02ly-gyw8bd7tyP1avUQLZVsR4ooRmhfbTDOWG8mEHez0DGygjp'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FUOGeQbd1lWY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Get artist object from Genius API\n",
        "def request_artist_info(artist_name, page):\n",
        "    base_url = 'https://api.genius.com'\n",
        "    headers = {'Authorization': 'Bearer ' + GENIUS_API_TOKEN}\n",
        "    search_url = base_url + '/search?per_page=10&page=' + str(page)\n",
        "    data = {'q': artist_name}\n",
        "    response = requests.get(search_url, data=data, headers=headers)\n",
        "    return response# Get Genius.com song url's from artist object\n",
        "    \n",
        "# DEMO\n",
        "# request_song_url('JKT48', 10)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JiiGMY_HMxxw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def request_song_url(artist_name, song_cap):\n",
        "    page = 1\n",
        "    songs = []\n",
        "    \n",
        "    while True:\n",
        "        response = request_artist_info(artist_name, page)\n",
        "        json = response.json()        # Collect up to song_cap song objects from artist\n",
        "        song_info = []\n",
        "        for hit in json['response']['hits']:\n",
        "            if artist_name.lower() in hit['result']['primary_artist']['name'].lower():\n",
        "                song_info.append(hit)\n",
        "    \n",
        "        # Collect song URL's from song objects\n",
        "        for song in song_info:\n",
        "            if (len(songs) < song_cap):\n",
        "                url = song['result']['url']\n",
        "                songs.append(url)\n",
        "            \n",
        "        if (len(songs) == song_cap):\n",
        "            break\n",
        "        else:\n",
        "            page += 1\n",
        "        \n",
        "    print('Preparing to scrape {} songs by {}'.format(len(songs), artist_name))\n",
        "    return songs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KXKfL4V27qsl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# For ignoring SSL certificate errors\n",
        "ctx = ssl.create_default_context()\n",
        "ctx.check_hostname = False\n",
        "ctx.verify_mode = ssl.CERT_NONE"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iARBlm3f7Zgc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "song_json = {}\n",
        "song_json[\"Primary Artist\"] = []\n",
        "song_json[\"Song Title\"] = []\n",
        "song_json[\"Lyrics\"] = []\n",
        "\n",
        "\n",
        "def scrape_from_link(url):\n",
        "\n",
        "    req = Request(url, headers = { 'User-Agent' : 'Mozilla/5.0' })\n",
        "    webpage = urlopen(req).read()\n",
        "    soup = BeautifulSoup(webpage, 'html.parser')\n",
        "    html = soup.prettify('utf-8')\n",
        "\n",
        "    for div in soup.findAll('h1', attrs = {'class': 'header_with_cover_art-primary_info-title'}):\n",
        "        song_json[\"Song Title\"].append(div.text.strip());\n",
        "        print(\"    - Scrapped: \" + song_json[\"Song Title\"][-1])\n",
        "\n",
        "    for div in soup.findAll('a', attrs = {'class': 'header_with_cover_art-primary_info-primary_artist'}):\n",
        "        song_json[\"Primary Artist\"].append(div.text.strip());    \n",
        "\n",
        "    for div in soup.findAll('div', attrs = {'class': 'lyrics'}):\n",
        "        song_json[\"Lyrics\"].append(div.text.strip().split(\"\\n\"));\n",
        "\n",
        "    # print(song_json[\"Song Title\"])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nQEAUoaz8VnR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# scrape_from_link('https://genius.com/Jkt48-only-today-lyrics')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xxEpESIZGsfs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# song_json"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-x-lVxMW9T98",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "bd153edd-bbf9-4807-93d2-0551c1df74fb"
      },
      "source": [
        "import json\n",
        "\n",
        "def write_lyrics_to_file(artist_name, song_count):\n",
        "    # song_json[\"Primary Artist\"].clear()\n",
        "    # song_json[\"Song Title\"].clear()\n",
        "    # song_json[\"Lyrics\"].clear()\n",
        "\n",
        "    with open(artist_name.lower() + '.json', 'w') as outfile:\n",
        "        urls = request_song_url(artist_name, song_count)\n",
        "        for url in urls:\n",
        "            # print(url)\n",
        "            scrape_from_link(url)\n",
        "      \n",
        "        json.dump(song_json, outfile, indent = 4, ensure_ascii = False)\n",
        "        print(\"\\nFinished Succesfully\")\n",
        "  \n",
        "# DEMO  \n",
        "write_lyrics_to_file('jkt48', 100)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Preparing to scrape 100 songs by jkt48\n",
            "    - Scrapped: Heavy Rotation\n",
            "    - Scrapped: Fortune Cookie Yang Mencinta (Fortune Cookie In Love)\n",
            "    - Scrapped: RIVER\n",
            "    - Scrapped: Pesawat Kertas 365 Hari\n",
            "    - Scrapped: Everyday, Kachuusha\n",
            "    - Scrapped: Dirimu Melody (Kimi Wa Melody)\n",
            "    - Scrapped: Only Today\n",
            "    - Scrapped: Baby! Baby! Baby!\n",
            "    - Scrapped: First Rabbit\n",
            "    - Scrapped: Sebagian Besar Kenangan (Omoide No Hotondo)\n",
            "    - Scrapped: Rapsodi\n",
            "    - Scrapped: Seesaw Game Penuh Air Mata (Namida no Seesaw Game)\n",
            "    - Scrapped: Kereta Kedewasaan (Otona Ressha)\n",
            "    - Scrapped: So Long!\n",
            "    - Scrapped: Bersama Kamu, Pelangi Dan Mentari (Kimi To Niji To Taiyou To)\n",
            "    - Scrapped: Beginner\n",
            "    - Scrapped: High Tension\n",
            "    - Scrapped: Kita Pernah Di Sini (Koko Ni Itakoto)\n",
            "    - Scrapped: Refrain Penuh Harapan (Kibouteki Refrain)\n",
            "    - Scrapped: UZA\n",
            "    - Scrapped: Sweet & Bitter\n",
            "    - Scrapped: BINGO!\n",
            "    - Scrapped: Di Bawah Langit Berwarna Sakura\n",
            "    - Scrapped: Siapa yang Mempertemukan Kita Berdua? (Dare Ga Futari Wo Deawaseta No Ka?)\n",
            "    - Scrapped: Wahai Kesatria (Tsuyokimono Yo)\n",
            "    - Scrapped: Cahaya Panjang (Nagai Hikari)\n",
            "    - Scrapped: Menanti\n",
            "    - Scrapped: Relax!\n",
            "    - Scrapped: Kamulah Yang Memberitahu Aku (Kimi ga Oshiete Kureta)\n",
            "    - Scrapped: Indahnya Senyum Manismu DST. (Suzukake Nancharaa)\n",
            "    - Scrapped: Indahnya Senyum Manismu DST. (Suzukake Nancharaa) (English Version)\n",
            "    - Scrapped: Cinta Pertama di Jam 7 Lewat 12 (7ji 12fun no Hatsukoi)\n",
            "    - Scrapped: Bukan Alpukat... (Avogado Janeeshi)\n",
            "    - Scrapped: Mimpi Yang Hina (Gesu Na Yume)\n",
            "    - Scrapped: Tunas Di Balik Seragam (Seifuku No Me)\n",
            "    - Scrapped: Cowok Durian (Durian Shounen)\n",
            "    - Scrapped: Wasshoi J!\n",
            "    - Scrapped: Back Hip Circle (Saka Agari)\n",
            "    - Scrapped: Kenyataan yang Ternoda (Kegarete Iru Shinjitsu)\n",
            "    - Scrapped: Betapa Indahnya Waktu Tempat Kita Lahir (Nante Suteki na Sekai ni Umareta no Darou)\n",
            "    - Scrapped: Musim Yang Selanjutnya (Tsugi No Season)\n",
            "    - Scrapped: Cinta Pertama Butterfly (Hatsukoi Butterfly)\n",
            "    - Scrapped: Majisuka Rock N’ Roll\n",
            "    - Scrapped: Faint\n",
            "    - Scrapped: Di Sini Rhodes, Lompatlah! (Koko Ga Rhodes Da, Koko De Tobe!)\n",
            "    - Scrapped: Sampai Musim Semi Tiba (Haru ga Kuru Made)\n",
            "    - Scrapped: Kamonegix\n",
            "    - Scrapped: Rasa Sayang Yang Dulu Aku Remehkan (Keibetsu Shiteita Aijou)\n",
            "    - Scrapped: Aishiteraburu\n",
            "    - Scrapped: Sekarang Sedang Jatuh Cinta (Tadaima Renaichuu)\n",
            "    - Scrapped: Virginity\n",
            "    - Scrapped: After Rain\n",
            "    - Scrapped: Birth\n",
            "    - Scrapped: Sekarang Para Para (Imapara)\n",
            "    - Scrapped: Fajar Sang Idola (Idol No Yoake)\n",
            "    - Scrapped: Sedikit Saja I Love You! (Hikaeme I Love You!)\n",
            "    - Scrapped: Bukan Salah Air Mata (Namida No Sei Janai)\n",
            "    - Scrapped: Pundak Kanan (Migikata)\n",
            "    - Scrapped: Kalender Yang Dipercepat (Hayaokuri Calendar)\n",
            "    - Scrapped: Boneka Teddy Bear (Kuma no Nuigurumi)\n",
            "    - Scrapped: Laptime Masa Remaja (Seishun no Laptime)\n",
            "    - Scrapped: Kebun Binatang Saat Hujan (Ame no Doubutsuen)\n",
            "    - Scrapped: Teacher Teacher\n",
            "    - Scrapped: Kesucian Hati Hingga Umur 19 Tahun (Junjou U-19)\n",
            "    - Scrapped: So Long! (English Version)\n",
            "    - Scrapped: Hanya Lihat Ke Depan (Mae Shika Mukanee)\n",
            "    - Scrapped: Pulang Kampung (Kikyou)\n",
            "    - Scrapped: Baby! Baby! Baby! (Passionate Prayer Version)\n",
            "    - Scrapped: Rasanya Malas (Darui Kanji)\n",
            "    - Scrapped: Seragam Ini Sangat Mengganggu (Seifuku ga Jama wo Suru)\n",
            "    - Scrapped: Luar Biasa (Saikou Kayo)\n",
            "    - Scrapped: Sakura Kita Makan Bersama (Sakura, Minna De Tabeta)\n",
            "    - Scrapped: LOVE CHASE\n",
            "    - Scrapped: Cinta Yang Tulus Crescendo (Junai no Crescendo)\n",
            "    - Scrapped: Mr. Kissman\n",
            "    - Scrapped: Musim Panas Sounds Good! (Manatsu No Sounds Good!)\n",
            "    - Scrapped: Sakura No Hanabiratachi (Kelopak-Kelopak Bunga Sakura)\n",
            "    - Scrapped: Viva! Hurricane\n",
            "    - Scrapped: Scrap and Build\n",
            "    - Scrapped: Value Milikku Saja (Boku Dake No Value)\n",
            "    - Scrapped: Sambil Menggandeng Erat Tanganku (Te Wo Tsunaginagara)\n",
            "    - Scrapped: Gonna Jump\n",
            "    - Scrapped: Sungai Impian (Yume No Kawa)\n",
            "    - Scrapped: Karena Kau Ada Untukku (Anata ga Ite Kureta Kara)\n",
            "    - Scrapped: Salah Dengar Rock (Soramimi Rock)\n",
            "    - Scrapped: Bel Sekolah Adalah Love Song  (Chime wa Love Song)\n",
            "    - Scrapped: Kapasitas Ikan Migrasi (Kaiyuugyou no Capacity)\n",
            "    - Scrapped: Summer Love Sounds Good (English Version)\n",
            "    - Scrapped: Di Tempat Yang Jauh pun (Tooku ni Ite mo)\n",
            "    - Scrapped: Adrenalin Masa Puber (Shishunki no Adrenaline)\n",
            "    - Scrapped: Luar Biasa (Saikou Kayo) (English Version)\n",
            "    - Scrapped: Karena Kusuka Dirimu (Kimi No Koto Ga Suki Dakara)\n",
            "    - Scrapped: Hubungan Kau Dan Aku (Kimi To Boku No Kankei)\n",
            "    - Scrapped: Refrain Full of Hope (English Version)\n",
            "    - Scrapped: HA!\n",
            "    - Scrapped: Mammoth\n",
            "    - Scrapped: Bel Terakhir Berbunyi (Saishuu Bell Ga Naru)\n",
            "    - Scrapped: Dewi Theater (Theater No Megami)\n",
            "    - Scrapped: Natasha yang Ku Cinta (Itoshiki Natasha)\n",
            "    - Scrapped: Berandalan di Surga (Tengoku Yarou)\n",
            "\n",
            "Finished Succesfully\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5kSNTSe5-279",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rAuVgjYeSP3q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_json(r'jkt48.json')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LCAx6zGtSlJu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ff4a15a9-57cd-49cc-9ca4-67affc5ca290"
      },
      "source": [
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Primary Artist</th>\n",
              "      <th>Song Title</th>\n",
              "      <th>Lyrics</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>JKT48</td>\n",
              "      <td>Heavy Rotation</td>\n",
              "      <td>[[Intro], One! Two! Three! Four, , [Chorus 1],...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>JKT48</td>\n",
              "      <td>Fortune Cookie Yang Mencinta (Fortune Cookie I...</td>\n",
              "      <td>[[Verse 1], Walaupun diri ini menyukaimu, Kamu...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>JKT48</td>\n",
              "      <td>RIVER</td>\n",
              "      <td>[[Intro], JKT... 48!, , [Refrain], Majulah ke ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>JKT48</td>\n",
              "      <td>Pesawat Kertas 365 Hari</td>\n",
              "      <td>[[Verse 1], Kutatap langit di pagi hari, Kuawa...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>JKT48</td>\n",
              "      <td>Everyday, Kachuusha</td>\n",
              "      <td>[[Verse 1], Saat mentari mulai bersinar, Lebih...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95</th>\n",
              "      <td>JKT48</td>\n",
              "      <td>Mammoth</td>\n",
              "      <td>[[Intro], We're mammoth!, We’re mammoth!, , [V...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>JKT48</td>\n",
              "      <td>Bel Terakhir Berbunyi (Saishuu Bell Ga Naru)</td>\n",
              "      <td>[[Verse 1], Larut malam peron stasiun kereta, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>JKT48</td>\n",
              "      <td>Dewi Theater (Theater No Megami)</td>\n",
              "      <td>[[Verse 1], Aku dengar dari orang-orang, Bahwa...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>JKT48</td>\n",
              "      <td>Natasha yang Ku Cinta (Itoshiki Natasha)</td>\n",
              "      <td>[[Verse 1], Ada seorang pria diam-diam menyeli...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>JKT48</td>\n",
              "      <td>Berandalan di Surga (Tengoku Yarou)</td>\n",
              "      <td>[[Verse 1], Longgarkanlah sekrup di kepalamu, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>100 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   Primary Artist  ...                                             Lyrics\n",
              "0           JKT48  ...  [[Intro], One! Two! Three! Four, , [Chorus 1],...\n",
              "1           JKT48  ...  [[Verse 1], Walaupun diri ini menyukaimu, Kamu...\n",
              "2           JKT48  ...  [[Intro], JKT... 48!, , [Refrain], Majulah ke ...\n",
              "3           JKT48  ...  [[Verse 1], Kutatap langit di pagi hari, Kuawa...\n",
              "4           JKT48  ...  [[Verse 1], Saat mentari mulai bersinar, Lebih...\n",
              "..            ...  ...                                                ...\n",
              "95          JKT48  ...  [[Intro], We're mammoth!, We’re mammoth!, , [V...\n",
              "96          JKT48  ...  [[Verse 1], Larut malam peron stasiun kereta, ...\n",
              "97          JKT48  ...  [[Verse 1], Aku dengar dari orang-orang, Bahwa...\n",
              "98          JKT48  ...  [[Verse 1], Ada seorang pria diam-diam menyeli...\n",
              "99          JKT48  ...  [[Verse 1], Longgarkanlah sekrup di kepalamu, ...\n",
              "\n",
              "[100 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h-GbaU7eSmGq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}