Wikipedia_Scrape.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Wikipedia Scrape.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jianzhiw/Web-Scrape/blob/master/Wikipedia_Scrape.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a78jJlKX_o_e",
        "colab_type": "text"
      },
      "source": [
        "# Wikipedia Scrape #\n",
        "\n",
        "To scrape text from wikipedia, we are going to use [BeautifulSoup](https://pypi.org/project/BeautifulSoup/) library to extract all the elements with starts with \\<p\\>.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X6cnaU1s-D8a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from bs4 import BeautifulSoup\n",
        "import requests\n",
        "import re\n",
        "\n",
        "# change the URL here\n",
        "url = \"https://en.wikipedia.org/wiki/Chatbot\"\n",
        "# change the output file here\n",
        "output_file = \"chatbot.txt\"\n",
        "\n",
        "\n",
        "respond = requests.get(url)\n",
        "\n",
        "soup = BeautifulSoup(respond.text)\n",
        "l = soup.find_all('p')\n",
        "text = \"\"\n",
        "#f = open(\"chatbot.txt\", mode=\"a\")\n",
        "for item in l:\n",
        "  #f.write(item.text)\n",
        "  text += item.text\n",
        "\n",
        "# remove reference (eg: [1])\n",
        "text = re.sub(r'\\[\\d*\\]', \"\", text)\n",
        "\n",
        "# write files\n",
        "f = open(output_file, mode=\"w\")\n",
        "f.write(text)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}