main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### INTRODUKTION (15 min, 17:00 - 17:15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ALT DET BASALE (30 min, 17:15 - 17:45)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Python Scripts / IPython Notebooks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# COMMENTS COMMENTS COMMENTS...\n",
    "\n",
    "# ALWAYS NICE TO WRITE COMMENTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Hello')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('''\n",
    "A picture of a dog...\n",
    " 0____\n",
    "  ||||\n",
    "''')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assignment og variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = 42\n",
    "\n",
    "b = [1, 2, 3, 4, 5, 5]\n",
    "\n",
    "c = range(3)\n",
    "\n",
    "d = 'PROSA'\n",
    "\n",
    "e = ['Peter', 'Rosa', 'Oscar', 'Sebastian', 'Adam']\n",
    "\n",
    "f = ['D', 4, 'M', e]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(a)\n",
    "print(c)\n",
    "print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f[1] = 'Just changed this'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fejl håndtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "1/0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(prosa)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Udvælge elementer i lister. Indeks og slice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(b[1])\n",
    "\n",
    "print(b[-2])\n",
    "\n",
    "print(b[2:4])\n",
    "\n",
    "print(b[-3:-1])\n",
    "\n",
    "print(f[3][2:4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f[5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dic = {\n",
    "    'integer': 42,\n",
    "    'float': 23.23,\n",
    "    'string': 'Python rules!',\n",
    "    'list': ['nested', 'list'],\n",
    "    'dictionary': {'inception': 'dictionary'}\n",
    "}\n",
    "\n",
    "person1 = {\n",
    "    'navn': 'Jess Alfredsen',\n",
    "    'vægt': 100,\n",
    "    'iq': 'høj'\n",
    "}\n",
    "person2 = {\n",
    "    'navn': 'Tobias Slott',\n",
    "    'vægt': 35,\n",
    "    'iq': 'lavere'\n",
    "}\n",
    "people = [person1, person2]\n",
    "print(people)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dic['float'])\n",
    "print(dic['dictionary'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dic['float'] = 12.34"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dic['ukendt key']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "names1 = set(['John', 'Jane', 'Jack', 'Janice'])\n",
    "\n",
    "print(names1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "names1.add('Tobias')\n",
    "\n",
    "print(names1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "names1.add('Tobias')\n",
    "\n",
    "print(names1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "names2 = set(['Anders', 'Line', 'Martin', 'Amalie', 'John', 'Jane'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(names1.union(names2))\n",
    "\n",
    "print(names1.intersection(names2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_____________________\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx in c:\n",
    "    print(e[idx])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For loop og dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = []\n",
    "for person in people:\n",
    "    weights.append(person['vægt'])\n",
    "print(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key in dic.keys():\n",
    "    print(dic[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for val in dic.values():\n",
    "    print(val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### List comprehension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = [number**2 for number in range(21)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = [number for number in x if number % 2 == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_letters = [word[0] for word in e]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(first_letters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Funktioner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(y))\n",
    "print(sum(y))\n",
    "print(max(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "z = 12.3456789\n",
    "\n",
    "print(round(z, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(10):\n",
    "    print(i)\n",
    "    if i > 5:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(names1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Funktioner vi selv laver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def broken_calculator_addition(x, y):\n",
    "    return x + y + 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "broken_calculator_addition(2, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_result(x, y):\n",
    "    res = broken_calculator_addition(x, y)\n",
    "    s = 'When you add %d and %d, you get %d' % (x, y, res)\n",
    "    print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_result(25, 15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Person:\n",
    "    \n",
    "    def __init__(self, first_name, last_name, age):\n",
    "        self.first_name = first_name\n",
    "        self.last_name = last_name\n",
    "        self.age = age\n",
    "        self.skills = []\n",
    "        self.workplace = None\n",
    "    \n",
    "    def add_skill(self, skill):\n",
    "        self.skills.append(skill)\n",
    "        \n",
    "    def set_workplace(self, workplace):\n",
    "        self.workplace = workplace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opret en person\n",
    "person1 = Person(first_name='Monty', last_name='Python', age=24)\n",
    "\n",
    "print(person1.first_name)\n",
    "print(person1.workplace)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opdater et object\n",
    "person1.set_workplace('Prosa')\n",
    "person1.add_skill('Knitting')\n",
    "\n",
    "print(person1.workplace)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "person2 = Person(first_name='Jens', last_name='Jensen', age=49)\n",
    "person3 = Person(first_name='Henning', last_name='Havørn', age=35)\n",
    "\n",
    "persons = [person1, person2, person3]\n",
    "\n",
    "for person in persons:\n",
    "    person.add_skill('Python programming')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(person1.skills)\n",
    "print(person2.skills)\n",
    "print(person3.skills)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bruge pakker fra Python community / Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "\n",
    "print(datetime.date.today())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime as dt\n",
    "\n",
    "print(dt.date.today())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import date\n",
    "\n",
    "print(date.today())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import hello\n",
    "\n",
    "hello.hello_world()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from termcolor import cprint \n",
    "from pyfiglet import figlet_format\n",
    "\n",
    "the_dog = '''the dog...\n",
    "      0____       \n",
    "         | | | |       '''\n",
    "cprint(figlet_format(the_dog, font='starwars'), 'green', 'on_grey', attrs=['bold'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import braces"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dataframes med Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "temp = {'Name': ['Monty Python', 'Jens Jensen', 'Henning Havørn'], 'Age': [24, 49, 35]}\n",
    "\n",
    "df1 = pd.DataFrame(data=temp)\n",
    "\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1['Name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.iloc[0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1[df1['Age'] > 30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1['Age (squared)'] = df1['Age']**2\n",
    "\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = {'Name': ['Monty Python', 'Jens Jensen', 'Henning Havørn', 'Martin Højland'],\n",
    "        'Is funny': [True, False, True, False]}\n",
    "\n",
    "df2 = pd.DataFrame(data=temp)\n",
    "\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = pd.merge(df1, df2, on='Name', how='right')\n",
    "\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3[df3['Is funny'] == True]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Vektorer og matricer med Numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "x = np.array([1, 2, 3])\n",
    "y = np.array([4, 5, 6])\n",
    "z = np.array([7, 8, 9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(x + y + z)\n",
    "\n",
    "print(x * 10)\n",
    "\n",
    "print(x**2)\n",
    "\n",
    "print(x * y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([x, y, z])\n",
    "\n",
    "print(X)\n",
    "\n",
    "print(X.flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Matrix multiplication\n",
    "np.matmul(X, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mu = 10\n",
    "sigma = 2\n",
    "\n",
    "# Let's simulate some data\n",
    "sample = np.random.normal(loc=mu, scale=sigma, size=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(sample)\n",
    "print(np.sort(sample))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Udvælge elementer i vektor med index og booleans\n",
    "print(np.argwhere(sample > mu))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample[np.argwhere(sample > mu)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ØVELSER: ALT DET BASALE (15 min, 17:45 - 18:00)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SPISEPAUSE (30 min, 18:00 - 18:30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PYTHON STACKEN (15 min, 18:30-18.45)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Databaser\n",
    "\n",
    "|Database|library|\n",
    "|:---|:---|\n",
    "|PostgreSQL|`psycopg2`|\n",
    "|Elasticsearch|`elasticsearch`|\n",
    "|MongoDB|`pymongo`|\n",
    "|neo4j|`py2neo`|\n",
    "|Microsoft SQL|`pyodbc` eller `pymssql`|"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## API'er\n",
    "\n",
    "* Kalde api'er: `requests`, `urllib`\n",
    "* Bygge API'er selv: `flask`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "pip install requests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scrape data fra hjemmesider"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "url = 'https://www.prosa.dk/arrangementer/'\n",
    "response = requests.get(url)\n",
    "\n",
    "soup = BeautifulSoup(response.text, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "events = []\n",
    "\n",
    "for link in soup.find_all('span'):\n",
    "    if link.text:\n",
    "        events.append(link.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for event in events:\n",
    "    print(event)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DATA ANALYSE (30 min, 18:45-19:15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DATA INDSAMLING"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Indlæsning af data (txt, csv, Stata, SAS, SPSS, osv.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_demo = pd.read_csv('Data/df_demography.csv', sep=',', dtype={'cpr': str})\n",
    "df_code = pd.read_csv('Data/df_coding_skills.csv', sep=',', dtype={'cpr': str})\n",
    "df_pers = pd.read_csv('Data/df_personal_characteristic.csv', sep=',', dtype={'cpr': str})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_code.to_pickle('Data/df_coding_skills.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_code = pd.read_pickle('Data/df_coding_skills.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge data frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(df_demo, df_code, on='cpr')\n",
    "df = pd.merge(df, df_pers, on='cpr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.iloc[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['gender'] = df['cpr'].str[9].apply(int)\n",
    "df['gender'] = np.where(df['gender'] % 2 == 0, 'woman', 'man')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['year'] = df['cpr'].str[4:6]\n",
    "df['year'] = '19' + df['year']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['month'] = df['cpr'].str[2:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['day'] = df['cpr'].str[0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['birthday'] = pd.to_datetime(df['day'] + df['month'] + df['year'], format='%d%m%Y')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "now = pd.Timestamp(dt.datetime.now())\n",
    "df['age'] = (now - df['birthday'])\n",
    "df['age'] = df['age'].astype('<m8[Y]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.iloc[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lineær regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = df['weight'].values\n",
    "y = df['height'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import stats\n",
    "\n",
    "# stats modulet i scipy har mange andre gode funktioner som bruges i forbindelse med statistik og analyse - hint hint ;)\n",
    "\n",
    "slope, intercept, r_value, p_value, std_err = stats.linregress(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('slope', slope, 'intercept', intercept, 'r_value', r_value, 'p_value', p_value, 'std_err', std_err, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('For hver kg du stiger i vægt, stiger du typisk {:.1f} cm i højde'.format(slope * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ØVELSE (30 min, 19:00 - 19:30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AVANCERET DATA ANALYSE OG MACHINE LEARNING (30 MIN, 19:30-20:00)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Supervised learning (Logistisk regression med Scikit-learn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df[['weight', 'height']].values\n",
    "y = df['gender'].values\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_reg = LogisticRegression()\n",
    "log_reg.fit(X_train, y_train)\n",
    "\n",
    "print(log_reg.coef_)\n",
    "print(log_reg.intercept_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = log_reg.predict(X_test)\n",
    "\n",
    "print(confusion_matrix(y_test, y_pred))\n",
    "print(accuracy_score(y_test, y_pred))\n",
    "print(sum(y_test == y_pred) / len(y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unsupervised learning (Kmeans med Scikit-learn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "from sklearn.preprocessing import scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_scale = scale(X, axis=0)\n",
    "X_train_scale = scale(X_train)\n",
    "X_test_scale = scale(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kmeans = KMeans(n_clusters=2)\n",
    "kmeans.fit(X_train_scale)\n",
    "\n",
    "print(kmeans.cluster_centers_[0])\n",
    "print(kmeans.cluster_centers_[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cluster = pd.DataFrame()\n",
    "df_cluster['height'] = X_train[:, 0]\n",
    "df_cluster['weight'] = X_train[:, 1]\n",
    "df_cluster['cluster'] = kmeans.labels_\n",
    "\n",
    "print(df_cluster[df_cluster['cluster'] == 0].mean())\n",
    "\n",
    "print(df_cluster[df_cluster['cluster'] == 1].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Kunne de to cluster være børn/voksne? Mænd/kvinder? Hunde/Heste?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster_pred = kmeans.predict(X_test_scale)\n",
    "print(cluster_pred)\n",
    "\n",
    "cluster_labels = y_test == 'woman' #man/woman\n",
    "print(cluster_labels)\n",
    "\n",
    "print(accuracy_score(cluster_labels, cluster_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DATA VISUALISERING (15 min, 20:00 - 20:15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* seaborn\n",
    "* ggplot\n",
    "* elaborate?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Visualisering med Matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 1500\n",
    "df_gender = df[df['gender'] == 'man']\n",
    "\n",
    "heights = df_gender['height'].values\n",
    "heights = np.random.choice(heights, size=N)\n",
    "\n",
    "weights = df_gender['weight'].values\n",
    "weights = np.random.choice(weights, size=N)\n",
    "\n",
    "slope, intercept, r_value, p_value, std_err = stats.linregress(weights, heights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.figure(figsize=(7,7))\n",
    "plt.plot(weights, heights, 'b.', weights, slope * weights + intercept, 'r-')\n",
    "plt.axis([40, 110, 1.4, 2.2])\n",
    "plt.xlabel('weight')\n",
    "plt.ylabel('height')\n",
    "plt.title('Hvordan er data fordelt?')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Scatter plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kmeans = KMeans(n_clusters=2)\n",
    "kmeans.fit(X_scale)\n",
    "\n",
    "df['cluster'] = kmeans.labels_\n",
    "\n",
    "#df.plot.scatter(x='height', y='weight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = X_scale[:, 0]\n",
    "heights = X_scale[:, 1]\n",
    "\n",
    "color_true = np.where(df['gender'] == 'man', 'b', 'r')\n",
    "color_pred = np.where(df['cluster'] == 1, 'c', 'm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(14,7))\n",
    "\n",
    "plt.scatter(weights, heights, c=color_true, marker='.')\n",
    "plt.xlabel('weight (scaled)')\n",
    "plt.ylabel('height (scaled)')\n",
    "plt.title('Observed data')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(14,7))\n",
    "\n",
    "plt.scatter(weights, heights, c=color_pred, marker='.')\n",
    "plt.plot(kmeans.cluster_centers_[:,0], kmeans.cluster_centers_[:,1], 'kX', markersize=10)\n",
    "plt.xlabel('weight (scaled)')\n",
    "plt.ylabel('height (scaled)')\n",
    "plt.title('K-means clustering')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Bar plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gns højde og vægt for mænd og kvinder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = 25\n",
    "\n",
    "plt.hist(df[df['gender'] == 'man']['weight'], bins=bins, alpha=0.75, facecolor='b')\n",
    "plt.hist(df[df['gender'] == 'woman']['weight'], bins=bins, alpha=0.75, facecolor='r')\n",
    "plt.xlabel('weight')\n",
    "plt.ylabel('frequency')\n",
    "plt.title('Histograms for weights')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# [SHOWCASE ET LÆKKERT PLOT MED SEARBORN]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.year = df.year.apply(lambda x: int(x))\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "plt.figure(figsize=(16,10))\n",
    "cmap = sns.cubehelix_palette(dark=.3, light=.8, as_cmap=True)\n",
    "ax = sns.scatterplot(x=\"weight\", y=\"height\",\n",
    "                     hue=\"age\", size=\"salary\",\n",
    "                     palette=\"Set2\",\n",
    "                     # palette=cmap,\n",
    "                     sizes=(10,800),\n",
    "                     data=df.sample(200))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TWITTER DATA (15 MIN, 20:15-20:30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import twitter\n",
    "\n",
    "CONSUMER_KEY = 'GPAzAQ5XsSFrsSUNorLQ12sN8'\n",
    "CONSUMER_SECRET = 'TJ0du2yXVZvQd65ZBLUpZMuvRzNqgNe6IriTtIJ8iq4LQC23oY'\n",
    "OAUTH_TOKEN = '770960920939986945-N9XOcS38M6p90Or5ckEWcQpw5vYqHTD'\n",
    "OAUTH_TOKEN_SECRET = 'zFUGw1EdGSmCqrwkFuhE5HNLekUOGNzQpeLlEyfxS663t'\n",
    "\n",
    "auth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET, CONSUMER_KEY, CONSUMER_SECRET)\n",
    "\n",
    "twitter_api = twitter.Twitter(domain='api.twitter.com', api_version='1.1', auth=auth)\n",
    "\n",
    "# Print twitter_api to test if there is an authorized twitter_api as expected\n",
    "print(twitter_api)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The Yahoo! Where On Earth ID for the entire world is 1\n",
    "# The Yahoo! Where On Earth ID for the US is 23424977\n",
    "# The Yahoo! Where On Earth ID for Denmark is 23424796\n",
    "WORLD_WOE_ID = 1 \n",
    "US_WOE_ID = 23424977\n",
    "DK_WOE_ID = 23424796\n",
    "\n",
    "# Find trends for each WOEID\n",
    "world_trends = twitter_api.trends.place(_id=WORLD_WOE_ID)\n",
    "us_trends = twitter_api.trends.place(_id=US_WOE_ID)\n",
    "dk_trends = twitter_api.trends.place(_id=DK_WOE_ID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Structure trends\n",
    "world_trends_set = set([trend['name'] for trend in world_trends[0]['trends']])\n",
    "us_trends_set = set([trend['name'] for trend in us_trends[0]['trends']])\n",
    "dk_trends_set = set([trend['name'] for trend in dk_trends[0]['trends']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(us_trends_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use Twittersearch\n",
    "\n",
    "q = '#dkpol'\n",
    "count = 100\n",
    "\n",
    "search_results = twitter_api.search.tweets(q=q, count=count)\n",
    "statuses = search_results['statuses']\n",
    "\n",
    "num_tweets = len(statuses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract tweet text and find number of retweets.\n",
    "\n",
    "# Extract tweet texts using list comprehension\n",
    "tweet_texts = [statuses[i].get(\"text\") for i in range(num_tweets)]\n",
    "\n",
    "# Extract number of retweets texts using list comprehension\n",
    "num_retweets = [statuses[i].get(\"retweet_count\") for i in range(num_tweets)]\n",
    "\n",
    "# Find most popular tweet by finding maximum number of retweets in list \"num_retweets\"\n",
    "max_retweets = np.argmax(np.array(num_retweets))\n",
    "max_indices = np.array(num_retweets).argsort()[-5:][::-1]\n",
    "\n",
    "# Print text of most retweeted/most popular tweet\n",
    "#print(tweet_texts[max_retweets])\n",
    "\n",
    "print(np.array(tweet_texts)[max_indices])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Python developer community\n",
    "__Every problem you will come across has already been solved!__\n",
    "\n",
    "* Google -> http://bfy.tw/2I5U\n",
    "* Stack Overflow\n",
    "* [Codecademy](https://www.codecademy.com/catalog/language/python)\n",
    "* [CodeCombat](https://codecombat.com/)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ØVELSE (15 min, 20:30 - 20:45)\n",
    "\n",
    "Kig evt. på exercises_solution.ipynb og stil spørgsmål"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SPØRGSMÅL / BUFFERTID (15 min, 20:45 - 21:00)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://surveys.enalyzer.com?pid=gn5rb8f8\n",
    "\n",
    "https://bit.ly/303QMac"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}