Facebook_ad_api_collector.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Facebook ad api collector",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMYKlL0DyolueAcgly5OWXc",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/qut-dmrc/fbadsearch/blob/main/Facebook_ad_api_collector.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CmgcAtuJLTr5"
      },
      "source": [
        "# First, set up authorisation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IBvwOp7ONwHu"
      },
      "source": [
        "from google.colab import auth\r\n",
        "auth.authenticate_user()\r\n",
        "\r\n",
        "# fetch the python api wrapper from github\r\n",
        "!pip3 install -e git+https://github.com/qut-dmrc/fbadsearch.git#egg=fbadsearch"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hD9x52kKn07b"
      },
      "source": [
        "# Restart the runtime before proceeding, and run the first cell again."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AyTgrS-LVqba"
      },
      "source": [
        "# restart the runtime before this step here\r\n",
        "from fbadsearch import FacebookAdSearch, save_data, fetch_ads, load_to_bq"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "rQqNecbPLX78"
      },
      "source": [
        "#@title If you already have a long-lived token, enter it here:\r\n",
        "LONG_LIVED_ACCESS_TOKEN = '' #@param {type: 'string'}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yxmxukec1_26"
      },
      "source": [
        "#@title Otherwise, get your access token here https://developers.facebook.com/tools/explorer/297028494528207/\r\n",
        "ACCESS_TOKEN = ''  #@param {type: 'string'}\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MjzkKeyag9LK"
      },
      "source": [
        "#@title GCS path and BigQuery table to save to:\r\n",
        "GC_PROJECT = '' #@param {type: 'string'}\r\n",
        "GCS_BUCKET = ''#@param {type: 'string'}\r\n",
        "GCS_DIR = \"\" #@param {type: 'string'}\r\n",
        "BQ_TABLE = '' #@param {type: 'string'}\r\n",
        "\r\n",
        "import datetime\r\n",
        "timestamp = datetime.datetime.utcnow().isoformat()\r\n",
        "GCS_DIR = f'{GCS_DIR}/{timestamp}'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w-Lb3lOaKR4p"
      },
      "source": [
        "if not LONG_LIVED_ACCESS_TOKEN:\r\n",
        "  # Get long-lived token\r\n",
        "  url = \"https://graph.facebook.com/v9.0/oauth/access_token\"\r\n",
        "  params = {'grant_type': 'fb_exchange_token',          \r\n",
        "            'client_id': 'YOUR_CLIENT_ID',\r\n",
        "            'client_secret': 'YOUR_CLIENT_SECRET',\r\n",
        "            'fb_exchange_token': ACCESS_TOKEN}\r\n",
        "\r\n",
        "  r = requests.get(url, params=params)\r\n",
        "  data = r.json()\r\n",
        "\r\n",
        "  LONG_LIVED_ACCESS_TOKEN = data['access_token']\r\n",
        "  print(f'Got long lived token:\\n{LONG_LIVED_ACCESS_TOKEN}')\r\n",
        "else:\r\n",
        "  print(f'Using long-lived token provided.')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uIeKOhZRpuCE"
      },
      "source": [
        "import logging\r\n",
        "\r\n",
        "log_file = f'ad_collector_{timestamp}.log'\r\n",
        "log_formatter = \"%(asctime)s [%(filename)-20.20s:%(lineno)-4.4s - %(funcName)-20.20s() [%(levelname)-8.8s]  %(message).5000s\"\r\n",
        "\r\n",
        "logging.basicConfig(level=logging.DEBUG, filename=log_file, filemode='a', format=log_formatter)\r\n",
        "\r\n",
        "# Create a custom logger\r\n",
        "logger = logging.getLogger('fbadsearch')\r\n",
        "\r\n",
        "# Create handlers\r\n",
        "c_handler = logging.StreamHandler()\r\n",
        "c_handler.setLevel(logging.INFO)\r\n",
        "\r\n",
        "# Create formatters and add it to handlers\r\n",
        "c_format = logging.Formatter(log_formatter)\r\n",
        "c_handler.setFormatter(c_format)\r\n",
        "\r\n",
        "# Add handlers to the logger\r\n",
        "logger.addHandler(c_handler)\r\n",
        "\r\n",
        "logger.info(f\"Logging to {log_file}\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uBqceqbSMaiQ"
      },
      "source": [
        "# Configuration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sVOWpVEF4Ryg"
      },
      "source": [
        "# NOTE: I replaced 'AmandaCamm4Senate' with page ID '832302137132814' for https://www.facebook.com/AmandaCammMP/\r\n",
        "\r\n",
        "# this is all page IDs from '141020_Qld_Election_Final Candidates_FB_Twitter.xlsx', which was derived from the electoral roll\r\n",
        "pol_page_ids = [785196888157078, 1785579221658334, 106607231221280, 100506221799325, 104524101187625, 105553407983008, 109518917420445, 103650197726925, \r\n",
        "                909423379175033, 1693010970976516, 1832562330303269, 1777080942596298, 109959002750968, 407720832712714, 2220634441369101, 338368899845370, \r\n",
        "                117656986742891, 100636498106021, 567004083977539, 114001797119430, 104299284722230, 350660715317781, 112908010419196, 104221031461259, \r\n",
        "                100156858417093, 389125434502747, 103035638207098, 106380891185520, 115406840293372, 523586527672529, 736734193067726, 114150970305956, \r\n",
        "                103396634852846, 108650754237728, 116501956844030, 576823859628727, 107301347688602, 105913364458834, 263665090825710, 255774254930728, \r\n",
        "                101457271423792, 104106704739897, 705975076183979, 101709431690846, 1514961152071432, 1897562910520016, 657663438179343, 545743365504124,\r\n",
        "                336275459877992, 1534219783464358, 100164035048916, 104845828020597, 102803318250426, 110055630739867, 1915186665431831, 641418173146075, \r\n",
        "                101229475097691, 1729147630711565, 115495863575291, 539690296167013, 112370233821131, 114523806906481, 467602216674884, 113204303738366, \r\n",
        "                113313763488937, 109449280530584, 111225464067837, 108467557636953, 103481191448837, 1863325200581981, 105224701294391, 113474843753954, \r\n",
        "                395201170875812, 108660834188644, 1256779447739574, 114200597031740, 289973794486663, 461093597424764, 1167376913283953, 1509842982624646, \r\n",
        "                292763430848105, 142712312762123, 111003630759377, 106556794289046, 104761038015826, 100136771873274, 432474437121377, 107231951131409, \r\n",
        "                102121481379494, 1169942783020880, 110725204092217, 249840622106964, 1725156551117500, 520765594773519, 263462127087524, 1208758175964023, \r\n",
        "                104078437826807, 113096467226142, 100166321803017, 109251870932868, 101417321688609, 101185374929025, 1410167955931527, 567814099965388,\r\n",
        "                213302842097486, 105893381227684, 113275300532255, 227843448632578, 119529791992309, 149428025406938, 106479674497366, 105883917945171,\r\n",
        "                113209043869509, 115362773499528, 221723521179531, 581527922536548, 100100165143301, 233777047885666, 121738193028105, 109800067441452, \r\n",
        "                106101341249183, 106403251214496, 111124140659278, 110966290591055, 207989436076755, 112450560339784, 576766316323347, 102096831555854, \r\n",
        "                328287840654244, 109343407457327, 111903673975158, 111707477258429, 112793727216427, 106725931187761, 100996221715253, 107765634022348, \r\n",
        "                101731018226021, 106213717906020, 1196449963861543, 587118181959376, 352801041811373, 100568588460862, 102429501580044, 324660077589275, \r\n",
        "                105624107957969, 106307547865969, 102488524573456, 606278869581152, 103635297783684, 283965465110918, 363184810447890, 106496577802582, \r\n",
        "                1675215042771280, 513154475468041, 1598711303739855, 100190838538821, 109943174169252, 231818983678379, 101335338377069, 101383795020479,\r\n",
        "                709617325724631, 113230193550827, 775462745806585, 1845950472089687, 107410651126691, 106440287879664, 554268431430489, 740083362711653, \r\n",
        "                788385304507966, 100499011732656, 303244313358772, 615734159083360, 229474257118871, 105098314587657, 103072239832731, 103109601504421, \r\n",
        "                108155947685109, 1524300257809805, 114108316662776, 113642363810676, 111647547204231, 101015264723836, 113153533805269, 2055206548040133,\r\n",
        "                126948162124511, 111334597214496, 1568113776764144, 656903121164867, 109853183890303, 110344233985845, 113957616692644, 1985122728369782,\r\n",
        "                1453598948193351, 423996701839665, 101351228230215, 101470824951637, 103683874718454, 168475823483405, 111556703767159, 102398231198500,\r\n",
        "                608640325907244, 686500954867746, 836025246434769, 107664064169906, 103403601543186, 102092101394038, 101149561446436, 109126144006684,\r\n",
        "                101906174920141, 774386052738210, 220677281357963, 297511687070414, 103086624806616, 107008534510908, 522394997927024, 102385998294579, \r\n",
        "                104967071007331, 1374427989490497, 105372087602238, 101728244995980, 113030643782486, 112793877175216, 107376851136972, 236444377561800,\r\n",
        "                106674377818894, 1408935539376139, 474104612632953, 1657049431237380, 233782593477030, 476400975832131, 159541697531443, 2338085742938967, \r\n",
        "                624349244885583, 113581900482266, 1430189740388666, 121967566067770, 203933633277603, 554347721374792, 102145868266091, 847722658587647,\r\n",
        "                417714538385668, 112436110439247, 1134296200015702, 1492121741050481, 344226585660387, 396892697109784, 101046031735037, 110315050799043,\r\n",
        "                612853245438547, 100614331807415, 104192634278, 1448950485153928, 108865297506178, 1233955626642726, 1496534207028271, 106194461192104,\r\n",
        "                100402335145235, 861586993882334, 116058153567277, 1910997169220735, 117245719917565, 2231610370442955, 247062522139479, 1804660016424957, \r\n",
        "                1718173078398192, 1563990510511621, 1954574018107631, 101355847935667, 1114525825238202, 100986351788907, 982578781943135, 114095057091976, \r\n",
        "                108724607614834, 484472008399989, 114859093693324, 760212544116382, 106936114474003, 394508164021609, 109558924169959, 576746423001703,\r\n",
        "                254204537980710, 105778420979705, 100752448317793, 101221321379731, 101227101730425, 108330807693201, 850215955156264, 220443132720971,\r\n",
        "                109115337563838, 113281133715746, 110039737541952, 1325182927495629, 102316024969687, 590171327666970, 101511358342349, 102047474939736,\r\n",
        "                451516984861373, 372985473548850, 105352941226837, 109312463945087, 104042988033763, 100235271790601, 900067690141232, 100161275099274,\r\n",
        "                355838727797763, 543159962822554, 101560051565626, 104776624407280, 504738263066099, 117236096581797, 100207568515359, 295613770481116,\r\n",
        "                1856952217890753, 632027813515639, 116553630206367, 113620733840833, 100410168353799, 1533558346882013, 126980844673171, 106181434300985,\r\n",
        "                105856501251723, 662955787059972, 180764285343226, 516047178492348, 2344361252476442, 135521496939589, 109879534099888, 116716883502488,\r\n",
        "                103834321490119, 583679722297960, 1818359645047406, 109032010961656, 212140395561960, 102553308232651, 123650765790366, 373143696085132,\r\n",
        "                435436303151180, 1524698857748303, 111245177414703, 286377921487406, 110889697057629, 101071261582099, 1408801042745655, 101994945016870,\r\n",
        "                1947325512179992, 223247229116486, 106812531111470, 109056410952157, 109656674159830, 104677414717000, 104532491409741, 584965482184059,\r\n",
        "                100537861830777, 113279090531389, 602894639867090, 418860658142824, 131130543754756, 112750263934699, 104364204671011, 101606898126747,\r\n",
        "                109755660886785, 103124224780415, 742367169159127, 103622581416252, 110232210680243, 331007484332138, 1296089260454472, 100689074952671,\r\n",
        "                104764508061113, 102018878207644, 113555240489914, 224778514198891, 191008468082204, 124454401575649, 210143127101006, 133032931544819,\r\n",
        "                102151004917932, 104804378054435, 101444028195549, 395174637282859, 109759564122839, 832302137132814]\r\n",
        "\r\n",
        "pol_page_ids = [ str(id) for id in pol_page_ids]              "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gqOH-XJq3DBM"
      },
      "source": [
        "config_text = \"\"\"\r\n",
        "ad_active_status: ALL \r\n",
        "ad_type: POLITICAL_AND_ISSUE_ADS\r\n",
        "ad_reached_countries: \r\n",
        "  - AU\r\n",
        "# Fields for querying all possible metadata from the API\r\n",
        "query_fields:\r\n",
        "  - ad_creation_time\r\n",
        "  - ad_creative_body\r\n",
        "  - ad_creative_link_caption\r\n",
        "  - ad_creative_link_description\r\n",
        "  - ad_creative_link_title\r\n",
        "  - ad_delivery_start_time\r\n",
        "  - ad_delivery_stop_time\r\n",
        "  - ad_snapshot_url\r\n",
        "  - demographic_distribution\r\n",
        "  - funding_entity\r\n",
        "  - impressions\r\n",
        "  - page_id\r\n",
        "  - page_name\r\n",
        "  - region_distribution\r\n",
        "  - spend\r\n",
        "\r\n",
        "start_date: 2020-08-01\r\n",
        "end_date: 2020-10-31\r\n",
        "\r\n",
        "\"\"\"\r\n",
        "\r\n",
        "import yaml\r\n",
        "from more_itertools import chunked\r\n",
        "\r\n",
        "config = yaml.safe_load(config_text)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WR-jKPZBhszt"
      },
      "source": [
        "# Fetch all data\r\n",
        "For each batch of ten page IDs, get all their ads for the timeperiod, save to Google Cloud Storage, and load to BigQuery"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qrn6_3MbPspw"
      },
      "source": [
        "from google.cloud import storage\r\n",
        "gcs_client = storage.Client(project=GC_PROJECT)\r\n",
        "fetch_and_save_ads(pol_page_ids, LONG_LIVED_ACCESS_TOKEN, config, gcs_client, GCS_BUCKET, GCS_DIR)        "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l4ZQY45AnUTQ"
      },
      "source": [
        "# upload the datafile to bigquery\r\n",
        "from google.cloud import bigquery\r\n",
        "bq_client = bigquery.Client(project=GC_PROJECT)\r\n",
        "\r\n",
        "uri = f'gs://{GCS_BUCKET}/{GCS_DIR}/batch_*.json'\r\n",
        "\r\n",
        "load_to_bq(bq_client, BQ_TABLE, uri)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VqkiB3_Wt4kR"
      },
      "source": [
        "# finally, save our log file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hjYS972meb_"
      },
      "source": [
        "# save log file\r\n",
        "save_log_file(gcs_client, GCS_BUCKET, GCS_DIR, log_file)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "thHCbvnrt1DI"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}