NLI_mask_token_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import argparse\n",
    "import csv\n",
    "import logging\n",
    "import os\n",
    "import random\n",
    "import sys\n",
    "import pickle\n",
    "import time\n",
    "import numpy as np\n",
    "from termcolor import colored\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from bert_util import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnli_processor = MnliProcessor()\n",
    "hans_processor = HansProcessor()\n",
    "train_examples = mnli_processor.get_train_examples('data/', 10000)\n",
    "test_examples = hans_processor.get_test_examples('data/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n",
      "\u001b[34mThe athlete by the doctors encouraged the senator .\u001b[0m \u001b[31mThe athlete encouraged the senator .\u001b[0m entailment\n"
     ]
    }
   ],
   "source": [
    "interested_test_idx = 7\n",
    "\n",
    "for te in test_examples:\n",
    "    if te.guid == interested_test_idx:\n",
    "        print(te.guid)\n",
    "        print(colored(te.text_a, \"blue\"), colored(te.text_b, \"red\"), te.label)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tok_sal_list, train_tok_sal_lists, test_pred_status = pickle.load(open(\"from_aws/NLI_influence_output_bert_e3/saliency_test_\" + str(interested_test_idx) + \".pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS]\t0.004317282\n",
      "the\t-0.007485319\n",
      "athlete\t-0.14760923\n",
      "by\t-0.049165454\n",
      "the\t-0.050391756\n",
      "doctors\t-0.14886299\n",
      "encouraged\t-0.088029\n",
      "the\t-0.032777905\n",
      "senator\t-0.055137962\n",
      ".\t-0.01177289\n",
      "[SEP]\t0.003715336\n",
      "\u001b[32mthe\t0.01097851\u001b[0m\n",
      "athlete\t-0.04906088\n",
      "\u001b[32mencouraged\t0.08856703\u001b[0m\n",
      "the\t-0.009075683\n",
      "senator\t-0.18473382\n",
      ".\t-0.022565214\n",
      "\u001b[32m[SEP]\t0.035753675\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "top_sal_ix = [_ix for _ix, _ in sorted(enumerate(test_tok_sal_list), key=lambda x: -x[1][1])][:3]\n",
    "for ix, (tok, sal) in enumerate(test_tok_sal_list):\n",
    "    if ix in top_sal_ix:\n",
    "        print(colored(tok + '\\t' + str(sal), \"green\"))\n",
    "    else:\n",
    "        print(tok + '\\t' + str(sal))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "orig_influences = pickle.load(open(\"from_aws/NLI_influence_output_bert_e3/influences_test_\" + str(interested_test_idx) + \".pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mOne of the drawings on view at MoMA is a diagram of the races, with the Jews identified as circumscised [ sic ] cut off from Earth.\u001b[0m \u001b[31mJews think of drawings on the MoMA as a diagram of the races teaching circumcision. \u001b[0m entailment\n",
      "5.741442680358887\n",
      "\n",
      "\u001b[34mTwo pillars surviving from the Greek Temple of Apollo stand like a gateway, but the Spanish era has given it a charming 17th-century ambience of Baroque houses with iron balconies supported by floral carvings and an occasional stone nymph.\u001b[0m \u001b[31mIt's comprised of parts of the Greek Temple of Apollo and Spanish Baroque houses.\u001b[0m entailment\n",
      "5.344240665435791\n",
      "\n",
      "\u001b[34mbecause you're having fun\u001b[0m \u001b[31mBecause you're having fun\u001b[0m entailment\n",
      "4.964197158813477\n",
      "\n",
      "\u001b[34m9. INVERSE PRICE CAPS\u001b[0m \u001b[31m9 Inward caps on price. \u001b[0m entailment\n",
      "4.923686981201172\n",
      "\n",
      "\u001b[34mI don't know if I was in heaven or hell, said Lillian Carter, the president's mother, after a visit.\u001b[0m \u001b[31mThe president's mother visited.\u001b[0m entailment\n",
      "4.879209995269775\n",
      "\n",
      "\u001b[34mDo it now, think 'bout it later.\u001b[0m \u001b[31mDon't think about it now, just do it.\u001b[0m entailment\n",
      "4.244877815246582\n",
      "\n",
      "\u001b[34mThe participating lawyers say Ginsburg, a diehard Red Sox fan known for his trademark bow ties, is tough to turn down.\u001b[0m \u001b[31mGinsburg is exceptionally tough to refuse according to the lawyers.\u001b[0m entailment\n",
      "4.093533515930176\n",
      "\n",
      "\u001b[34mHowever, they noted that more detailed guidelines often provided answers to employees' questions and served as a tool for educating subordinate security managers and others who wanted a more thorough understanding of good security practices.\u001b[0m \u001b[31mThe remarked that thorough guidelines are useful for educating security managers in lower ranks. \u001b[0m entailment\n",
      "3.957554340362549\n",
      "\n",
      "\u001b[34mIn 1511 Diego de Velazquez sailed from neighboring Hispaniola with some 300 conquistadores (conquerors).\u001b[0m \u001b[31mDiego de Velazquez sailed from Hispaniola in 1511.\u001b[0m entailment\n",
      "3.744591474533081\n",
      "\n",
      "\u001b[34mThese would be special cases of the more general case analyzed here.\u001b[0m \u001b[31mThis analysis is more general than these special cases.\u001b[0m entailment\n",
      "3.6332457065582275\n",
      "\n"
     ]
    }
   ],
   "source": [
    "order_of_interest = \"max\"\n",
    "label_of_interest = \"both\"\n",
    "num_of_interest = 10\n",
    "\n",
    "train_idx_sorted = list(np.argsort(orig_influences))\n",
    "if order_of_interest == \"max\":\n",
    "    train_idx_sorted.reverse()\n",
    "if label_of_interest == \"both\":\n",
    "    valid_labels = [\"entailment\", \"non-entailment\"]\n",
    "else:\n",
    "    valid_labels = [label_of_interest]\n",
    "\n",
    "cnt = 0\n",
    "for idx in train_idx_sorted:\n",
    "    te = train_examples[idx]\n",
    "    if te.label in valid_labels:\n",
    "        print(colored(te.text_a, \"blue\"), colored(te.text_b, \"red\"), te.label)\n",
    "        print(orig_influences[idx])\n",
    "#         top_sal_ix = [_ix for _ix, _ in sorted(enumerate(train_tok_sal_lists[idx]), key=lambda x: -x[1][1])][:3]\n",
    "#         for ix, (tok, sal) in enumerate(train_tok_sal_lists[idx]):\n",
    "#             if ix in top_sal_ix:\n",
    "#                 print(colored(tok + '\\t' + str(sal), \"green\"))\n",
    "#             else:\n",
    "#                 print(tok + '\\t' + str(sal))\n",
    "        print()\n",
    "        cnt += 1\n",
    "    if cnt >= num_of_interest:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "diff_influences_list, mask_ix = pickle.load(open(\"from_aws/NLI_influence_output_bert_e3/diff_mask_influences_test_\" + str(interested_test_idx) + \".pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "w.r.t. token: encouraged\n",
      "\n",
      "\u001b[34mOne of the drawings on view at MoMA is a diagram of the races, with the Jews identified as circumscised [ sic ] cut off from Earth.\u001b[0m \u001b[31mJews think of drawings on the MoMA as a diagram of the races teaching circumcision. \u001b[0m entailment\n",
      "18.779272079467773\n",
      "\n",
      "\u001b[34mTwo pillars surviving from the Greek Temple of Apollo stand like a gateway, but the Spanish era has given it a charming 17th-century ambience of Baroque houses with iron balconies supported by floral carvings and an occasional stone nymph.\u001b[0m \u001b[31mIt's comprised of parts of the Greek Temple of Apollo and Spanish Baroque houses.\u001b[0m entailment\n",
      "18.099933624267578\n",
      "\n",
      "\u001b[34mI don't know if I was in heaven or hell, said Lillian Carter, the president's mother, after a visit.\u001b[0m \u001b[31mThe president's mother visited.\u001b[0m entailment\n",
      "16.18392562866211\n",
      "\n",
      "\u001b[34m9. INVERSE PRICE CAPS\u001b[0m \u001b[31m9 Inward caps on price. \u001b[0m entailment\n",
      "14.431646347045898\n",
      "\n",
      "\u001b[34mDo it now, think 'bout it later.\u001b[0m \u001b[31mDon't think about it now, just do it.\u001b[0m entailment\n",
      "12.819534301757812\n",
      "\n",
      "\u001b[34mbecause you're having fun\u001b[0m \u001b[31mBecause you're having fun\u001b[0m entailment\n",
      "12.689254760742188\n",
      "\n",
      "\u001b[34mthat's Major Dad at eight i think it is\u001b[0m \u001b[31mI think Major Dad is on at eight o'clock.\u001b[0m entailment\n",
      "12.103283882141113\n",
      "\n",
      "\u001b[34mIn 1511 Diego de Velazquez sailed from neighboring Hispaniola with some 300 conquistadores (conquerors).\u001b[0m \u001b[31mDiego de Velazquez sailed from Hispaniola in 1511.\u001b[0m entailment\n",
      "12.007287979125977\n",
      "\n",
      "\u001b[34mHowever, they noted that more detailed guidelines often provided answers to employees' questions and served as a tool for educating subordinate security managers and others who wanted a more thorough understanding of good security practices.\u001b[0m \u001b[31mThe remarked that thorough guidelines are useful for educating security managers in lower ranks. \u001b[0m entailment\n",
      "11.697555541992188\n",
      "\n",
      "\u001b[34mFiction has its glories, but concealment is merely squalid.\u001b[0m \u001b[31mHiding things is just dirty, whereas there is glory in fiction\u001b[0m entailment\n",
      "11.545598983764648\n",
      "\n"
     ]
    }
   ],
   "source": [
    "remove_token_type = 2\n",
    "\n",
    "influences = diff_influences_list[remove_token_type]\n",
    "print(\"w.r.t. token:\", test_tok_sal_list[mask_ix[remove_token_type]][0])\n",
    "print()\n",
    "\n",
    "order_of_interest = \"max\"\n",
    "label_of_interest = \"both\"\n",
    "num_of_interest = 10\n",
    "\n",
    "train_idx_sorted = list(np.argsort(influences))\n",
    "if order_of_interest == \"max\":\n",
    "    train_idx_sorted.reverse()\n",
    "if label_of_interest == \"both\":\n",
    "    valid_labels = [\"entailment\", \"non-entailment\"]\n",
    "else:\n",
    "    valid_labels = [label_of_interest]\n",
    "\n",
    "cnt = 0\n",
    "for idx in train_idx_sorted:\n",
    "    te = train_examples[idx]\n",
    "    if te.label in valid_labels:\n",
    "        print(colored(te.text_a, \"blue\"), colored(te.text_b, \"red\"), te.label)\n",
    "        print(influences[idx])\n",
    "        print()\n",
    "        cnt += 1\n",
    "    if cnt >= num_of_interest:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp = 0.001\n",
    "\n",
    "idx_range = range(1, 31)\n",
    "influence_distance_lists = ([], [], [], [], [])\n",
    "for iti in idx_range:\n",
    "    orig_inf = pickle.load(open(\"from_aws/NLI_influence_output_bert_e3/influences_test_\" + str(iti) + \".pkl\", \"rb\"))\n",
    "    diff_inf_list, m_ix = pickle.load(open(\"from_aws/NLI_diff_influence_output_bert_e3/diff_mask_influences_test_\" + str(iti) + \".pkl\", \"rb\"))\n",
    "    for i, influences in enumerate(diff_inf_list):\n",
    "        influence_distance_lists[i].append(influence_distance(orig_inf, influences, top_percentage=tp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.32999999999999996"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(influence_distance_lists[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7933333333333332"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(influence_distance_lists[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.46"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(influence_distance_lists[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fe714440d30>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VPW9//HXJ3tYwhqQJayyhS1gWKy71StqheLC5lIUsfaWumDppWq5Ymv9VavorXSRxQVFoFYrVm5RW6oVJRIkEcIaNgnIEggEsi/f3x8z5kYIZIBJTjJ5Px8PHubMnMy8H53Mu2fOnO/3a845REQktIR5HUBERIJP5S4iEoJU7iIiIUjlLiISglTuIiIhSOUuIhKCVO4iIiFI5S4iEoJU7iIiISjCqydu3bq169Kli1dPLyJSL61ZsybbORdf3X6elXuXLl1ITU316ulFROolM9sVyH46LSMiEoJU7iIiIUjlLiISglTuIiIhSOUuIhKCVO4iIiFI5S4iEoI8u85dpC66fV4KWTkFdGwRy4JJw7yOI3LWVO4ilWTlFLAjO8/rGCLnTKdlRERCkMpdRCQEqdxFREKQyl1EJASp3EVEQpDKXUQkBKncRURCkMpdRCQEqdxFREKQyl1EJAQFVO5mNsLMNptZpplNr+L+Tma2wszWmtmXZnZd8KOKiEigqi13MwsHZgPXAonAeDNLPGG3R4ElzrlBwDjg98EOKiIigQvkyH0okOmc2+6cKwYWAaNO2McBcf6fmwF7gxdRpOblFZXy7Pub+epwPgBfHc7nf/6xlcKSMo+TiZydQGaF7ADsrrSdBZw4F+pjwPtm9hOgMXBVUNKJ1IKC4jImzE0hffeRitvKyh3PfrCFT7dl88pdQ4mOCPcwociZC+TI3aq4zZ2wPR542TnXEbgOWGBmJz22md1jZqlmlnrw4MEzTytSA175bOe3ir2yVdsPsyQ1q3YDiQRBIOWeBSRU2u7IyaddJgFLAJxznwExQOsTH8g596JzLtk5lxwfH392iUWC7O0v9lRzv8pd6p9Ayn010MPMuppZFL4vTJeesM9XwHcBzKwPvnLXobnUC4fyis/pfpG6qNpyd86VAlOA5cBGfFfFZJjZ42Y20r/bQ8BkM0sH3gAmOudOPHUjUid1a934nO4XqYsCWmbPObcMWHbCbTMq/bwBuCi40URqx4RhCXy+8/Ap728bF1OLaUSCQyNUpcErLT/9h8xFq3ezYNWuWkojEhwqd2nQdh/O57GlGwDfZWExkb63ROPocO6+uGvFfr/463oWr/7Ki4giZ0XlLg1WWbnjwcVpHC8qBeDey7vTrlksAG2axvDo9xJ5fFTfiv2nv7WOv6zRlTNSP6jcpcH640fbSN2VA0Df9nE8eFXPk/a548IuPHp9HwCcg2lvpvNO2ukvnRSpC1Tu0iB9mXWEWR9sASA6IoznxyURFVH12+HuS7ox/dreAJQ7mLoknWXrvq61rCJnQ+UuDU5BcRkPLE6r+CL1kev7cH6bpqf9nXsv685DV/uO7MvKHfe9sZb3M/bVeFaRs6Vylwbn18s2sv1gHgCX9Yzn9uGdA/q9n3y3B/ddeT7gu8Lmxwu/4J+b9tdYTpFzoXKXBmXFpgMVlzW2bBzF07cMwKyq6ZOq9uDVPbn3su4AlJQ57l3wBR9t0WBsqXtU7tJgZB8vYtqb6RXbT97YnzZNz2yAkpnxXyN6Mcl/mWRxWTn3vJrKp5nZQc0qcq5U7tIgOOeY/pd1ZB/3zRMzNjmBa/qed1aPZWY8en0f7rjQdzqnqLScSa+kkrL9UNDyipwrlbs0CItW7+bDjb7z451bNWLGDScuJnZmzIzHbujL+KGdACgoKePOl1ezZteppzEQqU0qdwl5O7LzePxd3yjU8DBj1tgkGkcHNK3SaYWFGU98vx83X9ARgPziMibOX03aKeaGF6lNKncJaSVl5TywOI0C/3J5U644n8GdWgTt8cPCjN/cNIDvJ7UH4FhRKXfMS2H9nqNBew6Rs6Fyl5D2u39mVqyylJTQnCn+SxmDKTzM+O0tA7l+QDsAcgtLuW1eChu/zg36c4kESuUuIWvNrhxmr8gEoFFUOLPGJhEZXjN/8hHhYTw3Nolr+rYF4Eh+CbfOTWHL/mM18nwi1VG5S0g6XlTK1CVplPlHoc74XiJda3jRjcjwMH43fjDf7d0GgMN5xUyYk8K2g8dr9HlFqqJyl5D0y3c3sOtQPgBXJ7Zl7JCEan7Dp2OLWLq2bkzHFrFn9bxREWH8/rbBXNbTt0Zw9vEiJsxZxc7svLN6PJGzZV6thpecnOxSU1M9eW4JbX9fv497X1sDQOsm0Sx/4BJaNYmu1QyFJWVMemU1KzN91763bxbD4h9eSELLRrWaQ0KPma1xziVXt5+O3CWkHMgt5OdvfVmx/fTNA2q92AFiIsOZe8cQhnVtCcDeo4WMn7OKPUcKaj2LNEwqdwkZzjl++uaX5OSXAHD78M5c4T//7YXYqHDmTxxCcmffpZdZOQVMmLOKfUcLPcskDYfKXULGq5/t4mP/JF7d4hvz8HV9PE4EjaMjeOnOISQlNAdg16F8JsxZxYFjKnipWSp3CQlb9x/j18s2AhARZjw/dhCxUeEep/JpGhPJK3cNpV+HOAC2Z+dx65wUso8XeZxMQpnKXeq94lLfKNSi0nLANy1v/47NPE71bc1iI3lt0jD6tPMV/NYDx7ltbgo5ecUeJ5NQpXKXem/Wh1vI2OsbDTqkS4uK+dbrmuaNonht0lB6tm0CwKZ9x7htXgpH/d8RiASTyl3qtZTth/jjR9sAaBIdwbNjkggPC3zxjdrWqkk0r989nO7xvgFVGXtzuWN+CrmFKngJLpW71Fu5hSVMXZLON0M1Hh/Vt15cRx7fNJqFk4fTpZUva3rWUSbO/5zjRaUeJ5NQElC5m9kIM9tsZplmNr2K+2eZWZr/3xYz05ynUuP++52MiuvGr+/fjtGDOnicKHBt42JYOHk4CS19I2G/+OoId720mvxiFbwER7XlbmbhwGzgWiARGG9m31rpwDn3oHMuyTmXBPwOeKsmwop8Y2n6Xt5euweAtnHRPDG63xmthVoXtG8ey8K7h9Ohua/gP995mEkvp1JQXOZxMgkFgRy5DwUynXPbnXPFwCJg1Gn2Hw+8EYxwIlXZe6SAR99eV7H9zC1JNG8U5WGis5fQshELJw/jvDjfWq6fbT/EPQtSKSxRwcu5CaTcOwC7K21n+W87iZl1BroC/zzF/feYWaqZpR48qBXj5cyVlzseWpJObqHv9MWki7tycY/WHqc6N51bNWbh5GHEN/VNk/Dvrdn86LU1FJWq4OXsBVLuVX3WPdVsY+OAN51zVf5VOudedM4lO+eS4+PjA80oUmH+yh185l+Iulfbpky7ppfHiYKjW3wT3pg8jFaNfZ9AVmw+yJSFaykpK/c4mdRXgZR7FlB5vtSOwN5T7DsOnZKRGrLx61ye+vtmAKLCw3huXBIxkXVjFGownN+mKa9PHkaLRpEAfLBhP/cvWkupCl7OQiDlvhroYWZdzSwKX4EvPXEnM+sFtAA+C25EEd8Uug8sSqPYX3Q/G9GrYrRnKOl9Xhyv3T2MZrG+gl+2bh9Tl6RXLDoiEqhqy905VwpMAZYDG4ElzrkMM3vczEZW2nU8sMh5NUG8hLSnl29ms3/Juu90b8VdF3X1OFHN6du+GQsmDaVpdATguzJo2p9V8HJmtFiH1HmfbM3mtnkpAMTFRLD8wUtp1+zsVkqqT774Kofb56aQ5780cmxyAk/e2J+wOjwCV2qeFuuQkHAkv5iH/pxWsf3E6P4NotgBBndqwct3DaWRf3bLxam7+cU769GHYwmEyl3qLOccD7+9jv25vqlxRw/qwA0D23ucqnYN6dKSeT8YQkyk7636espXzHx3gwpeqqVylzrrrS/2sGzdPgA6NI9l5qi+HifyxoXdWzH3jiFERfjeri9/upNfL9uogpfTUrlLnbT7cD7/vTQDADN4dsxA4mIiPU7lnYt7tObF2y8gKtz3lp3z7x08vXyzCl5OSeUudU5ZuWPqkrSKWRLvvaw7w7q18jiV9y7v1Ybf3zqYCP8Xqr//1zae/8dWj1NJXaVylzrnjx9tY/XOHAD6to/jwat6epyo7rgqsS0vTBhUMWf9cx9uZfaKTI9TSV2kcpc6ZV3WUWZ9sAWA6Igwnh+XVHGuWXxG9GvHc2OT+OaKyKeXb+bFj7d5G0rqHL1rpM4oKC7j/sVrKfUP1nnk+j6c36apx6nqphsGtueZMQP5ZpbjXy/bxEsrd3gbSuoUlbvUGb9etpHtB/MAuKxnPLcP7+xxorpt9KCO/OamARXbM9/dwIJVuzxMJHWJyl3qhBWbDlQUU4tGkTx984B6t/iGF8YkJ/Dr0f0rtn/x1/UsXv2Vh4mkrlC5i+eyjxcx7c30iu0nbxxAG//iFVK9CcM6MXPk/40BmP7WOv6yJsvDRFIXqNzFU845pv9lHdnHiwHf/Ckj+p3ncar65wff6cKj1/cBwDmY9mY676Tt8TiVeEnlLp5avHo3H27cD0DnVo2YcUNiNb8hp3L3Jd34rxG9ASh3MHVJOsvWfe1xKvGKyl08syM7j5nvbgAgPMyYNTaJxv5pbuXs/Ojy7ky92jcuoKzccd8ba3k/Y5/HqcQLKnfxRElZOQ8sTqPAvxD0lCvOZ3CnFh6nCg33fbcHP7nyfABKyx0/XvgFKzYd8DiV1DaVu3jihX9mkr77CABJCc2Z4i8jCY6pV/fkh5d1A6CkzPHD19bw8RYtSt+QqNyl1q3ZlcML/iHzsZHhzBqbRGS4/hSDycyYPqJ3xYpVxaXlTH41lU8zsz1OJrVF7yipVceLSpm6JK1iybgZNyTStXVjj1OFJjPjF9/rUzEYrKi0nEmvpPL5jsMeJ5PaoHKXWvXLdzew61A+AFf1acu4IQkeJwptZsbMkX0ZP9T3v3NBSRl3vvQ5a3bleJxMaprKXWrN8ox9LE7dDUDrJtH85qb+GoVaC8LCjCe+35+bL+gIQF5xGRPnf17xnYeEJpW71IoDuYVM/8uXFdtP3zyAVk2iPUzUsISFGb+5aQCjknzLFB4rKuX2eSms33PU42RSU1TuUuOcc0x780ty8ksAuH14Z67o3cbjVA1PeJjxzC0Dub5/OwByC0u5bV4KG7/O9TiZ1ASVu9S4Bat28ZH/Mrxu8Y15+Lo+HidquCLCw3huXBL/kdgWgCP5Jdw2N4Wt+495nEyCTeUuNSrzwDGeeG8jABFhxvNjBxEbFe5xqoYtMjyMFyYM5kr/p6dDecWMn5PCtoPHPU4mwaRylxpTXFrO/YvSKCotB+DBq3vSv2Mzj1MJQFREGL+/dTCX9owHfDNzTpizip3ZeR4nk2AJqNzNbISZbTazTDObfop9xpjZBjPLMLOFwY0p9dGsD7eQsdd3PndIlxbce1l3jxNJZTGR4bx4+wV8p7tv8fH9ub6C33043+NkEgzVlruZhQOzgWuBRGC8mSWesE8P4OfARc65vsADNZBV6pGU7Yf440e+dT2bREfw7JikikWdpe6IiQxn7g+SGdq1JQB7jxYyfs4q9hwp8DiZnKtAjtyHApnOue3OuWJgETDqhH0mA7OdczkAzjnNUtSA5RaWMHVJOs43CJWZI/uS0LKRt6HklBpFRTB/4hAu6OybuC0rp4AJc1ax72ihx8nkXARS7h2A3ZW2s/y3VdYT6GlmK81slZmNCFZAqX8eeyej4sjv+v7tuHHwiX8uUtc0iY7gpTuHMDChOQC7DuUzYc4qDhxTwddXgZR7VZ+l3QnbEUAP4HJgPDDXzJqf9EBm95hZqpmlHjyoGepC0bvpe3lrrW8FoLZx0Twxup9GodYTcTGRvHrnUPp1iANge3Yet85JIft4kcfJ5GwEUu5ZQOUJQDoCe6vY5x3nXIlzbgewGV/Zf4tz7kXnXLJzLjk+Pv5sM0sd9fXRAh55e13F9jO3JNG8UZSHieRMNWsUyYK7htH7vKYAbD1wnNvmppCTV+xxMjlTgZT7aqCHmXU1syhgHLD0hH3+ClwBYGat8Z2m2R7MoFK3lZc7HlqSTm5hKQCTLu7KxT1ae5xKzkaLxlG8fvcwerRpAsCmfce4bV4KR/0jjKV+qLbcnXOlwBRgObARWOKcyzCzx81spH+35cAhM9sArACmOecO1VRoqXvmr9zBp9t8L3mvtk2Zdk0vjxPJuWjVJJrXJw+jW7xvOuaMvbncMT+F3EIVfH1hzp14+rx2JCcnu9TUVE+eW4Jr49e5jHphJcVl5USFh/HOlIvo0y7O61gSBPtzCxn7p8/Y6Z+meXCn5rw6aRhNtNatZ8xsjXMuubr9NEJVzklhSRkPLEqjuMw3CnXaNb1U7CGkbVwMCycPJ6FlLABffHWEu15aTX5xqcfJpDoqdzknTy/fzGb/pFMXdmvFpIu7epxIgq1981gW3j2cDs19Bf/5zsNMejmVguIyj5PJ6ajc5ax9sjWbeZ/sACAuJoJnxgwkTKNQQ1JCy0YsnDyM8+JiAPhs+yHuWZBKYYkKvq5SuctZOZJfzE//nF6x/cTo/rT3H9lJaOrcqjELJw8jvqlvkZV/b83mR6+toahUBV8XqdzljDnneOTt9ezL9Y1eHD2oAzcMbO9xKqkN3eKb8MbkYbRq7Bu/sGLzQaYsXEuJ/zsXqTtU7nLG3l67h/fWfQ1Ah+axzBzV1+NEUpvOb9OU1ycPo0WjSAA+2LCf+xetpVQFX6eo3OWM7D6cz4x3MgAwg2fHDCQuJtLjVFLbep8Xx2t3D6NZrO+1X7ZuH1OXpFNW7s2l1XIylbsErKzcMXVJGseLfJfB3XtZd4Z1a+VxKvFK3/bNWDBpKE3917wvTd/LtDfTKVfB1wkqdwnYHz/axuqdOQD0bR/Hg1f19DiReG1Ax+a8Mmkojf1LJ771xR4efnudCr4OULlLQNZlHWXWB1sAiI4I47mxSURF6M9HYHCnFrx811BiI30Fv2j1bmYsXY9Xo9/FR+9OqVZBcRn3L15Lqf9o7OHr+tCjbVOPU0ldMqRLS+ZPHEJMpK9SXlv1FY//bYMK3kMqd6nWk/+7ke0HfQsnX9Yznjsu7OxxIqmLLuzeijl3JFd8ontp5U6e/N9NKniPqNzltFZsOsCrn+0CoEWjSJ6+eYAW35BTuqRHPH+6/QKiwn3V8uLH2/nt+5tV8B5QucspHTpexLQ3v6zYfvLGAbTxDz8XOZUrerVh9q2DifBPRTF7xTb+5x+ZHqdqeFTuUiXnHNPfWlexxNrY5ARG9DvP41RSX1yd2JbfjR9EuL/gZ324hdkrVPC1SeUuVVq8ejcfbNgPQOdWjZhxQ6LHiaS+ubZ/O2aNTeKbueSeXr6ZOR9rgbbaonKXk+zIzmPmuxsACDN4dkwSjbU4g5yFkQPb89tbBvLN1zRPLNvISyt3eBuqgVC5y7eUlJXzwOI0CvxTuU65sgcXdG7hcSqpz24c3JHf3DigYnvmuxtYsGqXh4kaBpW7fMsL/8wkffcRAAYmNOcnV57vcSIJBWOGJPDE6H4V27/463oWr/7Kw0ShT+UuFb74KocX/F96xUaG89zYJCLD9SciwXHrsM48Vum7m+lvreMva7I8TBTadCJVAMgrKuXBxWkVs/rNuCGRrq0be5xKQs3Ei7pSWu741XsbcQ6mvZlORLgxKqlDUB7/9nkpZOUU0LFFLAsmDQvKY9ZXKncB4Jd/28Au/wr3V/Vpy7ghCR4nklB19yXdKC4r56m/b6bcwdQl6USGh3Fd/3bn/NhZOQXsyM4LQsr6T5+5heUZ+1i0ejcArZtE85ub+msUqtSo/7z8/IpZRcvKHfe9sZb3M/Z5nCq0qNwbuAO5hUz/y/+NQn365gG0ahLtYSJpKO777vlMucL3hX1puePHC79gxaYDHqcKHSr3Bsw5x7Q3vyQnvwSA24d35orebTxOJQ2FmfHQf/Tkh5d2A6CkzPHD19bw8ZaDHicLDQGVu5mNMLPNZpZpZtOruH+imR00szT/v7uDH1WCbcGqXXzkfyN1i2/Mw9f18TiRNDRmxvRre3PnRV0AKC4tZ/KrqXyame1tsBBQbbmbWTgwG7gWSATGm1lVY9EXO+eS/P/mBjmnBFnmgWM88d5GACLCjOfGJhHrX01HpDaZGTO+l8htwzsBUFRazqRXUvl8x2GPk9VvgRy5DwUynXPbnXPFwCJgVM3GkppUXOobhVpU6lut/sGrezKgY3OPU0lDZmY8PrJfxVVaBSVl3PnS56zZleNxsvorkHLvAOyutJ3lv+1EN5nZl2b2ppnpOro67LkPt7B+Ty4AyZ1bcO9l3T1OJAJhYcavR/fnpsEdAcgrLmPi/M8rRkzLmQmk3Ku6Ju7EmfffBbo45wYAHwKvVPlAZveYWaqZpR48qC9NvPD5jsP84aNtADSJjmDW2KSKaVlFvBYWZjx18wBGJbUH4FhRKbfPS2H9nqMeJ6t/Ain3LKDykXhHYG/lHZxzh5xzRf7NOcAFVT2Qc+5F51yycy45Pj7+bPLKOcgtLOHBxWl8syjOzJF9SWjZyNtQIicIDzOeuWUg1/X3rR+QW1jKbfNS2Ph1rsfJ6pdAyn010MPMuppZFDAOWFp5BzOrPLRsJLAxeBElWB57J4M9RwoAuL5/O24cHJwh3yLBFhEexvPjBnF1YlsAjuSXcNvcFLbuP+Zxsvqj2nJ3zpUCU4Dl+Ep7iXMuw8weN7OR/t3uM7MMM0sH7gMm1lRgOTvvpu/lrbV7AGgbF80To/tpFKrUaZHhYbwwYRBX+sdeHMorZvycFLYdPO5xsvohoOvcnXPLnHM9nXPdnXNP+G+b4Zxb6v/55865vs65gc65K5xzm2oytJyZr48W8Mjb6yq2n7klieaNojxMJBKY6Ihwfn/rYC7p0RqA7ONFTJizip2aP6ZaGqEa4srLHQ8tSSe3sBSAuy7qysX+N4pIfRATGc6cO5L5TvdWAOzP9RX87sP5Hier21TuIW7+yh18uu0QAD3bNuFnI3p5nEjkzMVEhjP3B8kM7doSgL1HCxk/ZxV7/d8hyclU7iFs075cnvr7ZgCiwsN4buwgYiI1ClXqp0ZREcyfOKRi2cesnALGz1nFvqOFHierm1TuIaqwpIwHFqVRXOYbhTrtml4kto/zOJXIuWkSHcFLdw5hYIJvRPWuQ/lMmLuKA8dU8CdSuYeo3y7fzKZ9vsvGLuzWikkXd/U4kUhwxMVE8uqdQ+nXwXewsv1gHuNfXMUf/pVZcRR/JL+Y7ONFp3uYkKdyD0ErM7OZ+8kOAOJiInhmzEDCNApVQkizRpEsuGsYvc9rCsC2g3n85u+bKSgpAyAnv4Srn/2IdVkNd2Sryj3EHMkv5qEl6RXbT4zuT/vmsR4mEqkZLRpH8frdw4g9xfdIOfkl/Oj1NZT6T002NCr3EOKc45G317Mv1/fRdPSgDtwwsL3HqURqzpGCkoqj9apk5RTwr80Ncx4rlXsIeXvtHt5b9zUAHZrHMnNUX48TidSsbQeqH63aUEe0qtxDxO7D+cx4JwMAM3hmzEDiYiI9TiVSs1o1qX6kdYvGDXM0tso9BJT5R6EeL/KNQv3hpd0Z3q2Vx6lEat6ghBZ0Os3MprGR4VzT97xaTFR3qNxDwJ8+3sbnO31LkiW2i2Pq1T09TiRSO8LCjF99vx8Rp7ga7NHv9aFZbMP8BKtyr+fW7znKs+9vASA6IoznxyURFaGXVRqOS3vGs+TeCytmjwTfe2HeD5K5dVhnD5N5Sy1QjxUUl3H/orWUlvtW33j4uj70aNvU41QitW9wpxbMnziELq18p2jaN4/lu33aepzKWyr3euzJ/93ItoO+qU8v6xnPHRc23KMUEUBrFFSicq+nVmw6wKuf7QKgRaNInr55gP6wRaSCyr0eOnS8iGlvflmx/eSNA2gTF+NhIhGpa1Tu9YxzjulvrauYFGlMckdG9GuYl3qJyKmp3OuZJam7+WDDfgA6tWzEjBs0ClVETqZyr0d2Zucx890NAIQZzBqbRJPoCI9TiUhdpHKvJ0rLynlgcRr5xb5JkqZc2aNiRRoRkROp3OuJF1Zkkrb7CAADE5rzkyvP9ziRiNRlKvd64IuvcvjdPzMB31wZz41NIjJcL52InJoaoo7LKyrlwcVplPlHoc64IZGurRt7nEpE6jqVex33y79tYNehfACu6tOWcUMSPE4kIvWByr0OW56xj0WrdwPQukkU/++m/hqFKiIBCajczWyEmW02s0wzm36a/W42M2dmycGL2DAdOFbIz99aV7H91M0DaN0k2sNEIlKfVFvuZhYOzAauBRKB8WaWWMV+TYH7gJRgh2xonHP87M0vOZxXDMBtwztxZe+GPcOdiJyZQI7chwKZzrntzrliYBEwqor9fgk8BRQGMV+D9NqqXRWL+naLb8wj1530/6UiIqcVSLl3AHZX2s7y31bBzAYBCc65v53ugczsHjNLNbPUgwcb5ork1ck8cIxfvbcRgIgw47mxScRGhXucSkTqm0DGrlf1DZ6ruNMsDJgFTKzugZxzLwIvAiQnJ7tqdq+3bp+XQlZOAR1bxLJg0rCAf6+41DcKtai0HIAHr+7JgI7NayqmSMjp2CL2W/9tyAIp9yyg8vV3HYG9lbabAv2Af/mv5DgPWGpmI51zqcEKWp9k5RSwIzvvjH/vuQ+3sH5PLgDJnVtw72Xdgx1NJKSdycFUqAvktMxqoIeZdTWzKGAcsPSbO51zR51zrZ1zXZxzXYBVQIMt9rP1+Y7D/OGjbQA0iY5g1tgkwk+x6K+ISHWqLXfnXCkwBVgObASWOOcyzOxxMxtZ0wEbgtzCEh5cnIbzn6iaObIvCS0beRtKROq1gOaLdc4tA5adcNuMU+x7+bnHalgeeyeDPUcKALiu/3ncOLhDNb8hInJ6GqHqsb99uZe31u4BoG1cNE98X6NQReTcqdw99PXRAh55e33F9m9vGUiLxlEeJhKRUKFy90h5ueOnf07naEEJAHdd1JVLesR7nEpEQoXK3SPzV+5gZeYhAHq2bcLPRvTyOJGIhBKVuwc27cvlqb9vBiAqPIznxg5nWVHIAAAGi0lEQVQiJlKjUEUkeFTutaywpIwHFqVRXOYbhTrtml4kto/zOJWIhBqVey377fLNbNp3DIALu7Vi0sVdPU4kIqFI5V6LVmZmM/eTHQDExUTwzJiBhGkUqojUAJV7LTmaX8JDS9Irtn81uj/tm2tyIxGpGSr3WuCc4+G/rmNfrm+q++8ntWfkwPYepxKRUKZyrwV/TdvDe19+DUCH5rHMHNXP40QiEupU7jVs9+F8Zvw1AwAzeGbMQJrFRnqcSkRCncq9BjnneGhJOseKSgH44aXdGd6tlcepRKQhCGhWSAlccWk5xf6VlI4WlLDzUD4Aie3imHp1Ty+jiUgDonIPkvJyxx8+2sb8T3ZwKK8YgJx837wx0RFhPD8uiagIfVASkdqhcg+Sme9m8Mpnu6q8b+TA9vRo27SWE4lIQ6ZDySDYmZ13ymIH+HRbNmXlIbseuIjUQSr3IFiese+09+85Usj6PUdrKY2IiMo9KPKKywLYp7QWkoiI+Kjcg2Bgx2anvT8y3OhznmZ+FJHao3IPgst7taFbfONT3n/T4I5aPk9EapXKPQjCw4x5PxhCp5aNTrrvsp7xzLgh0YNUItKQqdyDpGvrxnww9VKeH5dE0xjfFabtmsXw8p1DaBSlK05FpHap3IMoOiKcUUkdaN0kGoCYyHDMNF+7iNQ+lbuISAgKqNzNbISZbTazTDObXsX995rZOjNLM7NPzEwnmUVEPFRtuZtZODAbuBZIBMZXUd4LnXP9nXNJwFPAs0FPKiIiAQvkyH0okOmc2+6cKwYWAaMq7+Ccy6202RjQWHsREQ8FchlHB2B3pe0sYNiJO5nZj4GpQBRwZVUPZGb3APcAdOrU6UyziohIgAI5cq/qco+Tjsydc7Odc92B/wIereqBnHMvOueSnXPJ8fHxZ5ZUREQCFki5ZwEJlbY7AntPs/8i4PvnEkpERM5NIOW+GuhhZl3NLAoYByytvIOZ9ai0eT2wNXgRRUTkTFV7zt05V2pmU4DlQDgw3zmXYWaPA6nOuaXAFDO7CigBcoAf1GRoERE5vYDGxTvnlgHLTrhtRqWf7w9yLhEROQcaoSoiEoJU7iIiIUjlLiISglTuIiIhSOUuIhKCVO4iIiFI5S4iEoK0/lsN6Ngi9lv/FRGpbSr3GrBg0kmTZoqI1CqdlhERCUEqdxGREKRyFxEJQSp3EZEQpHIXEQlBKncRkRCkchcRCUHm3ElrXdfOE5sdBHZ58uS1ozWQ7XUIOSt67eq3UH/9Ojvn4qvbybNyD3VmluqcS/Y6h5w5vXb1m14/H52WEREJQSp3EZEQpHKvOS96HUDOml67+k2vHzrnLiISknTkLiISglTuZ8nMjnudQWqOmY00s+mnuE+vvQfMrIuZra/i9sfN7KpqfvcxM/tpzaWrezSfu0gVnHNLgaVe55DqOedmeJ2hLtKR+zkyn6fNbL2ZrTOzsf7bf29mI/0/v21m8/0/TzKzX3mZuaHzHwFuMrO5/tftdTO7ysxWmtlWMxtqZhPN7AX//l3N7DMzW21mv/Q6fwMXbmZzzCzDzN43s1gze9nMbgYws+v8r+0nZvY/Zva3Sr+baGb/MrPtZnafR/lrjcr93N0IJAEDgauAp82sHfAxcIl/nw5Aov/ni4F/13ZIOcn5wPPAAKA3MAHfa/NT4OET9n0e+INzbgiwrzZDykl6ALOdc32BI8BN39xhZjHAn4BrnXMXAyeO4uwNXAMMBf7bzCJrJ7I3VO7n7mLgDedcmXNuP/ARMARfgV9iZonABmC/v/QvBD71LK18Y4dzbp1zrhzIAP7hfJeOrQO6nLDvRcAb/p8X1F5EqcIO51ya/+c1fPu16g1sd87t8G+/wbe955wrcs5lAweAtjWa1GM6537urKobnXN7zKwFMALfUXxLYAxw3Dl3rBbzSdWKKv1cXmm7nKrfF7pmuG6o/LqVAZVXoa/yvXia3w3p/tOR+7n7GBhrZuFmFg9cCnzuv+8z4AH/Pv/G95Ffp2Tqn5XAOP/Pt3oZRE5rE9DNzLr4t8d6F8V7Kvdz9zbwJZAO/BP4mXPum/Oy/wYinHOZwBf4jt5V7vXP/cCPzWw10MzrMFI151wB8J/A383sE2A/cNTbVN7RCFURCRlm1sQ5d9zMDJgNbHXOzfI6lxd05C4ioWSymaXh+5K8Gb6rZxokHbmLiIQgHbmLiIQglbuISAhSuYuIhCCVu4hICFK5i4iEIJW7iEgI+v99xKkI/oUjUgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.pointplot(x=[\"low\"]*len(idx_range)+[\"mid\"]*len(idx_range)+[\"high\"]*len(idx_range),\n",
    "              y=influence_distance_lists[0]+influence_distance_lists[1]+influence_distance_lists[2], ci=68)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}