redundancy.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python392jvsc74a57bd08d8d1de40fa9b1a75ae60eb32c95bab4992b78279d685b8955e4427681e20e6f",
   "display_name": "Python 3.9.2 64-bit ('pyro-1.6': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Union, Dict\n",
    "from itertools import combinations\n",
    "from collections import defaultdict, Counter\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import multinomial, binom, hypergeom, bernoulli, multivariate_hypergeom\n",
    "from scipy.special import comb, gammaln"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_tu(topics, n=10):\n",
    "    \"\"\"\n",
    "    Topic uniqueness measure from\n",
    "    https://www.aclweb.org/anthology/P19-1640.pdf\n",
    "    \"\"\"\n",
    "    tu_results = []\n",
    "    for topics_i in topics:\n",
    "        w_counts = 0\n",
    "        for w in topics_i[:n]:\n",
    "            w_counts += 1 / np.sum([w in topics_j[:n] for topics_j in topics]) # count(k, l)\n",
    "        tu_results.append((1 / n) * w_counts)\n",
    "    return tu_results\n",
    "\n",
    "\n",
    "def compute_tr(topics, n=10):\n",
    "    \"\"\"\n",
    "    Compute topic redundancy score from \n",
    "    https://jmlr.csail.mit.edu/papers/volume20/18-569/18-569.pdf\n",
    "    \"\"\"\n",
    "    tr_results = []\n",
    "    k = len(topics)\n",
    "    for i, topics_i in enumerate(topics):\n",
    "        w_counts = 0\n",
    "        for w in topics_i[:n]:\n",
    "            w_counts += np.sum([w in topics_j[:n] for j, topics_j in enumerate(topics) if j != i]) # count(k, l)\n",
    "        tr_results.append((1 / (k - 1)) * w_counts)\n",
    "    return tr_results\n",
    "\n",
    "\n",
    "def compute_td(topics, n=25):\n",
    "    \"\"\"\n",
    "    Compute topic diversity from \n",
    "    https://doi.org/10.1162/tacl_a_00325\n",
    "    \"\"\"\n",
    "    words = [w for topic in topics for w in topic[:n]]\n",
    "    return len(set(words)) / len(words)\n",
    "\n",
    "\n",
    "def compute_te(prob_w_given_topic, n=20, topics_sorted=None):\n",
    "    \"\"\"\n",
    "    Compute topic exclusivity from \n",
    "    https://icml.cc/Conferences/2012/papers/113.pdf\n",
    "    \"\"\"\n",
    "    # the should be normed (e.g., via softmax)\n",
    "    assert np.allclose(prob_w_given_topic.sum(1), 1)\n",
    "    num_topics = prob_w_given_topic.shape[0]\n",
    "    prob_w_normed = prob_w_given_topic.sum(0)\n",
    "    nonzero = prob_w_normed > 0\n",
    "    exclusivity = np.zeros_like(prob_w_given_topic)\n",
    "    exclusivity[:, nonzero] = prob_w_given_topic[:, nonzero] / prob_w_normed[nonzero]\n",
    "\n",
    "    if topics_sorted is None:\n",
    "        topics_sorted = np.flip(prob_w_given_topic.argsort(-1), -1)[:, :n]\n",
    "\n",
    "    return [np.mean(exclusivity[k, topics_sorted[k, :n]]) for k in range(num_topics)]\n",
    "\n",
    "\n",
    "def compute_to(topics, n=10):\n",
    "    \"\"\"\n",
    "    Calculate topic overlap (number of unique topic pairs sharing words)\n",
    "    \"\"\"\n",
    "    k = len(topics)\n",
    "    overlaps = np.zeros((k, k), dtype=float)\n",
    "    common_terms = np.zeros((k, k), dtype=float)\n",
    "    words = Counter([w for topic in topics for w in topic[:n]])\n",
    "\n",
    "    for i, t_i in enumerate(topics):\n",
    "        for j, t_j in enumerate(topics[i+1:], start=i+1):\n",
    "            if i != j:\n",
    "                overlap_ij = set(t_i[:n]) & set(t_j[:n])\n",
    "                overlaps[i, j] = len(overlap_ij) \n",
    "                common_terms[i, j] = sum(words[w] for w in overlap_ij)\n",
    "    \n",
    "    return overlaps.sum()\n",
    "\n",
    "\n",
    "def compute_tr_weighted(topics, n=10):\n",
    "    \"\"\"\n",
    "    Compute weighted topic redundancy score:\n",
    "    each additional word from the same topic counts more\n",
    "    than the previous\n",
    "\n",
    "    however, it fails the 3 overlapping topics ==\n",
    "    2 pairs of two topics test\n",
    "    \"\"\"\n",
    "    tr_results = []\n",
    "    k = len(topics)\n",
    "    for i, topics_i in enumerate(topics):\n",
    "        w_counts = 0\n",
    "        j_counts = defaultdict(int)\n",
    "        for w in topics_i[:n]:\n",
    "            for j, topics_j in enumerate(topics):\n",
    "                if j != i:\n",
    "                    if w in topics_j[:n]:\n",
    "                        j_counts[j] += 1\n",
    "                        w_counts += j_counts[j]\n",
    "\n",
    "        tr_results.append((1 / (k - 1)) * w_counts)\n",
    "    return np.array(tr_results)\n",
    "\n",
    "\n",
    "def compute_corrected_tr_weighted(topics, n=10, weight=0.9):\n",
    "    \"\"\"\n",
    "    Compute corrected topic redundancy score:\n",
    "    each additional word from the same topic counts more\n",
    "    than the previous, and words are downweighted\n",
    "    to account for double-counting\n",
    "    \"\"\"\n",
    "    tr_results = []\n",
    "    k = len(topics)\n",
    "    words = Counter([w for topic in topics for w in topic[:n]])\n",
    "    i_norm = ((k - 1) * sum(i for i in range(1, n+1)))\n",
    "    c_norm = (n * (k - 1))\n",
    "    w_norm = (n * k) * (k - 1)\n",
    "    w_c_norm = c_norm / w_norm\n",
    "\n",
    "    for i, topics_i in enumerate(topics):\n",
    "        i_counts = 0.\n",
    "        c_counts = 0.\n",
    "        j_counts = defaultdict(float)\n",
    "        w_counts = 0.\n",
    "        for w in topics_i[:n]:\n",
    "            for j, topics_j in enumerate(topics):\n",
    "                if j != i:\n",
    "                    if w in topics_j[:n]:\n",
    "                        j_counts[j] += 1\n",
    "                        i_counts += j_counts[j] * weight # TODO: some weighting\n",
    "                        c_counts += 1\n",
    "                        w_counts += words[w]\n",
    "        if c_counts == 0:\n",
    "            tr_results.append(0)\n",
    "        else:\n",
    "            #tr_results.append((1 / (k - 1)) * (i_counts) * (c_counts / w_counts))\n",
    "            #tr_results.append((i_counts / i_norm) * ((c_counts / w_counts) / w_c_norm) * 100)\n",
    "            tr_results.append((i_counts / i_norm) * (c_counts / w_counts) * 100)\n",
    "\n",
    "    return np.array(tr_results)\n",
    "\n",
    "def compute_corrected_to(topics, n=10, multiplier=2):\n",
    "    \"\"\"\n",
    "    A sensible overlap / redundancy measure. Words from a topic\n",
    "    are only counted once per \"edge\"\n",
    "    \"\"\"\n",
    "    k = len(topics)\n",
    "    # create de-duplicated adjacency matrix\n",
    "    # for each topic A_i, sorted by total number of overlaps:\n",
    "    #    create set of sets S = {S_{ij} = A_i \\cap A_j st. j=i+1,...,k}\n",
    "    #    sort sets in S by their cardinality in descending order\n",
    "    #    initialize a set W = {}\n",
    "    #    For each S_{ij}' in S:\n",
    "    #        if words are not already part of an edge, i.e., |W \\cap S_{ij}'| is 0:\n",
    "    #           create an edge between A_i and A_j with weight w = |S_{ij}'|\n",
    "    #           augment the list of words used in an edge, W = W \\cup S_{ij}'\n",
    "    overlap_counts = np.zeros((k, k), dtype=int)\n",
    "    overlap_dedup = np.zeros((k, k), dtype=int)\n",
    "    overlap_words = {}\n",
    "\n",
    "    # first count all the overlaps between topics\n",
    "    for i, topic_i in enumerate(topics):\n",
    "        for j, topic_j in enumerate(topics[i+1:], start=i+1):\n",
    "            words_ij = set(topic_i[:n]) & set(topic_j[:n])\n",
    "            overlap_counts[[i, j], [j, i]] = len(words_ij)\n",
    "            overlap_words[frozenset([i, j])] = words_ij\n",
    "\n",
    "    # sort topics by those with most overlaps\n",
    "    sort_idx = overlap_counts.sum(0).argsort()[::-1]\n",
    "    overlap_counts = overlap_counts[sort_idx, :][:, sort_idx]\n",
    "    for i, counts in enumerate(overlap_counts):\n",
    "        counted_words = set()\n",
    "        start = i + 1\n",
    "        for j in (counts[start:].argsort()[::-1] + start):\n",
    "            words_ij = overlap_words[frozenset([i, j])]\n",
    "            if overlap_counts[i, j] > 0 and len(counted_words & words_ij) == 0:\n",
    "                overlap_dedup[i, j] = overlap_counts[i, j]\n",
    "                counted_words |= words_ij\n",
    "\n",
    "    # TODO: incorporate equivalencies / clean up the below to be neater\n",
    "    # how many single word overlaps are equivalent to a full topic overlap\n",
    "    increments = np.linspace(1/multiplier, n, num=n)\n",
    "    redundancy = increments[overlap_dedup[overlap_dedup > 0] - 1].sum() / (n * (k - 1))\n",
    "    # redundancy = sum(increments[o - 1] for o in overlap_dedup[overlap_dedup > 0]) / (n * (k - 1))\n",
    "\n",
    "    # old redundancy = (overlap_dedup * (overlap_dedup/n)).sum() / ((n * (k - 1)))\n",
    "    # overlaps = np.zeros((k, k), dtype=int)\n",
    "    # for i, topic_i in enumerate(topics[:-1]):\n",
    "    #     intersections = [\n",
    "    #         (j, set(topic_i[:n]) & set(topic_j[:n])) for j, topic_j in enumerate(topics[i+1:], start=i+1)\n",
    "    #     ]\n",
    "    #     intersections = sorted(intersections, key=lambda x: len(x[1]), reverse=True)\n",
    "    #     counted_words = set()\n",
    "    #     for j, int_j in intersections:\n",
    "    #         if len(int_j) > 0 and len(int_j & counted_words) == 0:\n",
    "    #             overlaps[i, j] = len(int_j)\n",
    "    #             counted_words |= int_j\n",
    "    # redundancy = (overlaps * overlaps/n).sum() / ((n * (k - 1)))\n",
    "    # TODO: also return overlap counts\n",
    "    return redundancy\n",
    "\n",
    "\n",
    "def compute_all_redundancies(\n",
    "    beta: np.ndarray = None,\n",
    "    topics_sorted: np.ndarray = None,\n",
    "    n: Union[int, Dict[str, int]] = {\"tu\": 10, \"tr\": 10, \"td\": 25, \"te\": 20, \"to\": 10},\n",
    "    print_out: bool = True,\n",
    "    ):\n",
    "    if isinstance(n, int):\n",
    "        n = {\"tu\": n, \"tr\": n, \"td\": n, \"te\": n, \"to\": n}\n",
    "    \n",
    "    if topics_sorted is None:\n",
    "        topics_sorted = np.flip(beta.argsort(-1), -1)\n",
    "\n",
    "    metrics = {\n",
    "        \"tu\": np.mean(compute_tu(topics_sorted, n[\"tu\"])),\n",
    "        \"tr\": np.mean(compute_tr(topics_sorted, n[\"tr\"])),\n",
    "        \"tr_w\": np.mean(compute_corrected_tr_weighted(topics_sorted, n[\"tr\"])),\n",
    "        \"td\": compute_td(topics_sorted, n[\"td\"]),\n",
    "        \"to\": np.mean(compute_corrected_to(topics_sorted, n[\"to\"])),\n",
    "    }\n",
    "    if beta is not None:\n",
    "        metrics[\"te\"] = np.mean(compute_te(beta, n[\"te\"], topics_sorted))\n",
    "    if print_out:\n",
    "        print(*[f\"{k}: {v:0.3f}\" for k, v in metrics.items()])\n",
    "\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_topics(topics, transpose=True):\n",
    "    cols = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'\n",
    "    k, n = topics.shape\n",
    "    if k > len(cols):\n",
    "        raise ValueError(\"Too many topics\")\n",
    "    df = pd.DataFrame(topics, index=list(cols[:k]))\n",
    "    if transpose:\n",
    "        df = df.T\n",
    "    print(df.to_string(index=not transpose, header=transpose))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = 5000\n",
    "k = 20\n",
    "n = 10\n",
    "\n",
    "np.random.seed(11235)\n",
    "rand_idx = lambda k: np.random.choice(k, k, replace=False)\n",
    "\n",
    "unif_param = np.full(v, 1 / v)\n",
    "biased_param = np.random.dirichlet(unif_param) + 1e-6\n",
    "\n",
    "beta_unif = np.array([np.random.dirichlet(unif_param) for _ in range(k)])\n",
    "beta_biased = np.array([np.random.dirichlet(biased_param) for _ in range(k)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.975 tr: 0.026 tr_w: 0.022 td: 0.946 to: 0.013 te: 0.951\n"
     ]
    }
   ],
   "source": [
    "_ = compute_all_redundancies(beta=beta_unif, print_out=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.485 tr: 3.553 tr_w: 0.799 td: 0.222 to: 0.428 te: 0.202\n"
     ]
    }
   ],
   "source": [
    "scores = compute_all_redundancies(beta=beta_biased, print_out=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.485 tr: 3.553 tr_w: 0.799 td: 0.222 to: 0.428 te: 0.202\ntu: 0.485 tr: 3.553 tr_w: 0.799 td: 0.485 to: 0.428 te: 0.400\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "metadata": {},
     "execution_count": 8
    }
   ],
   "source": [
    "# some robustness checks\n",
    "shuff_beta = beta_biased[rand_idx(k)]\n",
    "scores == compute_all_redundancies(beta=shuff_beta, print_out=True)\n",
    "shuff_topics = np.flip(shuff_beta.argsort(1), -1)[:, :n]\n",
    "shuff_topics = shuff_topics[:, rand_idx(n)]\n",
    "scores == compute_all_redundancies(topics_sorted=shuff_topics, beta=shuff_beta, print_out=True)"
   ]
  },
  {
   "source": [
    "## Synthetic overlap tests\n",
    "k > n will yield equality between all metrics\n",
    "\n",
    "n >= k means that \"overlap\" does worse (undesirable)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# synthetic overlaps\n",
    "n = 5\n",
    "k = 6\n",
    "\n",
    "rand_idx = lambda k: np.random.choice(k, k, replace=False)\n",
    "helper_idx = int(np.power(10, np.ceil(np.log10(n)))) # helps visualize better\n",
    "topics = np.array([helper_idx*i+np.arange(n) for i in range(k)])\n",
    "\n",
    "max_idx = topics.max()\n",
    "gen_beta = lambda topics: np.array([np.bincount(topic, minlength=max_idx + 1) / n for topic in topics])\n",
    "beta = gen_beta(topics)\n",
    "\n",
    "duplicate_terms = topics[0]\n",
    "duplicate_probs = beta[0]\n",
    "# worst cast scenario: all repeated\n",
    "topics_all_repeats = [np.arange(n) for _ in range(k)]\n",
    "\n",
    "# one topic copied entirely\n",
    "topics_overlapping = np.vstack([duplicate_terms, topics[:k-1]])\n",
    "beta_overlapping = gen_beta(topics_overlapping)\n",
    "\n",
    "# duplicate words are distributed evenly across topics \n",
    "idx = min(n, k -1)\n",
    "topics_distributed = np.copy(topics)\n",
    "topics_distributed[1:idx+1, 0] = duplicate_terms[:idx]\n",
    "beta_distributed = gen_beta(topics_distributed)\n",
    "\n",
    "# single duplicate word distributed across topics\n",
    "topics_redundant_word = np.copy(topics)\n",
    "topics_redundant_word[0:idx+1, 0] = topics_redundant_word[0, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([[ 0,  1,  2,  3,  4],\n",
       "       [10, 11, 12, 13, 14],\n",
       "       [20, 21, 22, 23, 24],\n",
       "       [30, 31, 32, 33, 34],\n",
       "       [40, 41, 42, 43, 44],\n",
       "       [50, 51, 52, 53, 54]])"
      ]
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([[ 0,  1,  2,  3,  4],\n",
       "       [ 0,  1,  2,  3,  4],\n",
       "       [10, 11, 12, 13, 14],\n",
       "       [20, 21, 22, 23, 24],\n",
       "       [30, 31, 32, 33, 34],\n",
       "       [40, 41, 42, 43, 44]])"
      ]
     },
     "metadata": {},
     "execution_count": 21
    }
   ],
   "source": [
    "topics_overlapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([[ 0,  1,  2,  3,  4],\n",
       "       [ 0, 11, 12, 13, 14],\n",
       "       [ 1, 21, 22, 23, 24],\n",
       "       [ 2, 31, 32, 33, 34],\n",
       "       [ 3, 41, 42, 43, 44],\n",
       "       [ 4, 51, 52, 53, 54]])"
      ]
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "source": [
    "topics_distributed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 1.000 tr: 0.000 tr_w: 0.000 td: 1.000 to: 0.000 te: 1.000\n"
     ]
    }
   ],
   "source": [
    "_ = compute_all_redundancies(beta=beta, topics_sorted=topics, n=n) # best value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.167 tr: 5.000 tr_w: 15.000 td: 0.167 to: 1.000\n"
     ]
    }
   ],
   "source": [
    "_ = compute_all_redundancies(topics_sorted=topics_all_repeats, n=n) # worst value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.833 tr: 0.333 tr_w: 3.000 td: 0.833 to: 0.200 te: 0.833\n"
     ]
    }
   ],
   "source": [
    "_ = compute_all_redundancies(beta=beta_overlapping, topics_sorted=topics_overlapping, n=n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.833 tr: 0.333 tr_w: 1.000 td: 0.833 to: 0.100 te: 0.833\n"
     ]
    }
   ],
   "source": [
    "_ = compute_all_redundancies(beta=beta_distributed, topics_sorted=topics_distributed, n=n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.952 tr: 0.262 tr_w: 0.039 td: 0.952 to: 0.025\n"
     ]
    }
   ],
   "source": [
    "_ = compute_all_redundancies(topics_sorted=topics_redundant_word, n=n)"
   ]
  },
  {
   "source": [
    "Exploring the distributed/overlap a bit further"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "1 topic(s) with 10 repeats each\n",
      "tu: 0.952 tr: 0.048 tr_w: 0.214 td: 0.952 to: 0.050 te: 0.952\n",
      "2 topic(s) with 5 repeats each\n",
      "tu: 0.952 tr: 0.048 tr_w: 0.117 td: 0.952 to: 0.047 te: 0.952\n",
      "5 topic(s) with 2 repeats each\n",
      "tu: 0.952 tr: 0.048 tr_w: 0.058 td: 0.952 to: 0.039 te: 0.952\n",
      "10 topic(s) with 1 repeats each\n",
      "tu: 0.952 tr: 0.048 tr_w: 0.039 td: 0.952 to: 0.025 te: 0.952\n"
     ]
    }
   ],
   "source": [
    "for topics_with_repeats in range(1, min(n, k - 1)+1):\n",
    "    if len(duplicate_terms) % topics_with_repeats == 0:\n",
    "        bucket_size = len(duplicate_terms) // topics_with_repeats\n",
    "        topics_distributed_i = np.copy(topics)\n",
    "        for i in range(topics_with_repeats):\n",
    "            # duplicate words are distributed evenly across topics \n",
    "            topics_distributed_i[i+1, 0:bucket_size] = duplicate_terms[i*bucket_size:(i+1)*bucket_size]\n",
    "        print(f\"{topics_with_repeats} topic(s) with {bucket_size} repeats each\")\n",
    "        beta_distributed_i = gen_beta(topics_distributed_i)\n",
    "        compute_all_redundancies(beta=beta_distributed_i, topics_sorted=topics_distributed_i, n=n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "metadata": {},
     "execution_count": 27
    }
   ],
   "source": [
    "topics_with_repeats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Overlap: \n",
      "tu: 0.952 tr: 0.048 tr_w: 0.214 td: 0.952 to: 0.050\n",
      "\n",
      "Dist: \n",
      "tu: 0.952 tr: 0.048 tr_w: 0.039 td: 0.952 to: 0.025\n",
      "\n",
      "Dist double: \n",
      "tu: 0.905 tr: 0.143 tr_w: 0.078 td: 0.905 to: 0.050\n"
     ]
    }
   ],
   "source": [
    "if k >= (1+2*len(duplicate_terms)):\n",
    "    idx = len(duplicate_terms)\n",
    "    topics_distributed_double = np.copy(topics)\n",
    "    topics_distributed_double[1:(2*idx)+1, 0] = np.concatenate([duplicate_terms, duplicate_terms])\n",
    "    print(\"Overlap: \")\n",
    "    _ = compute_all_redundancies(topics_sorted=topics_overlapping, n=n)\n",
    "    print(\"\\nDist: \")\n",
    "    _ = compute_all_redundancies(topics_sorted=topics_distributed, n=n)\n",
    "    print(\"\\nDist double: \")\n",
    "    _ = compute_all_redundancies(topics_sorted=topics_distributed_double, n=n)"
   ]
  },
  {
   "source": [
    "Other synthetic examples of overlap.\n",
    "\n",
    "Four completely overlapping topics should score worst\n",
    "\n",
    "But two pairs of two identical topics vs.\n",
    "three identical topics should be the same:\n",
    "in both cases, you have effectively k-2 total topics"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "t0 = topics[0]\n",
    "t1 = topics[1]\n",
    "\n",
    "three_of_a_kind = np.vstack([t0, t0, topics[:k-2]])\n",
    "two_pair = np.vstack([t0, t0, t1, t1, topics[4:]])\n",
    "\n",
    "four_of_a_kind = np.vstack([t0, t0, t0, topics[:k-3]])\n",
    "full_house = np.vstack([t0, t0, t0, t1, t1, topics[5:]])\n",
    "assert four_of_a_kind.shape == three_of_a_kind.shape == two_pair.shape == full_house.shape == (k, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.900 tr: 0.158 tr_w: 0.474 td: 0.900 to: 0.105\n"
     ]
    }
   ],
   "source": [
    "_ = compute_all_redundancies(topics_sorted=three_of_a_kind, n=n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.900 tr: 0.105 tr_w: 0.474 td: 0.900 to: 0.105\n"
     ]
    }
   ],
   "source": [
    "_ = compute_all_redundancies(topics_sorted=two_pair, n=n) # should be == to three_of_a_kind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.850 tr: 0.316 tr_w: 0.711 td: 0.850 to: 0.158\n"
     ]
    }
   ],
   "source": [
    "_ = compute_all_redundancies(topics_sorted=four_of_a_kind, n=n) # should be \"worse\" than three_of_a_kind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.850 tr: 0.211 tr_w: 0.711 td: 0.850 to: 0.158\n"
     ]
    }
   ],
   "source": [
    "_ = compute_all_redundancies(topics_sorted=full_house, n=n) # should be == four_of_a_kind"
   ]
  },
  {
   "source": [
    "This is another similar artificial test. I think this too should be the same, but it's not clear this will hold true for metrics that meet other criteria"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.625 tr: 1.333 tr_w: 8.812 td: 0.625 to: 0.472\ntu: 0.625 tr: 1.000 tr_w: 9.750 td: 0.625 to: 0.472\n"
     ]
    }
   ],
   "source": [
    "semi_three_of_a_kind = [\n",
    "    [ 1,  2,  3,  4],\n",
    "    [ 1,  2,  3,  4],\n",
    "    [ 1,  2, 23, 24],\n",
    "    [31, 32, 33, 34],\n",
    "]\n",
    "\n",
    "semi_two_pair = [\n",
    "    [ 1,  2,  3,  4],\n",
    "    [ 1,  2,  3,  4],\n",
    "    [31, 32, 23, 24],\n",
    "    [31, 32, 33, 34],\n",
    "]\n",
    "\n",
    "_ = compute_all_redundancies(topics_sorted=semi_three_of_a_kind, n=4)\n",
    "_ = compute_all_redundancies(topics_sorted=semi_two_pair, n=4)"
   ]
  },
  {
   "source": [
    "Here we do a similar thing but topics overlap only partially (7 words)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "three_of_a_kind_partial = np.copy(topics)\n",
    "three_of_a_kind_partial[:, :n-3] = three_of_a_kind[:, :n-3]\n",
    "two_pair_partial = np.copy(topics)\n",
    "two_pair_partial[:, :n-3] = two_pair[:, :n-3]\n",
    "\n",
    "four_of_a_kind_partial = np.copy(topics)\n",
    "four_of_a_kind_partial[:, :n-3] = four_of_a_kind[:, :n-3]\n",
    "full_house_partial = np.copy(topics)\n",
    "full_house_partial[:, :n-3] = full_house[:, :n-3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.927 tr: 0.109 tr_w: 0.327 td: 0.927 to: 0.065\ntu: 0.927 tr: 0.073 tr_w: 0.327 td: 0.927 to: 0.065\ntu: 0.891 tr: 0.218 tr_w: 0.491 td: 0.891 to: 0.098\ntu: 0.891 tr: 0.145 tr_w: 0.491 td: 0.891 to: 0.098\n"
     ]
    }
   ],
   "source": [
    "_ = compute_all_redundancies(topics_sorted=three_of_a_kind_partial, n=n)\n",
    "_ = compute_all_redundancies(topics_sorted=two_pair_partial, n=n)\n",
    "\n",
    "_ = compute_all_redundancies(topics_sorted=four_of_a_kind_partial, n=n)\n",
    "_ = compute_all_redundancies(topics_sorted=full_house_partial, n=n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.818 tr: 0.273 tr_w: 1.636 td: 0.818 to: 0.200\ntu: 0.891 tr: 0.218 tr_w: 0.491 td: 0.891 to: 0.098\ntu: 0.927 tr: 0.182 tr_w: 0.218 td: 0.927 to: 0.040\n"
     ]
    }
   ],
   "source": [
    "# TODO: \n",
    "# consider when four_of_a_kind_partial should be less\n",
    "# than tree_of_a_kind (full)\n",
    "\n",
    "# for n = 10, 30 total redundant words\n",
    "# NB that four-of-a-kind with j word overlap is same as five-of-a-kind with j-1 (?)\n",
    "five_of_a_kind = np.vstack([np.vstack([t0, t0, t0, t0, topics[:k-4]])])\n",
    "five_of_a_kind_partial_6 = np.copy(topics)\n",
    "five_of_a_kind_partial_6[:, :n-4] = five_of_a_kind[:, :n-4]\n",
    "_ = compute_all_redundancies(topics_sorted=three_of_a_kind, n=n)\n",
    "_ = compute_all_redundancies(topics_sorted=four_of_a_kind_partial, n=n)\n",
    "_ = compute_all_redundancies(topics_sorted=five_of_a_kind_partial_6, n=n)"
   ]
  },
  {
   "source": [
    "Now, consider sets of n//2 repeating words:\n",
    "\n",
    "1 - n//2 words from one topic are found in one other topic\n",
    "\n",
    "2 - the same set of n//2 words from one topic appear in two other topics\n",
    "\n",
    "3 - two different sets of n//2 words from one topic appear once each in two other topics\n",
    "\n",
    "We want (larger is better): 1 > 3 >= 2 (it's not a dealbreaker if 2 and 3, but not preferable)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "mid_idx = n // 2\n",
    "topics_half_overlap = np.copy(topics)\n",
    "topics_half_overlap[1, :mid_idx] = topics_half_overlap[0, :mid_idx]\n",
    "\n",
    "topics_half_overlap_2x = np.copy(topics)\n",
    "topics_half_overlap_2x[1, :mid_idx] = topics_half_overlap_2x[0, :mid_idx]\n",
    "topics_half_overlap_2x[2, :mid_idx] = topics_half_overlap_2x[0, :mid_idx]\n",
    "\n",
    "topics_seperate_half_overlap = np.copy(topics)\n",
    "topics_seperate_half_overlap[1, :mid_idx] = topics_seperate_half_overlap[0, :mid_idx]\n",
    "topics_seperate_half_overlap[2, :mid_idx] = topics_seperate_half_overlap[0, mid_idx:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tu: 0.975 tr: 0.026 tr_w: 0.065 td: 0.975 to: 0.025\ntu: 0.950 tr: 0.079 tr_w: 0.129 td: 0.950 to: 0.050\ntu: 0.950 tr: 0.053 tr_w: 0.129 td: 0.950 to: 0.050\n"
     ]
    }
   ],
   "source": [
    "_ = compute_all_redundancies(topics_sorted=topics_half_overlap, n=n)\n",
    "_ = compute_all_redundancies(topics_sorted=topics_half_overlap_2x, n=n)\n",
    "_ = compute_all_redundancies(topics_sorted=topics_seperate_half_overlap, n=n)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}