day 34 - aho-corasick.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import deque, defaultdict\n",
    "from itertools import count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def aho_corasick():\n",
    "    G = defaultdict(count(1).__next__)  # transitions\n",
    "    W = defaultdict(set)                # alphabet\n",
    "    F = defaultdict(lambda: 0)          # fallbacks\n",
    "    O = defaultdict(set)                # outputs\n",
    "    \n",
    "    # automaton\n",
    "    return G, W, F, O"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def add_word(word, G, W, F, O):\n",
    "    state = 0\n",
    "\n",
    "    # add transitions between states\n",
    "    for w in word:\n",
    "        W[state].add(w)\n",
    "        state = G[state, w]\n",
    "        \n",
    "    # add output\n",
    "    O[state].add(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def build_fsa(G, W, F, O):\n",
    "    # initial states\n",
    "    queue = deque(G[0, w] for w in W[0])\n",
    "    \n",
    "    while queue:\n",
    "        state = queue.popleft()\n",
    "        \n",
    "        # for each letter in alphabet\n",
    "        for w in W[state]:\n",
    "            # find fallback state\n",
    "            t = F[state]\n",
    "            while t and (t, w) not in G:\n",
    "                t = F[t]\n",
    "                \n",
    "            # for next state define its fallback and output\n",
    "            s = G[state, w]\n",
    "            F[s] = G[t, w] if (t, w) in G else 0\n",
    "            O[s] |= O[F[s]]\n",
    "            \n",
    "            queue.append(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def search(text, G, W, F, O):\n",
    "    state = 0\n",
    "    \n",
    "    for i, t in enumerate(text):\n",
    "        # fallback\n",
    "        while state and (state, t) not in G:\n",
    "            state = F[state]\n",
    "            \n",
    "        # transition\n",
    "        state = G[state, t] if (state, t) in G else 0\n",
    "        \n",
    "        # output\n",
    "        if O[state]:\n",
    "            print('@', i, O[state])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "AC = aho_corasick()\n",
    "add_word('bar', *AC)\n",
    "add_word('ara', *AC)\n",
    "add_word('bara', *AC)\n",
    "add_word('barbara', *AC)\n",
    "build_fsa(*AC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "@ 2 {'bar'}\n",
      "@ 5 {'bar'}\n",
      "@ 12 {'bar'}\n",
      "@ 15 {'bar'}\n",
      "@ 16 {'ara', 'barbara', 'bara'}\n",
      "@ 26 {'bar'}\n",
      "@ 27 {'ara', 'bara'}\n"
     ]
    }
   ],
   "source": [
    "search('barbarian barbara said: barabum', *AC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}