docs/Haystack_basic_usage.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's look at a basic usage example for Haystack.\n",
    "\n",
    "First we have a program which allocates structures on the HEAP:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/*\r\n",
      "    Simple test of multiple simple structures allocations.\r\n",
      "*/\r\n",
      "\r\n",
      "#include <stdio.h>\r\n",
      "#include <stdlib.h>\r\n",
      "#include <unistd.h>\r\n",
      "#include <sys/types.h>\r\n",
      "#include <dlfcn.h>\r\n",
      "#include <stdbool.h>\r\n",
      "\r\n",
      "struct test3\r\n",
      "{\r\n",
      "  unsigned int val1;\r\n",
      "  unsigned int val2;\r\n",
      "  unsigned int * me;\r\n",
      "  unsigned int val2b;\r\n",
      "  unsigned int val1b;\r\n",
      "};\r\n",
      "\r\n",
      "struct Node {\r\n",
      "  unsigned int val1;\r\n",
      "  void * ptr1;\r\n",
      "  void * ptr2;\r\n",
      "};\r\n",
      "\r\n",
      "int test3(){\r\n",
      "  struct test3 * t3;\r\n",
      "  t3 = (struct test3 *) malloc(sizeof(struct test3));\r\n",
      "  t3->val1 = 0xdeadbeef;\r\n",
      "  t3->val1b = 0xdeadbeef;\r\n",
      "  t3->val2 = 0x10101010;\r\n",
      "  t3->val2b = 0x10101010;\r\n",
      "  t3->me = (unsigned int *) t3;\r\n",
      "  printf(\"o: test3 %p\\n\",t3);\r\n",
      "\r\n",
      "  return 0;\r\n",
      "}\r\n",
      "\r\n",
      "int test1(){\r\n",
      "  struct Node * node;\r\n",
      "  node = (struct Node *) malloc(sizeof(struct Node));\r\n",
      "  node->val1 = 0xdeadbeef;\r\n",
      "  node->ptr1 = node;\r\n",
      "  node->ptr2 = node;\r\n",
      "  printf(\"o: test1 %p\\n\",node);\r\n",
      "\r\n",
      "  return 0;\r\n",
      "}\r\n",
      "\r\n",
      "\r\n",
      "int main(){\r\n",
      "\r\n",
      "  // TEST\r\n",
      "  test1();\r\n",
      "  test3();\r\n",
      "  test1();\r\n",
      "  test3();\r\n",
      "  test1();\r\n",
      "  test3();\r\n",
      "\r\n",
      "  printf(\"pid %u\\n\",getpid());\r\n",
      "  fflush(stdout);\r\n",
      "  sleep(-1);\r\n",
      "\r\n",
      "  return 0;\r\n",
      "}\r\n",
      "\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!cat ../test/src/test-ctypes3.c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then, using ctypeslib, we have generated python ctypes records for the same structures."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "find_library(\"clang-3.7\") libclang-3.7.so.1\n",
      "find_library(\"clang\") None\n",
      "# -*- coding: utf-8 -*-\n",
      "#\n",
      "# TARGET arch is: []\n",
      "# WORD_SIZE is: 8\n",
      "# POINTER_SIZE is: 8\n",
      "# LONGDOUBLE_SIZE is: 16\n",
      "#\n",
      "import ctypes\n",
      "\n",
      "\n",
      "# if local wordsize is same as target, keep ctypes pointer function.\n",
      "if ctypes.sizeof(ctypes.c_void_p) == 8:\n",
      "    POINTER_T = ctypes.POINTER\n",
      "else:\n",
      "    # required to access _ctypes\n",
      "    import _ctypes\n",
      "    # Emulate a pointer class using the approriate c_int32/c_int64 type\n",
      "    # The new class should have :\n",
      "    # ['__module__', 'from_param', '_type_', '__dict__', '__weakref__', '__doc__']\n",
      "    # but the class should be submitted to a unique instance for each base type\n",
      "    # to that if A == B, POINTER_T(A) == POINTER_T(B)\n",
      "    ctypes._pointer_t_type_cache = {}\n",
      "    def POINTER_T(pointee):\n",
      "        # a pointer should have the same length as LONG\n",
      "        fake_ptr_base_type = ctypes.c_uint64 \n",
      "        # specific case for c_void_p\n",
      "        if pointee is None: # VOID pointer type. c_void_p.\n",
      "            pointee = type(None) # ctypes.c_void_p # ctypes.c_ulong\n",
      "            clsname = 'c_void'\n",
      "        else:\n",
      "            clsname = pointee.__name__\n",
      "        if clsname in ctypes._pointer_t_type_cache:\n",
      "            return ctypes._pointer_t_type_cache[clsname]\n",
      "        # make template\n",
      "        class _T(_ctypes._SimpleCData,):\n",
      "            _type_ = 'L'\n",
      "            _subtype_ = pointee\n",
      "            def _sub_addr_(self):\n",
      "                return self.value\n",
      "            def __repr__(self):\n",
      "                return '%s(%d)'%(clsname, self.value)\n",
      "            def contents(self):\n",
      "                raise TypeError('This is not a ctypes pointer.')\n",
      "            def __init__(self, **args):\n",
      "                raise TypeError('This is not a ctypes pointer. It is not instanciable.')\n",
      "        _class = type('LP_%d_%s'%(8, clsname), (_T,),{}) \n",
      "        ctypes._pointer_t_type_cache[clsname] = _class\n",
      "        return _class\n",
      "\n",
      "c_int128 = ctypes.c_ubyte*16\n",
      "c_uint128 = c_int128\n",
      "void = None\n",
      "if ctypes.sizeof(ctypes.c_longdouble) == 16:\n",
      "    c_long_double_t = ctypes.c_longdouble\n",
      "else:\n",
      "    c_long_double_t = ctypes.c_ubyte*16\n",
      "\n",
      "\n",
      "\n",
      "class struct_test3(ctypes.Structure):\n",
      "    _pack_ = True # source:False\n",
      "    _fields_ = [\n",
      "    ('val1', ctypes.c_uint32),\n",
      "    ('val2', ctypes.c_uint32),\n",
      "    ('me', POINTER_T(ctypes.c_uint32)),\n",
      "    ('val2b', ctypes.c_uint32),\n",
      "    ('val1b', ctypes.c_uint32),\n",
      "     ]\n",
      "\n",
      "class struct_Node(ctypes.Structure):\n",
      "    _pack_ = True # source:False\n",
      "    _fields_ = [\n",
      "    ('val1', ctypes.c_uint32),\n",
      "    ('PADDING_0', ctypes.c_ubyte * 4),\n",
      "    ('ptr1', POINTER_T(None)),\n",
      "    ('ptr2', POINTER_T(None)),\n",
      "     ]\n",
      "\n",
      "__all__ = \\\n",
      "    ['struct_Node', 'struct_test3']\n"
     ]
    }
   ],
   "source": [
    "!clang2py ../test/src/test-ctypes3.c #-o ../test/src/clang3_gen64.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have run the compiled C program, and dumped its memory mappings to files:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 2,3M\n",
      "4,0K 0x0000000000400000-0x0000000000401000\n",
      "4,0K 0x0000000000600000-0x0000000000601000\n",
      "132K 0x000000000072c000-0x000000000074d000\n",
      "1,8M 0x00007fb7519db000-0x00007fb751b9b000\n",
      " 16K 0x00007fb751d9b000-0x00007fb751d9f000\n",
      "8,0K 0x00007fb751d9f000-0x00007fb751da1000\n",
      " 16K 0x00007fb751da1000-0x00007fb751da5000\n",
      " 12K 0x00007fb751da5000-0x00007fb751da8000\n",
      "4,0K 0x00007fb751fa7000-0x00007fb751fa8000\n",
      "4,0K 0x00007fb751fa8000-0x00007fb751fa9000\n",
      "144K 0x00007fb751fa9000-0x00007fb751fcd000\n",
      " 12K 0x00007fb7521a2000-0x00007fb7521a5000\n",
      " 12K 0x00007fb7521c9000-0x00007fb7521cc000\n",
      "4,0K 0x00007fb7521cc000-0x00007fb7521cd000\n",
      "4,0K 0x00007fb7521cd000-0x00007fb7521ce000\n",
      "4,0K 0x00007fb7521ce000-0x00007fb7521cf000\n",
      "132K 0x00007ffcf8c6e000-0x00007ffcf8c8f000\n",
      "4,0K mappings\n",
      "\n",
      "o: test1 0x72c010\n",
      "o: test3 0x72c030\n",
      "o: test1 0x72c050\n",
      "o: test3 0x72c070\n",
      "o: test1 0x72c090\n",
      "o: test3 0x72c0b0\n",
      "pid 5931\n"
     ]
    }
   ],
   "source": [
    "!ls -hsw 1 ../test/src/test-ctypes3.64.dump/\n",
    "print ''\n",
    "!cat ../test/src/test-ctypes3.64.stdout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's load that memory dump in haystack."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<MemoryHandler for /home/other/Compil/python-haystack/test/src/test-ctypes3.64.dump with 17 mappings>\n"
     ]
    }
   ],
   "source": [
    "import haystack\n",
    "memdumpname = '../test/src/test-ctypes3.64.dump'\n",
    "# we need a memory dump loader\n",
    "from haystack import dump_loader\n",
    "memory_handler = dump_loader.load(memdumpname)\n",
    "print memory_handler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the first feature of haystack is to search a specific structure in memory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['struct_Node', 'c_int128', '__all__', '__builtins__', '__file__', 'c_uint128', 'struct_test3', '__package__', 'ctypes', 'void', 'POINTER_T', '__name__', 'c_long_double_t', '__doc__']\n"
     ]
    }
   ],
   "source": [
    "# we need to add our test path to the env\n",
    "import sys\n",
    "sys.path.append('../test/src/')\n",
    "py_modulename = 'ctypes3_gen64'\n",
    "\n",
    "# load this module with haystack\n",
    "my_model = memory_handler.get_model()\n",
    "test3 = my_model.import_module(\"ctypes3_gen64\")\n",
    "print test3.__dict__.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can search for the structure in memory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(<ctypes3_gen64.struct_test3 object at 0x7f5cea101e60>, 7520272), (<ctypes3_gen64.struct_test3 object at 0x7f5cea101ef0>, 7520304L), (<ctypes3_gen64.struct_test3 object at 0x7f5cea106050>, 7520336L), (<ctypes3_gen64.struct_test3 object at 0x7f5cea106170>, 7520368L), (<ctypes3_gen64.struct_test3 object at 0x7f5cea106290>, 7520400L), (<ctypes3_gen64.struct_test3 object at 0x7f5cea1063b0>, 7520432L)]\n"
     ]
    }
   ],
   "source": [
    "py_class = test3.struct_test3\n",
    "results = haystack.search_record(memory_handler, py_class)\n",
    "print results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The output is a list of ctypes records and their memory offset.\n",
    "We can also get a better formatted string output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[# --------------- 0x72c010 \n",
      "{ # <struct_test3 at 0x72c010>\n",
      "\"val1\": 3735928559L, # c_uint\n",
      "\"val2\": 0L, # c_uint\n",
      "\"me\": 3735928559, # c_uint 0xdeadbeefL,\n",
      "\"val2b\": 7520272L, # c_uint\n",
      "\"val1b\": 0L, # c_uint\n",
      "}# --------------- 0x72c030 \n",
      "{ # <struct_test3 at 0x72c030>\n",
      "\"val1\": 3735928559L, # c_uint\n",
      "\"val2\": 269488144L, # c_uint\n",
      "\"me\": 3735928559, # c_uint 0xdeadbeefL,\n",
      "\"val2b\": 269488144L, # c_uint\n",
      "\"val1b\": 3735928559L, # c_uint\n",
      "}# --------------- 0x72c050 \n",
      "{ # <struct_test3 at 0x72c050>\n",
      "\"val1\": 3735928559L, # c_uint\n",
      "\"val2\": 0L, # c_uint\n",
      "\"me\": 3735928559, # c_uint 0xdeadbeefL,\n",
      "\"val2b\": 7520336L, # c_uint\n",
      "\"val1b\": 0L, # c_uint\n",
      "}# --------------- 0x72c070 \n",
      "{ # <struct_test3 at 0x72c070>\n",
      "\"val1\": 3735928559L, # c_uint\n",
      "\"val2\": 269488144L, # c_uint\n",
      "\"me\": 3735928559, # c_uint 0xdeadbeefL,\n",
      "\"val2b\": 269488144L, # c_uint\n",
      "\"val1b\": 3735928559L, # c_uint\n",
      "}# --------------- 0x72c090 \n",
      "{ # <struct_test3 at 0x72c090>\n",
      "\"val1\": 3735928559L, # c_uint\n",
      "\"val2\": 0L, # c_uint\n",
      "\"me\": 3735928559, # c_uint 0xdeadbeefL,\n",
      "\"val2b\": 7520400L, # c_uint\n",
      "\"val1b\": 0L, # c_uint\n",
      "}# --------------- 0x72c0b0 \n",
      "{ # <struct_test3 at 0x72c0b0>\n",
      "\"val1\": 3735928559L, # c_uint\n",
      "\"val2\": 269488144L, # c_uint\n",
      "\"me\": 3735928559, # c_uint 0xdeadbeefL,\n",
      "\"val2b\": 269488144L, # c_uint\n",
      "\"val1b\": 3735928559L, # c_uint\n",
      "}]\n"
     ]
    }
   ],
   "source": [
    "out = haystack.output_to_string(memory_handler, results)\n",
    "print out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Or even a translation of these records to plain old python dynamically generated objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(<haystack.model.ctypes3_gen64.struct_test3_py object at 0x7f5cea0dbd90>, 7520272), (<haystack.model.ctypes3_gen64.struct_test3_py object at 0x7f5cea0dbd10>, 7520304L), (<haystack.model.ctypes3_gen64.struct_test3_py object at 0x7f5cea0e3d90>, 7520336L), (<haystack.model.ctypes3_gen64.struct_test3_py object at 0x7f5cea0e3f50>, 7520368L), (<haystack.model.ctypes3_gen64.struct_test3_py object at 0x7f5cea0e3f90>, 7520400L), (<haystack.model.ctypes3_gen64.struct_test3_py object at 0x7f5cea0e3fd0>, 7520432L)]\n"
     ]
    }
   ],
   "source": [
    "out = haystack.output_to_python(memory_handler, results)\n",
    "print out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "@0x72c010 val1: 0xdeadbeef , val1b: 0x0 \n",
      "@0x72c030 val1: 0xdeadbeef , val1b: 0xdeadbeef \n",
      "@0x72c050 val1: 0xdeadbeef , val1b: 0x0 \n",
      "@0x72c070 val1: 0xdeadbeef , val1b: 0xdeadbeef \n",
      "@0x72c090 val1: 0xdeadbeef , val1b: 0x0 \n",
      "@0x72c0b0 val1: 0xdeadbeef , val1b: 0xdeadbeef \n"
     ]
    }
   ],
   "source": [
    "for x in out:\n",
    "    print \"@0x%x val1: 0x%x , val1b: 0x%x \"%(x[1], x[0].val1, x[0].val1b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There should only be 3 of struct_test3.\n",
    "The true instance are at 0x202a030, 0x202a070 and 0x202a0b0\n",
    "\n",
    "oh-oh, there is a bit too many chunks in memory which could be possible struct_test3.\n",
    "\n",
    "Let's apply more constraints to the search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r\n",
      "[struct_test3]\r\n",
      "val1: [0xdeadbeef]\r\n",
      "val1b: [0xdeadbeef]\r\n",
      "val2: [0x10101010]\r\n",
      "val2b: [0x10101010]\r\n",
      "\r\n",
      "\r\n",
      "[struct_Node]\r\n",
      "val1: [0xdeadbeef]\r\n",
      "ptr2: [NotNull]\r\n"
     ]
    }
   ],
   "source": [
    "!cat ../test/src/ctypes3.constraints"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we say that val1 and val1b fields should both be 0xdeadbeef.\n",
    "Let's apply these constraints to the search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[# --------------- 0x72c030 \n",
      "{ # <struct_test3 at 0x72c030>\n",
      "\"val1\": 3735928559L, # c_uint\n",
      "\"val2\": 269488144L, # c_uint\n",
      "\"me\": 3735928559, # c_uint 0xdeadbeefL,\n",
      "\"val2b\": 269488144L, # c_uint\n",
      "\"val1b\": 3735928559L, # c_uint\n",
      "}# --------------- 0x72c070 \n",
      "{ # <struct_test3 at 0x72c070>\n",
      "\"val1\": 3735928559L, # c_uint\n",
      "\"val2\": 269488144L, # c_uint\n",
      "\"me\": 3735928559, # c_uint 0xdeadbeefL,\n",
      "\"val2b\": 269488144L, # c_uint\n",
      "\"val1b\": 3735928559L, # c_uint\n",
      "}# --------------- 0x72c0b0 \n",
      "{ # <struct_test3 at 0x72c0b0>\n",
      "\"val1\": 3735928559L, # c_uint\n",
      "\"val2\": 269488144L, # c_uint\n",
      "\"me\": 3735928559, # c_uint 0xdeadbeefL,\n",
      "\"val2b\": 269488144L, # c_uint\n",
      "\"val1b\": 3735928559L, # c_uint\n",
      "}]\n"
     ]
    }
   ],
   "source": [
    "from haystack import constraints\n",
    "handler = constraints.ConstraintsConfigHandler()\n",
    "my_constraints = handler.read('../test/src/ctypes3.constraints')\n",
    "\n",
    "# now use them\n",
    "results = haystack.search_record(memory_handler, py_class, my_constraints)\n",
    "print haystack.output_to_string(memory_handler, results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "@0x72c030 val1: 0xdeadbeef , val1b: 0xdeadbeef \n",
      "@0x72c070 val1: 0xdeadbeef , val1b: 0xdeadbeef \n",
      "@0x72c0b0 val1: 0xdeadbeef , val1b: 0xdeadbeef \n"
     ]
    }
   ],
   "source": [
    "for x in results:\n",
    "    print \"@0x%x val1: 0x%x , val1b: 0x%x \"%(x[1], x[0].val1, x[0].val1b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "The constraints did reduce the number of results, and haystack only returns allocated chunks of memory that match these constraints."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}