Merge branch 'release-0.27.1'

k0retux · Mar 1, 2019 · e89f255 · e89f255
2 parents 90de14e + 9dd093b
commit e89f255
Show file tree

Hide file tree

Showing 42 changed files with 1,289 additions and 718 deletions.
diff --git a/data_models/file_formats/jpg.py b/data_models/file_formats/jpg.py
@@ -49,26 +49,13 @@ class JPG_DataModel(DataModel):
     file_extension = 'jpg'
     name = 'jpg'
 
-    def create_node_from_raw_data(self, data, idx, filename):
-        nm = 'jpg_{:0>2d}'.format(idx)
-        jpg = self.jpg.get_clone(nm, new_env=True)
-        jpg.set_current_conf('ABS', recursive=True)
-        status, off, size, name = jpg.absorb(data, constraints=AbsNoCsts(size=True, struct=True,
-                                                                         contents=True))
-
-        print('{:s} Absorb Status: {!r}, {:d}, {:d}, {:s}'.format(nm, status, off, size, name))
-        print(' \_ length of original jpg: {:d}'.format(len(data)))
-        print(' \_ remaining: {!r}'.format(data[size:size+1000]))
-
-        if status == AbsorbStatus.FullyAbsorbed:
-            x = jpg['.*/SOF_hdr/X'].get_raw_value()
-            y = jpg['.*/SOF_hdr/Y'].get_raw_value()
-            d_priv = {'height':y, 'width':x}
-            jpg.set_private(d_priv)
-            print("--> Create {:s} from provided JPG sample [x:{:d}, y:{:d}].".format(nm, x, y))
-            return jpg
-        else:
-            return None
+    def _atom_absorption_additional_actions(self, atom):
+        x = atom['.*/SOF_hdr/X'].get_raw_value()
+        y = atom['.*/SOF_hdr/Y'].get_raw_value()
+        d_priv = {'height':y, 'width':x}
+        atom.set_private(d_priv)
+        msg = "add private data: size [x:{:d}, y:{:d}]".format(x, y)
+        return atom, msg
 
     def build_data_model(self):
 
@@ -170,10 +157,15 @@ def build_data_model(self):
          ]}
 
         mb = NodeBuilder(delayed_jobs=True)
-        self.jpg = mb.create_graph_from_desc(jpg_desc)
+        jpg = mb.create_graph_from_desc(jpg_desc)
 
-        self.jpg_dict = self.import_file_contents(extension='jpg')
-        self.register(self.jpg, *self.jpg_dict.values())
+        self.register(jpg)
+
+        jpg_abs = jpg.get_clone(new_env=True)
+        jpg_abs.set_current_conf('ABS', recursive=True)
+        self.register_atom_for_absorption(jpg_abs,
+                                          absorb_constraints=AbsNoCsts(size=True, struct=True,
+                                                                       contents=True))
 
 
 data_model = JPG_DataModel()

diff --git a/data_models/file_formats/pdf.py b/data_models/file_formats/pdf.py
@@ -1175,7 +1175,7 @@ def build_data_model(self):
 
         PDFObj.external_pdf_objs = gather_pdf_objects()
 
-        e_jpg = self.get_external_atom(dm_name='jpg', data_id='jpg_00')
+        e_jpg = self.get_external_atom(dm_name='jpg', data_id='JPG_00')
 
         PDFObj.jpg_node = e_jpg
 
@@ -1213,6 +1213,7 @@ def build_data_model(self):
 
     from framework.plumbing import *
     fmk = FmkPlumbing()
+    fmk.start()
 
     dm = data_model
     dm.load_data_model(fmk._name2dm)

diff --git a/data_models/file_formats/png.py b/data_models/file_formats/png.py
@@ -31,22 +31,6 @@ class PNG_DataModel(DataModel):
     file_extension = 'png'
     name = 'png'
 
-    def create_node_from_raw_data(self, data, idx, filename):
-        nm = 'PNG_{:0>2d}'.format(idx)
-        png = self.png.get_clone(nm, new_env=True)
-        status, off, size, name = png.absorb(data, constraints=AbsNoCsts(size=True))
-
-        print('{:s} Absorb Status: {!r}, {:d}, {:d}, {:s}'.format(nm, status, off, size, name))
-        print(' \_ length of original png: {:d}'.format(len(data)))
-        print(' \_ remaining: {!r}'.format(data[size:size+1000]))
-
-        if status == AbsorbStatus.FullyAbsorbed:
-            print("--> Create {:s} from provided PNG samples.".format(nm))
-            return png
-        else:
-            return Node(nm, values=['PNG ABSORBSION FAILED'])
-
-
     def build_data_model(self):
 
         png_simple_desc = \
@@ -119,11 +103,10 @@ def build_data_model(self):
 
 
         mb = NodeBuilder()
-        self.png = mb.create_graph_from_desc(png_desc)
+        png = mb.create_graph_from_desc(png_desc)
 
-        self.png_dict = self.import_file_contents(extension='png')
-
-        self.register(*self.png_dict.values())
+        self.register(png)
+        self.register_atom_for_absorption(png, absorb_constraints=AbsNoCsts(size=True))
 
 
 data_model = PNG_DataModel()

diff --git a/data_models/file_formats/zip.py b/data_models/file_formats/zip.py
@@ -33,25 +33,6 @@ class ZIP_DataModel(DataModel):
     file_extension = 'zip'
     name = 'zip'
 
-    def create_node_from_raw_data(self, data, idx, filename):
-
-        nm = 'ZIP_{:0>2d}'.format(idx)
-        pkzip = self.pkzip.get_clone(nm, new_env=True)
-        pkzip.set_current_conf('ABS', recursive=True)
-        status, off, size, name = pkzip.absorb(data, constraints=AbsNoCsts(size=True,struct=True))
-        # pkzip.show(raw_limit=400)
-
-        print('{:s} Absorb Status: {!r}, {:d}, {:d}, {:s}'.format(nm, status, off, size, name))
-        print(' \_ length of original zip: {:d}'.format(len(data)))
-        print(' \_ remaining: {!r}'.format(data[size:size+1000]))
-
-        if status == AbsorbStatus.FullyAbsorbed:
-            print("--> Create {:s} from provided ZIP samples.".format(nm))
-            return pkzip
-        else:
-            return Node(nm, values=['ZIP ABSORBSION FAILED'])
-
-
     def build_data_model(self):
 
         MIN_FILE = 1
@@ -333,14 +314,15 @@ def build_data_model(self):
               ]}
         ]}
 
-
         mb = NodeBuilder(delayed_jobs=True)
-        self.pkzip = mb.create_graph_from_desc(zip_desc)
-
-        self.zip_dict = self.import_file_contents(extension='zip')
+        pkzip = mb.create_graph_from_desc(zip_desc)
 
-        self.register(self.pkzip, *self.zip_dict.values())
+        self.register(pkzip)
 
+        pkzip_abs = pkzip.get_clone(new_env=True)
+        pkzip_abs.set_current_conf('ABS', recursive=True)
+        self.register_atom_for_absorption(pkzip_abs,
+                                          absorb_constraints=AbsNoCsts(size=True, struct=True))
 
 data_model = ZIP_DataModel()
 
diff --git a/data_models/protocols/http.py b/data_models/protocols/http.py
@@ -29,9 +29,6 @@
 class HTTPModel(DataModel):
     name = 'HTTP'
 
-    def create_node_from_raw_data(self, data, idx, filename):
-        pass
-
     def build_data_model(self):
 
         OCTET = "\x00-\xFF"         # any 8-bit sequence of data

diff --git a/data_models/protocols/pppoe.py b/data_models/protocols/pppoe.py
@@ -29,9 +29,6 @@ class PPPOE_DataModel(DataModel):
 
     file_extension = 'bin'
 
-    def create_node_from_raw_data(self, data, idx, filename):
-        pass
-
     def build_data_model(self):
 
         # refer to RFC 2516

diff --git a/data_models/protocols/sms.py b/data_models/protocols/sms.py
@@ -29,9 +29,6 @@ class SMS_DataModel(DataModel):
 
     file_extension = 'sms'
 
-    def create_node_from_raw_data(self, data, idx, filename):
-        pass
-
     def build_data_model(self):
 
 

diff --git a/data_models/tutorial/myproto.py b/data_models/tutorial/myproto.py
@@ -1,3 +1,5 @@
+# -*- coding: utf8 -*-
+
 ################################################################################
 #
 #  Copyright 2018 Eric Lacombe <eric.lacombe@security-labs.org>
@@ -61,7 +63,7 @@ def build_data_model(self):
                              'contents': UINT32_be()},
                             {'name': 'content',
                              'sync_size_with': 'len',
-                             'contents': String(min_sz=20, max_sz=50, alphabet='éùijklm:;!',
+                             'contents': String(min_sz=20, max_sz=50, alphabet=u'éùijklm:;!',
                                                 codec='latin-1')},
                             {'name': 'crc32',
                              'contents': CRC(vt=UINT32_be),

diff --git a/data_models/tutorial/tuto.py b/data_models/tutorial/tuto.py
@@ -16,9 +16,6 @@ class MyDF_DataModel(DataModel):
     file_extension = 'df'
     name = 'mydf'
 
-    def create_node_from_raw_data(self, data, idx, filename):
-        pass
-
     def build_data_model(self):
 
         test_node_desc = \

diff --git a/data_models/tutorial/tuto_strategy.py b/data_models/tutorial/tuto_strategy.py
@@ -71,8 +71,7 @@ def before_data_processing_cbk(env, step):
 empty.connect_to(step4)
 step4.connect_to(step1, cbk_after_sending=cbk_transition2)
 
-sc1 = Scenario('ex1')
-sc1.set_anchor(step1)
+sc_tuto_ex1 = Scenario('ex1', anchor=step1)
 
 ### SCENARIO 2 ###
 step4 = Step(DataProcess(process=['tTYPE#2'], seed='shape'))
@@ -82,8 +81,7 @@ def before_data_processing_cbk(env, step):
 step2_copy.connect_to(step4, cbk_after_fbk=cbk_transition1)
 step4.connect_to(step_final)
 
-sc2 = Scenario('ex2')
-sc2.set_anchor(step1_copy)
+sc_tuto_ex2 = Scenario('ex2', anchor=step1_copy)
 
 ### SCENARIO 3 ###
 anchor = Step(DataProcess(process=['tTYPE#3'], seed='exist_cond'),
@@ -99,8 +97,7 @@ def before_data_processing_cbk(env, step):
 option1.connect_to(anchor)
 option2.connect_to(anchor)
 
-sc3 = Scenario('ex3')
-sc3.set_anchor(anchor)
+sc_tuto_ex3 = Scenario('ex3', anchor=anchor)
 
 ### SCENARIO 4 & 5 ###
 dp = DataProcess(['tTYPE#NOREG'], seed='exist_cond', auto_regen=False)
@@ -241,10 +238,10 @@ def before_sending(env, step):
 reinit = Step(Data(Node('reinit', vt=String(values=['REINIT']))))
 reinit.connect_to(init)
 
-sc_test_basic = Scenario('BASIC', anchor=init, reinit_anchor=reinit)
+sc_tuto_ex4 = Scenario('ex4', anchor=init, reinit_anchor=reinit)
 
-tactics.register_scenarios(sc1, sc2, sc3, sc4, sc5, sc_test, sc_test2, sc_test3, sc_test4,
-                           sc_test_basic)
+tactics.register_scenarios(sc_tuto_ex1, sc_tuto_ex2, sc_tuto_ex3, sc_tuto_ex4,
+                           sc4, sc5, sc_test, sc_test2, sc_test3, sc_test4)
 
 @generator(tactics, gtype="CBK")
 class g_test_callback_01(Generator):

diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -57,7 +57,7 @@
 # The short X.Y version.
 version = '0.27'
 # The full version, including alpha/beta/rc tags.
-release = '0.27.0'
+release = '0.27.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

diff --git a/docs/source/disruptors.rst b/docs/source/disruptors.rst
@@ -317,33 +317,71 @@ MOD - Modify Node Contents
 --------------------------
 
 Description:
-  Change the content of the nodes specified by the regexp path with
-  the value privided as a parameter (use *node absorption*
-  infrastructure). If no path is provided, the root node will be
-  used.
+    Perform modifications on the provided data. Two ways are possible:
+
+    - Either the change is performed on the content of the nodes specified by the `path`
+      parameter with the new `value` provided, and the optional constraints for the
+      absorption (use *node absorption* infrastructure);
+
+    - Or the changed is performed based on a dictionary provided through the parameter `multi_mod`
 
 Reference:
   :class:`framework.generic_data_makers.d_modify_nodes`
 
 Parameters:
   .. code-block:: none
 
-    parameters:
-      |_ path
-      |      | desc: graph path regexp to select nodes on which the disruptor should
-      |      |       apply
-      |      | default: None [type: str]
-      |_ clone_node
-      |      | desc: if True the dmaker will always return a copy of the node. (for
-      |      |       stateless disruptors dealing with big data it can be useful
-      |      |       to it to False)
-      |      | default: False [type: bool]
-      |_ value
-      |      | desc: the new value to inject within the data
-      |      | default: '' [type: str]
-      |_ constraints
-      |      | desc: constraints for the absorption of the new value
-      |      | default: AbsNoCsts() [type: AbsCsts]
+      parameters:
+        |_ path
+        |      | desc: Graph path regexp to select nodes on which the disruptor should
+        |      |       apply.
+        |      | default: None [type: str]
+        |_ value
+        |      | desc: The new value to inject within the data.
+        |      | default: '' [type: str]
+        |_ constraints
+        |      | desc: Constraints for the absorption of the new value.
+        |      | default: AbsNoCsts() [type: AbsCsts]
+        |_ multi_mod
+        |      | desc: Dictionary of <path>:<item> pairs to change multiple nodes with
+        |      |       diferent values. <item> can be either only the new <value> or
+        |      |       a tuple (<value>,<abscsts>) if new constraint for absorption
+        |      |       is needed
+        |      | default: None [type: dict]
+        |_ clone_node
+        |      | desc: If True the dmaker will always return a copy of the node. (For
+        |      |       stateless disruptors dealing with big data it can be useful
+        |      |       to it to False.)
+        |      | default: False [type: bool]
+
+
+CALL - Call Function
+--------------------
+
+Description:
+    Call the function provided with the first parameter being the :class:`framework.data.Data`
+    object received as input of this disruptor, and optionally with additional parameters
+    if `params` is set. The function should return a :class:`framework.data.Data` object.
+
+    The signature of the function should be compatible with:
+
+    ``func(data, *args) --> Data()``
+
+Reference:
+  :class:`framework.generic_data_makers.d_modify_nodes`
+
+Parameters:
+  .. code-block:: none
+
+      parameters:
+        |_ func
+        |      | desc: The function that will be called with a node as its first parameter,
+        |      |       and provided optionnaly with addtionnal parameters if @params
+        |      |       is set.
+        |      | default: lambda x: x [type: method, function]
+        |_ params
+        |      | desc: Tuple of parameters that will be provided to the function.
+        |      | default: None [type: tuple]
 
 
 

diff --git a/docs/source/images/sc_basic.png b/docs/source/images/sc_basic.png
diff --git a/docs/source/images/sc_basic_cond_fuzz_tc1.png b/docs/source/images/sc_basic_cond_fuzz_tc1.png
diff --git a/docs/source/images/sc_basic_cond_fuzz_tc2.png b/docs/source/images/sc_basic_cond_fuzz_tc2.png
diff --git a/docs/source/images/sc_basic_data_fuzz_tc1.png b/docs/source/images/sc_basic_data_fuzz_tc1.png
diff --git a/docs/source/images/sc_basic_data_fuzz_tc2.png b/docs/source/images/sc_basic_data_fuzz_tc2.png
diff --git a/docs/source/images/sc_basic_stutter.png b/docs/source/images/sc_basic_stutter.png
diff --git a/docs/source/images/sc_ex1_step1.png b/docs/source/images/sc_ex1_step1.png
diff --git a/docs/source/images/sc_ex1_step2.png b/docs/source/images/sc_ex1_step2.png