diff --git a/.gitignore b/.gitignore index cffe1bf..7f43d26 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ *.pyc *~ .idea -.DS_Store \ No newline at end of file +.DS_Store +venv \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..c53fb20 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,25 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.10" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# If using Sphinx, optionally build your docs in additional formats such as PDF +formats: + - pdf + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/requirements.txt diff --git a/README.rst b/README.rst index 8b0802b..42d7525 100755 --- a/README.rst +++ b/README.rst @@ -69,15 +69,17 @@ of ``fuddly`` itself. Usage is as follows: Miscellaneous ------------- -+ Don't forget to populate ``~/fuddly_data/imported_data/`` with sample files for data ++ Don't forget to populate ``/imported_data/`` with sample files for data models that need it ++ Note that when the fuddly shell is launched, the path of the fuddly data folder is displayed as + well as its configuration folder + Dependencies ------------ -+ Compatible with Python2 and Python3 ++ Compatible with Python3 + Mandatory: - - `six`_: Python 2/3 compatibility - `sqlite3`_: SQLite3 data base + Optional: @@ -88,11 +90,12 @@ Dependencies - `serial`_: For serial port access - `cups`_: Python bindings for libcups - `rpyc`_: Remote Python Call (RPyC), a transparent and symmetric RPC library + - `pyxdg`_: XDG Base Directory support + For testing: - `ddt`_: Used for data-driven tests - - `mock`_: Used for mocking (only needed in Python2) + - `mock`_: Used for mocking + For documentation generation: @@ -113,3 +116,4 @@ Dependencies .. _sphinx: http://sphinx-doc.org/ .. _texlive: https://www.tug.org/texlive/ .. _readthedocs theme: https://github.com/snide/sphinx_rtd_theme +.. _pyxdg: https://pypi.org/project/pyxdg/ diff --git a/TODO b/TODO index d064343..70c9b7f 100644 --- a/TODO +++ b/TODO @@ -1,11 +1,8 @@ [NEW FEATURES] +- Add new IA infrastructure supporting the creation of data models (automatic discovery of data structure from raw data) +- Enhance current post-analysis tooling and add new features supporting investigation (diagrams, statistics, research by pattern, etc.) - Add GDB/PIN/QEMU probes/managers -- Add FmkDB visualization tools -- Add support for automatic adaptation of fuzz test cases depending on - specific Target meta-data (HW architecture, programming language, ...) -- Implement new node types that leverage python-constraint, or more - powerfull constraint programming library [ENHANCEMENT] diff --git a/data_models/file_formats/jpg.py b/data_models/file_formats/jpg.py index f52392a..5bc9784 100644 --- a/data_models/file_formats/jpg.py +++ b/data_models/file_formats/jpg.py @@ -50,8 +50,8 @@ class JPG_DataModel(DataModel): name = 'jpg' def _atom_absorption_additional_actions(self, atom): - x = atom['.*/SOF_hdr/X'].get_raw_value() - y = atom['.*/SOF_hdr/Y'].get_raw_value() + x = atom['.*/SOF_hdr/X'][0].get_raw_value() + y = atom['.*/SOF_hdr/Y'][0].get_raw_value() d_priv = {'height':y, 'width':x} atom.set_private(d_priv) msg = "add private data: size [x:{:d}, y:{:d}]".format(x, y) @@ -163,9 +163,9 @@ def build_data_model(self): jpg_abs = jpg.get_clone(new_env=True) jpg_abs.set_current_conf('ABS', recursive=True) - self.register_atom_for_absorption(jpg_abs, - absorb_constraints=AbsNoCsts(size=True, struct=True, - contents=True)) + self.register_atom_for_decoding(jpg_abs, + absorb_constraints=AbsNoCsts(size=True, struct=True, + content=True)) data_model = JPG_DataModel() diff --git a/data_models/file_formats/json_dm.py b/data_models/file_formats/json_dm.py new file mode 100644 index 0000000..5b7341f --- /dev/null +++ b/data_models/file_formats/json_dm.py @@ -0,0 +1,29 @@ +import json + +from framework.data_model import * +from framework.value_types import * +from framework.dmhelpers.json import json_model_builder, json_builder + +class JSON_DataModel(DataModel): + + name = 'json' + file_extension = 'json' + + def _create_atom_from_raw_data_specific(self, data, idx, filename): + json_data = json.loads(data) + node_name = 'json_'+filename[:-len(self.file_extension)-1] + if '$schema' in json_data: + try: + return json_model_builder(node_name=node_name, schema=json_data, ignore_pattern=False) + except: + print('\n*** WARNING: Node creation attempt failed. New attempt, but now ignore ' + 'regex patterns from string JSON types.') + return json_model_builder(node_name=node_name, schema=json_data, ignore_pattern=True) + else: + return json_builder(node_name=node_name, sample=json_data) + + def build_data_model(self): + pass + + +data_model = JSON_DataModel() \ No newline at end of file diff --git a/data_models/file_formats/json_dm_strategy.py b/data_models/file_formats/json_dm_strategy.py new file mode 100644 index 0000000..186be8d --- /dev/null +++ b/data_models/file_formats/json_dm_strategy.py @@ -0,0 +1,3 @@ +from framework.tactics_helpers import * + +tactics = Tactics() \ No newline at end of file diff --git a/data_models/file_formats/pdf.py b/data_models/file_formats/pdf.py index 3c167d2..936b6f6 100644 --- a/data_models/file_formats/pdf.py +++ b/data_models/file_formats/pdf.py @@ -1310,9 +1310,9 @@ def build_data_model(self): with open(gr.workspace_folder + 'TEST_FUZZING_PDF-orig' + '.pdf', 'wb') as f: f.write(val) - leaf0 = pdf.get_node_by_path('PDF.*leaf_0-0$').to_bytes() + leaf0 = pdf.get_first_node_by_path('PDF.*leaf_0-0$').to_bytes() pdf.set_current_conf('ALT', root_regexp='PDF.*leaf_0-0$') - leaf1 = pdf.get_node_by_path('PDF.*leaf_0-0$').to_bytes() + leaf1 = pdf.get_first_node_by_path('PDF.*leaf_0-0$').to_bytes() print(leaf0) print(leaf1) diff --git a/data_models/file_formats/png.py b/data_models/file_formats/png.py index 424391c..b708075 100644 --- a/data_models/file_formats/png.py +++ b/data_models/file_formats/png.py @@ -106,7 +106,7 @@ def build_data_model(self): png = mb.create_graph_from_desc(png_desc) self.register(png) - self.register_atom_for_absorption(png, absorb_constraints=AbsNoCsts(size=True)) + self.register_atom_for_decoding(png, absorb_constraints=AbsNoCsts(size=True)) data_model = PNG_DataModel() diff --git a/data_models/file_formats/zip.py b/data_models/file_formats/zip.py index b245b35..16e4fe3 100644 --- a/data_models/file_formats/zip.py +++ b/data_models/file_formats/zip.py @@ -321,8 +321,8 @@ def build_data_model(self): pkzip_abs = pkzip.get_clone(new_env=True) pkzip_abs.set_current_conf('ABS', recursive=True) - self.register_atom_for_absorption(pkzip_abs, - absorb_constraints=AbsNoCsts(size=True, struct=True)) + self.register_atom_for_decoding(pkzip_abs, + absorb_constraints=AbsNoCsts(size=True, struct=True)) data_model = ZIP_DataModel() diff --git a/data_models/protocols/pppoe.py b/data_models/protocols/pppoe.py index 56bc1f0..db74a26 100644 --- a/data_models/protocols/pppoe.py +++ b/data_models/protocols/pppoe.py @@ -35,10 +35,10 @@ def build_data_model(self): def cycle_tags(tag): tag.freeze() - if tag['.*/type'].get_current_raw_val() == 0x102: - tag['.*/type'].unfreeze() + if tag['.*/type'][0].get_current_raw_val() == 0x102: + tag['.*/type'][0].unfreeze() tag.freeze() - tag['.*/type'].unfreeze() + tag['.*/type'][0].unfreeze() tag.unfreeze(reevaluate_constraints=True) return tag @@ -124,18 +124,18 @@ def cycle_tags(tag): tag_node = mb.create_graph_from_desc(tag_desc) tag_service_name = tag_node.get_clone('tag_sn') - tag_service_name['.*/type'].set_values(value_type=UINT16_be(values=[0x0101])) + tag_service_name['.*/type'][0].set_values(value_type=UINT16_be(values=[0x0101])) tag_host_uniq = tag_node.get_clone('tag_host_uniq') - tag_host_uniq['.*/type'].set_values(value_type=UINT16_be(values=[0x0103])) + tag_host_uniq['.*/type'][0].set_values(value_type=UINT16_be(values=[0x0103])) tag_host_uniq_pads = tag_host_uniq.get_clone() tag_ac_name = tag_node.get_clone('tag_ac_name') # Access Concentrator Name - tag_ac_name['.*/type'].set_values(value_type=UINT16_be(values=[0x0102])) + tag_ac_name['.*/type'][0].set_values(value_type=UINT16_be(values=[0x0102])) tag_sn_error = tag_node.get_clone('tag_sn_error') # Service Name Error - tag_sn_error['.*/type'].set_values(value_type=UINT16_be(values=[0x0202])) + tag_sn_error['.*/type'][0].set_values(value_type=UINT16_be(values=[0x0202])) tag_service_name_pads = tag_service_name.get_clone() tag_node_pads = tag_node.get_clone() @@ -258,21 +258,21 @@ def cycle_tags(tag): # pppoe_msg.make_random(recursive=True) padi = pppoe_msg.get_clone('padi') - padi['.*/mac_dst'].set_values(value_type=String(values=[u'\xff\xff\xff\xff\xff\xff'])) - padi['.*/code'].set_values(value_type=UINT8(values=[0x9])) + padi['.*/mac_dst'][0].set_values(value_type=String(values=[u'\xff\xff\xff\xff\xff\xff'])) + padi['.*/code'][0].set_values(value_type=UINT8(values=[0x9])) pado = pppoe_msg.get_clone('pado') - pado['.*/code'].set_values(value_type=UINT8(values=[0x7])) - # pado['.*/code'].clear_attr(MH.Attr.Mutable) + pado['.*/code'][0].set_values(value_type=UINT8(values=[0x7])) + # pado['.*/code'][0].clear_attr(MH.Attr.Mutable) padr = pppoe_msg.get_clone('padr') - padr['.*/code'].set_values(value_type=UINT8(values=[0x19])) + padr['.*/code'][0].set_values(value_type=UINT8(values=[0x19])) pads = pppoe_msg.get_clone('pads') - pads['.*/code'].set_values(value_type=UINT8(values=[0x65])) + pads['.*/code'][0].set_values(value_type=UINT8(values=[0x65])) padt = pppoe_msg.get_clone('padt') - padt['.*/code'].set_values(value_type=UINT8(values=[0xa7])) + padt['.*/code'][0].set_values(value_type=UINT8(values=[0xa7])) self.register(pppoe_msg, padi, pado, padr, pads, padt, tag_host_uniq) diff --git a/data_models/protocols/pppoe_strategy.py b/data_models/protocols/pppoe_strategy.py index ceed706..7d72473 100644 --- a/data_models/protocols/pppoe_strategy.py +++ b/data_models/protocols/pppoe_strategy.py @@ -26,6 +26,7 @@ from framework.global_resources import * from framework.data_model import MH from framework.target_helpers import * +from framework.data import DataProcess tactics = Tactics() @@ -43,7 +44,7 @@ def retrieve_X_from_feedback(env, current_step, next_step, feedback, x='padi', u elif x == 'padr': if current_step.content is not None: mac_src = current_step.content['.*/mac_src'] - env.mac_src = mac_src + env.mac_src = mac_src[0] if mac_src is not None else None else: mac_src = env.mac_src if mac_src is not None: @@ -71,8 +72,8 @@ def retrieve_X_from_feedback(env, current_step, next_step, feedback, x='padi', u if result[0] == AbsorbStatus.FullyAbsorbed: try: - service_name = msg_x['.*/value/v101'].to_bytes() - mac_src = msg_x['.*/mac_src'].to_bytes() + service_name = msg_x['.*/value/v101'][0].to_bytes() + mac_src = msg_x['.*/mac_src'][0].to_bytes() except: continue print(' [ {:s} received! ]'.format(x.upper())) @@ -82,7 +83,7 @@ def retrieve_X_from_feedback(env, current_step, next_step, feedback, x='padi', u host_uniq = msg_x['.*/value/v103'] if host_uniq is not None: - host_uniq = host_uniq.to_bytes() + host_uniq = host_uniq[0].to_bytes() env.host_uniq = host_uniq t_fix_pppoe_msg_fields.host_uniq = host_uniq @@ -90,8 +91,8 @@ def retrieve_X_from_feedback(env, current_step, next_step, feedback, x='padi', u next_step.content.freeze() try: next_step.content['.*/tag_sn/value/v101'] = service_name - next_step.content['.*/tag_sn$'].unfreeze(recursive=True, reevaluate_constraints=True) - next_step.content['.*/tag_sn$'].freeze() + next_step.content['.*/tag_sn$'][0].unfreeze(recursive=True, reevaluate_constraints=True) + next_step.content['.*/tag_sn$'][0].freeze() except: pass @@ -134,7 +135,7 @@ def disrupt_data(self, dm, target, prev_data): n['.*/mac_dst'] = self.mac_src prev_data.add_info("update 'mac_src'") if not self.reevaluate_csts: - n['.*/mac_dst'].unfreeze(dont_change_state=True) + n['.*/mac_dst'][0].unfreeze(dont_change_state=True) except: print(error_msg.format('mac_dst')) else: @@ -152,11 +153,11 @@ def disrupt_data(self, dm, target, prev_data): if self.host_uniq: try: - if not n['.*/tag_host_uniq/.*/v103'].is_attr_set(MH.Attr.LOCKED) and \ - not n['.*/tag_host_uniq/len'].is_attr_set(MH.Attr.LOCKED) and \ - not n['.*/tag_host_uniq/type'].is_attr_set(MH.Attr.LOCKED): + if not n['.*/tag_host_uniq/.*/v103'][0].is_attr_set(MH.Attr.LOCKED) and \ + not n['.*/tag_host_uniq/len'][0].is_attr_set(MH.Attr.LOCKED) and \ + not n['.*/tag_host_uniq/type'][0].is_attr_set(MH.Attr.LOCKED): n['.*/tag_host_uniq/.*/v103'] = self.host_uniq - tag_uniq = n['.*/tag_host_uniq$'] + tag_uniq = n['.*/tag_host_uniq$'][0] tag_uniq.unfreeze(recursive=True, reevaluate_constraints=True) tag_uniq.freeze() prev_data.add_info("update 'host_uniq' with: {!s}".format(self.host_uniq)) diff --git a/data_models/protocols/usb.py b/data_models/protocols/usb.py index 211d471..c3b5c4f 100644 --- a/data_models/protocols/usb.py +++ b/data_models/protocols/usb.py @@ -104,7 +104,7 @@ def build_data_model(self): {'name': 'wMaxPacketSize', 'contents': BitField(subfield_limits=[11,13,16], subfield_val_extremums=[None,[0,2],[0,0]], - subfield_values=[[2**x for x in range(1,12)],None,[0]], + subfield_values=[[2**x for x in range(1,11)],None,[0]], endian=VT.LittleEndian), 'random': True, 'alt': [ diff --git a/data_models/tutorial/example.py b/data_models/tutorial/example.py deleted file mode 100644 index 55b3984..0000000 --- a/data_models/tutorial/example.py +++ /dev/null @@ -1,406 +0,0 @@ -#!/usr/bin/env python - -################################################################################ -# -# Copyright 2014-2016 Eric Lacombe -# -################################################################################ -# -# This file is part of fuddly. -# -# fuddly is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# fuddly is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with fuddly. If not, see -# -################################################################################ - -import sys - -sys.path.append('.') - -from framework.data_model import * -from framework.value_types import * - -from framework.basic_primitives import * - -class Example_DataModel(DataModel): - - def build_data_model(self): - - tx = Node('TX') - tx_h = Node('h', values=['/TX']) - - ku = Node('KU') - kv = Node('KV') - - ku_h = Node('KU_h', values=[':KU:']) - kv_h = Node('KV_h', values=[':KV:']) - - tux_subparts_1 = ['POWN', 'TAILS', 'WORLD1', 'LAND321'] - tux_subparts_2 = ['YYYY', 'ZZZZ', 'XXXX'] - ku.set_values(tux_subparts_1) - kv.set_values(tux_subparts_2) - - - tux_subparts_3 = ['[<]MARCHONS', '[<]TESTONS'] - kv.add_conf('ALT') - kv.set_values(tux_subparts_3, conf='ALT') - - tux_subparts_4 = [u'[\u00c2]PLIP', u'[\u00c2]GLOUP'] - ku.add_conf('ALT') - ku.set_values(value_type=String(values=tux_subparts_4, codec='utf8'), conf='ALT') - - idx = Node('IDX') - idx.set_values(value_type=SINT16_be(min=4,max=40)) - - tx.set_subnodes_basic([tx_h, idx, ku_h, ku, kv_h, kv]) - tx_cpy = tx.get_clone('TX_cpy') - - tc = Node('TC') - tc_h = Node('h', values=['/TC']) - - ku2 = Node('KU', base_node=ku) - kv2 = Node('KV', base_node=kv) - - ku_h2 = Node('KU_h', base_node=ku_h) - kv_h2 = Node('KV_h', base_node=kv_h) - - tc.set_subnodes_basic([tc_h, ku_h2, ku2, kv_h2, kv2]) - - - mark3 = Node('MARK3', values=[' ~(X)~ ']) - self.mark3 = mark3 - - tc.add_conf('ALT') - tc.set_subnodes_basic([mark3, tc_h, ku2, kv_h2], conf='ALT') - tc_cpy1= tc.get_clone('TC_cpy1') - tc_cpy2= tc.get_clone('TC_cpy2') - - mark = Node('MARK', values=[' [#] ']) - - idx2 = Node('IDX2', base_node=idx) - tux = Node('TUX') - tux_h = Node('h', values=['TUX']) - - # set 'mutable' attribute to False - tux_h.clear_attr(NodeInternals.Mutable) - tux_h_cpy = tux_h.get_clone('h_cpy') - - tux.set_subnodes_with_csts([ - 100, ['u>', [tux_h, 1], [idx2, 1], [mark, 1], - 'u=+(1,2)', [tc_cpy2, 2], [tx_cpy, 1, 2], - 'u>', [mark, 1], [tx, 1], [tc_cpy1, 1], - 'u=..', [tux_h, 1], [idx2, 1]], - - 1, ['u>', [mark, 1], - 's=..', [tux_h_cpy, 1, 3], [tc, 3], - 'u>', [mark, 1], [tx, 1], [idx2, 1]], - - 15, ['u>', [mark, 1], - 'u=.', [tux_h_cpy, 1, 3], [tc, 3], - 'u=.', [mark, 1], [tx, 1], [idx2, 1]] - ]) - - - mark2 = Node('MARK2', values=[' ~(..)~ ']) - - tux.add_conf('ALT') - tux.set_subnodes_with_csts( - [1, ['u>', [mark2, 1], - 'u=+(4000,1)', [tux_h, 1], [mark, 1], - 'u>', [mark2, 1], - 'u=.', [tux_h, 1], [tc, 10], - 'u>', [mark, 1], [tx, 1], [idx2, 1]] - ], conf='ALT') - - - concat = Node('CONCAT') - length = Node('LEN') - node_ex1 = Node('EX1') - - fct = lambda x: b' @ ' + x + b' @ ' - concat.set_func(fct, tux) - - if sys.version_info[0] > 2: - fct = lambda x: b'___' + bytes(chr(x[1]), internal_repr_codec) + b'___' - else: - fct = lambda x: b'___' + x[1] + b'___' - - concat.add_conf('ALT') - concat.set_func(fct, tux, conf='ALT') - - fct2 = lambda x: len(x) - length.set_func(fct2, tux) - - node_ex1.set_subnodes_basic([concat, tux, length]) - - - evt1 = Node('EVT1') - evt1.set_values(value_type=SINT16_be(values=[-4])) - evt1.set_fuzz_weight(10) - - evt2 = Node('EVT2') - evt2.set_values(value_type=UINT16_le(min=50, max=2**16-1)) - # evt2.set_values(value_type=UINT16_le()) - evt2.set_fuzz_weight(9) - - sep1 = Node('sep1', values=["+"]) - sep2 = Node('sep2', values=["*"]) - - sub1 = Node('SUB1') - sub1.set_subnodes_with_csts([ - 1, ['u>', [sep1, 3], [evt1, 2], [sep1, 3]] - ]) - - sp = Node('S', values=[' ']) - - ssub = Node('SSUB') - ssub.set_subnodes_basic([sp, evt2, sp]) - - sub2 = Node('SUB2') - sub2.set_subnodes_with_csts([ - 1, ['u>', [sep2, 3], [ssub, 1], [sep2, 3]] - ]) - - sep = Node('sep', values=[' -=||=- ']) - prefix = Node('Pre', values=['[1] ', '[2] ', '[3] ', '[4] ']) - prefix.make_determinist() - - te3 = Node('EVT3') - te3.set_values(value_type=BitField(subfield_sizes=[4,4], endian=VT.LittleEndian, - subfield_values=[[0x5, 0x6], [0xF, 0xC]])) - te3.set_fuzz_weight(8) - # te3.make_determinist() - - te4 = Node('EVT4') - te4.set_values(value_type=BitField(subfield_sizes=[4,4], endian=VT.LittleEndian, - subfield_val_extremums=[[4, 8], [3, 15]])) - te4.set_fuzz_weight(7) - # te4.make_determinist() - - te5 = Node('EVT5') - te5.set_values(value_type=INT_str(values=[9])) - te5.cc.set_specific_fuzzy_values([666]) - te5.set_fuzz_weight(6) - - te6 = Node('EVT6') - vt = BitField(subfield_limits=[2,6,8,10], subfield_values=[[4,2,1],[2,15,16,3],[2,3,0],[1]], - padding=0, lsb_padding=True, endian=VT.LittleEndian) - te6.set_values(value_type=vt) - te6.set_fuzz_weight(5) - # te6.make_determinist() - - - te7 = Node('EVT7') - vt = BitField(subfield_sizes=[4,4,4], - subfield_values=[[4,2,1], None, [2,3,0]], - subfield_val_extremums=[None, [3, 15], None], - padding=0, lsb_padding=False, endian=VT.BigEndian) - te7.set_values(value_type=vt) - te7.set_fuzz_weight(4) - # te7.make_determinist() - - suffix = Node('suffix', subnodes=[sep, te3, sep, te4, sep, te5, sep, te6, sep, te7]) - - typed_node = Node('TVE', subnodes=[prefix, sub1, sep, sub2, suffix]) - - # Simple - - tval1_bottom = Node('TV1_bottom') - vt = UINT16_be(values=[1,2,3,4,5,6]) - - tval1_bottom.set_values(value_type=vt) - tval1_bottom.make_determinist() - - sep_bottom = Node('sep_bottom', values=[' .. ']) - sep_bottom_alt = Node('sep_bottom_alt', values=[' ;; ']) - - tval2_bottom = Node('TV2_bottom') - vt = UINT16_be(values=[0x42,0x43,0x44]) - tval2_bottom.set_values(value_type=vt) - - alt_tag = Node('AltTag', values=[' |AltTag| ', ' +AltTag+ ']) - alt_tag_cpy = alt_tag.get_clone('AltTag_cpy') - - bottom = Node('Bottom_NT') - bottom.set_subnodes_with_csts([ - 1, ['u>', [sep_bottom, 1], [tval1_bottom, 1], [sep_bottom, 1], [tval2_bottom, 1]] - ]) - - val1_bottom2 = Node('V1_bottom2', values=['=BOTTOM_2=', '**BOTTOM_2**', '~~BOTTOM_2~~']) - val1_bottom2.add_conf('ALT') - val1_bottom2.set_values(['=ALT_BOTTOM_2=', '**ALT_BOTTOM_2**', '~~ALT_BOTTOM_2~~', '__ALT_BOTTOM_2__'], conf='ALT') - val1_bottom2.add_conf('ALT_2') - val1_bottom2.set_values(['=2ALT2_BOTTOM_2=', '**2ALT2_BOTTOM_2**', '~~2ALT2_BOTTOM_2~~'], conf='ALT_2') - val1_bottom2.set_fuzz_weight(2) - - val1_bottom2_cpy = val1_bottom2.get_clone('V1_bottom2_cpy') - - bottom2 = Node('Bottom_2_NT') - bottom2.set_subnodes_with_csts([ - 5, ['u>', [sep_bottom, 1], [val1_bottom2, 1]], - 1, ['u>', [sep_bottom_alt, 1], [val1_bottom2_cpy, 2], [sep_bottom_alt, 1]] - ]) - bottom2.add_conf('ALT') - bottom2.set_subnodes_with_csts([ - 5, ['u>', [alt_tag, 1], [val1_bottom2, 1], [alt_tag, 1]], - 1, ['u>', [alt_tag_cpy, 2], [val1_bottom2_cpy, 2], [alt_tag_cpy, 2]] - ], conf='ALT') - - tval2_bottom3 = Node('TV2_bottom3') - vt = UINT32_be(values=[0xF, 0x7]) - tval2_bottom3.set_values(value_type=vt) - bottom3 = Node('Bottom_3_NT') - bottom3.set_subnodes_with_csts([ - 1, ['u>', [sep_bottom, 1], [tval2_bottom3, 1]] - ]) - - val1_middle = Node('V1_middle', values=['=MIDDLE=', '**MIDDLE**', '~~MIDDLE~~']) - sep_middle = Node('sep_middle', values=[' :: ']) - alt_tag2 = Node('AltTag-Mid', values=[' ||AltTag-Mid|| ', ' ++AltTag-Mid++ ']) - - val1_middle_cpy1 = val1_middle.get_clone('V1_middle_cpy1') - val1_middle_cpy2 = val1_middle.get_clone('V1_middle_cpy2') - - middle = Node('Middle_NT') - middle.set_subnodes_with_csts([ - 5, ['u>', [val1_middle, 1], [sep_middle, 1], [bottom, 1]], - 3, ['u>', [val1_middle_cpy1, 2], [sep_middle, 1], [bottom2, 1]], - 1, ['u>', [val1_middle_cpy2, 3], [sep_middle, 1], [bottom3, 1]] - ]) - middle.add_conf('ALT') - middle.set_subnodes_with_csts([ - 5, ['u>', [alt_tag2, 1], [val1_middle, 1], [sep_middle, 1], [bottom, 1], [alt_tag2, 1]] - ], conf='ALT') - # middle.make_determinist() - - val1_top = Node('V1_top', values=['=TOP=', '**TOP**', '~~TOP~~']) - sep_top = Node('sep_top', values=[' -=|#|=- ', ' -=|@|=- ']) - - prefix1 = Node('prefix1', values=[" ('_') ", " (-_-) ", " (o_o) "]) - prefix2 = Node('prefix2', values=[" |X| ", " |Y| ", " |Z| "]) - - e_simple = Node('Simple') - e_simple.set_subnodes_with_csts([ - 1, ['u>', [prefix1, 1], [prefix2, 1], [sep_top, 1], [val1_top, 1], [sep_top, 1], [middle, 1]] - ]) - - ### NonTerm - - e = Node('TV2') - vt = UINT16_be(values=[1,2,3,4,5,6]) - e.set_values(value_type=vt) - sep3 = Node('sep3', values=[' # ']) - nt = Node('Bottom_NT') - nt.set_subnodes_with_csts([ - 1, ['u>', [e, 1], [sep3, 1], [e, 1]] - ]) - - sep = Node('sep', values=[' # ']) - sep2 = Node('sep2', values=[' -|#|- ']) - - e_val1 = Node('V1', values=['A', 'B', 'C']) - e_val1_cpy = e_val1.get_clone('V1_cpy') - e_typedval1 = Node('TV1', value_type=UINT16_be(values=[1,2,3,4,5,6])) - e_val2 = Node('V2', values=['X', 'Y', 'Z']) - e_val3 = Node('V3', values=['<', '>']) - - e_val_random = Node('Rnd', values=['RANDOM']) - e_val_random2 = Node('Rnd2', values=['RANDOM']) - - e_nonterm = Node('NonTerm') - e_nonterm.set_subnodes_with_csts([ - 100, ['u>', [e_val1, 1, 6], [sep, 1], [e_typedval1, 1, 6], - [sep2, 1], - 'u=+(2,3,3)', [e_val1_cpy, 1], [e_val2, 1, 3], [e_val3, 1], - 'u>', [sep2, 1], - 'u=..', [e_val1, 1, 6], [sep, 1], [e_typedval1, 1, 6]], - 50, ['u>', [e_val_random, 0, 1], [sep, 1], [nt, 1]], - 90, ['u>', [e_val_random2, 3]] - ]) - - - test_node_desc = \ - {'name': 'TestNode', - 'contents': [ - # block 1 - {'section_type': MH.Ordered, - 'duplicate_mode': MH.Copy, - 'contents': [ - - {'contents': BitField(subfield_sizes=[21,2,1], endian=VT.BigEndian, - subfield_values=[None, [0b10], [0,1]], - subfield_val_extremums=[[500, 600], None, None]), - 'name': 'val1', - 'qty': (1, 5)}, - - {'name': 'val2'}, - - {'name': 'middle', - 'custo_set': MH.Custo.NTerm.FrozenCopy, - 'custo_clear': MH.Custo.NTerm.MutableClone, - 'separator': {'contents': {'name': 'sep', - 'contents': String(values=['\n'], absorb_regexp='\n+'), - 'absorb_csts': AbsNoCsts(regexp=True)}}, - 'contents': [{ - 'section_type': MH.Random, - 'contents': [ - - {'contents': String(values=['OK', 'KO'], size=2), - 'name': 'val2'}, - - {'name': 'val21', - 'clone': 'val1'}, - - {'name': 'USB_desc', - 'import_from': 'usb', - 'data_id': 'STR'}, - - {'type': MH.Generator, - 'contents': lambda x: Node('cts', values=[x[0].to_bytes() \ - + x[1].to_bytes()]), - 'name': 'val22', - 'node_args': [('val21', 2), 'val3']} - ]}]}, - - {'contents': String(max_sz = 10), - 'name': 'val3', - 'sync_qty_with': 'val1', - 'alt': [ - {'conf': 'alt1', - 'contents': SINT8(values=[1,4,8])}, - {'conf': 'alt2', - 'contents': UINT16_be(min=0xeeee, max=0xff56), - 'determinist': True}]} - ]}, - - # block 2 - {'section_type': MH.Pick, - 'weights': (10,5), - 'contents': [ - {'contents': String(values=['PLIP', 'PLOP'], size=4), - 'name': ('val21', 2)}, - - {'contents': SINT16_be(values=[-1, -3, -5, 7]), - 'name': ('val22', 2)} - ]} - ]} - - mb = NodeBuilder(dm=self) - test_node = mb.create_graph_from_desc(test_node_desc) - - self.register(node_ex1, tux, typed_node, e_nonterm, e_simple, - val1_middle, middle, test_node) - - - -data_model = Example_DataModel() diff --git a/data_models/tutorial/example_strategy.py b/data_models/tutorial/example_strategy.py deleted file mode 100644 index e5a9593..0000000 --- a/data_models/tutorial/example_strategy.py +++ /dev/null @@ -1,93 +0,0 @@ -################################################################################ -# -# Copyright 2014-2016 Eric Lacombe -# -################################################################################ -# -# This file is part of fuddly. -# -# fuddly is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# fuddly is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with fuddly. If not, see -# -################################################################################ - -import sys -import random -import array -from copy import * - -from framework.plumbing import * - -from framework.node import * -from framework.tactics_helpers import * -from framework.fuzzing_primitives import * -from framework.basic_primitives import * - -tactics = Tactics() - -@generator(tactics, gtype="EX1", weight=2) -class example_02(Generator): - - def setup(self, dm, user_input): - self.tux = dm.get_atom('TUX') - self.tux_h = self.tux.get_node_by_path('TUX/h$') - self.tx = self.tux.get_node_by_path('TUX/TX$') - self.tc = self.tux.get_node_by_path('TUX/TC$') - - self.delim = Node('DELIM', values=[' [@] ']) - - self.tux.set_subnodes_with_csts([ - 1, ['u>', [self.delim, 1], - 'u=+', [self.tux_h, 1, 3], [self.tc, 1], - 'u>', [self.delim, 1], [self.tx, 1], [self.delim, 1], - 'u=.', [self.tx, 1], [self.tc, 1]] - ]) - - return True - - def generate_data(self, dm, monitor, target): - exported_node = Node(self.tux.name, base_node=self.tux) - dm.update_atom(exported_node) - return Data(exported_node) - - - -@generator(tactics, gtype="TVE_w", weight=2) -class g_typed_value_example_01(Generator): - - def generate_data(self, dm, monitor, target): - return Data(dm.get_atom('TVE')) - - -@generator(tactics, gtype="TVE_w", weight=10) -class g_typed_value_example_02(Generator): - - def generate_data(self, dm, monitor, target): - return Data(dm.get_atom('TVE')) - - -@disruptor(tactics, dtype="TVE/basic", weight=4) -class t_fuzz_tve_01(Disruptor): - - def disrupt_data(self, dm, target, prev_data): - - val = b"NEW_" + rand_string(min=5, max=10, str_set='XYZRVW').encode('latin-1') - - prev_content = prev_data.content - if isinstance(prev_content, Node): - prev_content.get_node_by_path('TVE.*EVT1$').set_frozen_value(val) - - else: - print('DONT_PROCESS_THIS_KIND_OF_DATA') - - return prev_data diff --git a/data_models/tutorial/myproto.py b/data_models/tutorial/myproto.py index d543975..7851b3b 100644 --- a/data_models/tutorial/myproto.py +++ b/data_models/tutorial/myproto.py @@ -44,7 +44,7 @@ def build_data_model(self): {'name': 'init', 'exists_if': (BitFieldCondition(sf=1, val=[1]), 'header'), 'contents': TIMESTAMP("%H:%M:%S"), - 'absorb_csts': AbsFullCsts(contents=False)}, + 'absorb_csts': AbsFullCsts(content=False, similar_content=False)}, {'name': 'register', 'custo_clear': MH.Custo.NTerm.FrozenCopy, @@ -85,14 +85,14 @@ def build_data_model(self): req_atom = NodeBuilder(add_env=True).create_graph_from_desc(req_desc) init_atom = req_atom.get_clone('init', ignore_frozen_state=True) - init_atom['.*/header'].set_subfield(idx=1, val=1) + init_atom['.*/header'][0].set_subfield(idx=1, val=1) init_atom.unfreeze(recursive=True) register_atom = req_atom.get_clone('register', ignore_frozen_state=True) - register_atom['.*/header'].set_subfield(idx=1, val=10) + register_atom['.*/header'][0].set_subfield(idx=1, val=10) register_atom.unfreeze(recursive=True) zregister_atom = req_atom.get_clone('zregister', ignore_frozen_state=True) - zregister_atom['.*/header'].set_subfield(idx=1, val=20) - zregister_atom['.*/header'].set_subfield(idx=2, val=3) + zregister_atom['.*/header'][0].set_subfield(idx=1, val=20) + zregister_atom['.*/header'][0].set_subfield(idx=2, val=3) zregister_atom.unfreeze(recursive=True) self.register(req_atom, init_atom, register_atom, zregister_atom) diff --git a/data_models/tutorial/myproto_strategy.py b/data_models/tutorial/myproto_strategy.py index 76537a4..a6e360c 100644 --- a/data_models/tutorial/myproto_strategy.py +++ b/data_models/tutorial/myproto_strategy.py @@ -11,7 +11,7 @@ def cbk_check_crc_error(env, current_step, next_step, fbk): def set_init_v3(env, step): - step.content['.*/header'].set_subfield(2, 3) + step.content['.*/header'][0].set_subfield(2, 3) init_step = Step('init', fbk_timeout=0.5, do_before_sending=set_init_v3) diff --git a/data_models/tutorial/tuto.py b/data_models/tutorial/tuto.py index 70bad85..a2b8aa1 100644 --- a/data_models/tutorial/tuto.py +++ b/data_models/tutorial/tuto.py @@ -10,6 +10,7 @@ from framework.dmhelpers.json import * from framework.dmhelpers.xml import tag_builder as xtb from framework.dmhelpers.xml import xml_decl_builder +from framework.constraint_helpers import Constraint class MyDF_DataModel(DataModel): @@ -193,10 +194,10 @@ def keycode_helper(blob, constraints, node_internals): 'shape_type': MH.Ordered, 'contents': [ {'name': 'opcode', - 'contents': String(values=['A1', 'A2', 'A3'], determinist=True)}, + 'contents': String(values=['A1', 'A2', 'A3'], determinist=True, case_sensitive=False)}, {'name': 'command_A1', - 'contents': String(values=['AAA', 'BBBB', 'CCCCC']), + 'contents': String(values=['AAA', 'BBBB', 'CCCCC'], case_sensitive=False), 'exists_if': (RawCondition('A1'), 'opcode'), 'qty': 3}, @@ -226,11 +227,11 @@ def keycode_helper(blob, constraints, node_internals): ]}, {'name': 'A31_payload', - 'contents': String(values=['$ A31_OK $', '$ A31_KO $'], determinist=False), + 'contents': String(values=['$ A31_OK $', '$ A31_KO $'], determinist=False, case_sensitive=False), 'exists_if': (BitFieldCondition(sf=2, val=[6,12]), 'A3_subopcode')}, {'name': 'A32_payload', - 'contents': String(values=['$ A32_VALID $', '$ A32_INVALID $'], determinist=False), + 'contents': String(values=['$ A32_VALID $', '$ A32_INVALID $'], determinist=False, case_sensitive=False), 'exists_if': (BitFieldCondition(sf=[0, 1, 2], val=[[500, 501], [1, 2], 5]), 'A3_subopcode')} ]} @@ -298,12 +299,12 @@ def keycode_helper(blob, constraints, node_internals): {'name': 'tstamp', 'contents': TIMESTAMP("%H%M%S"), - 'absorb_csts': AbsCsts(contents=False)}, + 'absorb_csts': AbsCsts(content=False)}, {'name': 'crc', 'contents': CRC(UINT32_be), 'node_args': ['tstamp', 'int32_qty'], - 'absorb_csts': AbsCsts(contents=False)} + 'absorb_csts': AbsCsts(content=False)} ]} @@ -368,7 +369,7 @@ def keycode_helper(blob, constraints, node_internals): {'name': 'crc', 'contents': CRC(vt=UINT32_be, after_encoding=False), 'node_args': ['enc_data', 'data2'], - 'absorb_csts': AbsFullCsts(contents=False) }, + 'absorb_csts': AbsFullCsts(content=False, similar_content=False)}, {'name': 'enc_data', 'encoder': GZIP_Enc(6), 'set_attrs': [NodeInternals.Abs_Postpone], @@ -376,7 +377,7 @@ def keycode_helper(blob, constraints, node_internals): {'name': 'len', 'contents': LEN(vt=UINT8, after_encoding=False), 'node_args': 'data1', - 'absorb_csts': AbsFullCsts(contents=False)}, + 'absorb_csts': AbsFullCsts(content=False, similar_content=False)}, {'name': 'data1', 'contents': String(values=['Test!', 'Hello World!'], codec='utf-16-le') }, ]}, @@ -398,7 +399,7 @@ def keycode_helper(blob, constraints, node_internals): 'mutable': False, 'contents': LEN(vt=UINT8, after_encoding=False), 'node_args': 'data1', - 'absorb_csts': AbsFullCsts(contents=False)}, + 'absorb_csts': AbsFullCsts(content=False, similar_content=False)}, {'name': 'data1', 'contents': String(values=['Test!', 'Hello World!']) }, @@ -529,12 +530,132 @@ def keycode_helper(blob, constraints, node_internals): 'debug': True } + + nested_desc = \ + {'name': 'nested', + 'custo_clear': MH.Custo.NTerm.MutableClone, + 'contents': [ + {'name' : 'line', + 'qty': (0,50), 'default_qty': 2, + 'contents': [ + {'name': 'sep', 'contents': String(values=['..'])}, + {'name': 'wrapper', + 'contents': [ + {'name': 'point', + 'contents': [ + + {'weight':50, + 'contents': [ + {'name': 'lat', + 'contents': [ + {'name': 'lat_dir', + 'contents': String(values=['N', 'S'])}, + {'name': 'lat_deg', + 'contents': INT_str(min=0, max=90, min_size=2)}, + {'name': 'lat_min', + 'qty': (0,1), + 'contents': INT_str(min=0, max=59, min_size=2)}, + ]}, + ]}, + + {'weight':40, + 'contents': [ + {'name': 'lon', + 'contents': [ + {'name': 'lon_dir', + 'contents': String(values=['E', 'W'])}, + {'name': 'lon_deg', + 'contents': INT_str(min=0, max=180, min_size=3)}, + {'name': 'lon_min', + 'qty': (0,1), + 'contents': INT_str(min=0, max=59, min_size=2)}, + ]}, + ]} + + ]} + ]} + ]} + ]} + + + csp_desc = \ + {'name': 'csp', + 'constraints': [Constraint(relation=lambda d1, d2: d1[1]+1 == d2[0] or d1[1]+2 == d2[0], + vars=('delim_1', 'delim_2')), + Constraint(relation=lambda x, y, z: x == 3*y + z, + vars=('x_val', 'y_val', 'z_val'))], + 'constraints_highlight': True, + 'contents': [ + {'name': 'equation', + 'contents': String(values=['x = 3y + z'])}, + {'name': 'delim_1', 'contents': String(values=[' [', ' ('])}, + {'name': 'variables', + 'separator': {'contents': {'name': 'sep', 'contents': String(values=[', '])}, + 'prefix': False, 'suffix': False}, + 'contents': [ + {'name': 'x', + 'contents': [ + {'name': 'x_symbol', + 'contents': String(values=['x:', 'X:'])}, + {'name': 'x_val', + 'contents': INT_str(min=120, max=130)} ]}, + {'name': 'y', + 'contents': [ + {'name': 'y_symbol', + 'contents': String(values=['y:', 'Y:'])}, + {'name': 'y_val', + 'contents': INT_str(min=30, max=40)}]}, + {'name': 'z', + 'contents': [ + {'name': 'z_symbol', + 'contents': String(values=['z:', 'Z:'])}, + {'name': 'z_val', + 'contents': INT_str(min=1, max=3)}]}, + ]}, + {'name': 'delim_2', 'contents': String(values=['-', ']', ')'])}, + ]} + + + csp_ns_desc = \ + {'name': 'csp_ns', + 'constraints': [Constraint(relation=lambda lat_deg, lon_deg: lat_deg == lon_deg + 1, + vars=('lat_deg', 'lon_deg'), + var_to_varns={'lat_deg': ('deg', 'lat'), + 'lon_deg': ('deg', 'lon')}), + Constraint(lambda lat_min, lon_deg: lat_min == lon_deg + 10, + vars=('lat_min', 'lon_deg'), + var_to_varns={'lon_deg': ('deg', 'lon'), + 'lat_min': ('min', 'lat')})], + 'contents': [ + {'name': 'latitude', + 'namespace': 'lat', + 'contents': [ + {'name': 'dir', + 'contents': String(values=['N', 'S'])}, + {'name': 'deg', + 'contents': INT_str(min=0, max=90, min_size=2)}, + {'name': 'min', + 'contents': INT_str(min=0, max=59, min_size=2)}, + ]}, + {'name': 'longitude', + 'namespace': 'lon', + 'contents': [ + {'name': 'dir', + 'contents': String(values=['E', 'W'])}, + {'name': 'deg', + 'contents': INT_str(min=0, max=180, min_size=3)}, + {'name': 'min', + 'contents': INT_str(min=0, max=59, min_size=2)}, + ]}, + ]} + self.register(test_node_desc, abstest_desc, abstest2_desc, separator_desc, sync_desc, len_gen_desc, misc_gen_desc, offset_gen_desc, shape_desc, for_network_tg1, for_network_tg2, for_net_default_tg, basic_intg, enc_desc, example_desc, regex_desc, xml1_desc, xml2_desc, xml3_desc, xml4_desc, xml5_desc, - json1_desc, json2_desc, file_desc) + json1_desc, json2_desc, file_desc, nested_desc, + csp_desc, csp_ns_desc) data_model = MyDF_DataModel() diff --git a/data_models/tutorial/tuto_strategy.py b/data_models/tutorial/tuto_strategy.py index 84b4403..376f16f 100644 --- a/data_models/tutorial/tuto_strategy.py +++ b/data_models/tutorial/tuto_strategy.py @@ -2,12 +2,12 @@ from framework.tactics_helpers import * from framework.global_resources import * from framework.scenario import * -from framework.data import Data +from framework.data import Data, DataProcess from framework.value_types import * tactics = Tactics() -def cbk_transition1(env, current_step, next_step, feedback): +def check_answer(env, current_step, next_step, feedback): if not feedback: print("\n\nNo feedback retrieved. Let's wait for another turn") current_step.make_blocked() @@ -30,7 +30,7 @@ def cbk_transition1(env, current_step, next_step, feedback): print("*** The next node won't be modified!") return True -def cbk_transition2(env, current_step, next_step): +def check_switch(env, current_step, next_step): if env.user_context.switch: return False else: @@ -48,8 +48,7 @@ def before_data_processing_cbk(env, step): step.content.show() return True -periodic1 = Periodic(DataProcess(process=[('C', UI(nb=1)), 'tTYPE'], seed='enc'), - period=5) +periodic1 = Periodic(DataProcess(process=[('C', UI(nb=1)), 'tTYPE'], seed='enc'), period=5) periodic2 = Periodic(Data('2nd Periodic (3s)\n'), period=3) ### SCENARIO 1 ### @@ -57,15 +56,15 @@ def before_data_processing_cbk(env, step): do_before_sending=before_sending_cbk, vtg_ids=0) step2 = Step('separator', fbk_timeout=2, clear_periodic=[periodic1], vtg_ids=1) empty = NoDataStep(clear_periodic=[periodic2]) -step4 = Step('off_gen', fbk_timeout=0) +step4 = Step('off_gen', fbk_timeout=0, refresh_atoms=False) step1_copy = copy.copy(step1) # for scenario 2 step2_copy = copy.copy(step2) # for scenario 2 step1.connect_to(step2) -step2.connect_to(empty, cbk_after_fbk=cbk_transition1) +step2.connect_to(empty, cbk_after_fbk=check_answer) empty.connect_to(step4) -step4.connect_to(step1, cbk_after_sending=cbk_transition2) +step4.connect_to(step1, cbk_after_sending=check_switch) sc_tuto_ex1 = Scenario('ex1', anchor=step1, user_context=UI(switch=False)) @@ -74,7 +73,7 @@ def before_data_processing_cbk(env, step): step_final = FinalStep() step1_copy.connect_to(step2_copy) -step2_copy.connect_to(step4, cbk_after_fbk=cbk_transition1) +step2_copy.connect_to(step4, cbk_after_fbk=check_answer) step4.connect_to(step_final) sc_tuto_ex2 = Scenario('ex2', anchor=step1_copy) @@ -88,7 +87,7 @@ def before_data_processing_cbk(env, step): option2 = Step(Data('Option 2'), do_before_data_processing=before_data_processing_cbk) -anchor.connect_to(option1, cbk_after_sending=cbk_transition2) +anchor.connect_to(option1, cbk_after_sending=check_switch) anchor.connect_to(option2) option1.connect_to(anchor) option2.connect_to(anchor) diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..0efae1e --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +Sphinx==4.3.2 +sphinx-rtd-theme==1.2.0 +sphinxcontrib-napoleon==0.7 diff --git a/docs/source/conf.py b/docs/source/conf.py index 9bc6bdd..a4767d9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -55,9 +55,9 @@ # built documents. # # The short X.Y version. -version = '0.27' +version = '0.30' # The full version, including alpha/beta/rc tags. -release = '0.27.2' +release = '0.30' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/data_analysis.rst b/docs/source/data_analysis.rst index 4e332c5..fe1a3f6 100644 --- a/docs/source/data_analysis.rst +++ b/docs/source/data_analysis.rst @@ -5,13 +5,17 @@ Data Analysis Each data you send and all the related information (the way the data has been built, the feedback from the target, and so on) are stored within the ``fuddly`` database -(an SQLite database located at ``~/fuddly_data/fmkdb.db``). They all get a unique ID, +(an SQLite database located at ``/fmkdb.db``). They all get a unique ID, starting from 1 and increasing by 1 each time a data is sent. + +FmkDB Toolkit +============= + To interact with the database a convenient toolkit is provided (``/tools/fmkdb.py``). Usage Examples -============== +-------------- Let's say you want to look at all the information that have been recorded for one of the data you sent, with the ID 4. The following @@ -35,23 +39,22 @@ For further information refer to the help by issuing:: Fmkdb Toolkit Manual -==================== +-------------------- Hereunder is shown the output of ``/tools/fmkdb.py -h``. .. code-block:: none usage: fmkdb.py [-h] [--fmkdb PATH] [--no-color] [-v] [--page-width WIDTH] - [--fbk-src FEEDBACK_SOURCES] [--project PROJECT_NAME] [-s] - [-i DATA_ID] [--info-by-date START END] - [--info-by-ids FIRST_DATA_ID LAST_DATA_ID] [-wf] [-wd] - [--without-fmkinfo] [--without-analysis] [--limit LIMIT] - [--raw] [-dd] [-df] [--data-atom ATOM_NAME] - [--fbk-atom ATOM_NAME] [--force-fbk-decoder DATA_MODEL_NAME] + [--fbk-src FEEDBACK_SOURCES] [--project PROJECT_NAME] + [--fbk-status-formula STATUS_REF] [-s] [-i DATA_ID] + [--info-by-date START END] [-ids FIRST_DATA_ID LAST_DATA_ID] [-wf] [-wd] [-wa] + [--without-fmkinfo] [--without-analysis] [--limit LIMIT] [--raw] [-dd] + [-df] [--data-atom ATOM_NAME] [--fbk-atom ATOM_NAME] + [--force-fbk-decoder DATA_MODEL_NAME] [--export-data FIRST_DATA_ID LAST_DATA_ID] [-e DATA_ID] [--remove-data FIRST_DATA_ID LAST_DATA_ID] [-r DATA_ID] - [--data-with-impact] [--data-with-impact-raw] - [--data-without-fbk] + [--data-with-impact] [--data-with-impact-raw] [--data-without-fbk] [--data-with-specific-fbk FEEDBACK_REGEXP] [-a IMPACT COMMENT] [--disprove-impact FIRST_ID LAST_ID] @@ -68,57 +71,58 @@ Hereunder is shown the output of ``/tools/fmkdb.py -h``. Configuration Handles: --fbk-src FEEDBACK_SOURCES - Restrict the feedback sources to consider (through a - regexp). Supported by: --data-with-impact, --data- - without-fbk, --data-with-specific-fbk - --project PROJECT_NAME - Restrict the data to be displayed to a specific - project. Supported by: --info-by-date, --info-by-ids, - --data-with-impact, --data-without-fbk, --data-with- + Restrict the feedback sources to consider (through a regexp). + Supported by: --data-with-impact, --data-without-fbk, --data-with- specific-fbk + --project PROJECT_NAME + Restrict the data to be displayed to a specific project. Supported + by: --info-by-date, --info-by-ids, --data-with-impact, --data- + without-fbk, --data-with-specific-fbk + --fbk-status-formula STATUS_REF + Restrict the data to be displayed to specific feedback status. + This option provides the formula to be used for feedback status + filtering (the character "?" should be used in place of the status + value that will be checked). Supported by: --data-with-impact Fuddly Database Visualization: -s, --all-stats Show all statistics Fuddly Database Information: -i DATA_ID, --data-id DATA_ID - Provide the data ID on which actions will be - performed. Without any other parameters the default - action is to display information on the specified data - ID. + Provide the data ID on which actions will be performed. Without + any other parameters the default action is to display information + on the specified data ID. --info-by-date START END - Display information on data sent between START and END - (date format 'Year/Month/Day' or 'Year/Month/Day-Hour' - or 'Year/Month/Day-Hour:Minute') - --info-by-ids FIRST_DATA_ID LAST_DATA_ID - Display information on all the data included within - the specified data ID range + Display information on data sent between START and END (date + format 'Year/Month/Day' or 'Year/Month/Day-Hour' or + 'Year/Month/Day-Hour:Minute') + -ids FIRST_DATA_ID LAST_DATA_ID, --info-by-ids FIRST_DATA_ID LAST_DATA_ID + Display information on all the data included within the specified + data ID range -wf, --with-fbk Display full feedback (expect --data-id) -wd, --with-data Display data content (expect --data-id) + -wa, --with-async-data + Display any related async data (expect --data-id) --without-fmkinfo Do not display fmkinfo (expect --data-id) --without-analysis Do not display user analysis (expect --data-id) - --limit LIMIT Limit the size of what is displayed from the sent data - and the retrieved feedback (expect --with-data or - --with-fbk). + --limit LIMIT Limit the size of what is displayed from the sent data and the + retrieved feedback (expect --with-data or --with-fbk). --raw Display data and feedback in raw format Fuddly Decoding: - -dd, --decode-data Decode sent data based on the data model used for the - selected data ID or the atome name provided by --atom - -df, --decode-fbk Decode feedback based on the data model used for the - selected data ID or the atome name provided by --fbk- - atom + -dd, --decode-data Decode sent data based on the data model used for the selected + data ID or the atome name provided by --atom + -df, --decode-fbk Decode feedback based on the data model used for the selected data + ID or the atome name provided by --fbk-atom --data-atom ATOM_NAME - Atom of the data model to be used for decoding the - sent data. If not provided, the name of the sent data - will be used. - --fbk-atom ATOM_NAME Atom of the data model to be used for decoding - feedback. If not provided, the default data model - decoder will be used (if one exists), or the name of - the first registered atom in the data model + Atom of the data model to be used for decoding the sent data. If + not provided, the name of the sent data will be used. + --fbk-atom ATOM_NAME Atom of the data model to be used for decoding feedback. If not + provided, the default data model decoder will be used (if one + exists), or the name of the first registered atom in the data + model --force-fbk-decoder DATA_MODEL_NAME - Decode feedback with the decoder of the data model - specified + Decode feedback with the decoder of the data model specified Fuddly Database Operations: --export-data FIRST_DATA_ID LAST_DATA_ID @@ -126,28 +130,113 @@ Hereunder is shown the output of ``/tools/fmkdb.py -h``. -e DATA_ID, --export-one-data DATA_ID Extract data from the provided data ID --remove-data FIRST_DATA_ID LAST_DATA_ID - Remove data from provided data ID range and all - related information from fmkDB + Remove data from provided data ID range and all related + information from fmkDB -r DATA_ID, --remove-one-data DATA_ID Remove data ID and all related information from fmkDB Fuddly Database Analysis: - --data-with-impact Retrieve data that negatively impacted a target. - Analysis is performed based on feedback status and - user analysis if present + --data-with-impact Retrieve data that negatively impacted a target. Analysis is + performed based on feedback status and user analysis if present --data-with-impact-raw - Retrieve data that negatively impacted a target. - Analysis is performed based on feedback status + Retrieve data that negatively impacted a target. Analysis is + performed based on feedback status --data-without-fbk Retrieve data without feedback --data-with-specific-fbk FEEDBACK_REGEXP - Retrieve data with specific feedback provided as a - regexp + Retrieve data with specific feedback provided as a regexp -a IMPACT COMMENT, --add-analysis IMPACT COMMENT - Add an impact analysis to a specific data ID (expect - --data-id). IMPACT should be either 0 (no impact) or 1 - (impact), and COMMENT provide information + Add an impact analysis to a specific data ID (expect --data-id). + IMPACT should be either 0 (no impact) or 1 (impact), and COMMENT + provide information --disprove-impact FIRST_ID LAST_ID - Disprove the impact of a group of data present in the - outcomes of '--data-with-impact-raw'. The group is - determined by providing the smaller data ID (FIRST_ID) - and the bigger data ID (LAST_ID). + Disprove the impact of a group of data present in the outcomes of + '--data-with-impact-raw'. The group is determined by providing the + smaller data ID (FIRST_ID) and the bigger data ID (LAST_ID). + + + +Plotty +====== + +Plotty is a tool used to vizualize data from the fmkDB. +To interact with the database a convenient toolkit is provided (``/tools/plotty/*``). + +Usage Examples +-------------- + +A very common usage is just to plot the data relatively to the date it was sent at. +To do that, you can use the plotty CLI, at ``/tools/plotty/plotty.py``:: + + ./tools/plotty.py -ids '0..100|2' + +Plots the SEND_DATE in function of the ID of every message which has en even ID beetween 0 and 100. +A lot of display and formatting options are available to build your own plotting experience ♥‿♥ + +For further information refer to the help by issuing:: + + ./tools/plotty/plotty.py -h + + +Plotty Manual +------------- + +Hereunder is shown the output of ``/tools/plotty/plotty.py -h`` + +.. code-block:: none + + usage: plotty.py [-h] -ids ID_RANGE [-df DATE_FORMAT] [-db PATH [PATH ...]] [-f FORMULA] [-poi POINTS_OF_INTEREST] [-gm {all,poi,auto}] [-hp] + [-l ANNOTATIONS [ANNOTATIONS ...]] [-al ASYNC_ANNOTATIONS [ASYNC_ANNOTATIONS ...]] [-o OTHER_ID_RANGE] [-s VERTICAL_SHIFT] + + Arguments for Plotty + + options: + -h, --help show this help message and exit + + Main parameters: + -ids ID_RANGE, --id-range ID_RANGE + The ID range to take into account should be: either ..[|], or ..[|], ..., + ..[|] + -df DATE_FORMAT, --date-format DATE_FORMAT + Wanted date format, in a strftime format (1989 C standard). Default is %H:%M:%S.%f + -db PATH [PATH ...], --fmkdb PATH [PATH ...] + Path to any fmkDB.db files. There can be many if using the --other_id_range option. Default is fuddly/data/directory/fmkDB.db + + Display Options: + -f FORMULA, --formula FORMULA + The formula to plot, in the form "y ~ x" + -poi POINTS_OF_INTEREST, --points-of-interest POINTS_OF_INTEREST + How many point of interest the plot should show. Default is none + -gm {all,poi,auto}, --grid-match {all,poi,auto} + Should the plot grid specifically match some element. Possible options are 'all', 'poi' and 'auto'. Default is 'all' + -hp, --hide-points Should the graph display every point above the line, or just the line. Default is to display the points + + Labels Configuration: + -l ANNOTATIONS [ANNOTATIONS ...], --labels ANNOTATIONS [ANNOTATIONS ...] + Display the specified labels for each Data ID represented in the curve. ('t' for TYPE, 'g' for TARGET, 's' for SIZE, 'a' for + ACK_DATE) + -al ASYNC_ANNOTATIONS [ASYNC_ANNOTATIONS ...], --async-labels ASYNC_ANNOTATIONS [ASYNC_ANNOTATIONS ...] + Display the specified labels for each Async Data ID represented in the curve. ('i' for 'ID', 't' for TYPE, 'g' for TARGET, 's' for + SIZE) + + Multiple Curves Options: + -o OTHER_ID_RANGE, --other-id-range OTHER_ID_RANGE + Other ranges of IDs to plot against the main one. All other options apply to it + -s VERTICAL_SHIFT, --vertical-shift VERTICAL_SHIFT + When --other-id-range is used, specify the spacing between the curves. The shift is computed as the multiplication between the + original curve height and this value + + +Concrete output example +----------------------- + +.. _plotty_example: +.. figure:: images/plotty_example.png + :align: center + + Example of curve comparison using Plotty + +Given a simple command line:: + + ./tools/plotty/plotty.py -id '2..12' -l t -al t i -df "%M:%S.%f" -poi 3 + +It is already possible to vizualize trends, behaviors and anomalies. Comparison becomes easier ! \ No newline at end of file diff --git a/docs/source/disruptors.rst b/docs/source/data_makers.rst similarity index 61% rename from docs/source/disruptors.rst rename to docs/source/data_makers.rst index bb20845..d785708 100644 --- a/docs/source/disruptors.rst +++ b/docs/source/data_makers.rst @@ -1,18 +1,65 @@ +.. _gen:generic-generators: + +Generic Data Makers +******************* + +Generic Generators +================== + +The current generic generators are presented within the following +sections. + +GENP - Pattern Generation +------------------------- + +Description: + Generate basic data based on a pattern and different parameters. + +Reference: + :class:`framework.generic_data_makers.g_generic_pattern` + +Parameters: + .. code-block:: none + + |_ pattern + | | desc: Pattern to be used for generating data + | | default: b'1234567890' [type: bytes] + |_ prefix + | | desc: Prefix added to the pattern + | | default: b'' [type: bytes] + |_ suffix + | | desc: Suffix replacing the end of the pattern + | | default: b'' [type: bytes] + |_ size + | | desc: Size of the generated data. + | | default: None [type: int] + |_ eval + | | desc: The pattern will be evaluated before being used. Note that the + | | evaluation shall result in a byte string. + | | default: False [type: bool] + + +POPULATION - Generator for Evolutionary Fuzzing +----------------------------------------------- + +This generator is used only internally by the evolutionary fuzzing infrastructure. + + .. _dis:generic-disruptors: Generic Disruptors -****************** +================== The current generic disruptors are presented within the following sections. Stateful Disruptors -=================== +------------------- .. _dis:ttype: tTYPE - Advanced Alteration of Terminal Typed Node --------------------------------------------------- +++++++++++++++++++++++++++++++++++++++++++++++++++ Description: Perform alterations on typed nodes (one at a time) according to: @@ -35,31 +82,42 @@ Reference: Parameters: .. code-block:: none - parameters: |_ init | | desc: make the model walker ignore all the steps until the provided | | one | | default: 1 [type: int] - |_ runs_per_node - | | desc: maximum number of test cases for a single node (-1 means until - | | the end) - | | default: -1 [type: int] |_ max_steps | | desc: maximum number of steps (-1 means until the end) | | default: -1 [type: int] + |_ min_node_tc + | | desc: Minimum number of test cases per node (-1 means until the end) + | | default: -1 [type: int] + |_ max_node_tc + | | desc: Maximum number of test cases per node (-1 means until the end). + | | This value is used for nodes with a fuzz weight strictly greater + | | than 1. + | | default: -1 [type: int] |_ clone_node - | | desc: if True the dmaker will always return a copy of the node. (for + | | desc: if True, this operator will always return a copy of the node. (for | | stateless disruptors dealing with big data it can be useful | | to it to False) | | default: True [type: bool] |_ path - | | desc: graph path regexp to select nodes on which the disruptor should + | | desc: Graph path regexp to select nodes on which the disruptor should | | apply | | default: None [type: str] + |_ sem + | | desc: Semantics to select nodes on which the disruptor should apply. + | | default: None [type: str, list] |_ deep - | | desc: when set to True, if a node structure has changed, the modelwalker + | | desc: When set to True, if a node structure has changed, the modelwalker | | will reset its walk through the children nodes | | default: True [type: bool] + |_ full_combinatory + | | desc: When set to True, enable full-combinatory mode for non-terminal + | | nodes. It means that the non-terminal nodes will be customized + | | in "FullCombinatory" mode + | | default: False [type: bool] |_ ign_sep | | desc: when set to True, separators will be ignored if | | any are defined. @@ -81,20 +139,37 @@ Parameters: |_ fuzz_mag | | desc: order of magnitude for maximum size of some fuzzing test cases. | | default: 1.0 [type: float] - |_ determinism - | | desc: If set to 'True', the whole model will be fuzzed in a deterministic - | | way. Otherwise it will be guided by the data model determinism. + |_ make_determinist + | | desc: If set to 'True', the whole model will be set in determinist mode. + | | Otherwise it will be guided by the data model determinism. + | | default: False [type: bool] + |_ leaf_fuzz_determinism + | | desc: If set to 'True', each typed node will be fuzzed in a deterministic + | | way. If set to 'False' each typed node will be fuzzed in a random + | | way. Otherwise, if it is set to 'None', it will be guided by + | | the data model determinism. Note: this option is complementary + | | to 'determinism' as it acts on the typed node substitutions + | | that occur through this disruptor | | default: True [type: bool] |_ leaf_determinism - | | desc: If set to 'True', each typed node will be fuzzed in a deterministic - | | way. Otherwise it will be guided by the data model determinism. - | | Note: this option is complementary to 'determinism' is it acts - | | on the typed node substitutions that occur through this disruptor + | | desc: If set to 'True', all the typed nodes of the model will be set + | | to determinist mode prior to any fuzzing. If set to 'False', + | | they will be set to random mode. Otherwise, if set to 'None', + | | nothing will be done. + | | default: None [type: bool] + |_ consider_sibbling_change + | | desc: While walking through terminal nodes, if sibbling nodes are + | | no more the same because of existence condition for instance, + | | walk through the new nodes. | | default: True [type: bool] + |_ ign_mutable_attr + | | desc: Walk through all the nodes even if their Mutable attribute is + | | cleared. + | | default: False [type: bool] tSTRUCT - Alter Data Structure ------------------------------- +++++++++++++++++++++++++++++++ Description: Perform constraints alteration (one at a time) on each node that depends on another one @@ -112,7 +187,6 @@ Reference: Parameters: .. code-block:: none - parameters: |_ init | | desc: make the model walker ignore all the steps until the provided | | one @@ -124,6 +198,9 @@ Parameters: | | desc: graph path regexp to select nodes on which the disruptor should | | apply | | default: None [type: str] + |_ sem + | | desc: Semantics to select nodes on which the disruptor should apply. + | | default: None [type: str, list] |_ deep | | desc: if True, enable corruption of non-terminal node internals | | default: False [type: bool] @@ -145,7 +222,7 @@ Usage Example: tALT - Walk Through Alternative Node Configurations ---------------------------------------------------- ++++++++++++++++++++++++++++++++++++++++++++++++++++ Description: Switch the configuration of each node, one by one, with the provided @@ -157,9 +234,8 @@ Reference: Parameters: .. code-block:: none - parameters: |_ clone_node - | | desc: if True the dmaker will always return a copy of the node. (for + | | desc: if True, this operator will always return a copy of the node. (for | | stateless disruptors dealing with big data it can be useful | | to it to False) | | default: True [type: bool] @@ -170,9 +246,13 @@ Parameters: |_ max_steps | | desc: maximum number of steps (-1 means until the end) | | default: -1 [type: int] - |_ runs_per_node - | | desc: maximum number of test cases for a single node (-1 means until - | | the end) + |_ min_node_tc + | | desc: Minimum number of test cases per node (-1 means until the end) + | | default: -1 [type: int] + |_ max_node_tc + | | desc: Maximum number of test cases per node (-1 means until the end). + | | This value is used for nodes with a fuzz weight strictly greater + | | than 1. | | default: -1 [type: int] |_ conf | | desc: Change the configuration, with the one provided (by name), of @@ -181,8 +261,40 @@ Parameters: | | default: None [type: str, list, tuple] +tCONST - Alteration of Constraints +++++++++++++++++++++++++++++++++++ + +Description: + When the CSP (Constraint Satisfiability Problem) backend are used in the node description. + This operator negates the constraint one-by-one and output 1 or more samples for each negated + constraint. + +Reference: + :class:`framework.generic_data_makers.sd_constraint_fuzz` + +Parameters: + .. code-block:: none + + |_ const_idx + | | desc: Index of the constraint to begin with (first index is 1) + | | default: 1 [type: int] + |_ sample_idx + | | desc: Index of the sample for the selected constraint to begin with + | | (first index is 1) + | | default: 1 [type: int] + |_ clone_node + | | desc: If True, this operator will always return a copy of the node. + | | (for stateless diruptors dealing with big data it can be usefull + | | to set it to False) + | | default: True [type: bool] + |_ samples_per_cst + | | desc: Maximum number of samples to output for each negated constraint + | | (-1 means until the end) + | | default: -1 [type: int] + + tSEP - Alteration of Separator Node ------------------------------------ ++++++++++++++++++++++++++++++++++++ Description: Perform alterations on separators (one at a time). Each time a @@ -196,9 +308,8 @@ Reference: Parameters: .. code-block:: none - parameters: |_ clone_node - | | desc: if True the dmaker will always return a copy of the node. (for + | | desc: if True, this operator will always return a copy of the node. (for | | stateless disruptors dealing with big data it can be useful | | to it to False) | | default: True [type: bool] @@ -209,14 +320,21 @@ Parameters: |_ max_steps | | desc: maximum number of steps (-1 means until the end) | | default: -1 [type: int] - |_ runs_per_node - | | desc: maximum number of test cases for a single node (-1 means until - | | the end) + |_ min_node_tc + | | desc: Minimum number of test cases per node (-1 means until the end) + | | default: -1 [type: int] + |_ max_node_tc + | | desc: Maximum number of test cases per node (-1 means until the end). + | | This value is used for nodes with a fuzz weight strictly greater + | | than 1. | | default: -1 [type: int] |_ path | | desc: graph path regexp to select nodes on which the disruptor should | | apply | | default: None [type: str] + |_ sem + | | desc: Semantics to select nodes on which the disruptor should apply. + | | default: None [type: str, list] |_ order | | desc: when set to True, the fuzzing order is strictly guided by the | | data structure. Otherwise, fuzz weight (if specified in the @@ -230,7 +348,7 @@ Parameters: tWALK - Walk Through a Data Model ---------------------------------- ++++++++++++++++++++++++++++++++++ Description: Walk through the provided data and for each visited node, iterates @@ -238,14 +356,13 @@ Description: alteration* is performed by this disruptor. Reference: - :class:`framework.generic_data_makers.sd_iter_over_data` + :class:`framework.generic_data_makers.sd_walk_data_model` Parameters: .. code-block:: none - parameters: |_ clone_node - | | desc: if True the dmaker will always return a copy of the node. (for + | | desc: if True, this operator will always return a copy of the node. (for | | stateless disruptors dealing with big data it can be useful | | to it to False) | | default: True [type: bool] @@ -256,32 +373,119 @@ Parameters: |_ max_steps | | desc: maximum number of steps (-1 means until the end) | | default: -1 [type: int] - |_ runs_per_node - | | desc: maximum number of test cases for a single node (-1 means until - | | the end) + |_ min_node_tc + | | desc: Minimum number of test cases per node (-1 means until the end) + | | default: -1 [type: int] + |_ max_node_tc + | | desc: Maximum number of test cases per node (-1 means until the end). + | | This value is used for nodes with a fuzz weight strictly greater + | | than 1. | | default: -1 [type: int] |_ path | | desc: graph path regexp to select nodes on which the disruptor should | | apply | | default: None [type: str] + |_ sem + | | desc: Semantics to select nodes on which the disruptor should apply. + | | default: None [type: str, list] + |_ full_combinatory + | | desc: When set to True, enable full-combinatory mode for non-terminal + | | nodes. It means that the non-terminal nodes will be customized + | | in "FullCombinatory" mode + | | default: True [type: bool] + |_ leaf_determinism + | | desc: If set to 'True', all the typed nodes of the model will be set + | | to determinist mode prior to any fuzzing. If set to 'False', + | | they will be set to random mode. Otherwise, if set to 'None', + | | nothing will be done. + | | default: None [type: bool] |_ order | | desc: when set to True, the walking order is strictly guided by the | | data structure. Otherwise, fuzz weight (if specified in the | | data model) is used for ordering | | default: True [type: bool] + |_ nt_only + | | desc: walk through non-terminal nodes only + | | default: False [type: bool] + |_ deep + | | desc: when set to True, if a node structure has changed, the modelwalker + | | will reset its walk through the children nodes + | | default: True [type: bool] |_ fix_all | | desc: for each produced data, reevaluate the constraints on the whole | | graph | | default: True [type: bool] - |_ nt_only - | | desc: walk through non-terminal nodes only - | | default: False [type: bool] + |_ ign_mutable_attr + | | desc: Walk through all the nodes even if their Mutable attribute is + | | cleared. + | | default: True [type: bool] + + +tWALKcsp - Walk Through the Constraint of a Data Model +++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Description: + When the CSP (Constraint Satisfiability Problem) backend are used in the data description. + This operator walk through the solutions of the CSP. + +Reference: + :class:`framework.generic_data_makers.sd_walk_csp_solutions` + +Parameters: + .. code-block:: none + + |_ init + | | desc: Make the operator ignore all the steps until the provided one + | | default: 1 [type: int] + |_ clone_node + | | desc: If True, this operator will always return a copy of the node. + | | (for stateless diruptors dealing with big data it can be usefull + | | to set it to False) + | | default: True [type: bool] + |_ notify_exhaustion + | | desc: When all the solutions of the CSP have been walked through, + | | the disruptor will notify it if this parameter is set to True. + | | default: True [type: bool] + Stateless Disruptors -==================== +-------------------- + +ADD - Add Data Within a Node +++++++++++++++++++++++++++++ + +Description: + Add some data within the retrieved input. + +Reference: + :class:`framework.generic_data_makers.d_add_data` + +Parameters: + .. code-block:: none + + |_ path + | | desc: Graph path to select the node on which the disruptor should + | | apply. + | | default: None [type: str] + |_ after + | | desc: If True, the addition will be done after the selected node. + | | Otherwise, it will be done before. + | | default: True [type: bool] + |_ atom + | | desc: Name of the atom to add within the retrieved input. It is mutually + | | exclusive with @raw + | | default: None [type: str] + |_ raw + | | desc: Raw value to add within the retrieved input. It is mutually + | | exclusive with @atom. + | | default: b'' [type: bytes, str] + |_ name + | | desc: If provided, the added node will have this name. + | | default: None [type: str] + OP - Perform Operations on Nodes --------------------------------- +++++++++++++++++++++++++++++++++ Description: Perform an operation on the nodes specified by the regexp path. @op is an operation that @@ -294,7 +498,6 @@ Reference: Parameters: .. code-block:: none - parameters: |_ path | | desc: Graph path regexp to select nodes on which the disruptor should | | apply. @@ -302,10 +505,14 @@ Parameters: |_ op | | desc: The operation to perform on the selected nodes. | | default: [type: method, function] + |_ op_ref + | | desc: Predefined operation that can be referenced by name. The current + | | predefined function are: 'unfreeze', 'freeze', 'walk'. Take + | | precedence over @op if not None. + | | default: None [type: str] |_ params | | desc: Tuple of parameters that will be provided to the operation. - | | (default: MH.Attr.Mutable) - | | default: (2,) [type: tuple] + | | default: () [type: tuple] |_ clone_node | | desc: If True the dmaker will always return a copy of the node. (For | | stateless disruptors dealing with big data it can be useful @@ -314,7 +521,7 @@ Parameters: MOD - Modify Node Contents --------------------------- +++++++++++++++++++++++++++ Description: Perform modifications on the provided data. Two ways are possible: @@ -331,32 +538,39 @@ Reference: Parameters: .. code-block:: none - parameters: |_ path | | desc: Graph path regexp to select nodes on which the disruptor should | | apply. | | default: None [type: str] + |_ sem + | | desc: Semantics to select nodes on which the disruptor should apply. + | | default: None [type: str, list] |_ value | | desc: The new value to inject within the data. - | | default: '' [type: str] + | | default: b'' [type: bytes] |_ constraints | | desc: Constraints for the absorption of the new value. | | default: AbsNoCsts() [type: AbsCsts] |_ multi_mod - | | desc: Dictionary of : pairs to change multiple nodes with - | | diferent values. can be either only the new or - | | a tuple (,) if new constraint for absorption - | | is needed + | | desc: Dictionary of : pairs or : + | | pairs or : pairs to change multiple + | | nodes with different values. can be either only the new + | | or a tuple (,) if new constraint for + | | absorption is needed | | default: None [type: dict] + |_ unfold + | | desc: Resolve all the generator nodes within the input before performing + | | the @path/@sem research + | | default: False [type: bool] |_ clone_node | | desc: If True the dmaker will always return a copy of the node. (For | | stateless disruptors dealing with big data it can be useful - | | to it to False.) + | | to set it to False.) | | default: False [type: bool] CALL - Call Function --------------------- +++++++++++++++++++++ Description: Call the function provided with the first parameter being the :class:`framework.data.Data` @@ -373,7 +587,6 @@ Reference: Parameters: .. code-block:: none - parameters: |_ func | | desc: The function that will be called with a node as its first parameter, | | and provided optionnaly with addtionnal parameters if @params @@ -386,7 +599,7 @@ Parameters: NEXT - Next Node Content ------------------------- +++++++++++++++++++++++++ Description: Move to the next content of the nodes from input data or from only @@ -400,13 +613,12 @@ Reference: Parameters: .. code-block:: none - parameters: |_ path | | desc: graph path regexp to select nodes on which the disruptor should | | apply | | default: None [type: str] |_ clone_node - | | desc: if True the dmaker will always return a copy of the node. (for + | | desc: if True, this operator will always return a copy of the node. (for | | stateless disruptors dealing with big data it can be useful | | to it to False) | | default: False [type: bool] @@ -417,7 +629,7 @@ Parameters: FIX - Fix Data Constraints --------------------------- +++++++++++++++++++++++++++ Description: Release constraints from input data or from only a piece of it (if @@ -435,20 +647,19 @@ Reference: Parameters: .. code-block:: none - parameters: |_ path | | desc: graph path regexp to select nodes on which the disruptor should | | apply | | default: None [type: str] |_ clone_node - | | desc: if True the dmaker will always return a copy of the node. (for + | | desc: if True, this operator will always return a copy of the node. (for | | stateless disruptors dealing with big data it can be useful | | to it to False) | | default: False [type: bool] ALT - Alternative Node Configuration ------------------------------------- +++++++++++++++++++++++++++++++++++++ Description: Switch to an alternate configuration. @@ -459,7 +670,6 @@ Reference: Parameters: .. code-block:: none - parameters: |_ path | | desc: graph path regexp to select nodes on which the disruptor should | | apply @@ -476,7 +686,7 @@ Parameters: C - Node Corruption -------------------- ++++++++++++++++++++ Description: Corrupt bits on some nodes of the data model. @@ -487,7 +697,6 @@ Reference: Parameters: .. code-block:: none - parameters: |_ path | | desc: graph path regexp to select nodes on which the disruptor should | | apply @@ -505,7 +714,7 @@ Parameters: Cp - Corruption at Specific Position ------------------------------------- +++++++++++++++++++++++++++++++++++++ Description: Corrupt bit at a specific byte. @@ -516,7 +725,6 @@ Reference: Parameters: .. code-block:: none - parameters: |_ new_val | | desc: if provided change the selected byte with the new one | | default: None [type: str] @@ -529,7 +737,7 @@ Parameters: EXT - Make Use of an External Program -------------------------------------- ++++++++++++++++++++++++++++++++++++++ Description: Call an external program to deal with the data. @@ -540,7 +748,6 @@ Reference: Parameters: .. code-block:: none - parameters: |_ path | | desc: graph path regexp to select nodes on which the disruptor should | | apply @@ -555,7 +762,7 @@ Parameters: SIZE - Truncate ---------------- ++++++++++++++++ Description: Truncate the data (or part of the data) to the provided size. @@ -566,7 +773,6 @@ Reference: Parameters: .. code-block:: none - parameters: |_ sz | | desc: truncate the data (or part of the data) to the provided size | | default: 10 [type: int] @@ -577,7 +783,7 @@ Parameters: STRUCT - Shake Up Data Structure --------------------------------- +++++++++++++++++++++++++++++++++ Description: Disrupt the data model structure (replace ordered sections by @@ -589,7 +795,6 @@ Reference: Parameters: .. code-block:: none - parameters: |_ path | | desc: graph path regexp to select nodes on which the disruptor should | | apply @@ -598,7 +803,7 @@ Parameters: COPY - Shallow Copy Data ------------------------- +++++++++++++++++++++++++ Description: Shallow copy of the input data, which means: ignore its frozen diff --git a/docs/source/data_manip.rst b/docs/source/data_manip.rst index a70b498..ee90c45 100644 --- a/docs/source/data_manip.rst +++ b/docs/source/data_manip.rst @@ -253,7 +253,7 @@ Search for Nodes in a Graph Searching a graph for specific nodes can be performed in basically two ways. Depending on the criteria based on which you want to perform the search, you should use: -- :meth:`framework.node.Node.get_node_by_path`: will retrieve the first node that match the +- :meth:`framework.node.Node.iter_nodes_by_path`: iterator that walk through all the nodes that match the *graph path*---you provide as a parameter---from the node on which the method is called (or ``None`` if nothing is found). The syntax defined to represent paths is similar to the one of filesystem paths. Each path are represented by a python string, where node names are @@ -262,9 +262,10 @@ criteria based on which you want to perform the search, you should use: 'ex/data_group/len' - You can also use a regexp to describe a path. Also, if you need to retrieve all the nodes - matching a path regexp you should use the following method. + Note the path provided is interpreted as a regexp. +- :meth:`framework.node.Node.get_first_node_by_path`: use the previous iterator to provide the first + node that match the *graph path* or ``None`` if nothing is found - :meth:`framework.node.Node.get_reachable_nodes`: It is the more flexible primitive that enables to perform a search based on syntactic and/or semantic criteria. It can take several @@ -291,6 +292,9 @@ criteria based on which you want to perform the search, you should use: + ``owned_conf``: The name of a node configuration (refer to :ref:`dmanip:conf`) that the targeted nodes own. + .. note:: If the search is only path-based, :meth:`framework.node.Node.iter_nodes_by_path` is the + preferable solution as it is more efficient. + The following code snippet illustrates the use of such criteria for retrieving all the nodes coming from the ``data2`` description (refer to :ref:`dmanip:entangle`): @@ -302,10 +306,12 @@ criteria based on which you want to perform the search, you should use: from framework.value_types import * fmk = FmkPlumbing() + fmk.start() + fmk.run_project(name='tuto') - data = fmk.get_data(['EX']) # Return a Data container on the data model example - data.content.freeze() + ex_node = fmk.dm.get_atom('ex') + ex_node.freeze() ic = NodeInternalsCriteria(mandatory_attrs=[NodeInternals.Mutable], node_kinds=[NodeInternals_TypedValue], @@ -314,14 +320,21 @@ criteria based on which you want to perform the search, you should use: sc = NodeSemanticsCriteria(mandatory_criteria=['sem1', 'sem2']) - data.content.get_reachable_nodes(internals_criteria=ic, semantics_criteria=sc, - owned_conf='alt2') + ex_node.get_reachable_nodes(internals_criteria=ic, semantics_criteria=sc, + owned_conf='alt2') Obviously, you don't need all these criteria for retrieving such node. It's only for exercise. .. note:: For abstracting away the data model from the rest of the framework, ``fuddly`` uses the - specific class :meth:`framework.data.Data` which acts as a data container. + specific class :class:`framework.data.Data` which acts as a data container. + Thus, while interacting with the different part of the framework, Node-based data + (or string-based data) should be encapsulated + in a :class:`framework.data.Data` object. + + For instance ``Data(ex_node)`` will create an object that encapsulate ``ex_node``. + Accessing the node again is done through the property :attr:`framework.data.Data.content` + The Node Dictionary Interface @@ -340,10 +353,12 @@ following operation are possible on a node: As a ``key``, you can provide: - A *path regexp* (where the node on which the method is called is considered as the root) to the - node you want to reach. If multiple nodes match the path regexp, the first one will be returned - (or ``None`` if the path match nothing). It is equivalent to calling - :meth:`framework.node.Node.get_node_by_path` on the node and providing the parameter - ``path_regexp`` with your path. + nodes you want to reach. A list of the nodes will be returned for the reading operation + (or ``None`` if the path match nothing), and + for the writing operation all the matching nodes will get the new value. + The reading operation is equivalent to calling + :meth:`framework.node.Node.iter_nodes_by_path` on the node and providing the parameter + ``path_regexp`` with your path (except the method will return a python generator instead of a list). The following python code snippet illustrate the access to the node named ``len`` to retrieve its byte string representation: @@ -351,10 +366,10 @@ As a ``key``, you can provide: .. code-block:: python :linenos: - rnode['ex/data_group/len'].to_bytes() + rnode['ex/data_group/len'][0].to_bytes() # same as: - rnode.get_node_by_path('ex/data_group/len').to_bytes() + rnode.get_first_node_by_path('ex/data_group/len').to_bytes() - A :class:`framework.node.NodeInternalsCriteria` that match the internal @@ -431,11 +446,11 @@ it if you like ;). usb_str = fmk.dm.get_external_atom(dm_name='usb', data_id='STR') - data = fmk.get_data(['EX']) # Return a Data container on the data model example + ex_node = fmk.dm.get_atom('ex') - data.content['ex/data0'] = usb_str # Perform the substitution + ex_node['ex/data0'] = usb_str # Perform the substitution - data.show() # Data.show() will call .show() on the embedded node + ex_node.show() # Data.show() will call .show() on the embedded node The result is shown below: @@ -445,11 +460,74 @@ The result is shown below: :scale: 100 % -.. warning:: Releasing constraints (like a CRC, an offset, a length, ...) of an altered +.. note:: Releasing constraints (like a CRC, an offset, a length, ...) of an altered data can be useful if you want ``fuddly`` to automatically recomputes the constraint for you and still comply to the model. Refer to :ref:`dmanip:freeze`. +You can also add subnodes to non-terminal nodes through the usage of :meth:`framework.node.NodeInternals_NonTerm.add()`. +For instance the following code snippet will add a new node after the node ``data2``. + +.. code-block:: python + :linenos: + + data2_node = ex_node['ex/data_group/data2'][0] + ex_node['ex/data_group$'][0].add(Node('my_node', values=['New node added']), + after=data2_node) + +Thus, if ``ex_node`` before the modification is:: + + [0] ex [NonTerm] + \__(1) ex/data0 [String] size=4B + | \_ codec=iso8859-1 + | \_raw: b'Plip' + \__[1] ex/data_group [NonTerm] + | \__[2] ex/data_group/len [GenFunc | node_args: ex/data_group/data1] + | | \__(3) ex/data_group/len/cts [UINT8] size=1B + | | \_ 5 (0x5) + | | \_raw: b'\x05' + | \__(2) ex/data_group/data1 [String] size=5B + | | \_ codec=iso8859-1 + | | \_raw: b'Test!' + | \__(2) ex/data_group/data2 [UINT16_be] size=2B + | | \_ 10 (0xA) + | | \_raw: b'\x00\n' + | \__(2) ex/data_group/data3 [UINT8] size=1B + | \_ 30 (0x1E) + | \_raw: b'\x1e' + \__(1) ex/data4 [String] size=3B + \_ codec=iso8859-1 + \_raw: b'Red' + + +After the modification it will be:: + + [0] ex [NonTerm] + \__(1) ex/data0 [String] size=4B + | \_ codec=iso8859-1 + | \_raw: b'Plip' + \__[1] ex/data_group [NonTerm] + | \__[2] ex/data_group/len [GenFunc | node_args: ex/data_group/data1] + | | \__(3) ex/data_group/len/cts [UINT8] size=1B + | | \_ 5 (0x5) + | | \_raw: b'\x05' + | \__(2) ex/data_group/data1 [String] size=5B + | | \_ codec=iso8859-1 + | | \_raw: b'Test!' + | \__(2) ex/data_group/data2 [UINT16_be] size=2B + | | \_ 10 (0xA) + | | \_raw: b'\x00\n' + | \__(2) ex/data_group/my_node [String] size=14B + | | \_ codec=iso8859-1 + | | \_raw: b'New node added' + | \__(2) ex/data_group/data3 [UINT8] size=1B + | \_ 30 (0x1E) + | \_raw: b'\x1e' + \__(1) ex/data4 [String] size=3B + \_ codec=iso8859-1 + \_raw: b'Red' + + .. _dmanip:prop: Operations on Node Properties and Attributes @@ -545,17 +623,17 @@ In what follows, we illustrate some node configuration change based on our data rnode.freeze() # We consider there is at least 2 'data2' nodes # We change the configuration of the second 'data2' node - rnode['ex/data_group/data2:2'].set_current_conf('alt1', ignore_entanglement=True) - rnode['ex/data_group/data2:2'].unfreeze() + rnode['ex/data_group/data2:2'][0].set_current_conf('alt1', ignore_entanglement=True) + rnode['ex/data_group/data2:2'][0].unfreeze() rnode.show() # We change back 'data2:2' to the default configuration - rnode['ex/data_group/data2:2'].set_current_conf('MAIN', ignore_entanglement=True) + rnode['ex/data_group/data2:2'][0].set_current_conf('MAIN', ignore_entanglement=True) # We change the configuration of the first 'data2' node - rnode['ex/data_group/data2'].set_current_conf('alt1', ignore_entanglement=True) + rnode['ex/data_group/data2'][0].set_current_conf('alt1', ignore_entanglement=True) # This time we unfreeze directly the parent node - rnode['ex/data_group$'].unfreeze(dont_change_state=True) + rnode['ex/data_group$'][0].unfreeze(dont_change_state=True) rnode.show() diff --git a/docs/source/data_model.rst b/docs/source/data_model.rst index a0d9a67..fc47c6c 100644 --- a/docs/source/data_model.rst +++ b/docs/source/data_model.rst @@ -27,13 +27,34 @@ name in giving the same name to different nodes:: 'my_name' - ('my_name', 2) - ('my_name', 'of the command') + ('my_name', 'namespace_1') + ('my_name', 'namespace_2') These names serve as *node references* during data description. + Another option is to use the keyword ``namespace``. .. note:: The character ``/`` is reserved and shall not be used in a *name*. + +namespace + [For non-terminal nodes only] + + Specify a namespace that will be used for the ``name`` of all the nodes reachable + from the node declaring the namespace. It means that the subnodes will be automatically + described by a tuple with the namespace value as second item + (refer to the description of the keyword ``name``). + +from_namespace + If specified, and some node name are used as inputs (refer for instance to keywords ``clone`` + or ``node_args``), the specified namespace will be used to fetch the nodes. + It is equivalent to use a tuple expression for referring to the node with the ``from_namespace`` value + as second item. + + Note that referring to a node without specifying the namespace will be interpreted as using the + current namespace. By default (even when no namespaces have been declared), one namespace is + always defined from the root node, it can be referred to by ``NodeBuilder.RootNS``. + + contents Every node description has at least a ``name`` and a ``contents`` attributes (except if you refer to an already existing node, and in @@ -58,6 +79,15 @@ contents Note that for defining a *function node* and not a generator node, you have to state the type attribute to ``MH.Leaf``. +default + Default value for the node. Only compatible with typed nodes + (:class:`framework.node.NodeInternals_TypedValue`). It is directly linked to the + ``default`` parameter of each type constructor. Refer to :ref:`vt:value-types` for more information. + +description + Textual description of the node. Note this information is shown by the method + :meth:`framework.node.Node.show()`. + qty Specify the amount of nodes to generate from the description, or a tuple ``(min, max)`` specifying the minimum (which can be 0) and the @@ -71,6 +101,9 @@ qty get such kind of data, specify explicitly the maximum, or use a disruptor to do so (:ref:`tuto:disruptors`). +default_qty + Specify the default amount of nodes to generate from the description. + It should be within ````. clone Allows to make a full copy of an existing node by providing its @@ -131,6 +164,13 @@ custo_set, custo_clear have this attribute cleared to prevent generic disruptors from altering them. This mode aims at limiting the number of test cases, by pruning what is assumed to be redundant. + + - ``MH.Custo.NTerm.CycleClone``: By default, this mode is *disabled*. + When enabled, and when the subnodes need to be duplicated because of a ``qty`` greater than 1, + the non-terminal node will walk through each copy, in order to cycle among + the various shapes/values of the subnodes. Note this customization won't be effective + if an evolution function is provided through the keyword ``evolution_func``. + - ``MH.Custo.NTerm.FrozenCopy``: By default, this mode is *enabled*. When enabled, it means that for child nodes which can be instantiated many times (refer to ``qty`` attribute), the instantiation process will make a frozen copy @@ -138,6 +178,16 @@ custo_set, custo_clear the time of the copy. If disabled, the instantiation process will ignore the frozen state, and thus will release all the constraints. + - ``MH.Custo.NTerm.FullCombinatory``: By default, this mode is *disabled*. When enabled, + walking through a non-terminal node will generate all "possible" combination of forms for each + subnode. The various considered forms for a subnode are based on the ``qty`` and ``default_qty`` + parameter provided. Thus there are at most 3 different forms that boil down to the different amounts of + subnodes (max, min and default values), and at least 1 if all are the same. Other possible values + in the range ```` are reachable in ``random`` mode, or by changing the subnode quantity manually. + When this mode is disabled, walking through the non-terminal node won't generate all possible + combinations but a subset of it based on a simpler algorithm that will walk through each subnode and + iterate for their different forms without considering the previous subnodes forms. + .. note:: Note that if the node is not frozen at the time of the copy, this customization won't have any effect. The main interest is @@ -148,21 +198,59 @@ custo_set, custo_clear children will be frozen at that time, the reconstruction will take into account this customization mode. + - ``MH.Custo.NTerm.StickToDefault``: By default, this mode is *disabled*. When enabled, + walking through a non-terminal node *won't* generate all "possible" combination of forms for each + subnode. Only the default quantity (refer to keyword ``default_qty``) is leveraged. Walking through such nodes will + generate new forms only if different shapes have been defined (refer to keyword ``shape_type`` + and ``section_type``). + - ``MH.Custo.NTerm.CollapsePadding``: By default, this mode is *disabled*. When enabled, every time two adjacent ``BitField`` 's (within its scope) are found, they - will be merged in order to remove any padding in between. This is done + will be merged in order to remove any padding in-between. This is done "recursively" until any inner padding is removed. .. note:: To be compatible with an *absorption* operation, the non-terminal set with this customization should comply with the following requirements: - - It shall only contains ``BitField`` 's (which implies that no *separators* shall be used) - The ``lsb_padding`` parameter shall be set to ``True`` on every related ``BitField`` 's. - The ``endian`` parameter shall be set to ``VT.BigEndian`` on every related ``BitField`` 's. - the ``qty`` keyword should not be used on the children except if it is equal to ``1``, or ``(1,1)``. + - ``MH.Custo.NTerm.DelayCollapsing``: By default, this mode is *disabled*. + To be used in + conjunction with ``MH.Custo.NTerm.CollapsePadding`` when the collapse operation should not + be performed in the current non-terminal node but in the parent node. + Refer to the code snippet below for an example: + + .. code-block:: python + + {'name': 'request', + 'custo_set': MH.Custo.NTerm.CollapsePadding, + 'contents': [ + {'name': 'header', + 'contents': BitField(subfield_sizes=[3,1], endian=VT.BigEndian, + subfield_val_extremums=[[0,7], [0,1]])}, + + {'name': 'payload', + 'custo_set': [MH.Custo.NTerm.CollapsePadding, MH.Custo.NTerm.DelayCollapsing], + 'contents': [ + {'name': 'status', + 'contents': BitField(subfield_sizes=[1,3], endian=VT.BigEndian, + subfield_values=[None,[0,1,2]])}, + {'name': 'count', + 'contents': UINT16_be()} + ]}, + + # [...] + } + + Without this mode, when resolving the `request` node to get the byte-string + the `payload` subnode will be resolved too early and will produce a byte-string without + any collapse operation. + + For *generator* node, the customizable behavior modes are: - ``MH.Custo.Gen.ForwardConfChange``: By default, this mode is *enabled*. @@ -324,9 +412,10 @@ separator 'contents': String(values=['\n'])}, 'prefix': False, 'suffix': False, - 'unique': True}, + 'unique': True, + 'always': False}, - The keys ``prefix``, ``suffix`` and ``unique`` are optional. They are + The keys ``prefix``, ``suffix``, ``unique`` and ``always`` are optional. They are described below. .. seealso:: Refer to :ref:`dm:pattern:separator` for an example using @@ -347,6 +436,12 @@ unique node copy). Otherwise, the separators will be references to a unique node (zero copy). +always + Used optionally within a *separator descriptor*. If set to ``True``, + the separator will be always generated even if the + subnodes it separates are not generated because their evaluated quantity is 0. + + encoder If specified, an encoder instance should be provided. The *encoding* will be applied transparently when the binary value of the non terminal node will be retrieved @@ -367,7 +462,6 @@ encoder Keywords to Describe Generator Node ----------------------------------- - node_args List of node parameters to be provided to a *generator* node or a *function* node. @@ -424,6 +518,10 @@ mutable Make the node mutable. It is a shortcut for the node attribute ``MH.Attr.Mutable``. +highlight + Make the node highlighted. It is a shortcut for the node attribute + ``MH.Attr.Highlight``. + set_attrs List of attributes to set on the node. The current generic attributes are: @@ -440,6 +538,13 @@ set_attrs - ``MH.Attr.Mutable``: If set, generic disruptors will consider the node as being mutable, meaning that it can be altered (default behavior). Otherwise, it will be ignored. + When a non-terminal node has this attribute, generic disruptors using + the ModelWalker algorithm (like ``tWALK`` and ``tTYPE``) will stick to + its default form (meaning default quantity will be used for each subnodes + and if the node has multiple shapes, the higher weighted one will be used. + Likewise for `Pick` sections). + Also, the method :meth:`framework.node.Node.unfreeze()` won't perform any + changes on non-terminal nodes which are not mutable. - ``MH.Attr.Determinist``: This attribute can be set directly through the keywords ``determinist`` or ``random``. Refer to them for details. By default, it is set. @@ -452,6 +557,9 @@ set_attrs - ``MH.Attr.Separator``: Used to distinguish a separator. Some disruptors can leverage this attribute to perform their alteration. + - ``MH.Attr.Highlight``: If set, make the framework display the node in color + when printed on the console. This attribute is also used by some disruptors to show the + location of their modification. .. note:: Most of the generic stateful disruptors will recursively @@ -592,6 +700,29 @@ charset - ``MH.Charset.UNICODE`` +.. _dm:node_cst_keywords: + +Keywords to Describe Constraints +-------------------------------- + +constraints + List of node constraints specified through :class:`framework.constraint_helpers.Constraint` objects. They will be added to a CSP (Constraint + Satisfiability Problem) associated to the currently described data, and resolved when + :meth:`Node.freeze` is called with the parameter ``resolve_csp`` set to True (this is performed by default by the operator ``tWALK``). + It should always be associated to a non-terminal node. + Refer to :ref:`dm:pattern:csp` for details on how to leverage such feature. + + Specific operators have been defined to handle CSP: + + - ``tWALKcsp`` that walk through the solutions of the CSP. + - ``tCONST`` that negates the constraint one-by-one and output 1 or more samples for each negate constraint. + +constraints_highlight + If set to ``True``, the value of the nodes implied in a CSP (that could be specified through the + keyword ``constraint``) are highlighted in the console, given the Logger parameter + ``highlight_marked_nodes`` is set to True. + + .. _vt:value-types: Value Types @@ -629,7 +760,8 @@ following parameters: Maximum valid value for the node backed by this *Integer object*. ``default`` [optional, default value: **None**] - If ``values`` is not provided, this value if provided will be used as the default one. + If not None, this value will be provided by default at first + when :meth:`framework.value_types.INT.get_value()` is called. ``determinist`` [default value: **True**] If set to ``True`` generated values will be in a deterministic @@ -684,6 +816,8 @@ For :class:`framework.value_types.INT_str`, additional parameters are available: ``reverse`` [optional, default value: **False**] Reverse the order of the string if set to ``True``. + + String ------ @@ -724,6 +858,14 @@ following parameters: Codec to use for encoding the string (e.g., 'latin-1', 'utf8'). Note that depending on the charset, additional fuzzing cases are defined. +``case_sensitive`` [default value: **True**] + If the string is set to be case sensitive then specific additional + test cases will be generated in fuzzing mode. + +``default`` [optional, default value: **None**] + If not None, this value will be provided by default at first + when :meth:`framework.value_types.String.get_value()` is called. + ``extra_fuzzy_list`` [optional, default value: **None**] During data generation, if this parameter is specified with some specific values, they will be part of the test cases generated by @@ -739,6 +881,10 @@ following parameters: provided. Also use during absorption to validate the contents. It is checked if there is no ``values``. +``values_desc`` [optional, default value: **None**] + Dictionary that maps string values to their descriptions (character strings). Leveraged for + display purpose. Even if provided, all values do not need to be described. + ``max_encoded_sz`` [optional, default value: **None**] Only relevant for subclasses that leverage the encoding infrastructure. Enable to provide the maximum legitimate size for an encoded string. @@ -756,16 +902,16 @@ that enables to handle transparently any encoding scheme: - Some test cases may be defined on the encoding scheme itself. .. note:: - To define a ``String`` subclass handling a specific encoding, you have to overload - the methods: :meth:`framework.value_types.String.encode` and :meth:`framework.value_types.String.decode`. - You may optionally overload: :meth:`framework.value_types.String.encoding_test_cases` if you want - to define encoding-related test cases. And if you need to initialize the encoding scheme you - should overload the method :meth:`framework.value_types.String.init_encoding_scheme`. + To define a ``String`` subclass handling a specific encoding, you first have to define + an encoder class that inherits from :class:`framework.encoders.Encoder` (you may also use an + existing one, if it fits your needs). + Then you have to create a subclass of String decorated by :func:`framework.value_types.from_encoder` + with your encoder class in parameter. + Additionally, you can overload :meth:`framework.value_types.String.encoding_test_cases` if you want + to implement specific test cases related to your encoding. They will be automatically added to + the set of test cases to be triggered by the disruptor ``tTYPE``. - Alternatively and preferably, you should define a subclass of :class:`framework.encoders.Encoder` - and then create a subclass of String decorated by :func:`framework.value_types.from_encoder` - with the your encoder subclass in parameter. By doing so, you enable your encoder to be also - usable by a non-terminal node. + Note that the encoder you defined can also be used by a non-terminal node (refer to :ref:`dm:pattern:encoder`). Below the different currently defined string types: @@ -774,6 +920,8 @@ Below the different currently defined string types: - :class:`framework.value_types.Filename`: Filename. Similar to the type ``String``, but some disruptors like ``tTYPE`` will generate more specific test cases. +- :class:`framework.value_types.FolderPath`: FolderPath. Similar to the type + ``Filename``, but generated test cases are slightly different. - :class:`framework.value_types.GZIP`: ``String`` compressed with ``zlib``. The parameter ``encoding_arg`` is used to specify the level of compression (0-9). - :class:`framework.value_types.GSM7bitPacking`: ``String`` encoded in conformity @@ -847,10 +995,10 @@ parameters: to do it at the node level by using the data model keyword ``determinist`` (refer to :ref:`dm:node_prop_keywords`). -``defaults`` [optional, default value: **None**] - List of default value for each sub-field. Used only when the related sub-field is - not described through ``subfield_values``. If ``subfield_values`` describes the related - sub-field, then a ``None`` item should be inserted at the corresponding position in the list. +``default`` [optional, default value: **None**] + If not None, it should be the list of default value for each sub-field. + They will be provided by default at first + when :meth:`framework.value_types.BitField.get_value()` is called. ``subfield_descs`` [optional, default value: **None**] List of descriptions (character strings) for each sub-field. To @@ -858,6 +1006,11 @@ parameters: others. This parameter is used for display purpose. Look at the following examples for usage. +``subfield_value_descs`` [optional, default value: **None**] + Dictionary providing descriptions (character strings) for values in each sub-field. More precisely, + the dictionary maps subfield indexes to other dictionaries whose provides the mapping between values and + descriptions. Leveraged for display purpose. Even if provided, all values do not need to be described. + Look at the following examples for usage. Let's take the following examples to make ``BitField`` usage obvious. On the first one, we specify the sub-fields of the @@ -893,7 +1046,7 @@ session. On the second example we specify the sub-fields of the ``BitField`` by their sizes. And the other parameters are described in the same way as the first example. We additionally specify the parameter -``subfield_descs``. Look at the output for the differences. +``subfield_descs`` and ``subfield_value_descs``. Look at the output for the differences. .. code-block:: python :linenos: @@ -903,13 +1056,14 @@ the first example. We additionally specify the parameter subfield_values=[[4,2,1], None, [10,13]], subfield_val_extremums=[None, [14, 15], None], padding=0, lsb_padding=False, endian=VT.BigEndian, - subfield_descs=['first', None, 'last']) + subfield_descs=['first', None, 'last'], + subfield_value_descs={0:{4:'optionA',2:'optionB'}}) t.pretty_print() # output of the previous call: # - # (+|padding: 0000 |2(last): 1101 |1: 1111 |0(first): 0100 |-) 2788 + # (+|padding: 0000 |2(last): 1101 |1: 1111 |0(first): 0100 [optionA] |-) 2788 .. seealso:: Methods are defined to help for modifying a @@ -967,6 +1121,8 @@ in :mod:`framework.dmhelpers.generic`): Return a *generator* that retrieves the value of another node, and then return a `vt` node with this value. +:meth:`framework.dmhelpers.generic.SELECT()` + Return a *generator* that select a subnode from a non-terminal node and return it .. _dm:builders: @@ -1845,3 +2001,79 @@ Example 5: Invalid regular expressions. # raise also an framework.error_handling.InconvertibilityError # because a quantifier (that requires the creation of a terminal node) # has been found within parenthesis. + + +.. _dm:pattern:csp: + +How to Describe Constraints of Data Formats +------------------------------------------- + +When some relations exist between various parts of the data format you want to describe you have +different possibilities within ``fuddly``: + +- either using some specific keywords that capture basic constraints (e.g., ``qty_from``, ``sync_size_with``, ``exists_if``, ...); +- or through Generator nodes (refer to :ref:`dm:generators`); +- or by specifying a CSP through the keyword ``constraint``, which leverage a constraint programming + backend (currently limited to the ``python-constraint`` module) + +The CSP specification case is described in more details in what follows. +To describe constraints in the form of a CSP, you should use the ``constraints`` keyword that allows you +to provide a list of :class:`framework.constraint_helpers.Constraint` objects, which are the +building blocks for specifying constraints between multiple nodes. + +For instance, let's analyse the following data description (extracted from the ``mydf`` data model in ``tuto.py``). + +.. code-block:: python + :linenos: + :emphasize-lines: 3-6, 11, 20-21 + + csp_desc = \ + {'name': 'csp', + 'constraints': [Constraint(relation=lambda d1, d2: d1[1]+1 == d2[0] or d1[1]+2 == d2[0], + vars=('delim_1', 'delim_2')), + Constraint(relation=lambda x, y, z: x == 3*y + z, + vars=('x_val', 'y_val', 'z_val'))], + 'constraints_highlight': True, + 'contents': [ + {'name': 'equation', + 'contents': String(values=['x = 3y + z'])}, + {'name': 'delim_1', 'contents': String(values=[' [', ' ('])}, + {'name': 'variables', + 'separator': {'contents': {'name': 'sep', 'contents': String(values=[', '])}, + 'prefix': False, 'suffix': False}, + 'contents': [ + {'name': 'x', + 'contents': [ + {'name': 'x_symbol', + 'contents': String(values=['x:', 'X:'])}, + {'name': 'x_val', + 'contents': INT_str(min=120, max=130)} ]}, + + [...] + +You can see that two constraints have been specified (l.3-6) through the specific +:class:`framework.constraint_helpers.Constraint` objects. The constructor take a mandatory ``relation`` parameter +expecting a boolean function that should express a relation between any nodes reachable +from the non-terminal node on which the ``constraints`` keyword is attached. +It takes also a ``vars`` parameter expecting a list of the names of the nodes +used in the boolean function (in the same order as the parameters of the function). + +.. note:: + + The ``constraints`` keyword can be used several times along the description, but all the specified + :class:`framework.constraint_helpers.Constraint` will eventually end up in a single CSP. + + +These constraints, will then be resolved at :meth:`framework.node.Node.freeze` time (depending if +the parameter ``resolve_csp`` is set to True). +Note also that before resolving the CSP it is possible to fix the value of some variables by freezing the related nodes +with the parameter ``restrict_csp``. This is what is performed by the :class:`framework.fuzzing_primitives.ModelWalker` +infrastructure when walking a specific node which is part of a CSP, so that the walked node won't be modified +further to the CSP solving process. + +.. note:: + + The constructor of :class:`framework.constraint_helpers.Constraint` takes also an optional parameter + ``var_to_varns`` in order to support namespaces (used to discriminate nodes having identical + name in the data description). Refer to ``namespace`` keyword for more details, and to the ``csp_ns`` node + description in the data model ``mydf`` (in ``tuto.py``). diff --git a/docs/source/evolutionary_fuzzing.rst b/docs/source/evolutionary_fuzzing.rst index 352ec68..69c7ca6 100644 --- a/docs/source/evolutionary_fuzzing.rst +++ b/docs/source/evolutionary_fuzzing.rst @@ -72,9 +72,8 @@ or if a maximum number of generation exceeds. * :meth:`_compute_probability_of_survival()`: simply normalize fitness scores between 0 and 1. * :meth:`_kill()`: rolls the dices ! * :meth:`_mutate()`: operates three bit flips on each individual using the stateless disruptor ``C``. -* :meth:`_crossover()`: compensates the kills through the use of the stateful disruptor ``tCOMB``. Of course, any - other disruptor could have been chosen (those introduced by the evolutionary fuzzing are described in - the next section). +* :meth:`_crossover()`: compensates the kills through the use of a crossover algorithm which + is configurable. Finally, to make an evolutionary process available to the framework, it has to be registered at project level (meaning inside a ``*_proj.py`` file), through :meth:`framework.Project.register_evolutionary_process`. @@ -92,124 +91,75 @@ Here under is provided an example to register an evolutionary process (defined i from framework.evolutionary_helpers import DefaultPopulation + init_dp1 = DataProcess([('tTYPE', UI(fuzz_mag=0.2))], seed='exist_cond') + init_dp1.append_new_process([('tSTRUCT', UI(deep=True))]) + project.register_evolutionary_processes( - ('evol', + ('evol1', DefaultPopulation, - {'init_process': [('SEPARATOR', UI(random=True)), 'tTYPE'], - 'size': 10, - 'max_generation_nb': 10}) + {'init_process': init_dp1, + 'max_size': 80, + 'max_generation_nb': 3, + 'crossover_algo': CrossoverHelper.crossover_algo1}) ) Once loaded from ``Fuddly``, ``Scenario`` are created from registered evolutionary processes, which are callable (like any other scenarios) through their associated ``Generator``. In our example, only one process is -registered and will lead to the creation of the generator ``SC_EVOL``. +registered and will lead to the creation of the generator ``SC_EVOL1``. After each call to it, the evolutionary process will progress and a new test case will be produced. Note that the :class:`framework.evolutionary_helpers.DefaultPopulation` is used with this scenario. -It expects three parameters: +It expects the following parameters: - The first one describe the process to follow to generate the data in the initial population (refer to the API documentation for more information). In the example, - we use the generator ``SEPARATOR`` to produce data compliant to the model in a randome way, then we - apply the disruptor ``tTYPE``. -- The second specify the size of the population. + the process enables to generate altered data from the data type ``exist_cond`` thanks to the + the disruptors ``tTYPE`` and ``tSTRUCT``. +- The second specify the maximum size of the population. - The third is a criteria to stop the evolutionary process. It provides the maximum number of generation to reach +- The fourth is the crossover algorithm to be used. You can either provide your own implementation + or use the ones available in :class:`framework.evolutionary_helpers.CrossoverHelper`. Refer to + :ref:`ef:crossover-algos` for more information. -.. _ef:crossover-disruptors: - -Specific disruptors -=================== +.. _ef:crossover-algos: -The evolutionary fuzzing introduces two stateful disruptors that can be used within the crossover operation. +Crossover Algorithms +==================== +The evolutionary fuzzing introduces two crossover algorithms that can be used within the crossover operation. -tCROSS - Randomly swap some leaf nodes --------------------------------------- +Algo1 - Randomly swap some root nodes' children +----------------------------------------------- Description: - Produce two children by making two graphs swap a given percentages of their leaf nodes. + Produce two nodes by swapping some of the children of two given graphs roots. + -.. _sd-crossover-image: -.. figure:: images/sd_crossover.png +.. _algo1-image: +.. figure:: images/crossover_algo1.png :align: center :scale: 50 % - tCROSS example + Algo1 example + Reference: - :class:`framework.generic_data_makers.sd_crossover` - -Parameters: - .. code-block:: none - - generic args: - |_ clone_node - | | desc: if True the dmaker will always return a copy of the node. (for - | | stateless diruptors dealing with big data it can be usefull - | | to it to False) - | | default: True [type: bool] - |_ init - | | desc: make the model walker ignore all the steps until the provided - | | one - | | default: 1 [type: int] - |_ max_steps - | | desc: maximum number of steps (-1 means until the end) - | | default: -1 [type: int] - |_ runs_per_node - | | desc: maximum number of test cases for a single node (-1 means until - | | the end) - | | default: -1 [type: int] - specific args: - |_ node - | | desc: node to crossover with - | | default: None [type: Node] - |_ percentage_to_share - | | desc: percentage of the base node to share - | | default: 0.50 [type: float] - - - -tCOMB - Randomly swap some root nodes' children ------------------------------------------------ + :meth:`framework.evolutionary_helpers.CrossoverHelper.crossover_algo1` -Description: - Produce two nodes by swapping some of the children of two given graphs roots. +Algo2 - Randomly swap some leaf nodes +------------------------------------- +Description: + Produce two children by making two graphs swap a given percentages of their leaf nodes. -.. _sd-combine-image: -.. figure:: images/sd_combine.png +.. _algo2-image: +.. figure:: images/crossover_algo2.png :align: center :scale: 50 % - tCOMB example - + Algo2 example Reference: - :class:`framework.generic_data_makers.sd_combine` - -Parameters: - .. code-block:: none - - generic args: - |_ clone_node - | | desc: if True the dmaker will always return a copy of the node. (for - | | stateless diruptors dealing with big data it can be usefull - | | to it to False) - | | default: True [type: bool] - |_ init - | | desc: make the model walker ignore all the steps until the provided - | | one - | | default: 1 [type: int] - |_ max_steps - | | desc: maximum number of steps (-1 means until the end) - | | default: -1 [type: int] - |_ runs_per_node - | | desc: maximum number of test cases for a single node (-1 means until - | | the end) - | | default: -1 [type: int] - specific args: - |_ node - | | desc: node to combine with - | | default: None [type: Node] + :meth:`framework.evolutionary_helpers.CrossoverHelper.get_configured_crossover_algo2` diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 33e2801..c54fbd2 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -19,7 +19,7 @@ and let ``fuddly`` recalculate every constraints for you. abszip.set_current_conf('ABS', recursive=True) abszip.absorb(zip_buff, constraints=AbsNoCsts(size=True,struct=True) - abszip['ZIP/file_list/file:2/data'].absorb(b'TEST', constraints=AbsNoCsts()) + abszip['ZIP/file_list/file:2/data'][0].absorb(b'TEST', constraints=AbsNoCsts()) abszip.unfreeze(only_generators=True) abszip.get_value() diff --git a/docs/source/framework.rst b/docs/source/framework.rst index 89e1dd6..a30e6a3 100644 --- a/docs/source/framework.rst +++ b/docs/source/framework.rst @@ -123,6 +123,17 @@ framework.targets.sim module :special-members: :exclude-members: __dict__, __weakref__ +framework.targets.ssh module +---------------------------- + +.. automodule:: framework.targets.ssh + :members: + :undoc-members: + :show-inheritance: + :private-members: + :special-members: + :exclude-members: __dict__, __weakref__ + framework.targets.printer module -------------------------------- @@ -191,6 +202,19 @@ framework.monitor module :special-members: :exclude-members: __dict__, __weakref__ + +framework.comm_backends module +------------------------------ + +.. automodule:: framework.comm_backends + :members: + :undoc-members: + :show-inheritance: + :private-members: + :special-members: + :exclude-members: __dict__, __weakref__ + + framework.tactics_helpers module -------------------------------- @@ -315,6 +339,37 @@ framework.knowledge.information module :special-members: :exclude-members: __dict__, __weakref__ +framework.constraint_helpers module +----------------------------------- + +.. automodule:: framework.constraint_helpers + :members: + :undoc-members: + :show-inheritance: + :private-members: + :special-members: + :exclude-members: __dict__, __weakref__ + + +framework.plumbing module +------------------------- + +.. automodule:: framework.plumbing + :members: + :undoc-members: + :exclude-members: __dict__, __weakref__, FmkShell, EnforceOrder, FmkFeedback, is_python_file + + +libs.utils module +----------------- + +.. automodule:: libs.utils + :members: + :undoc-members: + :show-inheritance: __dict__, __weakref__ + + + .. framework.plumbing module ----------------------- diff --git a/docs/source/images/sd_combine.png b/docs/source/images/crossover_algo1.png similarity index 100% rename from docs/source/images/sd_combine.png rename to docs/source/images/crossover_algo1.png diff --git a/docs/source/images/crossover_algo2.png b/docs/source/images/crossover_algo2.png new file mode 100644 index 0000000..2c5b8cb Binary files /dev/null and b/docs/source/images/crossover_algo2.png differ diff --git a/docs/source/images/plotty_example.png b/docs/source/images/plotty_example.png new file mode 100644 index 0000000..e3e3944 Binary files /dev/null and b/docs/source/images/plotty_example.png differ diff --git a/docs/source/images/sc_ex1_step1.png b/docs/source/images/sc_ex1_step1.png index 1343b10..d36899e 100644 Binary files a/docs/source/images/sc_ex1_step1.png and b/docs/source/images/sc_ex1_step1.png differ diff --git a/docs/source/images/sc_ex1_step2.png b/docs/source/images/sc_ex1_step2.png index b3a455e..ae5dc67 100644 Binary files a/docs/source/images/sc_ex1_step2.png and b/docs/source/images/sc_ex1_step2.png differ diff --git a/docs/source/images/sc_ex4.png b/docs/source/images/sc_ex4.png new file mode 100644 index 0000000..e7189f8 Binary files /dev/null and b/docs/source/images/sc_ex4.png differ diff --git a/docs/source/images/sc_ex4_cond_fuzz_tc1.png b/docs/source/images/sc_ex4_cond_fuzz_tc1.png new file mode 100644 index 0000000..ca1f9bc Binary files /dev/null and b/docs/source/images/sc_ex4_cond_fuzz_tc1.png differ diff --git a/docs/source/images/sc_ex4_cond_fuzz_tc2.png b/docs/source/images/sc_ex4_cond_fuzz_tc2.png new file mode 100644 index 0000000..dc2ef5b Binary files /dev/null and b/docs/source/images/sc_ex4_cond_fuzz_tc2.png differ diff --git a/docs/source/images/sc_ex4_data_fuzz_tc1.png b/docs/source/images/sc_ex4_data_fuzz_tc1.png new file mode 100644 index 0000000..7c753fd Binary files /dev/null and b/docs/source/images/sc_ex4_data_fuzz_tc1.png differ diff --git a/docs/source/images/sc_ex4_data_fuzz_tc2.png b/docs/source/images/sc_ex4_data_fuzz_tc2.png new file mode 100644 index 0000000..39befc7 Binary files /dev/null and b/docs/source/images/sc_ex4_data_fuzz_tc2.png differ diff --git a/docs/source/images/sc_ex4_stutter.png b/docs/source/images/sc_ex4_stutter.png new file mode 100644 index 0000000..e8a7099 Binary files /dev/null and b/docs/source/images/sc_ex4_stutter.png differ diff --git a/docs/source/images/sd_crossover.png b/docs/source/images/sd_crossover.png deleted file mode 100644 index 9e06c06..0000000 Binary files a/docs/source/images/sd_crossover.png and /dev/null differ diff --git a/docs/source/index.rst b/docs/source/index.rst index 29e6ec5..3d73c31 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -28,7 +28,7 @@ Contents: evolutionary_fuzzing - disruptors + data_makers targets diff --git a/docs/source/probes.rst b/docs/source/probes.rst index a240fba..a0271f0 100644 --- a/docs/source/probes.rst +++ b/docs/source/probes.rst @@ -51,7 +51,7 @@ SSH_Backend ----------- Reference: - :class:`framework.monitor.SSH_Backend` + :class:`framework.comm_backends.SSH_Backend` Description: This generic backend enables you to interact with a monitored system through an @@ -62,7 +62,7 @@ Serial_Backend -------------- Reference: - :class:`framework.monitor.Serial_Backend` + :class:`framework.comm_backends.Serial_Backend` Description: This generic backend enables you to interact with a monitored system through an @@ -72,7 +72,7 @@ Shell_Backend ------------- Reference: - :class:`framework.monitor.Shell_Backend` + :class:`framework.comm_backends.Shell_Backend` Description: This generic backend enables you to interact with a local monitored system @@ -104,3 +104,11 @@ Description: It can be done by specifying a ``threshold`` and/or a ``tolerance`` ratio. +ProbeCmd +-------- + +Reference: + :class:`framework.monitor.ProbeCmd` + +Description: + Generic probe that enables you to execute shell commands and retrieve the output. diff --git a/docs/source/scenario.rst b/docs/source/scenario.rst index 350cc66..1e2ae95 100644 --- a/docs/source/scenario.rst +++ b/docs/source/scenario.rst @@ -62,9 +62,9 @@ Let's begin with a simple example that interconnect 3 steps in a loop without an step4 = Step('off_gen', fbk_timeout=0) step1.connect_to(step2) - step2.connect_to(step3, cbk_after_fbk=cbk_transition1) + step2.connect_to(step3, cbk_after_fbk=check_answer) step3.connect_to(step4) - step4.connect_to(step1, cbk_after_sending=cbk_transition2) + step4.connect_to(step1, cbk_after_sending=check_switch) sc1 = Scenario('ex1', anchor=step1, user_context=UI(switch=False)) @@ -90,7 +90,7 @@ to register scenarios as shown in line 20. From line 9 to 13 we define 4 :class:`framework.scenario.Step`: - The first one commands the framework to send a data of type ``exist_cond`` (which is the name of a data registered - in the data model ``mydf``) as well as starting 2 tasks (threaded entities of the framework) that + in the data model ``mydf``) as well as starting 2 periodic tasks (threaded entities of the framework) that will emit each one a specific data. The first one will send the specified string every 5 seconds while the other one will send another string only once. Additionaly, the callback ``before_sending_cbk`` is set and will be triggered when the framework @@ -205,6 +205,12 @@ periodic tasks. .. seealso:: Refer to the section :ref:`sc:example` for practical information on how to use such features. +A step can also start a periodic or a one-shot task whose content could be entirely specified by +the user. This is done by providing a list of :class:`libs.utils.Task` +to the parameter ``start_tasks``. And, in order to stop previously started periodic tasks, +the parameter ``stop_tasks`` have to be filled with a list of references on the relevant +periodic tasks. Details on tasks are provided here :ref:`tuto:tasks`. + In addition to the features provided by a step, some user-defined callbacks can be associated to a step and executed while the framework is handling the step (that is generating data as specified by the step and sending it): @@ -433,6 +439,27 @@ The execution of this scenario will follow the pattern:: anchor --> option1 --> anchor --> option2 --> anchor --> option2 --> ... +In addition to the callbacks, a transition can be guarded by booleans linked to specific conditions. +They have to be specified as parameters of the method :meth:`framework.scenario.Step.connect_to`. +The current defined condition is: + + - `DataProcess completed` (parameter is ``dp_completed_guard``): which means, for a step hosting + a :class:`framework.data.DataProcess`, that if no more data can be issued by it the + condition is satisfied, and thus the transition can be crossed. + This is illustrated by the following example: + + .. code-block:: python + :linenos: + + step1 = Step(DataProcess(process=['tTYPE'], seed='4tg1')) + step2 = Step(DataProcess(process=['tTYPE#2'], seed='4tg2')) + + step1.connect_to(step2, dp_completed_guard=True) + step2.connect_to(FinalStep(), dp_completed_guard=True) + + sc_proj3 = Scenario('proj3', anchor=step1) + + .. _sc:dataprocess: Data Generation Process @@ -445,14 +472,14 @@ is described by a `data descriptor` which can be: - a :class:`framework.data.Data`; -- a :class:`framework.scenario.DataProcess`. +- a :class:`framework.data.DataProcess`. -A :class:`framework.scenario.DataProcess` is composed of a chain of generators and/or disruptors +A :class:`framework.data.DataProcess` is composed of a chain of generators and/or disruptors (with or without parameters) and optionally a ``seed`` on which the chain of disruptor will be applied to (if no generator is provided at the start of the chain). -A :class:`framework.scenario.DataProcess` can trigger the end of the scenario if a disruptor in the +A :class:`framework.data.DataProcess` can trigger the end of the scenario if a disruptor in the chain yields (meaning it has terminated its job with the provided data: it is *exhausted*). If you prefer that the scenario goes on, then you have to set the ``auto_regen`` parameter to ``True``. In such a case, when the step embedding @@ -460,8 +487,8 @@ the data process will be reached again, the framework will rerun the chain. This the exhausted disruptor and make new data available to it (by pulling data from preceding data makers in the chain or by using the *seed* again). -Additional *data maker chains* can be added to a :class:`framework.scenario.DataProcess` thanks to -:meth:`framework.scenario.DataProcess.append_new_process`. Switching from the current process to the +Additional *data maker chains* can be added to a :class:`framework.data.DataProcess` thanks to +:meth:`framework.data.DataProcess.append_new_process`. Switching from the current process to the next one is carried out when the current one is interrupted by a yielding disruptor. Note that in the case the data process has its ``auto_regen`` parameter set to ``True``, the current interrupted chain won't be rerun until every other @@ -470,7 +497,7 @@ chain has also get a chance to be executed. .. seealso:: Refer to :ref:`tuto:dmaker-chain` for more information on disruptor chaining. .. note:: It follows the same pattern as the instructions that can set a virtual operator - (:ref:`tuto:operator`). It is actually what the method :meth:`framework.plumbing.FmkPlumbing.get_data` + (:ref:`tuto:operator`). It is actually what the method :meth:`framework.plumbing.FmkPlumbing.process_data` takes as parameters. Here under examples of steps leveraging the different ways to describe their data to send. @@ -486,6 +513,13 @@ Here under examples of steps leveraging the different ways to describe their dat Step( DataProcess(process=['C', 'tTYPE'], seed='enc') ) Step( DataProcess(process=['C'], seed=Data('my seed')) ) +Steps may be configured to change the process of data generation. The following methods are defined +for such purpose: + +- :meth:`framework.scenario.Step.make_blocked` and :meth:`framework.scenario.Step.make_free` +- :meth:`framework.scenario.Step.set_dmaker_reset` and :meth:`framework.scenario.Step.clear_dmaker_reset` + + Finally, it is possible for a ``Step`` to describe multiple data to send at once; meaning the framework will be ordered to use :meth:`framework.target.Target.send_multiple_data` (refer to :ref:`targets-def`). For that purpose, you have to provide the ``Step`` constructor with diff --git a/docs/source/targets.rst b/docs/source/targets.rst index c481092..7c381bc 100644 --- a/docs/source/targets.rst +++ b/docs/source/targets.rst @@ -139,11 +139,11 @@ Description: - :meth:`framework.targets.local.LocalTarget.terminate()` for doing specific actions at target termination. - Feedback: This target will automatically provide feedback if the application writes on ``stderr`` or returns a negative status or terminates/crashes. - + ``stdout`` can also be parsed looking for user-provided keywords that will trigger + some feedback with negative status or even parsed by a user-provided function. Supported Feedback Mode: - :const:`framework.target_helpers.Target.FBK_WAIT_UNTIL_RECV` @@ -179,6 +179,28 @@ Usage example: temporary files need to be created. +SSHTarget +========= + +Reference: + :class:`framework.targets.ssh.SSHTarget` + +Description: + This generic target enables you to interact with a remote target requiring an SSH connection. + +Feedback: + This target will automatically provide the results of the commands sent through SSH. + +Supported Feedback Mode: + - :const:`framework.target_helpers.Target.FBK_WAIT_FULL_TIME` + - :const:`framework.target_helpers.Target.FBK_WAIT_UNTIL_RECV` + +Usage Example: + .. code-block:: python + :linenos: + + tg = SSHTarget(host='192.168.0.1', port=22, username='test', password='test') + PrinterTarget ============= @@ -240,3 +262,30 @@ Usage Example: tg = SIMTarget(serial_port='/dev/ttyUSB3', baudrate=115200, pin_code='0000' targeted_tel_num='0123456789', zone='33') + + + +TestTarget +========== + +Reference: + :class:`framework.targets.debug.TestTarget` + +Description: + This generic target enables you to stimulate a virtual target that could be useful for test + preparation for instance. + Some parameters enable to change the behavior of this target. + +Feedback: + This target could provide random feedback, or feedback chosen from a provided sample list, or + it could repeat the received data as its feedback. + +Supported Feedback Mode: + - :const:`framework.target_helpers.Target.FBK_WAIT_FULL_TIME` + - :const:`framework.target_helpers.Target.FBK_WAIT_UNTIL_RECV` + +Usage Example: + .. code-block:: python + :linenos: + + tg = TestTarget(name='mytest_target', fbk_samples=['OK','ERROR']) diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 6248d9b..4e3641f 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -57,15 +57,16 @@ this: *** Found Project: 'usb' *** >>> Look for Projects within 'projects/generic' Directory *** Found Project: 'standard' *** - ================================================================================ + ============================================[ Fuddly Home Information ]== - -=[ Fuddly Shell ]=- (with Fuddly FmK 0.23) + --> data folder: ~/.local/share/fuddly/ + --> contains: - fmkDB.db, logs, imported/exported data, ... + - user projects and user data models, ... + --> config folder: ~/.config/fuddly/ - >> + -=[ Fuddly Shell ]=- (with Fuddly FmK 0.30) -.. note:: The ``help`` command shows you every defined command within - ``Fuddly Shell``. You can also look at a brief command description and - syntax by typing ``help `` + >> Note that ``fuddly`` looks for *Data Model* files (within ``data_models/``) and *Project* files (within ``projects/``) during @@ -73,6 +74,16 @@ its initialization. A *Project* file is used to describe the targets that can be tested, the logger behaviour, and optionally specific monitoring means as well as some scenarios and/or virtual operators. +.. note:: + + Projects and data models files are retrieved either from + ``/{projects,data_models}/`` or from + ``/{projects,data_models}/``. + + Note that when the Fuddly shell is launched, the path of the + fuddly data folder is displayed as well as its configuration folder. + + .. seealso:: To create a new project file, and to describe the associated components refer to :ref:`tuto:project`. @@ -103,7 +114,7 @@ You can look at the defined targets by issuing the following command: [0] EmptyTarget [ID: 307144] [1] LocalTarget [Program: display] [2] LocalTarget [Program: okular] - [3] LocalTarget [Program: unzip, Args: -d ~/fuddly_data/workspace/] + [3] LocalTarget [Program: unzip, Args: -d ~/.local/share/fuddly/workspace/] [4] PrinterTarget [IP: 127.0.0.1, Name: PDF] [5] NetworkTarget [localhost:54321, localhost:12345] @@ -180,24 +191,24 @@ issuing the following command: *** Target initialization: (0) EmptyTarget [ID: 307144] *** *** Monitor is started *** - *** [ Fuzz delay = 0 ] *** + *** [ Sending delay = 0.00s ] *** *** [ Number of data sent in burst = 1 ] *** - *** [ Target health-check timeout = 10 ] *** - >> + *** [ Target EmptyTarget [ID: 241984] health-check timeout = 10.0s ] *** + >> .. note:: Note that just after the project is launched, some internal parameters are displayed, namely: - - The fuzzing delay, which allows you to set a minimum delay between + - The sending delay, which allows you to set a minimum delay between two data emission. (Can be changed through the command ``set_delay``). - The maximum number of data that will be sent in burst, thus - ignoring the fuzzing delay. (Can be changed through the command + ignoring the sending delay. (Can be changed through the command ``set_burst``) - The timeout value for checking target's health. (Can be changed - through the command ``set_health_timeout``) + through the command ``set_health_check_timeout``) Finally, you may prefer to directly launch your project thanks to the command ``run_project``. Indeed, by using it, you will automatically trigger the commands we @@ -228,6 +239,11 @@ command:: ... [dm_name_n]``, if you want to interact with a target with different data models simultaneously. +.. note:: The ``help`` command shows you every defined command within + ``Fuddly Shell``. You can also look at a brief command description and + syntax by typing ``help `` + + Send Malformed ZIP Files to the Target (Manually) ------------------------------------------------- @@ -317,7 +333,7 @@ this case---the ZIP data model---the first one will generate modeled ZIP archive based uniquely on the data model, whereas the other ones (``ZIP_00``, ``ZIP_01``, ...) generate modeled ZIP archives based on the sample files available within the directory -``~/fuddly_data/imported_data/zip/``. +``/imported_data/zip/``. For each one of these generators, some parameters are associated: @@ -491,8 +507,6 @@ can see on lines 16 & 19. :align: center :scale: 60% - - .. note:: Parameters are given to data makers (generators/disruptors) through a tuple wrapped with the characters @@ -864,6 +878,16 @@ Finally, if you want to resend data from previous sessions, you can do it by loo That command will store these data to the `Data Bank`. From then on, you could use ``show_db`` and ``replay_db`` as previously explained. +.. note:: + You can use disruptors with a ``replay_*`` command. However if these disruptors are stateful, + you should issue the command only once. Then, if you want to walk through the stateful disruptor, + you only have to switch to a ``send``-like command, and use as generator name the string ``NOGEN`` + + For instance:: + + >> replay_last tTYPE + >> send_loop -1 NOGEN tTYPE + .. _fuddly-advanced: @@ -879,32 +903,36 @@ will need to issue the following commands: from framework.plumbing import * fmk = FmkPlumbing() + fmk.start() From now on you can use ``fuddly`` through the object ``fmk``. Every commands defined by ``Fuddly Shell`` (refer to :ref:`tuto:start-fuzzshell`) are backed by a method of the class :class:`framework.plumbing.FmkPlumbing`. -Here under some basic commands to start with: +Here under some basic commands to launch the project ``tuto``, a virtual testing target and the +``ZIP`` data model: .. code-block:: python :linenos: - # To show the available projects fmk.show_projects() # Contains the list of all the Project objects available fmk.prj_list - # Load the ``standard`` project by name - fmk.load_project(name='standard') + # Load the ``tuto`` project by name + fmk.load_project(name='tuto') + + # Reference to the currently launched project, in our case ``tuto`` + fmk.prj # Show available targets for this project fmk.show_targets() - # Select the target with ID ``3`` - fmk.load_targets(3) + # Select the target with ID ``7`` + fmk.load_targets(7) # To show all the available data models fmk.show_data_models() @@ -921,23 +949,72 @@ Here under some basic commands to start with: # Launch the project and all the related components fmk.launch() - # Reference to the currently launched project, in our case ``standard`` - fmk.prj +.. note:: + The previous commands used to load a project, targets and data models can be factorized + in one line thanks to the following command:: + + # To launch the ``tuto`` project with the targets ID ``7`` and ``8`` + # and the ZIP data model in one line + fmk.run_project(name='tuto', tg_ids=[7,8], dm_name='zip') + +You can also change the timeout value used to retrieved feedback from the targets, as well as +tuning the way this value has to be considered (a maximum value or a strict time slice). + +.. code-block:: python + :linenos: + + fmk.set_feedback_timeout(1, tg_id=7) + fmk.set_feedback_mode(Target.FBK_WAIT_UNTIL_RECV, tg_id=7) + fmk.set_feedback_timeout(2, tg_id=8) + fmk.set_feedback_mode(Target.FBK_WAIT_FULL_TIME, tg_id=8) + +The effect of this commands is summarized in a specific screen that can be displayed by issuing the +command ``fmk.show_fmk_internals()``:: + + + -=[ FMK Internals ]=- + + [ General Information ] + FmkDB enabled: True + Workspace enabled: True + Sending delay: 0.0 + Number of data sent in burst: 1 + Target(s) health-check timeout: 4.0 + + [ Target Specific Information - (7) TestTarget [ID: 792104] ] + Feedback timeout: 1 + Feedback mode: Wait until the target has sent something back to us + + [ Target Specific Information - (8) TestTarget [ID: 792160] ] + Feedback timeout: 2 + Feedback mode: Wait for the full time slot allocated for feedback retrieval - # To launch the ``standard`` project with the target number ``3`` - # and the ZIP data model in one line - fmk.run_project(name='standard', tg=3, dm_name='zip') + +Other commands allowing you to perform some user code changes either in the project file or +the data models and take them into account without restarting fuddly: + +.. code-block:: python + :linenos: # Reload all sub-systems and data model definitions and choose the target ``0`` fmk.reload_all(tg_num=0) + # Reload the data model definitions + fmk.reload_dm() + +Then, when everything is loaded, the following commands is an example on how target interaction +can be performed: + +.. code-block:: python + :linenos: + # Show a list of the registered data type within the data model fmk.show_atom_identifiers() # Or list(fmk.dm.atom_identifiers()) # Get an instance of the modeled data ZIP_00 which is made from the - # absorption of an existing ZIP archive within ~/fuddly_data/imported_data/zip/ + # absorption of an existing ZIP archive within /imported_data/zip/ dt = fmk.dm.get_atom('ZIP_00') # Display the raw contents of the first generated element of the data type `dt` @@ -961,55 +1038,75 @@ Here under some basic commands to start with: # cases and enforce the disruptor to strictly follow the ZIP structure # Finally truncate the output to 200 bytes action_list = [('tTYPE', UI(init=5, order=True)), ('SIZE', UI(sz=200))] - altered_data = fmk.get_data(action_list, data_orig=Data(dt)) + altered_data = fmk.process_data(action_list, seed=Data(dt)) # Send this new data and look at the actions that perform tTYPE and # SIZE through the console or the logs fmk.send_data_and_log(altered_data) - The last command will display something like this (with some color if you have the ``xtermcolor`` python library): .. code-block:: none - ========[ 2 ]==[ 11/09/2015 - 20:06:56 ]======================= - ### Target ack received at: None - ### Initial Generator (currently disabled): - |- generator type: None | generator name: None | User input: None - ... + ====[ 3 ]==[ 27/06/2019 - 12:07:19 ]============================================ ### Step 1: - |- disruptor type: tTYPE | disruptor name: d_fuzz_typed_nodes | User input: G=[init=5], S=[order=True] + |- disruptor type: tTYPE | disruptor name: sd_fuzz_typed_nodes | User input: [init=5,order=True] |- data info: - |_ model walking index: 5 - |_ |_ run: 1 / -1 (max) + |_ model walking index: 4 + |_ |_ run: 4 / -1 (max) |_ current fuzzed node: ZIP_00/file_list/file/header/common_attrs/version_needed - |_ |_ value type: - |_ |_ original node value: 1400 (ascii: ) - |_ |_ corrupt node value: 0080 (ascii: �) + |_ |_ value type: + |_ |_ original node value (hex): b'1403' + |_ | (ascii): b'\x14\x03' + |_ |_ corrupt node value (hex): b'0000' + |_ (ascii): b'\x00\x00' ### Step 2: - |- disruptor type: SIZE | disruptor name: d_max_size | User input: G=None, S=[sz=200] + |- disruptor type: SIZE | disruptor name: d_max_size | User input: [sz=200] |- data info: - |_ orig node length: 1054002 + |_ orig node length: 595 |_ right truncation |_ new node length: 200 ### Data size: 200 bytes ### Emitted data is stored in the file: - ./exported_data/zip/2015_09_11_200656_00.zip + /exported_data/zip/2019_06_27_120719_00.zip + ### FmkDB Data ID: 542 + ### Ack from 'TestTarget [ID: 725768]' received at: 2019-06-27 12:07:19.071751 + ### Feedback from 'TestTarget [ID: 725768]' (status=0): + CRC error + +The previous commands can be factorized through the method +:meth:`framework.plumbing.FmkPlumbing.process_data_and_send()` + +For instance fuzzing the targets 7 and 8 simultaneously (that handle ZIP format) until exhaustion +of test cases can be done thanks to the following lines: .. code-block:: python :linenos: - # And to terminate fuddly properly - fmk.stop() + # Hereunder the chosen fuzzing follow a 2-step approach: + # 1- the disruptor tTYPE is called on the seed and starts from the 5th test case + # 2- a trailer payload is added at the end of what is generated previsouly + + dp = DataProcess([('tTYPE', UI(deep=True, init=5)), + ('ADD', UI(raw='This is added at the end'))], + seed='ZIP_00') + + fmk.process_data_and_send(dp, max_loop=-1, tg_ids=[7,8]) +We did not discuss all the methods available from :class:`framework.plumbing.FmkPlumbing`but you +should now be more familiar with :class:`framework.plumbing.FmkPlumbing` and go on with its exploration. + +Finally, in order to exit the framework, the following method should be called (otherwise, +various threads would block the correct termination of the framework):: + + fmk.stop() For more information on how to manually make modification on data, refer to the section :ref:`tuto:disruptors` - Implementing a Data Model and Defining a Project Environment ============================================================ @@ -1197,7 +1294,7 @@ Defining the Imaginary MyDF Data Model Assuming we want to model an imaginary data format called `MyDF`. Two files need to be created either within ``/data_models/`` or within -``~/fuddly_data/user_data_models/`` (or within any subdirectory): +``/user_data_models/`` (or within any subdirectory): ``mydf.py`` Contain the implementation of the data model related to @@ -1255,12 +1352,12 @@ model, by calling :func:`framework.data_model.DataModel.register()` on them. .. note:: - If you want to import data samples that comply with your data model: + In the frame of your data model if you want to instantiate atoms from samples: - - Add your samples there: ``~/fuddly_data/imported_data//`` + - Add your samples there: ``/imported_data//`` - Within the method :meth:`framework.data_model.DataModel.build_data_model()`, and once you defined - your atoms, call the method :meth:`framework.data_model.DataModel.register_atom_for_absorption()` + your atoms, call the method :meth:`framework.data_model.DataModel.register_atom_for_decoding()` to register the atom that will be used to model your samples. (To perform this action the framework leverages the node absorption mechanism -- :ref:`tuto:dm-absorption`.) For a usage example, refer to the ZIP data model. @@ -1272,8 +1369,15 @@ model, by calling the method :meth:`framework.data_model.DataModel._atom_absorption_additional_actions()` as illsutrated by the JPG data model. + Finally, if you need even more flexibility in order to create atoms from samples, because + node absorption is not satisfactory in your context, then you could overload the method + :meth:`framework.data_model.DataModel._create_atom_from_raw_data_specific()`. + Refer to the JSON data model for an illustration, where this method is overloaded in order to create + either atoms that represent JSON schemas or atoms that model some JSON data; depending on the JSON + files provided in ``/imported_data/json``. + .. note:: - The method :meth:`framework.data_model.DataModel.register_atom_for_absorption()` is also leveraged + The method :meth:`framework.data_model.DataModel.register_atom_for_decoding()` is also leveraged by the decoding feature of the class :class:`framework.data_model.DataModel`, which is implemented by the method :meth:`framework.data_model.DataModel.decode()`. @@ -1580,9 +1684,14 @@ Currently, there is four kinds of constraints: attributes are specified within a terminal node---this constraint control it. -``contents`` +``content`` Only the values specified in the data model are accepted +``similar_content`` + This constraint is a lighter version of ``content``. It allows values similar to the one defined + in the data model to be accepted in absorption operations. This is especially leveraged by + String() to distinguish case sensitive from case incensitive strings. + ``regexp`` This constraint control if regular expression---that some terminal nodes can specify---should be complied to. @@ -2005,7 +2114,7 @@ Defining a Project Environment The environment---composed of at least one target, a logger, and optionnaly some monitoring means and virtual operators---is setup within a project file located within ``/projects/`` or within -`~/fuddly_data/user_projects/``. To illustrate that let's +``/user_projects/``. To illustrate that let's show the beginning of ``generic/standard_proj.py``: .. code-block:: python @@ -2024,7 +2133,7 @@ show the beginning of ``generic/standard_proj.py``: # project.default_dm = 'mydf' logger = Logger(record_data=False, explicit_data_recording=False, - export_orig=False, export_raw_data=False) + export_raw_data=False) printer1_tg = PrinterTarget(tmpfile_ext='.png') printer1_tg.set_target_ip('127.0.0.1') @@ -2062,8 +2171,9 @@ and optionally: - targets (referenced by a variable ``targets``, :ref:`targets-def`) - scenarios (:ref:`scenario-infra`) that can be registered into a project through the method :meth:`framework.project.Project.register_scenarios` -- probes (:ref:`tuto:probes`). -- operators (:ref:`tuto:operator`). +- probes (:ref:`tuto:probes`) +- tasks (:ref:`tuto:tasks`) +- operators (:ref:`tuto:operator`) A default data model or a list of data models can be added to the project through its attribute ``default_dm``. ``fuddly`` will use this @@ -2145,24 +2255,24 @@ and kept in sync with the log files. The outputs of the logger are of four types: -- ``~/fuddly_data/logs/*_logs``: the history of your +- ``/logs/*_logs``: the history of your test session for the project named ``project_name``. The files are prefixed with the test session starting time. A new one is created each time you run a new project or you reload the current one. Note these files are created only if the parameter ``enable_file_logging`` is set to True. -- ``~/fuddly_data/logs/*_stats``: some statistics of +- ``/logs/*_stats``: some statistics of the kind of data that has been emitted during the session. Note these files are created only if the parameter ``enable_file_logging`` is set to True. -- ``~/fuddly_data/exported_data//*./exported_data//*.``: the data emitted during a session are stored within the their data model directory. Each one is prefixed by the date of emission, and each one is uniquely identified within the log files. -- records within the database ``~/fuddly_data/fmkDB.db``. Every piece of +- records within the database ``/fmkDB.db``. Every piece of information from the previous files are recorder with this database. Some parameters allows to customize the behavior of the logger, such as: @@ -2535,3 +2645,46 @@ it returns, by setting a negative status and some feedback on it. .. note:: Setting a negative status through :class:`framework.operator_helpers.LastInstruction` will make ``fuddly`` act the same as for a negative status from a probe. In addition, the operator will be shutdown. + +.. _tuto:tasks: + +Defining Tasks +++++++++++++++ + +Contrary to probes (:ref:`tuto:probes`), Tasks are not sequenced by the framework, they run asynchronously. +They can be periodic or one-shot and their logic need to be defined entirely by the user. +They can be started either when a target is launched (see below) or by a step of a scenario (refer to :ref:`sc:steps`). + +To implement the logic of the task, you need to inherit from :class:`libs.utils.Task` and to +implement the :meth:`__call__` method. This method is then called either once or with a period that +is specified in the constructor. +When run by the framework this task has some attributes automatically filled that you can leverage +in your logic: + +- :attr:`libs.utils.Task.feedback_gate`: provide an access to the last 10 seconds of feedback. + (:class:`framework.database.FeedbackGate`) +- :attr:`libs.utils.Task.dm`: current loaded data model. +- :attr:`libs.utils.Task.targets`: enabled targets. +- :attr:`libs.utils.Task.fmkops`: provide access to some framework operations + (:class:`framework.plumbing.ExportableFMKOps`). + +Moreover, you could also print some information in another terminal window dedicated to the task. +For such case, you should set the parameter ``new_window`` of the :class:`libs.utils.Task` constructor to +``True``, then use a specific API composed of :meth:`libs.utils.Task.print` and :meth:`libs.utils.Task.print_nl`. + +Like with probes (:ref:`tuto:probes`), you can associate tasks to ``targets`` in order to execute +them when a target is enabled. But they need to be instantiated first (while probes are only referenced +by class name). +Thus, for instance, if you want to run one or more tasks when a target ``A`` is enabled by the framework +you have to put in the project ``targets`` list, a tuple containing first the target itself, then +all the needed instantiated tasks. + +Here under an example with the target ``A`` associated to a task of class name ``myTask``, and +the target ``B`` without tasks: + +.. code-block:: python + :linenos: + + targets = [(A, myTask()), B] + +.. note:: You can mix probes and tasks \ No newline at end of file diff --git a/framework/basic_primitives.py b/framework/basic_primitives.py index 4cf147d..54b0c58 100644 --- a/framework/basic_primitives.py +++ b/framework/basic_primitives.py @@ -34,8 +34,7 @@ def rand_string(size=None, min=1, max=10, str_set=string.printable): if size is None: size = random.randint(min, max) else: - # if size is not an int, TypeError is raised with python3, but not - # with python2 where the loop condition is always evaluated to True + # if size is not an int, TypeError is raised with python3 assert isinstance(size, int) while len(out) < size: diff --git a/framework/comm_backends.py b/framework/comm_backends.py new file mode 100644 index 0000000..2f41c59 --- /dev/null +++ b/framework/comm_backends.py @@ -0,0 +1,466 @@ +import datetime +import select +import socket +import subprocess +import sys +import threading +import time +import getpass + +from framework import error_handling as eh +from libs.external_modules import ssh_module, ssh, serial_module, serial + +class BackendError(Exception): + def __init__(self, msg, status=-1): + self.msg = msg + self.status = status + +class Backend(object): + + def __init__(self, codec='latin_1'): + """ + Args: + codec (str): codec used by the monitored system to answer. + """ + self._started = False + self.codec = codec + self._sync_lock = threading.Lock() + + def start(self): + with self._sync_lock: + if not self._started: + self._started = True + self._start() + + def stop(self): + with self._sync_lock: + if self._started: + self._started = False + self._stop() + + def exec_command(self, cmd): + with self._sync_lock: + return self._exec_command(cmd) + + def read_output(self, chan_desc): + """ + Args: + chan_desc: object returned by :meth:`Backend.exec_command` that enables to gather + output data + + Returns: + bytes: data retrieved through the communication channel + """ + raise NotImplementedError + + def _exec_command(self, cmd): + """ + Args: + cmd: command to execute through the communication channel + + Returns: list of file descriptors (e.g., stdout, stderr) + """ + raise NotImplementedError + + def _start(self): + pass + + def _stop(self): + pass + + +class SSH_Backend(Backend): + + NO_PASSWORD = 10 + ASK_PASSWORD = 20 + + """ + Backend to execute command through a serial line. + """ + def __init__(self, target_addr='localhost', port=22, bind_address=None, + username=None, password=None, pkey_path=None, pkey_password=NO_PASSWORD, + proxy_jump_addr=None, proxy_jump_bind_addr=None, proxy_jump_port=None, + proxy_jump_username=None, proxy_jump_password=None, + proxy_jump_pkey_path=None, proxy_jump_pkey_password=NO_PASSWORD, + codec='latin-1', + timeout=None, get_pty=False): + """ + Args: + target_addr (str): IP of the SSH server. + port (int): port of the SSH server. + username (str): username to connect with. + password (str): (optional) password related to the username. Could also be the special value + `SSHTarget.ASK_PASSWORD` that will prompt the user for the password at the time of connection. + pkey_path (str): (optional) path of the private key (if no password provided). + pkey_password: (optional) if the private key is encrypted, this parameter + can be either the password to decrypt it, or the special value `SSHTarget.ASK_PASSWORD` that will + prompt the user for the password at the time of connection. If the private key is + not encrypted, then this parameter should be set to `SSHTarget.NO_PASSWORD` + proxy_jump_addr: If a proxy jump has to be done before reaching the target, this parameter + should be provided with the proxy address to connect with. + proxy_jump_bind_addr: internal address of the proxy to communication with the target. + proxy_jump_port: port on which the SSH server of the proxy listen to. + proxy_jump_username: username to use for the connection with the proxy. + proxy_jump_password: (optional) password related to the username. Could also be the special value + `SSHTarget.ASK_PASSWORD` that will prompt the user for the password at the time of connection. + proxy_jump_pkey_path: (optional) path to the private key related to the username. + proxy_jump_pkey_password: (optional) if the private key is encrypted, this parameter + can be either the password to decrypt it, or the special value `SSHTarget.ASK_PASSWORD` that will + prompt the user for the password at the time of connection. If the private key is + not encrypted, then this parameter should be set to `SSHTarget.NO_PASSWORD`. + codec (str): codec used by the monitored system to answer. + timeout (float): timeout on blocking read/write operations. None disables + timeouts on socket operations + get_pty (bool): Request a pseudo-terminal from the server. It implies that processes + executed from this ssh session will be attached to the pty and will be killed + once the session is closed. (Otherwise they could remain on the server.) + """ + Backend.__init__(self, codec=codec) + if not ssh_module: + raise eh.UnavailablePythonModule('Python module for SSH is not available!') + self.host = target_addr + self.port = port + self.bind_address = bind_address + self.username = username + + assert password is None or pkey_path is None + self.password = password + self.pkey_path = pkey_path + self.pkey_password = pkey_password + + assert proxy_jump_password is None or proxy_jump_pkey_path is None + self.proxy_jump_addr = proxy_jump_addr + self.proxy_jump_bind_addr = proxy_jump_bind_addr + self.proxy_jump_port = proxy_jump_port + self.proxy_jump_username = proxy_jump_username + self.proxy_jump_password = proxy_jump_password + self.proxy_jump_pkey_path = proxy_jump_pkey_path + self.proxy_jump_pkey_password = proxy_jump_pkey_password + + self.timeout = timeout + self.get_pty = get_pty + self.client = None + + @staticmethod + def _create_pkey(pkey_path, pkey_password, prompt='PKey Password:'): + if pkey_path is not None: + if pkey_password == SSH_Backend.NO_PASSWORD: + pkey_password = None + elif pkey_password == SSH_Backend.ASK_PASSWORD: + pkey_password = getpass.getpass(prompt=f'{prompt}') + else: + pass + pkey = ssh.RSAKey.from_private_key_file(pkey_path, password=pkey_password) + else: + pkey = None + + return pkey + + def _start(self): + + tg_info = f'[addr:{self.host},user:{self.username}]' + if self.password is not None: + if self.password == SSH_Backend.ASK_PASSWORD: + self.password = getpass.getpass(prompt=f'Target Password {tg_info}:') + elif self.password == SSH_Backend.NO_PASSWORD: + self.password = None + else: + pass + + self.pkey = self._create_pkey(self.pkey_path, self.pkey_password, + prompt=f'Target PKey Password {tg_info}:') + + pj_info = f'[addr:{self.proxy_jump_addr},user:{self.proxy_jump_username}]' + if self.proxy_jump_password is not None: + if self.proxy_jump_password == SSH_Backend.ASK_PASSWORD: + self.proxy_jump_password = getpass.getpass(prompt=f'ProxyJump Password {pj_info}:') + elif self.proxy_jump_password == SSH_Backend.NO_PASSWORD: + self.proxy_jump_password = None + else: + pass + + self.proxy_jump_pkey = self._create_pkey(self.proxy_jump_pkey_path, self.proxy_jump_pkey_password, + prompt=f'ProxyJump PKey Password {pj_info}:') + + if self.proxy_jump_addr is not None: + jumpbox = ssh.SSHClient() + jumpbox.set_missing_host_key_policy(ssh.AutoAddPolicy()) + jumpbox.connect(self.proxy_jump_addr, username=self.proxy_jump_username, pkey=self.proxy_jump_pkey) + + jumpbox_transport = jumpbox.get_transport() + + if self.proxy_jump_addr: + src_addr = (self.proxy_jump_addr, self.proxy_jump_port) + dest_addr = (self.host, self.port) + sock = jumpbox_transport.open_channel("direct-tcpip", dest_addr, src_addr) + else: + sock = None + + elif self.bind_address: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind((self.bind_address, 0)) + sock.connect((self.host, self.port)) + else: + sock = None + + self.client = ssh.SSHClient() + self.client.set_missing_host_key_policy(ssh.AutoAddPolicy()) + + self.client.connect(hostname=self.host, port=self.port, username=self.username, + password=self.password, pkey=self.pkey, sock=sock) + + def _stop(self): + self.client.close() + + def _exec_command(self, cmd): + try: + ssh_in, ssh_out, ssh_err = self.client.exec_command(cmd, timeout=self.timeout, get_pty=self.get_pty) + except ssh.ssh_exception.SSHException: + raise BackendError('SSH connection not active anymore. Need {} reset'.format(self.__class__.__name__), + status=-3) + + return ssh_out, ssh_err + + def read_output(self, chan_desc): + ssh_out, ssh_err = chan_desc + out_data = err_data = '' + out_exception = err_exception = None + + try: + out_data = self.read_stdout(chan_desc) + except BackendError as err: + out_exception = err + + try: + err_data = self.read_stderr(chan_desc) + except BackendError as err: + err_exception = err + + if not out_data and not err_data: + excp_msg = '' + excp_status_code = 0 + if out_exception: + excp_msg += 'stdout: ' + str(out_exception) + excp_status_code += out_exception.status + if err_exception: + excp_msg += ' | ' if excp_msg else '' + excp_msg += 'stderr: ' + str(err_exception) + excp_status_code += err_exception.status + + raise BackendError(excp_msg, status=excp_status_code) + + elif out_data and err_data: + output = out_data + '\n' + err_data + elif out_data: + output = out_data + else: + output = err_data + + return output + + def read_stdout(self, chan_desc): + return self._read_fd(chan_desc[0]) + + def read_stderr(self, chan_desc): + return self._read_fd(chan_desc[1]) + + def _read_fd(self, fdesc): + data = '' + new_line = 'POISON' + timeout = False + + try: + while new_line: + new_line = fdesc.readline() + data += new_line + except socket.timeout: + timeout = True + + if not data: + if timeout: + raise BackendError('Read timeout', status=-1) + else: + raise BackendError('No more data to read', status=-2) + + return data + + def set_timeout(self, timeout): + self.timeout = timeout + +class Serial_Backend(Backend): + """ + Backend to execute command through a serial line. + """ + def __init__(self, serial_port, baudrate=115200, bytesize=8, parity='N', stopbits=1, + xonxoff=False, rtscts=False, dsrdtr=False, + username=None, password=None, slowness_factor=5, + cmd_notfound=b'command not found', codec='latin-1'): + """ + Args: + serial_port (str): path to the tty device file. (e.g., '/dev/ttyUSB0') + baudrate (int): baud rate of the serial line. + bytesize (int): number of data bits. (5, 6, 7, or 8) + parity (str): parity checking. ('N', 'O, 'E', 'M', or 'S') + stopbits (int): number of stop bits. (1, 1.5 or 2) + xonxoff (bool): enable software flow control. + rtscts (bool): enable hardware (RTS/CTS) flow control. + dsrdtr (bool): enable hardware (DSR/DTR) flow control. + username (str): username to connect with. If None, no authentication step will be attempted. + password (str): password related to the username. + slowness_factor (int): characterize the slowness of the monitored system. The scale goes from + 1 (fastest) to 10 (slowest). This factor is a base metric to compute the time to wait + for the authentication step to terminate (if `username` and `password` parameter are provided) + and other operations involving to wait for the monitored system. + cmd_notfound (bytes): pattern used to detect if the command does not exist on the + monitored system. + codec (str): codec used to send/receive information through the serial line + """ + Backend.__init__(self, codec=codec) + if not serial_module: + raise eh.UnavailablePythonModule('Python module for Serial is not available!') + + self.serial_port = serial_port + self.baudrate = baudrate + self.bytesize = bytesize + self.parity = parity + self.stopbits= stopbits + self.xonxoff = xonxoff + self.rtscts = rtscts + self.dsrdtr = dsrdtr + self.slowness_factor = slowness_factor + self.cmd_notfound = cmd_notfound + self.username = bytes(username, self.codec) + self.password = bytes(password, self.codec) + + self.client = None + + def _start(self): + self.ser = serial.Serial(self.serial_port, self.baudrate, bytesize=self.bytesize, + parity=self.parity, stopbits=self.stopbits, + xonxoff=self.xonxoff, dsrdtr=self.dsrdtr, rtscts=self.rtscts, + timeout=self.slowness_factor*0.1) + if self.username is not None: + assert self.password is not None + self.ser.flushInput() + self.ser.write(self.username+b'\r\n') + time.sleep(0.1) + self.ser.readline() # we read login echo + pass_prompt = self.ser.readline() + retry = 0 + eot_sent = False + while pass_prompt.lower().find(b'password') == -1: + retry += 1 + if retry > 3 and eot_sent: + self.stop() + raise BackendError('Unable to establish a connection with the serial line.') + elif retry > 3: + # we send an EOT if ever the console was not in its initial state + # (already logged, or with the password prompt, ...) when we first write on + # the serial line. + self.ser.write(b'\x04\r\n') + time.sleep(self.slowness_factor*0.8) + self.ser.flushInput() + self.ser.write(self.username+b'\r\n') + time.sleep(0.1) + self.ser.readline() # we consume the login echo + pass_prompt = self.ser.readline() + retry = 0 + eot_sent = True + else: + chunks = self._read_serial(duration=self.slowness_factor*0.2) + pass_prompt = b''.join(chunks) + time.sleep(0.1) + self.ser.write(self.password+b'\r\n') + time.sleep(self.slowness_factor*0.7) + + def _stop(self): + self.ser.write(b'\x04\r\n') # we send an EOT (Ctrl+D) + self.ser.close() + + def _exec_command(self, cmd): + if not self.ser.is_open: + raise BackendError('Serial port not open') + + cmd = bytes(cmd, self.codec) + cmd += b'\r\n' + self.ser.flushInput() + self.ser.write(cmd) + time.sleep(0.1) + return self.ser + + def read_output(self, chan_desc): + chan_desc.readline() # we consume the 'writing echo' from the input + try: + result = self._read_serial(serial_chan=chan_desc, duration=self.slowness_factor * 0.8) + except serial.SerialException: + raise BackendError('Exception while reading serial line') + else: + # We have to remove the new prompt line at the end. + # But in our testing environment, the two last entries had to be removed, namely + # 'prompt_line \r\n' and 'prompt_line ' !? + # print('\n*** DBG: ', result) + result = result[:-2] + ret = b''.join(result) + if ret.find(self.cmd_notfound) != -1: + raise BackendError('The command does not exist on the target_addr') + else: + return ret + + def _read_serial(self, serial_chan, duration): + result = [] + t0 = datetime.datetime.now() + delta = -1 + while delta < duration: + now = datetime.datetime.now() + delta = (now - t0).total_seconds() + res = serial_chan.readline() + if res == b'': + break + result.append(res) + return result + + +class Shell_Backend(Backend): + """ + Backend to execute shell commands locally + """ + def __init__(self, timeout=None, codec='latin-1'): + """ + Args: + timeout (float): timeout in seconds for reading the result of the command + codec (str): codec used by the monitored system to answer. + """ + Backend.__init__(self, codec=codec) + self._timeout = timeout + self._app = None + + def _start(self): + pass + + def _stop(self): + pass + + def _exec_command(self, cmd): + self._app = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ready_to_read, ready_to_write, in_error = \ + select.select([self._app.stdout, self._app.stderr], [], [], self._timeout) + + return ready_to_read, in_error + + def read_output(self, chan_desc): + ready_to_read, in_error = chan_desc + if in_error: + # the command does not exist on the system + raise BackendError('Issue with file descriptors') + elif ready_to_read: + if len(ready_to_read) == 2: + err = ready_to_read[1].read() + if err.strip(): + raise BackendError('ERROR: {!s}'.format(ready_to_read[1].read())) + if ready_to_read[0]: + return ready_to_read[0].read() + else: + raise BackendError('BUG') + else: + return b'' + diff --git a/framework/config.py b/framework/config.py index 8e2ffe7..44350ee 100644 --- a/framework/config.py +++ b/framework/config.py @@ -57,9 +57,6 @@ def _unindent(self, multiline): return self.__unindent.sub('', multiline) def add(self, name, doc): - if sys.version_info[0] <= 2: - doc = unicode(doc) - self.configs[name] = self._unindent(doc) @@ -69,14 +66,42 @@ def add(self, name, doc): [global] config_name: FmkPlumbing -[defvalues] -fuzz.delay = 0.01 +[misc] +fuzz.delay = 0 fuzz.burst = 1 -;; [defvalues.doc] +;; [misc.doc] ;; self: (default values used when the framework resets) ;; fuzz.delay: Default value (> 0) for fuzz_delay -;; fuzz.burst: Default value (>= 1)for fuzz_burst +;; fuzz.burst: Default value (>= 1) for fuzz_burst + +[targets] +empty_tg.verbose = False + +;; [targets.doc] +;; self: configuration related to targets +;; empty_tg.verbose: Enable verbose mode (if True) on the default EmptyTarget() + +[terminal] +external_term = False +name=x-terminal-emulator +title_arg=--title +hold_arg=--hold +exec_arg=-e +exec_arg_type=string +extra_args= + +;; [terminal.doc] +;; self: Configuration applicable to the external terminal +;; external_term: Use an external terminal +;; name: Command to call the terminal +;; title_arg: Option used by the terminal to set the title +;; hold_arg: Options to keep the terminal open after the commands exits +;; exec_arg: Option to specify the program to be run by the terminal +;; exec_arg_type: How the command should be passed on the command line, can be + string if the command and it's arguments are to be passed as one string or + list if they are to be individual arguments +;; extra_args: Extra argument to pass on the command line ''') @@ -101,7 +126,7 @@ def add(self, name, doc): used to display the helpers. [send_loop] -aligned: True +aligned: False aligned_options.batch_mode: False aligned_options.hide_cursor: True aligned_options.prompt_height: 3 @@ -117,6 +142,26 @@ def add(self, name, doc): ''') +default.add('Database', u''' +[global] +config_name: Database + +[async_data] +before_data_id = 5 +after_data_id = 60 + +;; [async_data.doc] +;; self: Configuration applicable to ASYNC DATA. +;; +;; before_data_id: an async_data (without any associated data_id) will be considered to be related + to a data sent afterwards if the number of seconds that separates it from that data is less + than the amount specified in this parameter. +;; after_data_id: if after the last registered data by the framework, an async data is sent after + more than the amount of seconds specified in this parameter, it won't be considered to be + related to this last registered data. + +''') + def check_type(name, attr, value): original = value @@ -207,13 +252,11 @@ def config_write(that, stream=sys.stdout): def sectionize(that, parent): - if sys.version_info[0] > 2: - unicode = str if parent is not None: that.config_name = parent try: - name = unicode(that.config_name) + name = str(that.config_name) except BaseException: name = that.config_name @@ -489,10 +532,7 @@ def __init__(self, parent, path=['.'], ext=['.ini', '.conf', '.cfg']): if verbose: sys.stderr.write('Loading {}...\n'.format(filename)) with open(filename, 'r') as cfile: - if sys.version_info[0] > 2: - self.parser.read_file(cfile, source=filename) - else: - self.parser.readfp(cfile, filename=filename) + self.parser.read_file(cfile, source=filename) loaded = True except BaseException as e: if verbose: @@ -501,22 +541,13 @@ def __init__(self, parent, path=['.'], ext=['.ini', '.conf', '.cfg']): continue if loaded or name in default.configs: - if not loaded and sys.version_info[0] > 2: + if not loaded: if verbose: sys.stderr.write( 'Loading default config for {}...\n'.format(name)) self.parser.read_string(default.configs[name], 'default_' + name) loaded = True - elif not loaded: - if verbose: - sys.stderr.write( - 'Loading default config for {}...\n'.format(name)) - stream = io.StringIO() - stream.write(default.configs[name]) - stream.seek(0) - self.parser.readfp(stream, 'default_' + name) - loaded = True if not self.parser.has_section('global'): raise AttributeError( diff --git a/framework/constraint_helpers.py b/framework/constraint_helpers.py new file mode 100644 index 0000000..b708122 --- /dev/null +++ b/framework/constraint_helpers.py @@ -0,0 +1,243 @@ +################################################################################ +# +# Copyright 2022 Eric Lacombe +# +################################################################################ +# +# This file is part of fuddly. +# +# fuddly is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# fuddly is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with fuddly. If not, see +# +################################################################################ + +import copy +from typing import Tuple, List + +from libs.external_modules import * +if csp_module: + from constraint import * + +class ConstraintError(Exception): pass + +class Constraint(object): + + relation = None + vars = None + _var_domain = None + _orig_relation = None + + def __init__(self, relation, vars: Tuple, var_to_varns: dict = None): + """ + + Args: + relation: boolean function that define the constraints between variables + vars (list): list of the names of the nodes used in the boolean function in `relation` + (in the same order as the parameters of the function). + var_to_varns (dict): dictionary that associates for each name in `vars`, the comprehensive + reference to the related node, which is a tuple of its name and its namespace. + """ + + self.relation = self._orig_relation = relation + self.vars = vars + self.var_to_varns = var_to_varns + + @property + def var_domain(self): + return self._var_domain + + @var_domain.setter + def var_domain(self, var_domain): + self._var_domain = var_domain + + def negate(self): + self.relation = self._negated_relation + + def reset_to_original(self): + self.relation = self._orig_relation + + def _negated_relation(self, *args): + return not self._orig_relation(*args) + + def __copy__(self): + new_cst = type(self)(self._orig_relation, self.vars, self.var_to_varns) + new_cst.__dict__.update(self.__dict__) + return new_cst + +class CSP(object): + + _constraints = None + _vars = None + _var_to_varns = None + _var_node_mapping = None + _var_domain = None + _var_domain_updated = False + _orig_var_domain = None + _problem = None + _solutions = None + _model = None + _exhausted_solutions = None + _is_solution_queried = False + highlight_variables = None + + def __init__(self, constraints: Constraint or List[Constraint] = None, highlight_variables=False): + assert csp_module, "the CSP backend is disabled because python-constraint module is not installed!" + + if isinstance(constraints, Constraint): + c_copy = copy.copy(constraints) + self._vars = c_copy.vars + self._constraints = [c_copy] + self._var_to_varns = copy.copy(c_copy.var_to_varns) + else: + self._constraints = [] + self._vars = () + for r in constraints: + r_copy = copy.copy(r) + self._constraints.append(r_copy) + self._vars += r_copy.vars + if r_copy.var_to_varns: + if self._var_to_varns is None: + self._var_to_varns = {} + self._var_to_varns.update(r_copy.var_to_varns) + for v in r_copy.vars: + if v not in r_copy.var_to_varns: + self._var_to_varns[v] = v + + self._var_node_mapping = {} + self._var_domain = {} + self._var_domain_updated = False + + self.highlight_variables = highlight_variables + + def reset(self): + self._problem = Problem() + self._solutions = None + self._model = None + self._exhausted_solutions = False + self._is_solution_queried = False + + def iter_vars(self): + for v in self._vars: + yield v + + def from_var_to_varns(self, var): + return var if self._var_to_varns is None else self._var_to_varns[var] + + @property + def var_domain_updated(self): + return self._var_domain_updated + + def set_var_domain(self, var, domain): + assert bool(domain) + self._var_domain[var] = copy.copy(domain) + self._var_domain_updated = True + + def save_current_var_domains(self): + self._orig_var_domain = copy.copy(self._var_domain) + self._var_domain_updated = False + + def restore_var_domains(self): + self._var_domain = copy.copy(self._orig_var_domain) + self._var_domain_updated = False + + def map_var_to_node(self, var, node): + self._var_node_mapping[var] = node + + @property + def var_mapping(self): + return self._var_node_mapping + + def get_solution(self): + if not self._model: + self.next_solution() + + self._is_solution_queried = True + + return self._model + + def _solve_constraints(self): + for c in self._constraints: + for v in c.vars: + try: + self._problem.addVariable(v, self._var_domain[v]) + except ValueError: + # most probable cause: duplicated variable is attempted to be inserted + # (other cause is empty domain which is checked at init) + pass + self._problem.addConstraint(c.relation, c.vars) + + self._solutions = self._problem.getSolutionIter() + + def next_solution(self): + if self._solutions is None or self._exhausted_solutions: + self.reset() + self._solve_constraints() + try: + mdl = next(self._solutions) + except StopIteration: + raise ConstraintError(f'No solution found for this CSP\n --> variables: {self._vars}') + else: + self._model = mdl + else: + try: + mdl = next(self._solutions) + except StopIteration: + self._exhausted_solutions = True + else: + self._model = mdl + + self._is_solution_queried = False + + def negate_constraint(self, idx): + assert 0 <= idx < self.nb_constraints + c = self._constraints[idx] + c.negate() + self.reset() + + def reset_constraint(self, idx): + assert 0 <= idx < self.nb_constraints + c = self._constraints[idx] + c.reset_to_original() + self.reset() + + def get_all_constraints(self): + return self._constraints + + def get_constraint(self, idx): + assert 0 <= idx < self.nb_constraints + return self._constraints[idx] + + @property + def nb_constraints(self): + return len(self._constraints) + + @property + def is_current_solution_queried(self): + return self._is_solution_queried + + @property + def exhausted_solutions(self): + return self._exhausted_solutions + + def __copy__(self): + new_csp = type(self)(constraints=self._constraints) + new_csp.__dict__.update(self.__dict__) + new_csp._var_domain = copy.copy(self._var_domain) + new_csp._var_node_mapping = copy.copy(self._var_node_mapping) + new_csp._solutions = None # the generator cannot be copied + new_csp._model = copy.copy(self._model) + new_csp._constraints = [] + for c in self._constraints: + new_csp._constraints.append(copy.copy(c)) + + return new_csp diff --git a/framework/cosmetics.py b/framework/cosmetics.py index 520943f..d41680d 100644 --- a/framework/cosmetics.py +++ b/framework/cosmetics.py @@ -108,35 +108,19 @@ def reinit(self): self.countp = 0 -if sys.version_info[0] > 2: +class stdout_wrapper(wrapper, io.TextIOWrapper): + """Wrap stdout and handle cosmetic issues.""" - class stdout_wrapper(wrapper, io.TextIOWrapper): - """Wrap stdout and handle cosmetic issues.""" + def __init__(self): + io.TextIOWrapper.__init__(self, + io.BytesIO(), sys.__stdout__.encoding) + wrapper.__init__(self, io.TextIOWrapper) - def __init__(self): - io.TextIOWrapper.__init__(self, - io.BytesIO(), sys.__stdout__.encoding) - wrapper.__init__(self, io.TextIOWrapper) - - def reinit(self): - wrapper.reinit(self) - io.TextIOWrapper.__init__(self, io.BytesIO()) - - stdout_wrapped = stdout_wrapper() -else: - - class stdout_wrapper(wrapper, io.BytesIO): - """Wrap stdout and handle cosmetic issues.""" - - def __init__(self): - io.BytesIO.__init__(self) - wrapper.__init__(self, io.BytesIO) - - def reinit(self): - wrapper.reinit(self) - io.BytesIO.__init__(self) + def reinit(self): + wrapper.reinit(self) + io.TextIOWrapper.__init__(self, io.BytesIO()) - stdout_wrapped = stdout_wrapper() +stdout_wrapped = stdout_wrapper() if not import_error: @@ -149,10 +133,7 @@ def reinit(self): civis = curses.tigetstr("civis") cvvis = curses.tigetstr("cvvis") else: - if sys.version_info[0] > 2: - el, ed, cup, civis, cvvis = ('', ) * 5 - else: - el, ed, cup, civis, cvvis = (b'', ) * 5 + el, ed, cup, civis, cvvis = ('', ) * 5 def get_size( @@ -215,11 +196,8 @@ def buffer_content(): Returns: The whole content of stdout_wrapped.""" - if sys.version_info[0] > 2: - stdout_wrapped.seek(0) - payload = stdout_wrapped.buffer.read() - else: - payload = stdout_wrapped.getvalue() + stdout_wrapped.seek(0) + payload = stdout_wrapped.buffer.read() return payload @@ -255,10 +233,7 @@ def disp( Args: payload (bytes): the bytes displayed on screen.""" - if sys.version_info[0] > 2: - stdout_unwrapped.buffer.write(payload) - else: - stdout_unwrapped.write(payload) + stdout_unwrapped.buffer.write(payload) def tty_noecho(): diff --git a/framework/data.py b/framework/data.py index 6053a6f..96f44bf 100644 --- a/framework/data.py +++ b/framework/data.py @@ -75,7 +75,7 @@ def get_length(self): class NodeBackend(DataBackend): - def update_from(self, obj): + def update_from(self, obj: Node): self._node = obj if obj.env is None: obj.set_env(Env()) @@ -97,6 +97,9 @@ def data_maker_name(self): def to_str(self): return self._node.to_str() + def to_formatted_str(self): + return self._node.to_formatted_str() + def to_bytes(self): return self._node.to_bytes() @@ -149,6 +152,9 @@ def data_maker_name(self): def to_str(self): return unconvert_from_internal_repr(self._content) + def to_formatted_str(self): + return self.to_str() + def to_bytes(self): return self._content @@ -187,6 +193,9 @@ def data_maker_name(self): def to_str(self): return 'Empty Backend' + def to_formatted_str(self): + return 'Empty Backend' + def to_bytes(self): return b'Empty Backend' @@ -197,49 +206,123 @@ def get_length(self): return 0 +class AttrGroup(object): + + def __init__(self, attrs_desc): + self._attrs = attrs_desc + + def set(self, name): + if name not in self._attrs: + raise ValueError + self._attrs[name] = True + + def clear(self, name): + if name not in self._attrs: + raise ValueError + self._attrs[name] = False + + def is_set(self, name): + if name not in self._attrs: + raise ValueError + return self._attrs[name] + + def copy_from(self, attr_group): + assert isinstance(attr_group, AttrGroup) + self._attrs = copy.copy(attr_group._attrs) + + def __copy__(self): + new_attrgr = type(self)() + new_attrgr.__dict__.update(self.__dict__) + new_attrgr._attrs = copy.copy(self._attrs) + return new_attrgr + + +class DataAttr(AttrGroup): + + Reset_DMakers = 1 + + def __init__(self, attrs_to_set=None, attrs_to_clear=None): + iv = { + DataAttr.Reset_DMakers: False + } + AttrGroup.__init__(self, iv) + if attrs_to_set: + for a in attrs_to_set: + self.set(a) + if attrs_to_clear: + for a in attrs_to_clear: + self.clear(a) + + class Data(object): _empty_data_backend = EmptyBackend() - def __init__(self, data=None, altered=False, tg_ids=None): - self.altered = altered - self._backend = None + def __init__(self, content=None, altered=False, tg_ids=None, description=None): + + self.description = description - self._type = None + self.estimated_data_id = None self._data_id = None - self._recordable = False - self._unusable = False - self._blocked = False + self._backend = None - self.feedback_timeout = None - self.feedback_mode = None + self.set_basic_attributes() + self.altered = altered self.info_list = [] self.info = {} + self._history = None - self.scenario_dependence = None + self.tg_ids = tg_ids # targets ID # callback related self._callbacks = {} self._pending_ops = {} - self._history = None + if content is None: + self._backend = self._empty_data_backend + elif isinstance(content, Node): + self._backend = NodeBackend(content) + else: + self._backend = RawBackend(content) + + self._type = (self._backend.data_maker_type, self._backend.data_maker_name, None) + + def set_basic_attributes(self, from_data=None): + + self.attrs = DataAttr() if from_data is None else copy.copy(from_data.attrs) + self._type = None if from_data is None else from_data._type + + self.feedback_timeout = None if from_data is None else from_data.feedback_timeout + self.feedback_mode = None if from_data is None else from_data.feedback_mode + + self.sending_delay = None if from_data is None else from_data.sending_delay + + self.altered = False if from_data is None else from_data.altered + + self._recordable = False if from_data is None else from_data._recordable + self._unusable = False if from_data is None else from_data._unusable # Used to provide information on the origin of the Data(). # If it comes from a scenario _origin point to the related scenario. - self._origin = None + self._origin = None if from_data is None else from_data._origin - self.tg_ids = tg_ids # targets ID + self._blocked = False if from_data is None else from_data._blocked + + self.scenario_dependence = None if from_data is None else from_data.scenario_dependence + + # If True, the data will not interrupt the framework while processing + # the data even if the data is unusable, The framework will just go on + # to its next task without handing over to the end user. + # Used especially by the Scenario Infrastructure. + self.on_error_handover_to_user = True if from_data is None else from_data.on_error_handover_to_user # This attribute is set to True when the Data content has been retrieved from the fmkDB - self.from_fmkdb = False + self.from_fmkdb = False if from_data is None else from_data.from_fmkdb - if data is None: - self._backend = self._empty_data_backend - elif isinstance(data, Node): - self._backend = NodeBackend(data) - else: - self._backend = RawBackend(data) + def set_attributes_from(self, attr_group): + assert isinstance(attr_group, DataAttr) + self.attrs = attr_group @property def content(self): @@ -294,6 +377,9 @@ def to_bytes(self): def to_str(self): return self._backend.to_str() + def to_formatted_str(self): + return self._backend.to_formatted_str() + def make_blocked(self): self._blocked = True @@ -309,6 +395,12 @@ def make_unusable(self): def is_unusable(self): return self._unusable + def has_node_content(self): + return isinstance(self._backend, NodeBackend) + + def has_raw_content(self): + return isinstance(self._backend, RawBackend) + # Only taken into account if the Logger has been set to # record data only when requested (explicit_data_recording == True) def make_recordable(self): @@ -317,17 +409,18 @@ def make_recordable(self): def is_recordable(self): return self._recordable - def generate_info_from_content(self, original_data=None, origin=None, additional_info=None): + def generate_info_from_content(self, data=None, origin=None, additional_info=None): dmaker_type = self._backend.data_maker_type dmaker_name = self._backend.data_maker_name + initial_gen_user_input = None - if original_data is not None: - self.altered = original_data.altered - if original_data.origin is not None: - self.add_info("Data instantiated from: {!s}".format(original_data.origin)) - if original_data.info: + if data is not None: + self.set_basic_attributes(from_data=data) + if data.origin is not None: + self.add_info("Data instantiated from: {!s}".format(data.origin)) + if data.info: info_bundle_to_remove = [] - for key, info_bundle in original_data.info.items(): + for key, info_bundle in data.info.items(): info_bundle_to_remove.append(key) for chunk in info_bundle: for info in chunk: @@ -336,27 +429,23 @@ def generate_info_from_content(self, original_data=None, origin=None, additional for key in info_bundle_to_remove: self.remove_info_from(*key) + initial_gen_user_input = data.get_initial_dmaker()[2] + elif origin is not None: self.add_info("Data instantiated from: {!s}".format(origin)) else: - pass + return + if additional_info is not None: for info in additional_info: self.add_info(info) + self.remove_info_from(dmaker_type, dmaker_name) self.bind_info(dmaker_type, dmaker_name) - initial_generator_info = [dmaker_type, dmaker_name, None] + initial_generator_info = [dmaker_type, dmaker_name, initial_gen_user_input] self.set_initial_dmaker(initial_generator_info) self.set_history([initial_generator_info]) - def copy_info_from(self, data): - print(self.info_list, self.info, self._type, self._history) - print(data.info_list, data.info, data._type, data._history) - self.info_list = data.info_list - self.info = data.info - self._type = data._type - self._history = data._history - def add_info(self, info_str): self.info_list.append(info_str) @@ -389,19 +478,44 @@ def read_info(self, dmaker_type, data_maker_name): info_l = self.info[key] except KeyError: print("\n*** The key " \ - "({:s}, {:s}) does not exist! ***\n".format(dmaker_type, data_maker_name)) + "({:s}, {:s}) does not exist! ***".format(dmaker_type, data_maker_name)) print("self.info contents: ", self.info) return for info in info_l: yield info + def take_info_ownership(self, keep_previous_info=True): + if not keep_previous_info: + self.info = {} + return + + if self.info: + legacy_info = self.info + self.info = {} + legacy_info_list = [] + for key, info_container in legacy_info.items(): + dmaker_type, data_maker_name = key + legacy_info_list.append('=== INFO FROM {} ==='.format(dmaker_type)) + for info_l in info_container: + info_l = ['| '+ m for m in info_l] + legacy_info_list += info_l + if legacy_info_list and not self.info_list: + self.info_list = legacy_info_list + elif legacy_info_list and self.info_list: + self.info_list = legacy_info_list + self.info_list + else: + pass + def set_history(self, hist): self._history = hist def get_history(self): return self._history + def reset_history(self): + self._history= None + def get_length(self): return self._backend.get_length() @@ -448,6 +562,8 @@ def run_callbacks(self, feedback=None, hook=HOOK.after_fbk): self._pending_ops[hook] = [] if cbk_ops is not None: self._pending_ops[hook].append(cbk_ops.get_operations()) + if cbk_ops.is_flag_set(CallBackOps.ForceDataHandling): + self.on_error_handover_to_user = False if cbk_ops.is_flag_set(CallBackOps.RemoveCB): del new_cbks[cbk_id] if cbk_ops.is_flag_set(CallBackOps.StopProcessingCB): @@ -491,6 +607,7 @@ def __copy__(self): new_data._pending_ops = {} # we do not copy pending_ops new_data._backend = copy.copy(self._backend) new_data._targets = copy.copy(self._targets) + new_data.attrs = copy.copy(self.attrs) return new_data def __str__(self): @@ -505,23 +622,29 @@ class CallBackOps(object): # Flags RemoveCB = 1 # If True, remove this callback after execution StopProcessingCB = 2 # If True, any callback following this one won't be processed + ForceDataHandling = 3 # Instructions Add_PeriodicData = 10 # ask for sending periodically a data Del_PeriodicData = 11 # ask for stopping a periodic sending + Start_Task = 12 # ask for sending periodically a data + Stop_Task = 13 # ask for stopping a periodic sending Set_FbkTimeout = 21 # set the time duration for feedback gathering for the further data sending Replace_Data = 30 # replace the data by another one - def __init__(self, remove_cb=False, stop_process_cb=False): + def __init__(self, remove_cb=False, stop_process_cb=False, ignore_no_data=False): self.instr = { CallBackOps.Add_PeriodicData: {}, CallBackOps.Del_PeriodicData: [], + CallBackOps.Start_Task: {}, + CallBackOps.Stop_Task: [], CallBackOps.Set_FbkTimeout: None, CallBackOps.Replace_Data: None } self.flags = { CallBackOps.RemoveCB: remove_cb, - CallBackOps.StopProcessingCB: stop_process_cb + CallBackOps.StopProcessingCB: stop_process_cb, + CallBackOps.ForceDataHandling: ignore_no_data } def set_flag(self, name): @@ -539,7 +662,10 @@ def add_operation(self, instr_type, id=None, param=None, period=None): if instr_type == CallBackOps.Add_PeriodicData: assert id is not None and param is not None self.instr[instr_type][id] = (param, period) - elif instr_type == CallBackOps.Del_PeriodicData: + elif instr_type == CallBackOps.Start_Task: + assert id is not None and param is not None + self.instr[instr_type][id] = (param, period) + elif instr_type == CallBackOps.Del_PeriodicData or instr_type == CallBackOps.Stop_Task: assert id is not None self.instr[instr_type].append(id) elif instr_type == CallBackOps.Set_FbkTimeout: @@ -553,3 +679,136 @@ def add_operation(self, instr_type, id=None, param=None, period=None): def get_operations(self): return self.instr + + +class DataProcess(object): + def __init__(self, process, seed=None, tg_ids=None, auto_regen=False): + """ + Describe a process to generate a data. + + Args: + process (list): List of disruptors (possibly complemented by parameters) to apply to + a ``seed``. However, if the list begin with a generator, the disruptor chain will apply + to the outcome of the generator. The generic form for a process is: + ``[action_1, (action_2, UI_2), ... action_n]`` + where ``action_N`` can be either: ``dmaker_type_N`` or ``(dmaker_type_N, dmaker_name_N)`` + seed: (Optional) Can be a registered :class:`framework.data_model.Node` name or + a :class:`framework.data_model.Data`. Will be provided to the first disruptor in + the disruptor chain (described by the parameter ``process``) if it does not begin + with a generator. + tg_ids (list): Virtual ID list of the targets to which the outcomes of this data process will be sent. + If ``None``, the outcomes will be sent to the first target that has been enabled. + In the context of scenario, it embeds virtual IDs. + auto_regen (boolean): If ``True``, the data process will be in a state requesting the framework to + rerun the data maker chain after a disruptor yielded (meaning it is exhausted with + the data provided to it). + It will make the chain going on with new data coming either from the first + non-exhausted disruptor (preceding the exhausted one), or from the generator if + all disruptors are exhausted. + If ``False``, the data process won't be in this state and the + framework won't rerun the data maker chain once a disruptor yield. + It means the framework will alert about this issue to the end-user, or when used + within a Scenario, it will redirect the decision to the scenario itself (this condition + may trigger a transition in the scenario). + """ + self.seed = seed + self.auto_regen = auto_regen + self.auto_regen_cpt = 0 + self.outcomes = None + self.feedback_timeout = None + self.feedback_mode = None + self.tg_ids = tg_ids + + self.dp_completed = False + + self._process = [process] + self._process_idx = 0 + self._blocked = False + + def append_new_process(self, process): + """ + Append a new process to the list. + """ + self._process.append(process) + + def next_process(self): + if self._process_idx == len(self._process) - 1: + self._process_idx = 0 + return False + else: + self._process_idx += 1 + return True + + def reset(self): + self.auto_regen_cpt = 0 + self.outcomes = None + self._process_idx = 0 + + @property + def process(self): + return self._process[self._process_idx] + + @process.setter + def process(self, value): + self._process[self._process_idx] = value + + @property + def process_qty(self): + return len(self._process) + + def make_blocked(self): + self._blocked = True + if self.outcomes is not None: + self.outcomes.make_blocked() + + def make_free(self): + self._blocked = False + if self.outcomes is not None: + self.outcomes.make_free() + + def formatted_str(self, oneliner=False): + desc = '' + suffix = ', proc=' if oneliner else '\n' + if None in self._process: + suffix = ', no process ' if oneliner else ' ' + + if isinstance(self.seed, str): + desc += "seed='" + self.seed + "'" + suffix + elif isinstance(self.seed, Data): + if isinstance(self.seed.content, Node): + seed_str = self.seed.content.name + else: + seed_str = "Data('{!r}'...)".format(self.seed.to_str()[:10]) + desc += "seed='{:s}'".format(seed_str) + suffix + else: + desc += suffix[2:] + + for proc in self._process: + if proc is None: + break + for d in proc: + if isinstance(d, (list, tuple)): + desc += '{!s}/'.format(d[0]) + else: + assert isinstance(d, str) + desc += '{!s}/'.format(d) + desc = desc[:-1] + desc += ',' if oneliner else '\n' + desc = desc[:-1] # if oneliner else desc[:-1] + + return desc + + def __repr__(self): + return self.formatted_str(oneliner=True) + + def __copy__(self): + new_datap = type(self)(self.process) + new_datap.__dict__.update(self.__dict__) + new_datap._process = copy.copy(self._process) + new_datap.reset() + return new_datap + + +class EmptyDataProcess(object): + def __init__(self, seed=None, tg_ids=None, auto_regen=False): + DataProcess.__init__(self, process=None, seed=seed, tg_ids=tg_ids, auto_regen=auto_regen) \ No newline at end of file diff --git a/framework/data_model.py b/framework/data_model.py index 33935a6..3272e35 100644 --- a/framework/data_model.py +++ b/framework/data_model.py @@ -21,12 +21,14 @@ # ################################################################################ +import threading + import framework.global_resources as gr from framework.data import * from framework.dmhelpers.generic import * from framework.node_builder import NodeBuilder from libs.external_modules import * - +from libs.utils import Accumulator #### Data Model Abstraction @@ -82,17 +84,35 @@ def _atom_absorption_additional_actions(self, atom): """ return atom, '' + + def _create_atom_from_raw_data_specific(self, data, idx, filename): + """ + Overload this method when creating a node from binary strings need more actions than + performing a node absorption. + + Args: + data (bytes): file content + idx (int): index of the imported file + filename (str): name of the imported file + + Returns: + An atom or ``None`` + """ + raise NotImplementedError + + def create_atom_from_raw_data(self, data, idx, filename): """ This function is called for each files (with the right extension) present in ``imported_data/`` and absorb their content by leveraging the atoms of the data model registered for absorption or if none are - registered, wrap their content in a :class:`framework.node.Node`. + registered, either call the method _create_atom_from_raw_data_specific() if it is defined or + wrap their content in a :class:`framework.node.Node`. Args: - filename (str): name of the imported file data (bytes): file content idx (int): index of the imported file + filename (str): name of the imported file Returns: An atom or ``None`` @@ -119,12 +139,14 @@ def create_atom_from_raw_data(self, data, idx, filename): else: return None else: - return Node('RAW_{:s}'.format(filename[:-len(self.file_extension)-1]), - values=[data]) - + try: + return self._create_atom_from_raw_data_specific(data, idx, filename) + except NotImplementedError: + return Node('RAW_{:s}'.format(filename[:-len(self.file_extension)-1]), + values=[data]) - def register_atom_for_absorption(self, atom, absorb_constraints=AbsFullCsts(), - decoding_scope=None): + def register_atom_for_decoding(self, atom, absorb_constraints=AbsFullCsts(), + decoding_scope=None): """ Register an atom that will be used by the DataModel when an operation requiring data absorption is performed, like self.decode(). @@ -132,9 +154,11 @@ def register_atom_for_absorption(self, atom, absorb_constraints=AbsFullCsts(), Args: atom: Atom to register for absorption absorb_constraints: Constraints to be used for the absorption - decoding_scope: Should be either an atom name or a list of the atom name that can be - absorbed by the registered atom. If set to None, the atom will be the default one - used for absorption operation if no other nodes exist with a specific scope. + decoding_scope: Should be either an atom name that can be + absorbed by the registered atom, or a textual description of the scope, or + a list of the previous elements. + If set to None, the atom will be the default one + used for decoding operation if no other nodes exist with a specific scope. """ @@ -149,55 +173,88 @@ def register_atom_for_absorption(self, atom, absorb_constraints=AbsFullCsts(), assert isinstance(decoding_scope, (list, tuple)) - for atom_name in decoding_scope: - self._atoms_for_abs[atom_name] = (prepared_atom, absorb_constraints) - + for scope in decoding_scope: + self._atoms_for_abs[scope] = (prepared_atom, absorb_constraints) - def decode(self, data, atom_name=None, requested_abs_csts=None, colorized=True): + def decode(self, data, scope=None, atom_name=None, requested_abs_csts=None, colorized=True): + """ + Args: + data: + atom_name (str): requested atom name for the decoding (linked to self.register_atom_for_decoding) + scope (str): requested scope for the decoding (linked to self.register_atom_for_decoding) + requested_abs_csts: + colorized: - class Accumulator: - content = '' - def accumulate(self, msg): - self.content += msg + Returns: + tuple: + Node which is the result of the absorption or None and + Textual description of the result + """ a = Accumulator() accumulate = a.accumulate - if atom_name is None and self._default_atom_for_abs: + try: + atom, extra = self.absorb(data, scope=scope, atom_name=atom_name, requested_abs_csts=requested_abs_csts) + except ValueError as err: + msg = colorize(f'\n*** ERROR: {err} ***', rgb=Color.ERROR) + return None, msg + + status, off, size, name = extra + + if atom is None: + accumulate(colorize("\n*** DECODING ERROR [atom used: '{:s}'] ***", rgb=Color.ERROR) + .format(name)) + accumulate('\nAbsorption Status: {!r}, {:d}, {:d}'.format(status, off, size)) + accumulate('\n \_ length of original data: {:d}'.format(len(data))) + accumulate('\n \_ remaining: {!r}'.format(data[size:size+1000])) + else: + accumulate('\n') + atom.show(log_func=accumulate, display_title=False, pretty_print=colorized) + + return atom, a.content + + def absorb(self, data, scope=None, atom_name=None, requested_abs_csts=None): + """ + + Args: + data: + atom_name (str): requested atom name for the decoding (linked to self.register_atom_for_decoding) + scope (str): requested scope for the decoding (linked to self.register_atom_for_decoding) + requested_abs_csts: + + Returns: + Node: + Node which is the result of the absorption or None + """ + + if scope is None and atom_name is None and self._default_atom_for_abs: atom, abs_csts = self._default_atom_for_abs atom_for_abs = self._backend(atom).atom_copy(atom) atom_name = atom.name - elif atom_name is None: + elif scope is None and atom_name is None: atom_name = list(self._dm_hashtable.keys())[0] atom_for_abs = self.get_atom(atom_name) abs_csts = AbsFullCsts() else: try: - if self._atoms_for_abs and atom_name in self._atoms_for_abs: + if self._atoms_for_abs and scope in self._atoms_for_abs: + atom_for_abs, abs_csts = self.get_atom_for_absorption(scope) + elif self._atoms_for_abs and atom_name in self._atoms_for_abs: atom_for_abs, abs_csts = self.get_atom_for_absorption(atom_name) else: atom_for_abs = self.get_atom(atom_name) abs_csts = AbsFullCsts() except ValueError: - msg = colorize("\n*** ERROR: provided atom name is unknown: '{:s}' ***".format(atom_name), - rgb=Color.ERROR) - return msg + raise ValueError(f"provided atom name is unknown: '{atom_name}'") abs_csts_to_apply = abs_csts if requested_abs_csts is None else requested_abs_csts - status, off, size, name = atom_for_abs.absorb(data, constraints=abs_csts_to_apply) - if status == AbsorbStatus.FullyAbsorbed: - accumulate('\n') - atom_for_abs.show(log_func=accumulate, display_title=False, pretty_print=colorized) - else: - accumulate(colorize("\n*** DECODING ERROR [atom used: '{:s}'] ***", rgb=Color.ERROR) - .format(atom_for_abs.name)) - accumulate('\nAbsorption Status: {!r}, {:d}, {:d}'.format(status, off, size)) - accumulate('\n \_ length of original data: {:d}'.format(len(data))) - accumulate('\n \_ remaining: {!r}'.format(data[size:size+1000])) + atom = atom_for_abs if status == AbsorbStatus.FullyAbsorbed else None + + return atom, (status, off, size, name) - return a.content def cleanup(self): pass @@ -210,6 +267,8 @@ def __init__(self): self._atoms_for_abs = None self._default_atom_for_abs= None self._decoded_data = None + self._included_data_models = None + self._dm_access_lock = threading.Lock() def _backend(self, atom): if isinstance(atom, (Node, dict)): @@ -220,6 +279,14 @@ def _backend(self, atom): def __str__(self): return self.name if self.name is not None else 'Unnamed' + @property + def included_models(self): + return self._included_data_models + + def customize_node_backend(self, default_gen_custo=None, default_nonterm_custo=None): + self.node_backend.default_gen_custo = default_gen_custo + self.node_backend.default_nonterm_custo = default_nonterm_custo + def register(self, *atom_list): for a in atom_list: if a is None: continue @@ -227,11 +294,12 @@ def register(self, *atom_list): self._dm_hashtable[key] = prepared_atom def get_atom(self, hash_key, name=None): - if hash_key in self._dm_hashtable: - atom = self._dm_hashtable[hash_key] - return self._backend(atom).atom_copy(atom, new_name=name) - else: - raise ValueError('Requested atom does not exist!') + with self._dm_access_lock: + if hash_key in self._dm_hashtable: + atom = self._dm_hashtable[hash_key] + return self._backend(atom).atom_copy(atom, new_name=name) + else: + raise ValueError('Requested atom does not exist!') def get_atom_for_absorption(self, hash_key): @@ -261,11 +329,15 @@ def load_data_model(self, dm_db): self._built = True def merge_with(self, data_model): + if self._included_data_models is None: + self._included_data_models = {} + self._included_data_models[data_model] = [] for k, v in data_model._dm_hashtable.items(): if k in self._dm_hashtable: raise ValueError("the data ID {:s} exists already".format(k)) else: self._dm_hashtable[k] = v + self._included_data_models[data_model].append(k) self.node_backend.merge_with(data_model.node_backend) @@ -337,6 +409,9 @@ def get_import_directory_path(self, subdir=None): class NodeBackend(object): + default_gen_custo = None + default_nonterm_custo = None + def __init__(self, data_model): self._dm = data_model self._confs = set() @@ -353,7 +428,8 @@ def prepare_atom(self, atom): raise UserWarning(msg) if isinstance(atom, dict): - mb = NodeBuilder(dm=self._dm) + mb = NodeBuilder(dm=self._dm, default_gen_custo=self.default_gen_custo, + default_nonterm_custo=self.default_nonterm_custo) desc_name = 'Unreadable Name' try: desc_name = atom['name'] @@ -362,7 +438,7 @@ def prepare_atom(self, atom): print('-'*60) traceback.print_exc(file=sys.stdout) print('-'*60) - msg = "*** ERROR: problem encountered with the '{desc:s}' descriptor!".format(desc=desc_name) + msg = "*** ERROR: problem encountered with the '{desc!s}' descriptor!".format(desc=desc_name) raise UserWarning(msg) if atom.env is None: @@ -377,7 +453,6 @@ def prepare_atom(self, atom): def atom_copy(self, orig_atom, new_name=None): name = orig_atom.name if new_name is None else new_name node = Node(name, base_node=orig_atom, ignore_frozen_state=False, new_env=True) - # self.update_knowledge_source(node) return node def update_atom(self, atom, existing_env=False): diff --git a/framework/database.py b/framework/database.py index 7d584b3..8a9aaab 100644 --- a/framework/database.py +++ b/framework/database.py @@ -26,13 +26,16 @@ import re import math import threading -from datetime import datetime +import copy +from datetime import datetime, timedelta +from typing import Optional import framework.global_resources as gr +from framework.config import config import libs.external_modules as em from framework.knowledge.feedback_collector import FeedbackSource from libs.external_modules import * -from libs.utils import ensure_dir, chunk_lines +from libs.utils import chunk_lines def regexp(expr, item): @@ -53,15 +56,16 @@ def regexp_bin(expr, item): class FeedbackGate(object): - def __init__(self, database): + def __init__(self, database, only_last_entries=True): """ Args: database (Database): database to be associated with """ self.db = database + self.last_fbk_entries = only_last_entries def __iter__(self): - for item in self.db.iter_last_feedback_entries(): + for item in self.db.iter_feedback_entries(last=self.last_fbk_entries): yield item def get_feedback_from(self, source): @@ -69,7 +73,7 @@ def get_feedback_from(self, source): source = FeedbackSource(source) try: - fbk = self.db.last_feedback[source] + fbk = self.db.last_feedback[source] if self.last_fbk_entries else self.db.feedback_trail[source] except KeyError: raise else: @@ -91,7 +95,7 @@ def iter_entries(self, source=None): - the 4-uplet: (source, status, timestamp, content) if `source` is `None` """ - for item in self.db.iter_last_feedback_entries(source=source): + for item in self.db.iter_feedback_entries(last=self.last_fbk_entries, source=source): yield item def sources_names(self): @@ -103,21 +107,22 @@ def sources_names(self): list: names of the feedback sources """ - return [str(fs) for fs in self.db.last_feedback.keys()] + fbk_db = self.db.last_feedback if self.last_fbk_entries else self.db.feedback_trail + return [str(fs) for fs in fbk_db.keys()] - # for python2 compatibility - def __nonzero__(self): - return bool(self.db.last_feedback) + @property + def size(self): + return len(list(self.db.iter_feedback_entries(last=self.last_fbk_entries))) - # for python3 compatibility def __bool__(self): - return bool(self.db.last_feedback) + return bool(self.db.last_feedback if self.last_fbk_entries else self.db.feedback_trail) class Database(object): DDL_fname = 'fmk_db.sql' + DEFAULT_DB_NAME = 'fmkDB.db' DEFAULT_DM_NAME = '__DEFAULT_DATAMODEL' DEFAULT_GTYPE_NAME = '__DEFAULT_GTYPE' DEFAULT_GEN_NAME = '__DEFAULT_GNAME' @@ -125,20 +130,30 @@ class Database(object): OUTCOME_ROWID = 1 OUTCOME_DATA = 2 + FEEDBACK_TRAIL_TIME_WINDOW = 10 # seconds + def __init__(self, fmkdb_path=None): - self.name = 'fmkDB.db' + + self.name = Database.DEFAULT_DB_NAME if fmkdb_path is None: self.fmk_db_path = os.path.join(gr.fuddly_data_folder, self.name) else: - self.fmk_db_path = fmkdb_path + self.fmk_db_path = os.path.expanduser(fmkdb_path) + + self._ref_names = {} + self.config = None self.enabled = False self.current_project = None self.last_feedback = {} + self.feedback_trail = {} # store feedback entries for self.feedback_trail_time_window + self.feedback_trail_init_ts = None + self.feedback_trail_time_window = self.FEEDBACK_TRAIL_TIME_WINDOW self._data_id = None + self._current_sent_date = None self._sql_handler_thread = None self._sql_handler_stop_event = threading.Event() @@ -155,6 +170,13 @@ def __init__(self, fmkdb_path=None): self._ok = None + self.fbk_timeout_re = re.compile('.*feedback timeout = (.*)s$') + + + @staticmethod + def get_default_db_path(): + return os.path.join(gr.fuddly_data_folder, Database.DEFAULT_DB_NAME) + def _is_valid(self, connection, cursor): valid = False with connection: @@ -169,10 +191,10 @@ def _is_valid(self, connection, cursor): tables = filter(lambda x: not x.startswith('sqlite'), tables) for t in tables: cur.execute('select * from {!s}'.format(t)) - ref_names = list(map(lambda x: x[0], cur.description)) + self._ref_names[t] = list(map(lambda x: x[0], cur.description)) cursor.execute('select * from {!s}'.format(t)) names = list(map(lambda x: x[0], cursor.description)) - if ref_names != names: + if self._ref_names[t] != names: valid = False break else: @@ -180,6 +202,9 @@ def _is_valid(self, connection, cursor): return valid + def column_names_from(self, table): + return self._ref_names[table] + def _sql_handler(self): if os.path.isfile(self.fmk_db_path): connection = sqlite3.connect(self.fmk_db_path, detect_types=sqlite3.PARSE_DECLTYPES) @@ -260,7 +285,7 @@ def _stop_sql_handler(self): self._sql_handler_thread.join() - def submit_sql_stmt(self, stmt, params=None, outcome_type=None, error_msg=''): + def submit_sql_stmt(self, stmt, params=None, outcome_type: Optional[int] = None, error_msg=''): """ This method is the only one that should submit request to the threaded SQL handler. It is also synchronized to guarantee request order (especially needed when you wait for @@ -285,7 +310,7 @@ def submit_sql_stmt(self, stmt, params=None, outcome_type=None, error_msg=''): # If we care about outcomes, then we are sure to get outcomes from the just # submitted SQL statement as this method is 'synchronized'. while not self._sql_stmt_handled.is_set(): - self._sql_stmt_handled.wait(0.1) + self._sql_stmt_handled.wait(0.001) self._sql_stmt_handled.clear() with self._sql_stmt_outcome_lock: @@ -294,6 +319,8 @@ def submit_sql_stmt(self, stmt, params=None, outcome_type=None, error_msg=''): return ret def start(self): + self.config = config("Database", path=[gr.config_folder]) + if self._sql_handler_thread is not None: return @@ -320,9 +347,15 @@ def enable(self): def disable(self): self.enabled = False + def is_enabled(self): + return self.enabled + + def flush_feedback(self): + self.last_feedback = {} + self.feedback_trail = {} + def flush_current_feedback(self): self.last_feedback = {} - self.last_feedback_sources_names = {} def execute_sql_statement(self, sql_stmt, params=None): return self.submit_sql_stmt(sql_stmt, params=params, outcome_type=Database.OUTCOME_DATA) @@ -368,15 +401,55 @@ def insert_data(self, dtype, dm_name, raw_data, sz, sent_date, ack_date, if self._data_id is None: d_id = self.submit_sql_stmt(stmt, params=params, outcome_type=Database.OUTCOME_ROWID, - error_msg=err_msg) + error_msg=err_msg) self._data_id = d_id else: self.submit_sql_stmt(stmt, params=params, error_msg=err_msg) self._data_id += 1 + self._current_sent_date = sent_date + return self._data_id + def get_next_data_id(self, prev_id=None): + if prev_id is None and self._data_id is None: + return None + elif prev_id is not None: + return prev_id + 1 + elif self._data_id is not None: + return self._data_id + 1 + + def insert_async_data(self, dtype, dm_name, raw_data, sz, sent_date, + target_ref, prj_name, current_data_id=None): + + if not self.enabled: + return None + + blob = sqlite3.Binary(raw_data) + + stmt = "INSERT INTO ASYNC_DATA(CURRENT_DATA_ID,TYPE,DM_NAME,CONTENT,SIZE,SENT_DATE,"\ + "TARGET,PRJ_NAME)"\ + " VALUES(?,?,?,?,?,?,?,?)" + + if self._current_sent_date is not None: + # We do not associate an async data to the last data_id if + # it is sent more than 60 seconds after + if (sent_date - self._current_sent_date).total_seconds() > self.config.async_data.after_data_id: + data_id = None + else: + data_id = self._data_id if current_data_id is None else current_data_id + else: + data_id = self._data_id if current_data_id is None else current_data_id + + params = (data_id, dtype, dm_name, blob, sz, sent_date, str(target_ref), prj_name) + err_msg = 'while inserting a value into table ASYNC_DATA!' + + self.submit_sql_stmt(stmt, params=params, outcome_type=None, error_msg=err_msg) + + return + + def insert_steps(self, data_id, step_id, dmaker_type, dmaker_name, data_id_src, user_input, info): if not self.enabled: @@ -394,19 +467,32 @@ def insert_steps(self, data_id, step_id, dmaker_type, dmaker_name, data_id_src, def insert_feedback(self, data_id, source, timestamp, content, status_code=None): + if self.feedback_trail_init_ts is None: + self.feedback_trail_init_ts = timestamp + + # timestamp could be None, in this case we ignore the following condition + if timestamp is not None and \ + timestamp - self.feedback_trail_init_ts > timedelta(seconds=self.feedback_trail_time_window): + self.feedback_trail = {} + self.feedback_trail_init_ts = timestamp + if not isinstance(source, FeedbackSource): source = FeedbackSource(source) if source not in self.last_feedback: self.last_feedback[source] = [] - self.last_feedback[source].append( - { - 'timestamp': timestamp, - 'content': content, - 'status': status_code - } - ) + if source not in self.feedback_trail: + self.feedback_trail[source] = [] + + fbk_entry = { + 'timestamp': timestamp, + 'content': content, + 'status': status_code + } + + self.last_feedback[source].append(fbk_entry) + self.feedback_trail[source].append(fbk_entry) if self.current_project: self.current_project.trigger_feedback_handlers(source, timestamp, content, status_code) @@ -423,17 +509,19 @@ def insert_feedback(self, data_id, source, timestamp, content, status_code=None) err_msg = 'while inserting a value into table FEEDBACK!' self.submit_sql_stmt(stmt, params=params, error_msg=err_msg) - def iter_last_feedback_entries(self, source=None): - last_fbk = self.last_feedback.items() + + def iter_feedback_entries(self, last=True, source=None): + feedback = copy.copy(self.last_feedback if last else self.feedback_trail) if source is None: - for src, fbks in last_fbk: + for src, fbks in feedback.items(): for item in fbks: status = item['status'] ts = item['timestamp'] content = item['content'] yield src, status, ts, content else: - for item in self.last_feedback[source]: + fbk_from_src = self.last_feedback[source] if last else self.feedback_trail[source] + for item in fbk_from_src: status = item['status'] ts = item['timestamp'] content = item['content'] @@ -477,7 +565,7 @@ def fetch_data(self, start_id=1, end_id=-1): ''' SELECT DATA.ID, DATA.CONTENT, DATA.TYPE, DMAKERS.NAME, DATA.DM_NAME FROM DATA INNER JOIN DMAKERS - ON DATA.TYPE = DMAKERS.TYPE AND DMAKERS.CLONE_TYPE IS NULL + ON DATA.TYPE = DMAKERS.TYPE AND DATA.DM_NAME = DMAKERS.DM_NAME AND DMAKERS.CLONE_TYPE IS NULL WHERE DATA.ID >= {sid:d} {ign_eid:s} AND DATA.ID <= {eid:d} UNION ALL SELECT DATA.ID, DATA.CONTENT, DMAKERS.CLONE_TYPE AS TYPE, DMAKERS.CLONE_NAME AS NAME, @@ -513,7 +601,7 @@ def check_data_existence(self, data_id, colorized=True): return data def display_data_info(self, data_id, with_data=False, with_fbk=False, with_fmkinfo=True, - with_analysis=True, + with_analysis=True, with_async_data=False, fbk_src=None, limit_data_sz=None, page_width=100, colorized=True, raw=False, decoding_hints=None, dm_list=None): @@ -533,11 +621,22 @@ def display_data_info(self, data_id, with_data=False, with_fbk=False, with_fmkin "ORDER BY STEP_ID ASC;".format(data_id=data_id) ) - if not steps: - print(colorize("*** BUG with data ID '{:d}' (data should always have at least 1 step) " - "***".format(data_id), - rgb=Color.ERROR)) - return + async_data = self.execute_sql_statement( + f"SELECT ID, CURRENT_DATA_ID, TYPE, DM_NAME, CONTENT, SIZE, SENT_DATE," + f" TARGET, PRJ_NAME FROM ASYNC_DATA " + f"WHERE PRJ_NAME == ?" + f" AND ( CURRENT_DATA_ID == ?" + # If current_data_id is null, we only keep async data which were + # sent no more than 5 seconds before the requested data_id. + # And we ignore the ones (current_data_id is null) that have been sent after + # (because it means the framework was reset) + f" OR ( CURRENT_DATA_ID IS NULL" + f" AND CAST((JulianDay(?)-JulianDay(SENT_DATE))*24*60*60 AS INTEGER) " + f" < {self.config.async_data.before_data_id}" + f" AND (JulianDay(?)-JulianDay(SENT_DATE)) >= 0 )" + f" );", + params=(prj, data_id, sent_date, sent_date) + ) if fbk_src: feedback = self.execute_sql_statement( @@ -566,17 +665,27 @@ def display_data_info(self, data_id, with_data=False, with_fbk=False, with_fmkin "ORDER BY ERROR DESC;".format(data_id=data_id) ) + fbk_timeout_info = self.execute_sql_statement( + 'SELECT CONTENT, DATE FROM FMKINFO ' + 'WHERE CONTENT REGEXP "feedback timeout =" ' + 'ORDER BY DATE DESC;' + ) + analysis_records = self.execute_sql_statement( "SELECT CONTENT, DATE, IMPACT FROM ANALYSIS " "WHERE DATA_ID == {data_id:d} " "ORDER BY DATE ASC;".format(data_id=data_id) ) - def search_dm(data_model_name): + def search_dm(data_model_name, load_arg): for dm in dm_list: if dm.name == data_model_name: dm.load_data_model(load_arg) - return dm.decode + + def decode_wrapper(*args, **kwargs): + return dm.decode(*args, **kwargs)[1] + + return decode_wrapper else: print(colorize("*** ERROR: No available data model matching this database entry " "[requested data model: '{:s}'] ***".format(data_model_name), @@ -589,15 +698,16 @@ def search_dm(data_model_name): fbk_decoder_func = None if decoding_hints is not None: load_arg, decode_data, decode_fbk, user_atom_name, user_fbk_atom_name, forced_fbk_decoder = decoding_hints - decoder_func = search_dm(dm_name) - if decoder_func is None: - decode_data, decode_fbk = False, False - if forced_fbk_decoder: - fbk_decoder_func = search_dm(forced_fbk_decoder) - if fbk_decoder_func is None: - decode_fbk = False - else: - fbk_decoder_func = decoder_func + if decode_data or decode_fbk: + decoder_func = search_dm(dm_name, load_arg) + if decoder_func is None: + decode_data = False + if decode_fbk: + if forced_fbk_decoder: + fbk_decoder_func = search_dm(forced_fbk_decoder, load_arg) + else: + fbk_decoder_func = decoder_func + decode_fbk = fbk_decoder_func is not None line_pattern = '-' * page_width data_id_pattern = " Data ID #{:d} ".format(data_id) @@ -636,6 +746,21 @@ def search_dm(data_model_name): prt(msg) + msg = '' + for idx, fbkt in enumerate(fbk_timeout_info, start=1): + content, tstamp = fbkt + if sent_date >= tstamp: + parsed = self.fbk_timeout_re.match(content) + val = parsed.group(1) + 's' if parsed else 'FmkDB ERROR' + break + else: + val = 'Not found in FmkDB (default value was most likely used)' + + msg += colorize("\n Feedback Timeout: ", rgb=Color.FMKINFO) + msg += colorize(f"{val}", rgb=Color.FMKSUBINFO) + msg += colorize('\n' + line_pattern, rgb=Color.NEWLOGENTRY) + prt(msg) + def handle_dmaker(dmk_pattern, info, dmk_type, dmk_name, name_sep_sz, ui, id_src=None): msg = '' @@ -661,37 +786,38 @@ def handle_dmaker(dmk_pattern, info, dmk_type, dmk_name, name_sep_sz, ui, id_src return msg - msg = '' - first_pass = True - prefix_sz = 7 - name_sep_sz = len(data_type) - for _, _, dmk_type, _, _, _, _ in steps: - dmk_type_sz = 0 if dmk_type is None else len(dmk_type) - name_sep_sz = dmk_type_sz if dmk_type_sz > name_sep_sz else name_sep_sz - sid = 1 - for _, step_id, dmk_type, dmk_name, id_src, ui, info in steps: - if first_pass: - if dmk_type is None: - assert (id_src is not None) - continue - else: - first_pass = False - msg += colorize("\n Step #{:d}:".format(sid), rgb=Color.FMKINFOGROUP) - if dmk_type != data_type: - msg += colorize("\n |_ Generator: ", rgb=Color.FMKINFO) - msg += colorize(str(data_type), rgb=Color.FMKSUBINFO) - sid += 1 + if steps: + msg = '' + first_pass = True + prefix_sz = 7 + name_sep_sz = len(data_type) + for _, _, dmk_type, _, _, _, _ in steps: + dmk_type_sz = 0 if dmk_type is None else len(dmk_type) + name_sep_sz = dmk_type_sz if dmk_type_sz > name_sep_sz else name_sep_sz + sid = 1 + for _, step_id, dmk_type, dmk_name, id_src, ui, info in steps: + if first_pass: + if dmk_type is None: + assert (id_src is not None) + continue + else: + first_pass = False msg += colorize("\n Step #{:d}:".format(sid), rgb=Color.FMKINFOGROUP) - msg += handle_dmaker('Disruptor', info, dmk_type, dmk_name, len(data_type), ui) + if dmk_type != data_type: + msg += colorize("\n |_ Generator: ", rgb=Color.FMKINFO) + msg += colorize(str(data_type), rgb=Color.FMKSUBINFO) + sid += 1 + msg += colorize("\n Step #{:d}:".format(sid), rgb=Color.FMKINFOGROUP) + msg += handle_dmaker('Disruptor', info, dmk_type, dmk_name, len(data_type), ui) + else: + msg += handle_dmaker('Generator', info, dmk_type, dmk_name, name_sep_sz, ui, + id_src=id_src) else: - msg += handle_dmaker('Generator', info, dmk_type, dmk_name, name_sep_sz, ui, - id_src=id_src) - else: - msg += colorize("\n Step #{:d}:".format(sid), rgb=Color.FMKINFOGROUP) - msg += handle_dmaker('Disruptor', info, dmk_type, dmk_name, name_sep_sz, ui) - sid += 1 - msg += colorize('\n' + line_pattern, rgb=Color.NEWLOGENTRY) - prt(msg) + msg += colorize("\n Step #{:d}:".format(sid), rgb=Color.FMKINFOGROUP) + msg += handle_dmaker('Disruptor', info, dmk_type, dmk_name, name_sep_sz, ui) + sid += 1 + msg += colorize('\n' + line_pattern, rgb=Color.NEWLOGENTRY) + prt(msg) msg = '' for idx, info in enumerate(analysis_records, start=1): @@ -763,6 +889,32 @@ def handle_dmaker(dmk_pattern, info, dmk_type, dmk_name, name_sep_sz, ui, id_src msg += data_content msg += colorize('\n' + line_pattern, rgb=Color.NEWLOGENTRY) + if with_async_data and async_data: + for async_data_id, curr_data_id, dtype, dm_name, content, size, async_date, target, prj in async_data: + add_info = f' --> sent before Data ID #{data_id}' if curr_data_id is None else '' + date_str = async_date.strftime("%d/%m/%Y - %H:%M:%S.%f") if async_date else 'None' + msg += colorize(f'\n Async Data ID #{async_data_id}', rgb=Color.FMKINFOGROUP) + msg += colorize(f' (', rgb=Color.FMKINFOGROUP) + msg += colorize(f'{date_str}', rgb=Color.DATE) + msg += colorize(f')', rgb=Color.FMKINFOGROUP) + msg += colorize(add_info, rgb=Color.WARNING) + msg += colorize(f'\n | Type:', rgb=Color.FMKINFO) + msg += colorize(f' {dtype}', rgb=Color.FMKSUBINFO) + msg += colorize(f' | DM:', rgb=Color.FMKINFO) + msg += colorize(f' {dm_name}', rgb=Color.FMKSUBINFO) + msg += colorize(f' | Size:', rgb=Color.FMKINFO) + msg += colorize(f' {size} Bytes', rgb=Color.FMKSUBINFO) + msg += colorize(f'\n | Target:', rgb=Color.FMKINFO) + msg += colorize(f' {target}\n', rgb=Color.FMKSUBINFO) + + content = gr.unconvert_from_internal_repr(content) + content = self._handle_binary_content(content, sz_limit=limit_data_sz, raw=raw, + colorized=colorized) + msg += content + + if async_data: + msg += colorize('\n' + line_pattern, rgb=Color.NEWLOGENTRY) + if with_fbk: for src, tstamp, status, content in feedback: formatted_ts = None if tstamp is None else tstamp.strftime("%d/%m/%Y - %H:%M:%S.%f") @@ -796,10 +948,7 @@ def handle_dmaker(dmk_pattern, info, dmk_type, dmk_name, name_sep_sz, ui, id_src def _handle_binary_content(self, content, sz_limit=None, raw=False, colorized=True): colorize = self._get_color_function(colorized) - if sys.version_info[0] > 2: - content = content if not raw else '{!a}'.format(content) - else: - content = content if not raw else repr(content) + content = content if not raw else '{!a}'.format(content) if sz_limit is not None and len(content) > sz_limit: content = content[:sz_limit] @@ -809,7 +958,7 @@ def _handle_binary_content(self, content, sz_limit=None, raw=False, colorized=Tr def display_data_info_by_date(self, start, end, with_data=False, with_fbk=False, with_fmkinfo=True, - with_analysis=True, + with_analysis=True, with_async_data=False, fbk_src=None, prj_name=None, limit_data_sz=None, raw=False, page_width=100, colorized=True, decoding_hints=None, dm_list=None): @@ -834,6 +983,7 @@ def display_data_info_by_date(self, start, end, with_data=False, with_fbk=False, self.display_data_info(data_id, with_data=with_data, with_fbk=with_fbk, with_fmkinfo=with_fmkinfo, with_analysis=with_analysis, + with_async_data=with_async_data, fbk_src=fbk_src, limit_data_sz=limit_data_sz, raw=raw, page_width=page_width, colorized=colorized, @@ -843,7 +993,7 @@ def display_data_info_by_date(self, start, end, with_data=False, with_fbk=False, rgb=Color.ERROR)) def display_data_info_by_range(self, first_id, last_id, with_data=False, with_fbk=False, with_fmkinfo=True, - with_analysis=True, + with_analysis=True, with_async_data=False, fbk_src=None, prj_name=None, limit_data_sz=None, raw=False, page_width=100, colorized=True, decoding_hints=None, dm_list=None): @@ -868,6 +1018,7 @@ def display_data_info_by_range(self, first_id, last_id, with_data=False, with_fb data_id = rec[0] self.display_data_info(data_id, with_data=with_data, with_fbk=with_fbk, with_fmkinfo=with_fmkinfo, with_analysis=with_analysis, + with_async_data=with_async_data, fbk_src=fbk_src, limit_data_sz=limit_data_sz, raw=raw, page_width=page_width, colorized=colorized, @@ -937,6 +1088,7 @@ def export_data(self, first, last=None, colorized=True): base_dir = gr.exported_data_folder prev_export_date = None export_cpt = 0 + data_id = first for rec in records: data_id, data_type, dm_name, sent_date, content = rec @@ -954,14 +1106,15 @@ def export_data(self, first, last=None, colorized=True): else: export_cpt += 1 - export_fname = '{typ:s}_{date:s}_{cpt:0>2d}.{ext:s}'.format( + export_fname = '{typ:s}_ID{did:d}_{date:s}_{cpt:0>2d}.{ext:s}'.format( date=current_export_date, cpt=export_cpt, ext=file_extension, - typ=data_type) + typ=data_type, + did=data_id) export_full_fn = os.path.join(base_dir, dm_name, export_fname) - ensure_dir(export_full_fn) + gr.ensure_dir(export_full_fn) with open(export_full_fn, 'wb') as fd: fd.write(content) @@ -969,6 +1122,8 @@ def export_data(self, first, last=None, colorized=True): print(colorize("Data ID #{:d} --> {:s}".format(data_id, export_full_fn), rgb=Color.FMKINFO)) + data_id += 1 + else: print(colorize("*** ERROR: The provided DATA IDs do not exist ***", rgb=Color.ERROR)) @@ -1007,6 +1162,11 @@ def remove_data(self, data_id, colorized=True): rgb=Color.FMKINFO)) + def shrink_db(self): + self.execute_sql_statement( + "VACUUM;" + ) + def get_project_record(self, prj_name=None): if prj_name: prj_records = self.execute_sql_statement( @@ -1023,22 +1183,24 @@ def get_project_record(self, prj_name=None): return prj_records - def get_data_with_impact(self, prj_name=None, fbk_src=None, display=True, verbose=False, + def get_data_with_impact(self, prj_name=None, fbk_src=None, fbk_status_formula='? < 0', + display=True, verbose=False, raw_analysis=False, colorized=True): + fbk_status_formula = fbk_status_formula.replace('?', 'STATUS') colorize = self._get_color_function(colorized) if fbk_src: fbk_records = self.execute_sql_statement( - "SELECT DATA_ID, STATUS, SOURCE FROM FEEDBACK " - "WHERE STATUS < 0 and SOURCE REGEXP ?;", + f"SELECT DATA_ID, STATUS, SOURCE FROM FEEDBACK " + f"WHERE {fbk_status_formula} and SOURCE REGEXP ?;", params=(fbk_src,) ) else: fbk_records = self.execute_sql_statement( - "SELECT DATA_ID, STATUS, SOURCE FROM FEEDBACK " - "WHERE STATUS < 0;" + f"SELECT DATA_ID, STATUS, SOURCE FROM FEEDBACK " + f"WHERE {fbk_status_formula};" ) diff --git a/framework/dmhelpers/generic.py b/framework/dmhelpers/generic.py index d7a3f49..f4531af 100644 --- a/framework/dmhelpers/generic.py +++ b/framework/dmhelpers/generic.py @@ -78,8 +78,12 @@ class Custo: # NonTerminal node custo class NTerm: MutableClone = NonTermCusto.MutableClone + CycleClone = NonTermCusto.CycleClone FrozenCopy = NonTermCusto.FrozenCopy CollapsePadding = NonTermCusto.CollapsePadding + DelayCollapsing = NonTermCusto.DelayCollapsing + FullCombinatory = NonTermCusto.FullCombinatory + StickToDefault = NonTermCusto.StickToDefault # Generator node (leaf) custo class Gen: @@ -107,6 +111,8 @@ class Attr: Separator = NodeInternals.Separator + Highlight = NodeInternals.Highlight + LOCKED = NodeInternals.LOCKED DEBUG = NodeInternals.DEBUG @@ -237,7 +243,7 @@ def timestamp(time_format, utc, set_attrs, clear_attrs): def CRC(vt=fvt.INT_str, poly=0x104c11db7, init_crc=0, xor_out=0xFFFFFFFF, rev=True, set_attrs=None, clear_attrs=None, after_encoding=True, freezable=False, - base=16, letter_case='upper', reverse_str=False): + base=16, letter_case='upper', min_sz=4, reverse_str=False): """ Return a *generator* that returns the CRC (in the chosen type) of all the node parameters. (Default CRC is PKZIP CRC32) @@ -257,13 +263,14 @@ def CRC(vt=fvt.INT_str, poly=0x104c11db7, init_crc=0, xor_out=0xFFFFFFFF, rev=Tr base (int): Relevant when ``vt`` is ``INT_str``. Numerical base to use for string representation letter_case (str): Relevant when ``vt`` is ``INT_str``. Letter case for string representation ('upper' or 'lower') + min_sz (int): Relevant when ``vt`` is ``INT_str``. Minimum size of the resulting string. reverse_str (bool): Reverse the order of the string if set to ``True``. """ class Crc(object): unfreezable = not freezable def __init__(self, vt, poly, init_crc, xor_out, rev, set_attrs, clear_attrs, - base=16, letter_case='upper', reverse_str=False): + base=16, letter_case='upper', min_sz=4, reverse_str=False): self.vt = vt self.poly = poly self.init_crc = init_crc @@ -273,6 +280,7 @@ def __init__(self, vt, poly, init_crc, xor_out, rev, set_attrs, clear_attrs, self.clear_attrs = clear_attrs self.base = base self.letter_case = letter_case + self.min_sz = min_sz self.reverse_str = reverse_str def __call__(self, nodes): @@ -295,7 +303,7 @@ def __call__(self, nodes): if issubclass(self.vt, fvt.INT_str): n = Node('cts', value_type=self.vt(values=[result], force_mode=True, base=self.base, letter_case=self.letter_case, - reverse=self.reverse_str)) + reverse=self.reverse_str, min_size=self.min_sz)) else: n = Node('cts', value_type=self.vt(values=[result], force_mode=True)) n.set_semantics(NodeSemantics(['crc'])) @@ -307,7 +315,7 @@ def __call__(self, nodes): vt = MH._validate_int_vt(vt) return Crc(vt, poly, init_crc, xor_out, rev, set_attrs, clear_attrs, - base=base, letter_case=letter_case, reverse_str=reverse_str) + base=base, letter_case=letter_case, min_sz=min_sz, reverse_str=reverse_str) @@ -414,7 +422,7 @@ def __call__(self, helper): raise NotImplementedError('Value type not supported') n = Node('cts', value_type=vtype) - MH._handle_attrs(n, set_attrs, clear_attrs) + MH._handle_attrs(n, self.set_attrs, self.clear_attrs) return n assert(not issubclass(vt, fvt.BitField)) @@ -498,7 +506,7 @@ def __call__(self, nodes, helper): off = nodes[-1].get_subnode_off(idx) n = Node('cts_off', value_type=self.vt(values=[base+off], force_mode=True)) - MH._handle_attrs(n, set_attrs, clear_attrs) + MH._handle_attrs(n, self.set_attrs, self.clear_attrs) return n vt = MH._validate_int_vt(vt) @@ -556,7 +564,11 @@ def __call__(self, node, helper): else: base_node = node - tg_node = base_node[self.path] + tg_node = list(base_node.iter_nodes_by_path(self.path, resolve_generator=True)) + if tg_node: + tg_node = tg_node[0] + else: + raise ValueError('incorrect path: {}'.format(self.path)) if tg_node.is_nonterm(): n = Node('cts', base_node=tg_node, ignore_frozen_state=False) @@ -576,10 +588,74 @@ def __call__(self, node, helper): n = Node('cts', value_type=vtype) n.set_semantics(NodeSemantics(['clone'])) - MH._handle_attrs(n, set_attrs, clear_attrs) + MH._handle_attrs(n, self.set_attrs, self.clear_attrs) return n assert(vt is None or not issubclass(vt, fvt.BitField)) return CopyValue(path, depth, vt, set_attrs, clear_attrs) + +def SELECT(idx=None, path=None, filter_func=None, fallback_node=None, clone=True, + set_attrs=None, clear_attrs=None): + """ + Return a *generator* that select a subnode from a non-terminal node and return it (or a copy + of it depending on the parameter `clone`). If the `path` parameter is provided, the previous + selected node is searched for the `path` in order to return the related subnode instead. The + non-terminal node is selected regarding various criteria provided as parameters. + + Args: + idx (int): if None, the node will be selected randomly, otherwise it should be given + the subnodes position in the non-terminal node, or in the subset of subnodes if the + `filter_func` parameter is provided. + path (str): if provided, it has to be a path identifying a subnode to clone from the + selected node. + filter_func: function to filter the subnodes prior to any selection. + fallback_node (Node): if 'path' does not exist, then the clone node will be the one provided + in this parameter. + clone (bool): [default: True] If True, the returned node will be cloned, otherwise + the original node will be returned. + set_attrs (list): attributes that will be set on the generated node (only if `clone` is True). + clear_attrs (list): attributes that will be cleared on the generated node (only if `clone` is True). + """ + class SelectNode(object): + + def __init__(self, idx=None, path=None, filter_func=None, fallback_node=None, clone=True, + set_attrs=None, clear_attrs=None): + self.set_attrs = set_attrs + self.clear_attrs = clear_attrs + self.idx = idx + self.fallback_node = Node('fallback_node', values=['!FALLBACK!']) if fallback_node is None else fallback_node + self.clone = clone + self.path = path + self.filter_func = filter_func + + def __call__(self, node): + assert node.is_nonterm() + node.freeze() + + if self.filter_func: + filtered_nodes = list(filter(self.filter_func, node.frozen_node_list)) + else: + filtered_nodes = node.frozen_node_list + + if filtered_nodes: + idx = random.randint(0, len(filtered_nodes)-1) if self.idx is None else self.idx + selected_node = filtered_nodes[idx] + if selected_node and self.path: + # print(f'{selected_node.name} {self.path}') + # selected_node.show() + selected_node = selected_node[self.path] + selected_node = self.fallback_node if selected_node is None else selected_node[0] + else: + selected_node = self.fallback_node + + if self.clone: + selected_node = selected_node.get_clone(ignore_frozen_state=False) + selected_node.set_semantics(NodeSemantics(['clone'])) + MH._handle_attrs(selected_node, self.set_attrs, self.clear_attrs) + + return selected_node + + return SelectNode(idx=idx, path=path, filter_func=filter_func, fallback_node=fallback_node, + clone=clone, set_attrs=set_attrs, clear_attrs=clear_attrs) diff --git a/framework/dmhelpers/json.py b/framework/dmhelpers/json.py index 26736e6..6b54cd6 100755 --- a/framework/dmhelpers/json.py +++ b/framework/dmhelpers/json.py @@ -1,5 +1,6 @@ ################################################################################ # +# Copyright 2019 Eric Lacombe # Copyright 2017 Rockwell Collins Inc. # ################################################################################ @@ -27,15 +28,209 @@ import framework.global_resources as gr import uuid -def json_builder(tag_name, sample=None, node_name=None, codec='latin-1', - tag_name_mutable=True, struct_mutable=True, determinist=True): + +def json_model_builder(node_name, schema, struct_mutable=True, determinist=False, + ignore_pattern=False, codec='latin-1', value_suffix='_value'): """ - Helper for modeling an JSON structure. + Helper for modeling an JSON structure from a JSON schema. + + Args: + node_name: name of the node to be created + schema (dict): the JSON schema to be translated to a fuddly node descriptor + struct_mutable (bool): if ``False`` the JSON structure "will not" be mutable, meaning + that each node related to the structure will have its ``Mutable`` attribute cleared. + determinist (bool): if ``False``, the attribute order could change from one retrieved + data to another. + ignore_pattern (bool): if ``True``, the ``pattern`` attribute of ``string`` types will be + ignored + codec (str): codec to be used for generating the JSON structure. + + Returns: + dict: Node-description of the JSON structure. + """ + + if schema is None: + raise DataModelDefinitionError + + sc_type = schema.get('type') + sc_desc = schema.get('description') + + if sc_type == 'object': + properties = schema.get('properties') + if properties is None: + raise DataModelDefinitionError + + required_keys = schema.get('required') + + tag_start = \ + {'name': ('obj_start', uuid.uuid1()), + 'contents': fvt.String(values=['{'], codec=codec), + 'mutable': struct_mutable, 'set_attrs': MH.Attr.Separator} + + tag_end = \ + {'name': ('obj_end', uuid.uuid1()), + 'contents': fvt.String(values=['}'], codec=codec), + 'mutable': struct_mutable, 'set_attrs': MH.Attr.Separator} + + obj_cts = [] + + for key, value in properties.items(): + sep_id = uuid.uuid1() + prop = [ + {'name': ('sep', sep_id), 'contents' : fvt.String(values=['"'], codec=codec), + 'set_attrs': MH.Attr.Separator, 'mutable': struct_mutable}, + {'name': ('key', uuid.uuid1()), 'contents' : fvt.String(values=[key], codec=codec)}, + {'name': ('sep', sep_id)}, + {'name': ('col', uuid.uuid1()), 'contents' : fvt.String(values=[':'], codec=codec), + 'set_attrs': MH.Attr.Separator, 'mutable': struct_mutable} + ] + + prop_value = json_model_builder(node_name=key + value_suffix, schema=value, determinist=determinist, + codec=codec, struct_mutable=struct_mutable, + ignore_pattern=ignore_pattern) + prop.append(prop_value) + if required_keys and key in required_keys: + prop_desc = {'name': (key, uuid.uuid1()), 'contents': prop} + else: + prop_desc = {'name': (key, uuid.uuid1()), 'contents': prop, 'qty': (0,1)} + obj_cts.append(prop_desc) + + obj_desc = \ + {'name': ('attrs', uuid.uuid1()), + 'shape_type': MH.Ordered if determinist else MH.FullyRandom, + 'random': not determinist, + 'separator': {'contents': {'name': ('obj_sep', uuid.uuid1()), + 'contents': fvt.String(values=[',\n'], max_sz=100, + absorb_regexp='\s*,\s*', codec=codec), + 'mutable': struct_mutable, + 'absorb_csts': AbsNoCsts(size=True, regexp=True)}, + 'prefix': False, 'suffix': False, 'unique': False}, + 'contents': obj_cts} + + node_desc = \ + {'name': (node_name, uuid.uuid1()), + 'contents': [tag_start, obj_desc, tag_end]} + + elif sc_type == 'string': + min_len = schema.get('minLength') + max_len = schema.get('maxLength', 30) + pattern = schema.get('pattern') + enum = schema.get('enum') + + format = schema.get('format') + if format == 'ipv4': + pattern = '\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}' + + if pattern is None or ignore_pattern: + str_desc = \ + {'name': ('string', uuid.uuid1()), + 'contents': fvt.String(values=enum, min_sz=min_len, max_sz=max_len, codec=codec, + absorb_regexp=pattern)} + if pattern is not None: + str_desc['absorb_csts'] = AbsNoCsts(size=True, regexp=True) + else: + str_desc = \ + {'name': ('string', uuid.uuid1()), + 'contents': pattern} + + str_desc['semantics'] = node_name[:-len(value_suffix)] + + sep_id = uuid.uuid1() + node_desc = \ + {'name': (node_name, uuid.uuid1()), + 'contents': [ + {'name': ('sep', sep_id), 'contents' : fvt.String(values=['"'], codec=codec), + 'set_attrs': MH.Attr.Separator, 'mutable': struct_mutable}, + str_desc, + {'name': ('sep', sep_id)}, + ]} + + elif sc_type == 'integer': + mini = schema.get('minimum') + ex_mini = schema.get('exclusiveMinimum') + if ex_mini is not None: + mini = ex_mini+1 + maxi = schema.get('maximum') + ex_maxi = schema.get('exclusiveMaximum') + if ex_maxi is not None: + maxi = ex_maxi-1 + + node_desc = \ + {'name': (node_name, uuid.uuid1()), + 'semantics': node_name[:-len(value_suffix)], + 'contents': fvt.INT_str(min=mini, max=maxi)} + + elif sc_type == 'boolean': + node_desc = \ + {'name': (node_name, uuid.uuid1()), + 'semantics': node_name[:-len(value_suffix)], + 'contents': fvt.String(values=['true', 'false'])} + + elif sc_type == 'null': + node_desc = \ + {'name': (node_name, uuid.uuid1()), + 'semantics': node_name[:-len(value_suffix)], + 'contents': fvt.String(values=['null'])} + + elif sc_type == 'array': + + tag_start = \ + {'name': ('array_start', uuid.uuid1()), + 'contents': fvt.String(values=['['], codec=codec), + 'mutable': struct_mutable, 'set_attrs': MH.Attr.Separator} + + tag_end = \ + {'name': ('array_end', uuid.uuid1()), + 'contents': fvt.String(values=[']'], codec=codec), + 'mutable': struct_mutable, 'set_attrs': MH.Attr.Separator} + + items_type= schema.get('items') + if items_type is not None: + item_desc = json_model_builder(node_name='item', schema=items_type, determinist=determinist, + codec=codec, struct_mutable=struct_mutable, + ignore_pattern=ignore_pattern) + else: + item_desc = {'name': ('item', uuid.uuid1()), 'contents': fvt.INT_str()} + + min_items = schema.get('minItems', 0) + max_items = schema.get('maxItems', -1) + + item_desc['qty'] = (min_items, max_items) + + array_desc = \ + {'name': ('items', uuid.uuid1()), + 'shape_type': MH.Ordered if determinist else MH.FullyRandom, + 'random': not determinist, + 'custo_clear': MH.Custo.NTerm.FrozenCopy, + 'separator': {'contents': {'name': ('obj_sep', uuid.uuid1()), + 'contents': fvt.String(values=[','], max_sz=100, + absorb_regexp='\s*,\s*', codec=codec), + 'mutable': struct_mutable, + 'absorb_csts': AbsNoCsts(size=True, regexp=True)}, + 'prefix': False, 'suffix': False, 'unique': False}, + 'contents': [item_desc]} + + node_desc = \ + {'name': (node_name, uuid.uuid1()), + 'contents': [tag_start, array_desc, tag_end]} + + + else: + raise DataModelDefinitionError + + node_desc['description'] = sc_desc + + return node_desc + + +def json_builder(node_name, sample=None, codec='latin-1', + tag_name_mutable=True, struct_mutable=True, determinist=True): + """ + Helper for modeling an JSON structure from JSON samples. Args: - tag_name (str): name of the JSON tag. - sample (dict): the JSON structure to be converted to a fuddly structure node_name (str): name of the node to be created. + sample (dict): the JSON structure to be converted to a fuddly structure codec (str): codec to be used for generating the JSON structure. tag_name_mutable (bool): if ``False``, the tag name will not be mutable, meaning that its ``Mutable`` attribute will be cleared. @@ -70,7 +265,7 @@ def json_builder(tag_name, sample=None, node_name=None, codec='latin-1', assert not isinstance(value, list) if isinstance(value, dict): # If the type of v is a dictionary, build a sub JSON structure for it. - modeled_v.append(json_builder(tag_name + "_" + str(idx)+str(subidx), sample=value)) + modeled_v.append(json_builder(node_name + "_" + str(idx) + str(subidx), sample=value)) else: checked_value = value if gr.is_string_compatible(value) else str(value) modeled_v.append( @@ -101,7 +296,7 @@ def json_builder(tag_name, sample=None, node_name=None, codec='latin-1', 'mutable': struct_mutable, 'name': 'suffix'+str(idx)} ]}) elif isinstance(v, dict): - params.append(json_builder(tag_name + "_" + str(idx), sample=v)) + params.append(json_builder(node_name + "_" + str(idx), sample=v)) elif gr.is_string_compatible(v): params += [ {'name': ('sep', sep_id)}, @@ -143,7 +338,7 @@ def json_builder(tag_name, sample=None, node_name=None, codec='latin-1', 'mutable': struct_mutable, 'set_attrs': MH.Attr.Separator} tag_start_desc = \ - {'name': tag_name if node_name is None else node_name, + {'name': node_name, 'contents': [tag_start_open_desc, tag_start_cts_desc, tag_start_close_desc]} return tag_start_desc diff --git a/framework/encoders.py b/framework/encoders.py index 86bd749..34c7711 100644 --- a/framework/encoders.py +++ b/framework/encoders.py @@ -31,16 +31,16 @@ class Encoder(object): def __init__(self, encoding_arg=None): - self._encoding_arg = encoding_arg + self.encoding_arg = encoding_arg self.reset() def reset(self): - self.init_encoding_scheme(self._encoding_arg) + self.init_encoding_scheme(self.encoding_arg) def __copy__(self): - new_data = type(self)(self._encoding_arg) + new_data = type(self)(self.encoding_arg) new_data.__dict__.update(self.__dict__) - new_data.encoding_arg = copy.copy(self._encoding_arg) + new_data.encoding_arg = copy.copy(self.encoding_arg) return new_data def encode(self, val): @@ -80,6 +80,7 @@ def init_encoding_scheme(self, arg): """ pass + class GZIP_Enc(Encoder): def init_encoding_scheme(self, arg=None): @@ -113,9 +114,8 @@ def init_encoding_scheme(self, arg): Can be individually set to None """ assert isinstance(arg, (tuple, list)) - if sys.version_info[0] > 2: - assert arg[0] is None or isinstance(arg[0], bytes) - assert arg[1] is None or isinstance(arg[1], bytes) + assert arg[0] is None or isinstance(arg[0], bytes) + assert arg[1] is None or isinstance(arg[1], bytes) self.prefix = b'' if arg[0] is None else arg[0] self.suffix = b'' if arg[1] is None else arg[1] self.prefix_sz = 0 if self.prefix is None else len(self.prefix) @@ -141,10 +141,6 @@ def decode(self, val): class GSM7bitPacking_Enc(Encoder): def encode(self, msg): - if sys.version_info[0] > 2: - ORD = lambda x: x - else: - ORD = ord msg_sz = len(msg) l = [] idx = 0 @@ -155,19 +151,15 @@ def encode(self, msg): if off == 0 and off_cpt > 0: c_idx = idx + 1 if c_idx+1 < msg_sz: - l.append((ORD(msg[c_idx])>>off)+((ORD(msg[c_idx+1])<<(7-off))&0x00FF)) + l.append((msg[c_idx]>>off)+((msg[c_idx+1]<<(7-off))&0x00FF)) elif c_idx < msg_sz: - l.append(ORD(msg[c_idx])>>off) + l.append(msg[c_idx]>>off) idx = c_idx + 1 off_cpt += 1 return b''.join(map(lambda x: struct.pack('B', x), l)) def decode(self, msg): - if sys.version_info[0] > 2: - ORD = lambda x: x - else: - ORD = ord msg_sz = len(msg) l = [] c_idx = 0 @@ -179,8 +171,8 @@ def decode(self, msg): l.append(lsb) lsb = 0 if c_idx < msg_sz: - l.append(((ORD(msg[c_idx])<>(7-off) + l.append(((msg[c_idx]<>(7-off) c_idx += 1 off_cpt += 1 diff --git a/framework/error_handling.py b/framework/error_handling.py index 06afd97..b2207a7 100644 --- a/framework/error_handling.py +++ b/framework/error_handling.py @@ -30,8 +30,11 @@ class UserInterruption(Exception): pass class PopulationError(Exception): pass class ExtinctPopulationError(PopulationError): pass +class CrossOverError(PopulationError): pass class DataModelDefinitionError(Exception): pass +class DataModelAccessError(Exception): pass + class ProjectDefinitionError(Exception): pass class RegexParserError(DataModelDefinitionError): pass diff --git a/framework/evolutionary_helpers.py b/framework/evolutionary_helpers.py index 0173cbe..ae43337 100644 --- a/framework/evolutionary_helpers.py +++ b/framework/evolutionary_helpers.py @@ -22,13 +22,17 @@ # ################################################################################ +import math +import re +import functools +import uuid from operator import attrgetter from framework.tactics_helpers import * from framework.global_resources import UI from framework.scenario import * -from framework.error_handling import ExtinctPopulationError, PopulationError - +from framework.error_handling import ExtinctPopulationError, PopulationError, CrossOverError +from framework.data import DataProcess class Population(object): """ Population to be used within an evolutionary scenario """ @@ -61,6 +65,9 @@ def is_final(self): """ Check if the population can still evolve or not """ raise NotImplementedError + def size(self): + return len(self._individuals) + def __len__(self): return len(self._individuals) @@ -85,54 +92,207 @@ def __next__(self): next = __next__ + def __repr__(self): + return self.__class__.__name__ + class Individual(object): """ Represents a population member """ - def __init__(self, fmk, node): + def __init__(self, fmk, data): self._fmk = fmk - self.node = node + self.data = data self.feedback = None + def mutate(self): + raise NotImplementedError + +class CrossoverHelper(object): + + class Operand(object): + + def __init__(self, node): + self.node = node + self.leafs = [] + + for path, node in self.node.iter_paths(): + if node.is_term() and path not in self.leafs: + self.leafs.append(path) + + self.shared = None + + def compute_sub_graphs(self, percentage): + random.shuffle(self.leafs) + self.shared = self.leafs[:int(round(len(self.leafs) * percentage))] + self.shared.sort() + + change = True + while change: + + change = False + index = 0 + length = len(self.shared) + + while index < length: + + current_path = self.shared[index] + + slash_index = current_path[::-1].find('/') + # check if we are dealing with the root node + if slash_index == -1: + index += 1 + continue + + parent_path = current_path[:-current_path[::-1].find('/') - 1] + children_nb = self._count_brothers(index, parent_path) + if children_nb == self.node.get_first_node_by_path(parent_path).cc.get_subnode_qty(): + self._merge_brothers(index, parent_path, children_nb) + change = True + index += 1 + length = len(self.shared) + else: + index += children_nb + + def _count_brothers(self, index, pattern): + count = 1 + p = re.compile(u'^' + pattern + '($|/*)') + for i in range(index + 1, len(self.shared)): + if re.match(p, self.shared[i]) is not None: + count += 1 + return count + + def _merge_brothers(self, index, pattern, length): + for _ in range(0, length, 1): + del self.shared[index] + self.shared.insert(index, pattern) + + @staticmethod + def _swap_nodes(node_1, node_2): + node_2_copy = node_2.get_clone() + node_1_copy = node_1.get_clone() + node_2.set_contents(node_1_copy) + node_1.set_contents(node_2_copy) + + @staticmethod + def _get_nodes(node): + while True: + nodes = [node] if node.is_term() else node.cc.frozen_node_list + if len(nodes) == 1 and not nodes[0].is_term(): + node = nodes[0] + else: + break + return nodes + + @staticmethod + def _add_default_crossover_info(ind_1, ind_2, crossover_desc=''): + crossover_desc = "[" + crossover_desc + "]" if crossover_desc else '' + + crossover_desc = 'Crossover between data ID {} and {} {}'\ + .format(ind_1._data_id, ind_2._data_id, crossover_desc) + + ind_1.altered = True + ind_1.add_info(crossover_desc) + ind_2.altered = True + ind_2.add_info(crossover_desc) + + + @classmethod + def crossover_algo1(cls, ind_1, ind_2): + ind_1_nodes = cls._get_nodes(ind_1.content) + ind_2_nodes = cls._get_nodes(ind_2.content) + + if len(ind_1_nodes) == 0 or len(ind_2_nodes) == 0: + raise CrossOverError + + swap_nb = len(ind_1_nodes) if len(ind_1_nodes) < len(ind_2_nodes) else len(ind_2_nodes) + swap_nb = int(math.ceil(swap_nb / 2.0)) + + random.shuffle(ind_1_nodes) + random.shuffle(ind_2_nodes) + + for i in range(swap_nb): + cls._swap_nodes(ind_1_nodes[i], ind_2_nodes[i]) + + cls._add_default_crossover_info(ind_1, ind_2, crossover_desc='algo1') + return ind_1, ind_2 + + @classmethod + def _crossover_algo2(cls, ind_1, ind_2, percentage_to_share): + ind_1_operand = cls.Operand(ind_1.content) + ind_1_operand.compute_sub_graphs(percentage_to_share) + random.shuffle(ind_1_operand.shared) + + ind_2_operand = cls.Operand(ind_2.content) + ind_2_operand.compute_sub_graphs(1.0 - percentage_to_share) + random.shuffle(ind_2_operand.shared) + + swap_nb = len(ind_1_operand.shared) if len(ind_1_operand.shared) < len(ind_2_operand.shared) else len(ind_2_operand.shared) + + for i in range(swap_nb): + node_1 = ind_1_operand.node.get_first_node_by_path(ind_1_operand.shared[i]) + node_2 = ind_2_operand.node.get_first_node_by_path(ind_2_operand.shared[i]) + cls._swap_nodes(node_1, node_2) + + cls._add_default_crossover_info(ind_1, ind_2, + crossover_desc='algo2, sharing%:{}'.format(percentage_to_share)) + return ind_1, ind_2 + + @classmethod + def get_configured_crossover_algo2(cls, percentage_to_share=None): + """ + Args: + percentage_to_share: Percentage of the nodes to share. + + Returns: func + """ + if percentage_to_share is None: + percentage_to_share = float(random.randint(3, 7)) / 10.0 + elif not (0 < percentage_to_share < 1): + print("Invalid percentage, a float between 0 and 1 need to be provided") + return None + + return functools.partial(cls._crossover_algo2, percentage_to_share=percentage_to_share) class DefaultIndividual(Individual): """ Provide a default implementation of the Individual class """ - def __init__(self, fmk, node): - Individual.__init__(self, fmk, node) + def __init__(self, fmk, data, mutation_order=1): + Individual.__init__(self, fmk, data) self.score = None self.probability_of_survival = None # between 0 and 1 + self.mutation_order = mutation_order - def mutate(self, nb): - data = self._fmk.get_data([('C', UI(nb=nb))], data_orig=Data(self.node)) + def mutate(self): + assert isinstance(self.data.content, Node) + data = self._fmk.process_data([('C', UI(nb=self.mutation_order))], seed=self.data) if data is None: raise PopulationError - assert isinstance(data.content, Node) - self.node = data.content + data.add_info('Mutation applied on data {}'.format(data._data_id)) + self.data = data class DefaultPopulation(Population): """ Provide a default implementation of the Population base class """ - def _initialize(self, init_process, size=100, max_generation_nb=50): + def _initialize(self, init_process, max_size=100, max_generation_nb=50, + crossover_algo=CrossoverHelper.crossover_algo1): """ Configure the population Args: init_process (string): individuals that compose this population will be built using - the provided process. The generic form for a process is: - ``[action_1, (action_2, UI_2), ... action_n]`` - where ``action_N`` can be either: ``dmaker_type_N`` or ``(dmaker_type_N, dmaker_name_N)`` - size (integer): size of the population to manipulate + the provided :class:`framework.data.DataProcess` + max_size (integer): maximum size of the population to manipulate max_generation_nb (integer): criteria used to stop the evolution process + crossover_algo (func): Crossover algorithm to use """ Population._initialize(self) self.DATA_PROCESS = init_process - self.SIZE = size + self.MAX_SIZE = max_size self.MAX_GENERATION_NB = max_generation_nb - self.generation = None + self.crossover_algo = crossover_algo def reset(self): """ Generate the first generation of individuals in a random way """ @@ -141,12 +301,16 @@ def reset(self): self.generation = 1 # individuals initialization - for _ in range(0, self.SIZE): - data = self._fmk.get_data(self.DATA_PROCESS) + cpt = 0 + while cpt < self.MAX_SIZE or self.MAX_SIZE == -1: + cpt += 1 + data = self._fmk.handle_data_desc(self.DATA_PROCESS, resolve_dataprocess=True, + save_generator_seed=False) if data is None: - raise PopulationError - node = data.content - self._individuals.append(DefaultIndividual(self._fmk, node)) + break + data.add_info('Data generated from the DataProcess provided for the population initialization:') + data.add_info(' |_ {!s}'.format(self.DATA_PROCESS)) + self._individuals.append(DefaultIndividual(self._fmk, data)) def _compute_scores(self): """ Compute the scores of each individuals """ @@ -174,29 +338,28 @@ def _kill(self): def _mutate(self): """ Operates three bit flips on each individual """ for individual in self._individuals: - individual.mutate(3) - print(str(individual) + "mutated !!") + individual.mutate() def _crossover(self): - """ Compensates the kills through the usage of the tCOMB disruptor """ + """ Compensates the kills through the usage of the COMB disruptor """ random.shuffle(self._individuals) current_size = len(self._individuals) i = 0 - while len(self._individuals) < self.SIZE and i < int(current_size / 2): - ind_1 = self._individuals[i].node - ind_2 = self._individuals[i+1].node.get_clone() - - - while True: - data = self._fmk.get_data([('tCOMB', UI(node=ind_2))], data_orig=Data(ind_1)) - if data is None or data.is_unusable(): - break - else: - self._individuals.append(DefaultIndividual(self._fmk, data.content)) - - i += 2 + while len(self._individuals) < self.MAX_SIZE and i < int(current_size / 2): + ind_1 = self._individuals[i].data + ind_2 = copy.copy(self._individuals[i+1].data) + + try: + ind_1, ind_2 = self.crossover_algo(ind_1, ind_2) + except CrossOverError: + continue + else: + self._individuals.append(DefaultIndividual(self._fmk, ind_1)) + self._individuals.append(DefaultIndividual(self._fmk, ind_2)) + finally: + i += 2 def evolve(self): """ Describe the evolutionary process """ @@ -216,6 +379,10 @@ def evolve(self): def is_final(self): return self.generation == self.MAX_GENERATION_NB + def __repr__(self): + return self.__class__.__name__ + '[max_sz={}, max_gen={}]'.format(self.MAX_SIZE, self.MAX_GENERATION_NB) + + class EvolutionaryScenariosFactory(object): @@ -236,14 +403,14 @@ def build(fmk, name, population_cls, args): population = population_cls(fmk, **args) def cbk_after(env, current_step, next_step, fbk_gate): - print("Callback after") - # set the feedback of the last played individual population[population.index - 1].feedback = list(fbk_gate) return True - step = Step(data_desc=DataProcess(process=[('POPULATION', UI(population=population))])) + generator_name = 'POPULATION#{!s}'.format(random.randint(1,100000)) + step = Step(data_desc=DataProcess(process=[(generator_name, + UI(population=population))])) step.connect_to(step, cbk_after_fbk=cbk_after) return Scenario(name, anchor=step) diff --git a/framework/fmk_db.sql b/framework/fmk_db.sql index e8c0452..8159bd0 100644 --- a/framework/fmk_db.sql +++ b/framework/fmk_db.sql @@ -107,6 +107,18 @@ CREATE TABLE ANALYSIS ( IMPACT BOOLEAN ); +CREATE TABLE ASYNC_DATA ( + ID INTEGER PRIMARY KEY ASC AUTOINCREMENT, + CURRENT_DATA_ID INTEGER REFERENCES DATA (ID), + TYPE TEXT, + DM_NAME TEXT REFERENCES DATAMODEL (NAME), + CONTENT BLOB, + SIZE INTEGER, + SENT_DATE TIMESTAMP, + TARGET TEXT, + PRJ_NAME TEXT REFERENCES PROJECT (NAME) +); + CREATE VIEW STATS AS SELECT TYPE, sum(CPT) as TOTAL FROM ( diff --git a/framework/fuzzing_primitives.py b/framework/fuzzing_primitives.py index 61162bd..d818276 100644 --- a/framework/fuzzing_primitives.py +++ b/framework/fuzzing_primitives.py @@ -61,10 +61,10 @@ def __init__(self, root_node, node_consumer, make_determinist=False, make_random self._root_node.make_finite(all_conf=True, recursive=True) if make_determinist: - assert(not make_random) + assert not make_random self._root_node.make_determinist(all_conf=True, recursive=True) elif make_random: - assert(not make_determinist) + assert not make_determinist self._root_node.make_random(all_conf=True, recursive=True) self._root_node.freeze() @@ -74,7 +74,12 @@ def __init__(self, root_node, node_consumer, make_determinist=False, make_random assert(self._max_steps > 0 or self._max_steps == -1) - self.ic = dm.NodeInternalsCriteria(mandatory_attrs=[dm.NodeInternals.Mutable, dm.NodeInternals.Finite]) + if node_consumer.ignore_mutable_attr: + mattr = [dm.NodeInternals.Finite] + else: + mattr = [dm.NodeInternals.Mutable, dm.NodeInternals.Finite] + + self.ic = dm.NodeInternalsCriteria(mandatory_attrs=mattr) self.triglast_ic = dm.NodeInternalsCriteria(mandatory_custo=[dm.GenFuncCusto.TriggerLast]) self.consumed_node_path = None @@ -91,24 +96,27 @@ def __iter__(self): self._cpt = 1 gen = self.walk_graph_rec([self._root_node], structure_has_changed=False, - consumed_nodes=set(), parent_node=self._root_node) + consumed_nodes=set(), parent_node=self._root_node, + consumer=self._consumer) for consumed_node, orig_node_val in gen: - self._root_node.freeze() + self._root_node.freeze(resolve_csp=True) - if self._cpt >= self._initial_step: - self.consumed_node_path = consumed_node.get_path_from(self._root_node) - if self.consumed_node_path == None: - # 'consumed_node_path' can be None if - # consumed_node is not part of the frozen rnode - # (it may however exist when rnode is not - # frozen). This situation can trigger in some - # specific situations related to the use of - # existence conditions within a data model. Thus, - # in this case we skip the just generated case as - # nothing is visible. - continue + self.consumed_node_path = consumed_node.get_path_from(self._root_node) + if self.consumed_node_path == None: + # 'consumed_node_path' can be None if + # consumed_node is not part of the frozen rnode + # (it may however exist when rnode is not + # frozen). This situation can trigger in some + # specific situations related to the use of + # existence conditions within a data model. Thus, + # in this case we skip the just generated case as + # nothing is visible. + continue + if self._cpt >= self._initial_step: yield self._root_node, consumed_node, orig_node_val, self._cpt + else: + pass if self._max_steps != -1 and self._cpt >= (self._max_steps+self._initial_step-1): self._cpt += 1 @@ -129,16 +137,17 @@ def __iter__(self): return - def _do_reset(self, node): - last_gen = self._root_node.get_reachable_nodes(internals_criteria=self.triglast_ic) + def _do_reset(self, node, consumer): + last_gen = self._root_node.get_reachable_nodes(internals_criteria=self.triglast_ic, + resolve_generator=True) for n in last_gen: n.unfreeze(ignore_entanglement=True) node.unfreeze(recursive=False) # self._root_node.unfreeze(recursive=True, dont_change_state=True) node.unfreeze(recursive=True, dont_change_state=True, ignore_entanglement=True) - self._consumer.do_after_reset(node) + consumer.do_after_reset(node) - def walk_graph_rec(self, node_list, structure_has_changed, consumed_nodes, parent_node): + def walk_graph_rec(self, node_list, structure_has_changed, consumed_nodes, parent_node, consumer): reset = False guilty = None @@ -168,7 +177,8 @@ def walk_graph_rec(self, node_list, structure_has_changed, consumed_nodes, paren # For each node we look for direct subnodes fnodes = node.get_reachable_nodes(internals_criteria=self.ic, exclude_self=True, - respect_order=self._consumer.respect_order, relative_depth=1) + respect_order=consumer.respect_order, + resolve_generator=True, relative_depth=1) if DEBUG: DEBUG_PRINT('--(2)-> Node:' + node.name + ', exhausted:' + repr(node.is_exhausted()), level=2) for e in fnodes: @@ -179,7 +189,7 @@ def walk_graph_rec(self, node_list, structure_has_changed, consumed_nodes, paren # call ourselves recursively with the list of subnodes if fnodes: generator = self.walk_graph_rec(fnodes, structure_has_changed, consumed_nodes, - parent_node=node) + parent_node=node, consumer=consumer) for consumed_node, orig_node_val in generator: yield consumed_node, orig_node_val # YIELD @@ -189,8 +199,14 @@ def walk_graph_rec(self, node_list, structure_has_changed, consumed_nodes, paren # for possible uses/modifications. This is performed within our # method node_consumer_helper(). if perform_second_step: + + if consumer.consider_side_effects_on_sibbling: + original_parent_node_list = None + if parent_node.is_nonterm(): + original_parent_node_list = set(parent_node.subnodes_set).intersection(set(parent_node.frozen_node_list)) + consumer_gen = self.node_consumer_helper(node, structure_has_changed, consumed_nodes, - parent_node=parent_node) + parent_node=parent_node, consumer=consumer) for consumed_node, orig_node_val, reset, ignore_node in consumer_gen: DEBUG_PRINT(" [ reset: {!r:s} | ignore_node: {!r:s} | " \ @@ -209,7 +225,7 @@ def walk_graph_rec(self, node_list, structure_has_changed, consumed_nodes, paren if ignore_node and reset: perform_second_step = False again = True - self._do_reset(node) + self._do_reset(node, consumer) break elif ignore_node and not reset: perform_second_step = False @@ -218,7 +234,7 @@ def walk_graph_rec(self, node_list, structure_has_changed, consumed_nodes, paren elif reset: perform_second_step = True again = True - self._do_reset(node) + self._do_reset(node, consumer) break else: perform_second_step = True @@ -226,26 +242,60 @@ def walk_graph_rec(self, node_list, structure_has_changed, consumed_nodes, paren yield consumed_node, orig_node_val # YIELD + if consumer.consider_side_effects_on_sibbling: + if parent_node.is_nonterm(): + parent_node.unfreeze(recursive=True, reevaluate_constraints=True, ignore_entanglement=True) + parent_node.freeze() + new_parent_node_list = set(parent_node.subnodes_set).intersection(set(parent_node.frozen_node_list)) + + if original_parent_node_list != new_parent_node_list: + fnodes = parent_node.get_reachable_nodes(internals_criteria=self.ic, exclude_self=True, + respect_order=consumer.respect_order, + resolve_generator=True, relative_depth=1) + if fnodes: + fnodes.remove(node) + # TODO: check if there is a need to instantiate a copy of the + # current consumer with a specific state. + # For BasicVisitor, there is no need, as the only state is + # the .firstcall value. And we don't need to reset it, because + # when we reach this code we already yield once the node value that + # we will walk through. Thus, firstcall need to stay to False, + # otherwise we will yield the same value twice. + # + # Needed for tTYPE: + # - new_consumer = copy.copy(consumer) with a special reset (TBC) + # - tTYPE or the walker need to be changed somehow so that it could discover + # other NT shapes linked to node existence. + generator = self.walk_graph_rec(fnodes, structure_has_changed, consumed_nodes, + parent_node=parent_node, consumer=consumer) + for consumed_node, orig_node_val in generator: + yield consumed_node, orig_node_val # YIELD + + # We reach this case if the consumer is not interested # with 'node'. Then if the node is not exhausted we # may have new cases where the consumer will find # something (assuming the consumer accepts to reset). - elif self._consumer.need_reset(node): # and not node.is_exhausted(): + elif consumer.need_reset(node) and node.is_attr_set(dm.NodeInternals.Mutable): again = False if node.is_exhausted() else True # Not consumed so we don't unfreeze() with recursive=True - self._do_reset(node) + self._do_reset(node, consumer) else: again = False + if consumer.consider_side_effects_on_sibbling: + parent_node.unfreeze(recursive=True, reevaluate_constraints=True, ignore_entanglement=True) + parent_node.freeze() + if node.is_nonterm(): structure_has_changed = node.cc.structure_will_change() - if structure_has_changed and self._consumer.need_reset_when_structure_change: + if structure_has_changed and consumer.need_reset_when_structure_change: structure_has_changed = False idx = node_list.index(node) - gen = self.walk_graph_rec(node_list[:idx], False, set(), parent_node=parent_node) + gen = self.walk_graph_rec(node_list[:idx], False, set(), parent_node=parent_node, consumer=consumer) for consumed_node, orig_node_val in gen: yield consumed_node, orig_node_val # YIELD @@ -261,7 +311,7 @@ def walk_graph_rec(self, node_list, structure_has_changed, consumed_nodes, paren # if n in consumed_nodes: # consumed_nodes.remove(n) - elif structure_has_changed and not self._consumer.need_reset_when_structure_change: + elif structure_has_changed and not consumer.need_reset_when_structure_change: structure_has_changed = False # print('--> ', node.name, node, node.is_attr_set(dm.NodeInternals.Mutable), 'exhausted: ', node.is_exhausted()) consumed_nodes = set() @@ -269,10 +319,11 @@ def walk_graph_rec(self, node_list, structure_has_changed, consumed_nodes, paren return - def node_consumer_helper(self, node, structure_has_changed, consumed_nodes, parent_node): + def node_consumer_helper(self, node, structure_has_changed, consumed_nodes, parent_node, + consumer): def _do_if_not_interested(node, orig_node_val): - reset = self._consumer.need_reset(node) + reset = consumer.need_reset(node) if reset and not node.is_exhausted(): return node, orig_node_val, True, True # --> x, x, reset, ignore_node elif reset and node.is_exhausted(): @@ -287,12 +338,12 @@ def _do_if_not_interested(node, orig_node_val): not_recovered = False consume_called_again = False - if self._consumer.interested_by(node): + if consumer.interested_by(node): if node in consumed_nodes: go_on = False else: - self._consumer.save_node(node) - go_on = self._consumer.consume_node(node) + consumer.save_node(node) + go_on = consumer.consume_node(node) else: go_on = False @@ -301,16 +352,17 @@ def _do_if_not_interested(node, orig_node_val): raise ValueError # We should never return here, otherwise its a bug we want to alert on consumed_nodes.add(node) - node.freeze() + node.freeze(restrict_csp=True, resolve_csp=True) not_recovered = True - max_steps = self._consumer.wait_for_exhaustion(node) + max_steps = consumer.wait_for_exhaustion(node) again = True # We enter this loop only if the consumer is interested by the # node. while again: - reset = self._consumer.need_reset(node) + consume_called_again = False + reset = consumer.need_reset(node) if reset and not node.is_exhausted(): @@ -323,14 +375,15 @@ def _do_if_not_interested(node, orig_node_val): elif node.is_exhausted(): # --> (reset and node.is_exhausted()) or (not reset and node.is_exhausted()) + # DEBUG_PRINT('*** node_consumer_helper(): exhausted') yield node, orig_node_val, False, False - if self._consumer.interested_by(node): - if self._consumer.still_interested_by(node): - self._consumer.consume_node(node) + if consumer.interested_by(node): + if consumer.still_interested_by(node): + consumer.consume_node(node) else: - self._consumer.recover_node(node) - if self._consumer.fix_constraints: + consumer.recover_node(node) + if consumer.fix_constraints: node.fix_synchronized_nodes() yield _do_if_not_interested(node, orig_node_val) raise ValueError # We should never return here, otherwise its a bug we want to alert on @@ -340,8 +393,8 @@ def _do_if_not_interested(node, orig_node_val): not_recovered = True else: if node in consumed_nodes: - self._consumer.recover_node(node) - if self._consumer.fix_constraints: + consumer.recover_node(node) + if consumer.fix_constraints: node.fix_synchronized_nodes() not_recovered = False return @@ -349,48 +402,27 @@ def _do_if_not_interested(node, orig_node_val): else: yield node, orig_node_val, False, False - # if max_steps != 0 and not consume_called_again: - # max_steps -= 1 - # # In this case we iterate only on the current node - # node.unfreeze(recursive=False, ignore_entanglement=True) - # node.freeze() - # if self._consumer.fix_constraints: - # node.fix_synchronized_nodes() - # elif not consume_called_again: - # if not_recovered and (self._consumer.interested_by(node) or node in consumed_nodes): - # self._consumer.recover_node(node) - # if self._consumer.fix_constraints: - # node.fix_synchronized_nodes() - # if not node.is_exhausted() and self._consumer.need_reset(node): - # yield None, None, True, True - # again = False - # - # else: - # consume_called_again = False - # node.freeze() if max_steps != 0: max_steps -= 1 if consume_called_again: - node.freeze() - consume_called_again = False + node.freeze(restrict_csp=True, resolve_csp=True) + # consume_called_again = False else: # In this case we iterate only on the current node node.unfreeze(recursive=False, ignore_entanglement=True) - node.freeze() - if self._consumer.fix_constraints: + node.freeze(restrict_csp=True, resolve_csp=True) + if consumer.fix_constraints: node.fix_synchronized_nodes() - - elif consume_called_again: - node.freeze() - consume_called_again = False - + elif consume_called_again and max_steps != 0: + node.freeze(restrict_csp=True, resolve_csp=True) + # consume_called_again = False else: - if not_recovered and (self._consumer.interested_by(node) or node in consumed_nodes): - self._consumer.recover_node(node) - if self._consumer.fix_constraints: + if not_recovered and (consumer.interested_by(node) or node in consumed_nodes): + consumer.recover_node(node) + if consumer.fix_constraints: node.fix_synchronized_nodes() - if not node.is_exhausted() and self._consumer.need_reset(node): + if not node.is_exhausted() and consumer.need_reset(node): yield None, None, True, True again = False @@ -401,10 +433,14 @@ def _do_if_not_interested(node, orig_node_val): class NodeConsumerStub(object): def __init__(self, max_runs_per_node=-1, min_runs_per_node=-1, respect_order=True, - fuzz_magnitude=1.0, fix_constraints=False, **kwargs): + fuzz_magnitude=1.0, fix_constraints=False, ignore_mutable_attr=False, + consider_side_effects_on_sibbling=False, + **kwargs): self.need_reset_when_structure_change = False + self.consider_side_effects_on_sibbling = consider_side_effects_on_sibbling self.fuzz_magnitude = fuzz_magnitude self.fix_constraints = fix_constraints + self.ignore_mutable_attr = ignore_mutable_attr self._internals_criteria = None self._semantics_criteria = None @@ -423,10 +459,15 @@ def __init__(self, max_runs_per_node=-1, min_runs_per_node=-1, respect_order=Tru self.__node_backup = None self.init_specific(**kwargs) - + + def reset_state(self): + """ + Called by the ModelWalker to reinitialize the disruptor. + """ def init_specific(self, **kwargs): self._internals_criteria = dm.NodeInternalsCriteria(negative_node_kinds=[dm.NodeInternals_NonTerm]) + self._semantics_criteria = dm.NodeSemanticsCriteria() def preload(self, root_node): """ @@ -535,7 +576,7 @@ def interested_by(self, node): else: cond1 = True - if self._semantics_criteria is not None: + if self._semantics_criteria is not None and self._semantics_criteria: if node.semantics is None: cond2 = False else: @@ -560,8 +601,14 @@ def interested_by(self, node): class BasicVisitor(NodeConsumerStub): - def init_specific(self, **kwargs): + def init_specific(self, reset_when_change=True): + self._reset_when_change = reset_when_change + self.reset_state() + + def reset_state(self): self._internals_criteria = dm.NodeInternalsCriteria(negative_node_kinds=[dm.NodeInternals_NonTerm]) + self._semantics_criteria = dm.NodeSemanticsCriteria() + self.need_reset_when_structure_change = self._reset_when_change self.firstcall = True def consume_node(self, node): @@ -599,9 +646,11 @@ def wait_for_exhaustion(self, node): class NonTermVisitor(BasicVisitor): - def init_specific(self, **kwargs): - self._internals_criteria = dm.NodeInternalsCriteria(node_kinds=[dm.NodeInternals_NonTerm]) - self.need_reset_when_structure_change = True + def init_specific(self, reset_when_change=True): + self._internals_criteria = dm.NodeInternalsCriteria(node_kinds=[dm.NodeInternals_NonTerm], + mandatory_attrs=[dm.NodeInternals.Mutable]) + self._semantics_criteria = dm.NodeSemanticsCriteria() + self.need_reset_when_structure_change = reset_when_change self.last_node = None self.current_node = None @@ -649,6 +698,7 @@ def init_specific(self, **kwargs): self.need_reset_when_structure_change = True self._internals_criteria = dm.NodeInternalsCriteria(mandatory_attrs=[dm.NodeInternals.Mutable]) + self._semantics_criteria = dm.NodeSemanticsCriteria() self._owned_confs = ['MAIN'] self.current_consumed_node = None self.orig_conf = None @@ -736,22 +786,25 @@ def wait_for_exhaustion(self, node): class TypedNodeDisruption(NodeConsumerStub): - def init_specific(self, ignore_separator=False, enforce_determinism=True): + def init_specific(self, ignore_separator=False, determinist=True): + mattr = None if self.ignore_mutable_attr else [dm.NodeInternals.Mutable] if ignore_separator: - self._internals_criteria = dm.NodeInternalsCriteria(mandatory_attrs=[dm.NodeInternals.Mutable], + self._internals_criteria = dm.NodeInternalsCriteria(mandatory_attrs=mattr, negative_attrs=[dm.NodeInternals.Separator], node_kinds=[dm.NodeInternals_TypedValue, dm.NodeInternals_GenFunc]) else: - self._internals_criteria = dm.NodeInternalsCriteria(mandatory_attrs=[dm.NodeInternals.Mutable], + self._internals_criteria = dm.NodeInternalsCriteria(mandatory_attrs=mattr, node_kinds=[dm.NodeInternals_TypedValue, dm.NodeInternals_GenFunc]) + self._semantics_criteria = dm.NodeSemanticsCriteria() + # self.orig_value = None self.current_fuzz_vt_list = None self.current_node = None self.orig_internal = None - self.enforce_determinism = enforce_determinism + self.determinist = determinist self._ignore_separator = ignore_separator self.sep_list = None @@ -760,7 +813,8 @@ def init_specific(self, ignore_separator=False, enforce_determinism=True): def preload(self, root_node): if not self._ignore_separator: ic = dm.NodeInternalsCriteria(mandatory_attrs=[dm.NodeInternals.Separator]) - self.sep_list = set(map(lambda x: x.to_bytes(), root_node.get_reachable_nodes(internals_criteria=ic))) + self.sep_list = set(map(lambda x: x.to_bytes(), + root_node.get_reachable_nodes(internals_criteria=ic, resolve_generator=True))) self.sep_list = list(self.sep_list) def consume_node(self, node): @@ -787,13 +841,19 @@ def consume_node(self, node): node.set_values(value_type=vt_obj, ignore_entanglement=True, preserve_node=True) node.make_finite() - if self.enforce_determinism: + if self.determinist is None: + pass + elif self.determinist: node.make_determinist() + else: + node.make_random() node.unfreeze(ignore_entanglement=True) # we need to be sure that the current node is freezable node.set_attr(dm.NodeInternals.Freezable) node.set_attr(dm.NodeInternals.LOCKED) + node.cc.highlight = True + return True else: raise ValueError @@ -826,7 +886,8 @@ def _add_separator_cases(self, vt_node): except ValueError: print("\n*** WARNING: separator not part of the initial set. (Could happen if " "separators are generated dynamically)") - self.current_fuzz_vt_list.insert(0, vtype.String(values=sep_l)) + if sep_l: + self.current_fuzz_vt_list.insert(0, vtype.String(values=sep_l)) else: sz = len(current_val) if sz > 1: @@ -863,6 +924,8 @@ def init_specific(self, separators=None): dm.NodeInternalsCriteria(mandatory_attrs=[dm.NodeInternals.Mutable, dm.NodeInternals.Separator], node_kinds=[dm.NodeInternals_Term]) + self._semantics_criteria = dm.NodeSemanticsCriteria() + self.values = [b''] if separators is not None: self.values += list(separators) @@ -895,12 +958,12 @@ def fuzz_data_tree(top_node, paths_regexp=None): node_kinds=[dm.NodeInternals_NonTerm]) if paths_regexp: - node_list = top_node.get_reachable_nodes(path_regexp=paths_regexp) + node_list = top_node.get_reachable_nodes(path_regexp=paths_regexp, resolve_generator=True) else: node_list = [top_node] for node in node_list: - l = node.get_reachable_nodes(internals_criteria=c) + l = node.get_reachable_nodes(internals_criteria=c, resolve_generator=True) for e in l: e.cc.change_subnodes_csts([('*', 'u=.')]) diff --git a/framework/generic_data_makers.py b/framework/generic_data_makers.py index 1e472f1..c659ecb 100755 --- a/framework/generic_data_makers.py +++ b/framework/generic_data_makers.py @@ -23,6 +23,8 @@ import types import subprocess +import uuid + from copy import * from framework.node import * @@ -33,6 +35,7 @@ from framework.value_types import * from framework.data_model import DataModel from framework.dmhelpers.generic import MH +from framework.node import NodeSemanticsCriteria as NSC, NodeInternalsCriteria as NIC # from framework.plumbing import * from framework.evolutionary_helpers import Population @@ -45,24 +48,72 @@ # GENERATORS # ####################### +@generator(tactics, gtype="GENP", weight=4, + args={'pattern': ('Pattern to be used for generating data', b'1234567890', bytes), + 'prefix': ('Prefix added to the pattern', b'', bytes), + 'suffix': ('Suffix replacing the end of the pattern', b'', bytes), + 'size': ('Size of the generated data.', None, int), + 'eval': ('The pattern will be evaluated before being used. Note that the evaluation ' + 'shall result in a byte string.', False, bool) + }) +class g_generic_pattern(Generator): + """ + Generate basic data based on a pattern and different parameters. + """ + def setup(self, dm, user_input): + if not self.pattern: + return False + return True + + def generate_data(self, dm, monitor, target): + if self.eval: + try: + pattern = eval(self.pattern) + except: + data = Data() + # data.make_unusable() + data.add_info('Invalid expression provided in @pattern. It will be used without evaluation.') + return data + else: + pattern = self.pattern + + if self.size is not None: + psize = len(pattern) + pattern = pattern * (self.size // psize) + pattern[:(self.size%psize)] + if self.prefix: + pattern = self.prefix + pattern[:-(len(self.prefix))] + + else: + pattern = self.prefix+pattern + + if self.suffix: + pattern = pattern[:-len(self.suffix)] + self.suffix + if self.size is not None and self.size < len(pattern): + pattern = pattern[:self.size] + + return Data(pattern) + + @generator(tactics, gtype='POPULATION', weight=1, - args={'population': ('The population to iterate over.', None, Population)}) + args={'population': ('The population to iterate over.', None, Population), + 'track': ('Keep trace of the changes that occurred on data, generation after generation', + False, bool)} + ) class g_population(Generator): """ Walk through the given population """ def setup(self, dm, user_input): assert self.population is not None - self.population.reset() - + self._pop_sz = self.population.size() + self._curr_generation = self.population.generation return True def generate_data(self, dm, monitor, target): reset = False try: - data = Data(self.population.next().node) + data = self.population.next().data except StopIteration: - # all individuals of the current population have been sent if self.population.is_final(): @@ -70,6 +121,8 @@ def generate_data(self, dm, monitor, target): else: try: self.population.evolve() + self._pop_sz = self.population.size() + self._curr_generation = self.population.generation except ExtinctPopulationError: reset = True else: @@ -80,6 +133,10 @@ def generate_data(self, dm, monitor, target): data.make_unusable() self.need_reset() + data.add_info('Generation: {}, Population size: {}'.format(self._curr_generation, self._pop_sz)) + data.add_info('Data index in the population: {}'.format(self.population.index)) + data.take_info_ownership(keep_previous_info=self.track) + return data @@ -96,18 +153,35 @@ def truncate_info(info, max_size=60): @disruptor(tactics, dtype="tWALK", weight=1, modelwalker_user=True, args={'path': ('Graph path regexp to select nodes on which' \ ' the disruptor should apply.', None, str), + 'sem': ('Semantics to select nodes on which' \ + ' the disruptor should apply.', None, (str, list)), + 'full_combinatory': ('When set to True, enable full-combinatory mode for ' + 'non-terminal nodes. It means that the non-terminal nodes ' + 'will be customized in "FullCombinatory" mode', False, bool), + 'leaf_determinism': ("If set to 'True', all the typed nodes of the model will be " + "set to determinist mode prior to any fuzzing. If set " + "to 'False', they will be set to random mode. " + "Otherwise, if set to 'None', nothing will be done.", None, bool), 'order': ('When set to True, the walking order is strictly guided ' \ 'by the data structure. Otherwise, fuzz weight (if specified ' \ 'in the data model) is used for ordering.', True, bool), 'nt_only': ('Walk through non-terminal nodes only.', False, bool), + 'deep': ('When set to True, if a node structure has changed, the modelwalker ' \ + 'will reset its walk through the children nodes.', True, bool), + 'consider_sibbling_change': + ('While walking through terminal nodes, if sibbling nodes are ' + 'no more the same because of existence condition for instance, walk through ' + 'the new nodes.', True, bool), + 'ign_mutable_attr': ('Walk through all the nodes even if their Mutable attribute ' + 'is cleared.', True, bool), 'fix_all': ('For each produced data, reevaluate the constraints on the whole graph.', True, bool)}) -class sd_iter_over_data(StatefulDisruptor): - ''' +class sd_walk_data_model(StatefulDisruptor): + """ Walk through the provided data and for each visited node, iterates over the allowed values (with respect to the data model). Note: *no alteration* is performed by this disruptor. - ''' + """ def setup(self, dm, user_input): return True @@ -119,11 +193,33 @@ def set_seed(self, prev_data): prev_content.make_finite(all_conf=True, recursive=True) + if self.full_combinatory: + nic = NodeInternalsCriteria(node_kinds=[NodeInternals_NonTerm]) + nl = prev_content.get_reachable_nodes(internals_criteria=nic, path_regexp=self.path, + ignore_fstate=True) + for n in nl: + n.cc.custo.full_combinatory_mode = True + + if self.leaf_determinism is not None: + nic = NodeInternalsCriteria(node_kinds=[NodeInternals_TypedValue]) + nl = prev_content.get_reachable_nodes(internals_criteria=nic, path_regexp=self.path, + ignore_fstate=True) + for n in nl: + if self.leaf_determinism: + n.make_determinist() + else: + n.make_random() + if self.nt_only: - consumer = NonTermVisitor(respect_order=self.order) + consumer = NonTermVisitor(respect_order=self.order, ignore_mutable_attr=self.ign_mutable_attr, + consider_side_effects_on_sibbling=self.consider_sibbling_change, + reset_when_change=self.deep, fix_constraints=self.fix_all) else: - consumer = BasicVisitor(respect_order=self.order) - consumer.set_node_interest(path_regexp=self.path) + consumer = BasicVisitor(respect_order=self.order, ignore_mutable_attr=self.ign_mutable_attr, + consider_side_effects_on_sibbling=self.consider_sibbling_change, + reset_when_change=self.deep, fix_constraints=self.fix_all) + sem_crit = NSC(optionalbut1_criteria=self.sem) + consumer.set_node_interest(path_regexp=self.path, semantics_criteria=sem_crit) self.modelwalker = ModelWalker(prev_content, consumer, max_steps=self.max_steps, initial_step=self.init) self.walker = iter(self.modelwalker) @@ -158,11 +254,16 @@ def disrupt_data(self, dm, target, data): @disruptor(tactics, dtype="tTYPE", weight=1, modelwalker_user=True, args={'path': ('Graph path regexp to select nodes on which' \ ' the disruptor should apply.', None, str), + 'sem': ('Semantics to select nodes on which' \ + ' the disruptor should apply.', None, (str, list)), 'order': ('When set to True, the fuzzing order is strictly guided ' \ 'by the data structure. Otherwise, fuzz weight (if specified ' \ 'in the data model) is used for ordering.', True, bool), 'deep': ('When set to True, if a node structure has changed, the modelwalker ' \ 'will reset its walk through the children nodes.', True, bool), + 'full_combinatory': ('When set to True, enable full-combinatory mode for non-terminal nodes. It ' + 'means that the non-terminal nodes will be customized in "FullCombinatory" mode', + False,bool), 'ign_sep': ('When set to True, separators will be ignored ' \ 'if any are defined.', False, bool), 'fix_all': ('For each produced data, reevaluate the constraints on the whole graph.', @@ -171,14 +272,26 @@ def disrupt_data(self, dm, target, data): " (only implemented for 'sync_size_with' and 'sync_enc_size_with').", True, bool), 'fuzz_mag': ('Order of magnitude for maximum size of some fuzzing test cases.', 1.0, float), - 'determinism': ("If set to 'True', the whole model will be fuzzed in " - "a deterministic way. Otherwise it will be guided by the " - "data model determinism.", True, bool), - 'leaf_determinism': ("If set to 'True', each typed node will be fuzzed in " - "a deterministic way. Otherwise it will be guided by the " + 'make_determinist': ("If set to 'True', the whole model will be set in determinist mode." + "Otherwise it will be guided by the data model determinism.", False, bool), + 'leaf_fuzz_determinism': ("If set to 'True', each typed node will be fuzzed in " + "a deterministic way. If set to 'False' each typed node " + "will be fuzzed in a random way. Otherwise, if it is set to " + "'None', it will be guided by the " "data model determinism. Note: this option is complementary to " - "'determinism' is it acts on the typed node substitutions " + "'determinism' as it acts on the typed node substitutions " "that occur through this disruptor", True, bool), + 'leaf_determinism': ("If set to 'True', all the typed nodes of the model will be " + "set to determinist mode prior to any fuzzing. If set " + "to 'False', they will be set to random mode. " + "Otherwise, if set to 'None', nothing will be done.", None, bool), + 'ign_mutable_attr': ('Walk through all the nodes even if their Mutable attribute ' + 'is cleared.', False, bool), + 'consider_sibbling_change': + ('[EXPERIMENTAL] While walking through terminal nodes, if sibbling nodes are ' + 'no more the same because of existence condition for instance, walk through ' + 'the new nodes. (Currently, work only with some specific data model construction.)', + False, bool), }) class sd_fuzz_typed_nodes(StatefulDisruptor): """ @@ -204,17 +317,47 @@ def set_seed(self, prev_data): prev_data.add_info('DONT_PROCESS_THIS_KIND_OF_DATA') return prev_data + if self.full_combinatory: + nic = NodeInternalsCriteria(node_kinds=[NodeInternals_NonTerm]) + nl = prev_content.get_reachable_nodes(internals_criteria=nic, path_regexp=self.path, + ignore_fstate=True) + for n in nl: + n.cc.custo.full_combinatory_mode = True + + if self.leaf_determinism is not None: + nic = NodeInternalsCriteria(node_kinds=[NodeInternals_TypedValue]) + nl = prev_content.get_reachable_nodes(internals_criteria=nic, path_regexp=self.path, + ignore_fstate=True) + for n in nl: + if self.leaf_determinism: + n.make_determinist() + else: + n.make_random() + self.consumer = TypedNodeDisruption(max_runs_per_node=self.max_runs_per_node, min_runs_per_node=self.min_runs_per_node, fuzz_magnitude=self.fuzz_mag, fix_constraints=self.fix, respect_order=self.order, + ignore_mutable_attr=self.ign_mutable_attr, + consider_side_effects_on_sibbling=self.consider_sibbling_change, ignore_separator=self.ign_sep, - enforce_determinism=self.leaf_determinism) + determinist=self.leaf_fuzz_determinism) self.consumer.need_reset_when_structure_change = self.deep - self.consumer.set_node_interest(path_regexp=self.path) + sem_crit = NSC(optionalbut1_criteria=self.sem) + self.consumer.set_node_interest(path_regexp=self.path, semantics_criteria=sem_crit) self.modelwalker = ModelWalker(prev_content, self.consumer, max_steps=self.max_steps, - initial_step=self.init, make_determinist=self.determinism) + initial_step=self.init, make_determinist=self.make_determinist) + + # After ModelWalker init, 'prev_content' is frozen. We can now check if 'self.path' exists in the + # node, because if it does not exist (e.g., user mistype) the ModelWalker will walk until the end of + # all the possible paths, and will finally yield nothing. This walk could take a lot of time depending on + # the model. Thus, in this situation we inform the user right away. + if self.path: + d = prev_content.get_nodes_by_paths(path_list=[self.path]) + if not d[self.path]: + raise ValueError(f'The provided path "{self.path}" does not exist.') + self.walker = iter(self.modelwalker) self.max_runs = None @@ -361,6 +504,8 @@ def disrupt_data(self, dm, target, data): @disruptor(tactics, dtype="tSEP", weight=1, modelwalker_user=True, args={'path': ('Graph path regexp to select nodes on which' \ ' the disruptor should apply.', None, str), + 'sem': ('Semantics to select nodes on which' \ + ' the disruptor should apply.', None, (str, list)), 'order': ('When set to True, the fuzzing order is strictly guided ' \ 'by the data structure. Otherwise, fuzz weight (if specified ' \ 'in the data model) is used for ordering.', True, bool), @@ -396,7 +541,8 @@ def set_seed(self, prev_data): respect_order=self.order, separators=sep_list) self.consumer.need_reset_when_structure_change = self.deep - self.consumer.set_node_interest(path_regexp=self.path) + sem_crit = NSC(optionalbut1_criteria=self.sem) + self.consumer.set_node_interest(path_regexp=self.path, semantics_criteria=sem_crit) self.modelwalker = ModelWalker(prev_content, self.consumer, max_steps=self.max_steps, initial_step=self.init) self.walker = iter(self.modelwalker) @@ -447,6 +593,8 @@ def disrupt_data(self, dm, target, data): 'max_steps': ('Maximum number of steps (-1 means until the end).', -1, int), 'path': ('Graph path regexp to select nodes on which' \ ' the disruptor should apply.', None, str), + 'sem': ('Semantics to select nodes on which' \ + ' the disruptor should apply.', None, (str, list)), 'deep': ('If True, enable corruption of non-terminal node internals', False, bool) }) class sd_struct_constraints(StatefulDisruptor): @@ -470,8 +618,14 @@ def set_seed(self, prev_data): self.seed = prev_content self.seed.make_finite(all_conf=True, recursive=True) + # self.seed.make_determinist(all_conf=True, recursive=True) self.seed.freeze() + # self.seed.unfreeze(recursive=True) + # self.seed.freeze() + + # print('\n*** original data:\n',self.seed.to_bytes()) + self.idx = 0 ic_exist_cst = NodeInternalsCriteria(required_csts=[SyncScope.Existence]) @@ -479,7 +633,10 @@ def set_seed(self, prev_data): ic_size_cst = NodeInternalsCriteria(required_csts=[SyncScope.Size]) ic_minmax_cst = NodeInternalsCriteria(node_kinds=[NodeInternals_NonTerm]) + sem_crit = None if self.sem is None else NSC(optionalbut1_criteria=self.sem) + self.exist_cst_nodelist = self.seed.get_reachable_nodes(internals_criteria=ic_exist_cst, path_regexp=self.path, + semantics_criteria=sem_crit, ignore_fstate=True) # print('\n*** NOT FILTERED nodes') # for n in self.exist_cst_nodelist: @@ -488,12 +645,21 @@ def set_seed(self, prev_data): # print('\n*** FILTERED nodes') # for n in self.exist_cst_nodelist: # print(' |_ ' + n.name) + + # print('\n***before:') + # for n in self.exist_cst_nodelist: + # print(' |_ ' + n.name) + nodelist = copy.copy(self.exist_cst_nodelist) for n in nodelist: if n.get_path_from(self.seed) is None: self.exist_cst_nodelist.remove(n) + # print('\n***after:') + # for n in self.exist_cst_nodelist: + # print(' |_ ' + n.name) self.qty_cst_nodelist_1 = self.seed.get_reachable_nodes(internals_criteria=ic_qty_cst, path_regexp=self.path, + semantics_criteria=sem_crit, ignore_fstate=True) # self.qty_cst_nodelist_1 = self.seed.filter_out_entangled_nodes(self.qty_cst_nodelist_1) nodelist = copy.copy(self.qty_cst_nodelist_1) @@ -504,7 +670,8 @@ def set_seed(self, prev_data): self.qty_cst_nodelist_2 = copy.copy(self.qty_cst_nodelist_1) self.size_cst_nodelist_1 = self.seed.get_reachable_nodes(internals_criteria=ic_size_cst, path_regexp=self.path, - ignore_fstate=True) + semantics_criteria=sem_crit, + ignore_fstate=True) nodelist = copy.copy(self.size_cst_nodelist_1) for n in nodelist: if n.get_path_from(self.seed) is None: @@ -513,6 +680,7 @@ def set_seed(self, prev_data): if self.deep: minmax_cst_nodelist = self.seed.get_reachable_nodes(internals_criteria=ic_minmax_cst, path_regexp=self.path, + semantics_criteria=sem_crit, ignore_fstate=True) self.minmax_cst_nodelist_1 = set() @@ -537,7 +705,8 @@ def set_seed(self, prev_data): self.max_runs = len(self.exist_cst_nodelist) + 2*len(self.size_cst_nodelist_1) + \ 2*len(self.qty_cst_nodelist_1) + 3*len(self.minmax_cst_nodelist_1) - + + # print('\n*** final setup:\n',self.seed.to_bytes()) def disrupt_data(self, dm, target, data): @@ -583,21 +752,21 @@ def disrupt_data(self, dm, target, data): new_mini = max(0, mini-1) self.seed.env.add_node_to_corrupt(consumed_node, corrupt_type=Node.CORRUPT_NODE_QTY, corrupt_op=lambda x, y: (new_mini, new_mini)) - op_performed = "set node amount to its minimum minus one" + op_performed = f"set node amount to its minimum minus one ({new_mini})" elif self.deep and self.minmax_cst_nodelist_2: consumed_node, mini, maxi = self.minmax_cst_nodelist_2.pop() if self.idx == step_idx: new_maxi = (maxi+1) self.seed.env.add_node_to_corrupt(consumed_node, corrupt_type=Node.CORRUPT_NODE_QTY, corrupt_op=lambda x, y: (new_maxi, new_maxi)) - op_performed = "set node amount to its maximum plus one" + op_performed = f"set node amount to its maximum plus one ({new_maxi})" elif self.deep and self.minmax_cst_nodelist_3: consumed_node, mini, maxi = self.minmax_cst_nodelist_3.pop() if self.idx == step_idx: new_maxi = (maxi*10) self.seed.env.add_node_to_corrupt(consumed_node, corrupt_type=Node.CORRUPT_NODE_QTY, corrupt_op=lambda x, y: (new_maxi, new_maxi)) - op_performed = "set node amount to a value way beyond its maximum" + op_performed = f"set node amount to a value way beyond its maximum ({new_maxi})" else: stop = True break @@ -609,12 +778,24 @@ def disrupt_data(self, dm, target, data): self.handover() return data + # print('\n***disrupt before:\n',self.seed.to_bytes()) corrupted_seed = Node(self.seed.name, base_node=self.seed, ignore_frozen_state=False, new_env=True) + corrupted_seed = self.seed.get_clone(ignore_frozen_state=False, new_env=True) self.seed.env.remove_node_to_corrupt(consumed_node) + # print('\n***disrupt source:\n',self.seed.to_bytes()) + # print('\n***disrupt clone 1:\n',corrupted_seed.to_bytes()) + # nt_nodes_crit = NodeInternalsCriteria(node_kinds=[NodeInternals_NonTerm]) + # ntlist = corrupted_seed.get_reachable_nodes(internals_criteria=nt_nodes_crit, ignore_fstate=False) + # for nd in ntlist: + # # print(nd.is_attr_set(NodeInternals.Finite)) + # nd.unfreeze(recursive=True, reevaluate_constraints=True, ignore_entanglement=True) + corrupted_seed.unfreeze(recursive=True, reevaluate_constraints=True, ignore_entanglement=True) corrupted_seed.freeze() + # print('\n***disrupt after:\n',corrupted_seed.to_bytes()) + data.add_info('sample index: {:d}'.format(self.idx)) data.add_info(' |_ run: {:d} / {:d}'.format(self.idx, self.max_runs)) data.add_info('current fuzzed node: {:s}'.format(consumed_node.get_path_from(self.seed))) @@ -626,190 +807,6 @@ def disrupt_data(self, dm, target, data): return data -# ADAPTED FOR THE EVOLUTIONARY PROCESS # - -class SwapperDisruptor(StatefulDisruptor): - """ - Merge two nodes to produce two children - """ - - def _swap_nodes(self, node_1, node_2): - node_2_copy = node_2.get_clone() - node_2.set_contents(node_1) - node_1.set_contents(node_2_copy) - - def set_seed(self, prev_data): - - self.count = 0 # number of sent element - - prev_content = prev_data.content - assert isinstance(prev_content, Node) - prev_content.freeze(recursive=True) - - if self.node is None: - self.node = prev_content.get_clone() - - def disrupt_data(self, dm, target, data): - - if self.count == 2: - data.make_unusable() - self.handover() - - elif self.count == 1: - data.update_from(self.node) - - self.count += 1 - data.altered = True - return data - - -@disruptor(tactics, dtype="tCROSS", weight=1, - args={'node': ('Node to crossover with.', None, Node), - 'percentage_to_share': ('Percentage of the base node to share.', 0.50, float)}) -class sd_crossover(SwapperDisruptor): - """ - Makes two graphs share a certain percentages of their leaf nodes in order to produce two children - """ - - class Operand(object): - - def __init__(self, node): - self.node = node - - self.leafs = [] - - for path, node in self.node.iter_paths(): - if node.is_term() and path not in self.leafs: - self.leafs.append(path) - - self.shared = None - - def compute_sub_graphs(self, percentage): - random.shuffle(self.leafs) - self.shared = self.leafs[:int(round(len(self.leafs) * percentage))] - self.shared.sort() - - change = True - while change: - - change = False - index = 0 - length = len(self.shared) - - while index < length: - - current_path = self.shared[index] - - slash_index = current_path[::-1].find('/') - - # check if we are dealing with the root node - if slash_index == -1: - index += 1 - continue - - parent_path = current_path[:-current_path[::-1].find('/') - 1] - children_nb = self._count_brothers(index, parent_path) - if children_nb == self.node.get_node_by_path(parent_path).cc.get_subnode_qty(): - self._merge_brothers(index, parent_path, children_nb) - change = True - index += 1 - length = len(self.shared) - else: - index += children_nb - - def _count_brothers(self, index, pattern): - count = 1 - p = re.compile(u'^' + pattern + '($|/*)') - for i in range(index + 1, len(self.shared)): - if re.match(p, self.shared[i]) is not None: - count += 1 - return count - - def _merge_brothers(self, index, pattern, length): - for _ in range(0, length, 1): - del self.shared[index] - self.shared.insert(index, pattern) - - def setup(self, dm, user_input): - if self.percentage_to_share is None: - self.percentage_to_share = float(random.randint(3, 7)) / 10.0 - elif not (0 < self.percentage_to_share < 1): - print("Invalid percentage, a float between 0 and 1 need to be provided") - return False - - return True - - def set_seed(self, prev_data): - prev_content = prev_data.content - if not isinstance(prev_content, Node): - prev_data.add_info('DONT_PROCESS_THIS_KIND_OF_DATA') - return prev_data - - SwapperDisruptor.set_seed(self, prev_data) - - source = self.Operand(prev_content) - source.compute_sub_graphs(self.percentage_to_share) - random.shuffle(source.shared) - - param = self.Operand(self.node) - param.compute_sub_graphs(1.0 - self.percentage_to_share) - random.shuffle(param.shared) - - swap_nb = len(source.shared) if len(source.shared) < len(param.shared) else len(param.shared) - - for i in range(swap_nb): - node_1 = source.node.get_node_by_path(path=source.shared[i]) - node_2 = param.node.get_node_by_path(path=param.shared[i]) - self._swap_nodes(node_1, node_2) - - -@disruptor(tactics, dtype="tCOMB", weight=1, - args={'node': ('Node to combine with.', None, Node)}) -class sd_combine(SwapperDisruptor): - """ - Merge two nodes by swapping some roots' children - """ - - def setup(self, dm, user_input): - return True - - def get_nodes(self, node): - while True: - nodes = [node] if node.is_term() else node.cc.frozen_node_list - - if len(nodes) == 1 and not nodes[0].is_term(): - node = nodes[0] - else: - break - - return nodes - - def set_seed(self, prev_data): - - SwapperDisruptor.set_seed(self, prev_data) - - prev_content = prev_data.content - if not isinstance(prev_content, Node): - prev_data.add_info('DONT_PROCESS_THIS_KIND_OF_DATA') - return prev_data - - source = self.get_nodes(prev_content) - param = self.get_nodes(self.node) - - if len(source) == 0 or len(param) == 0: - prev_data.add_info('DONT_PROCESS_THIS_KIND_OF_DATA') - return prev_data - - swap_nb = len(source) if len(source) < len(param) else len(param) - swap_nb = int(math.ceil(swap_nb / 2.0)) - - random.shuffle(source) - random.shuffle(param) - - for i in range(swap_nb): - self._swap_nodes(source[i], param[i]) - - ######################## # STATELESS DISRUPTORS # ######################## @@ -840,7 +837,7 @@ def _get_cmd(self): def disrupt_data(self, dm, target, prev_data): prev_content = prev_data.content if self.path and isinstance(prev_content, Node): - node = prev_content.get_node_by_path(path_regexp=self.path) + node = prev_content.get_first_node_by_path(path_regexp=self.path) if node is None: prev_data.add_info('INVALID INPUT') return prev_data @@ -1005,7 +1002,7 @@ def disrupt_data(self, dm, target, prev_data): prev_content = prev_data.content if isinstance(prev_content, Node): if self.path is not None: - node = prev_content.get_node_by_path(self.path) + node = prev_content.get_first_node_by_path(self.path) if node is None: node = prev_content else: @@ -1120,7 +1117,7 @@ def disrupt_data(self, dm, target, prev_data): @disruptor(tactics, dtype="Cp", weight=4, args={'idx': ('Byte index to be corrupted (from 1 to data length).', 1, int), - 'new_val': ('If provided change the selected byte with the new one.', None, str), + 'new_val': ('If provided change the selected byte with the new one.', None, bytes), 'ascii': ('Enforce all outputs to be ascii 7bits.', False, bool)}) class d_corrupt_bits_by_position(Disruptor): ''' @@ -1168,7 +1165,7 @@ def setup(self, dm, user_input): def disrupt_data(self, dm, target, prev_data): prev_content = prev_data.content if not isinstance(prev_content, Node): - prev_data.add_info('INVALID INPUT') + prev_data.add_info('UNSUPPORTED INPUT') return prev_data if self.path: @@ -1181,12 +1178,12 @@ def disrupt_data(self, dm, target, prev_data): for n in l: n.unfreeze(recursive=True, reevaluate_constraints=True, ignore_entanglement=True) - prev_data.add_info("release constraints from the node '{!s}'".format(n.name)) + prev_data.add_info("reevaluate constraints from the node '{!s}'".format(n.name)) n.freeze() else: prev_content.unfreeze(recursive=True, reevaluate_constraints=True, ignore_entanglement=True) - prev_data.add_info('release constraints from the root') + prev_data.add_info('reevaluate constraints from the root') prev_content.freeze() @@ -1218,7 +1215,7 @@ def disrupt_data(self, dm, target, prev_data): prev_content = prev_data.content if not isinstance(prev_content, Node): - prev_data.add_info('INVALID INPUT') + prev_data.add_info('UNSUPPORTED INPUT') return prev_data prev_content.freeze() @@ -1250,11 +1247,15 @@ def disrupt_data(self, dm, target, prev_data): @disruptor(tactics, dtype="OP", weight=4, args={'path': ('Graph path regexp to select nodes on which ' \ 'the disruptor should apply.', None, str), + 'sem': ('Semantics to select nodes on which' \ + ' the disruptor should apply.', None, (str, list)), 'op': ('The operation to perform on the selected nodes.', Node.clear_attr, - (types.MethodType, types.FunctionType)), # python3, python2 - 'params': ('Tuple of parameters that will be provided to the operation. (' - 'default: MH.Attr.Mutable)', - (MH.Attr.Mutable,), + types.MethodType), + 'op_ref': ("Predefined operation that can be referenced by name. The current " + "predefined function are: 'unfreeze', 'freeze', 'walk', 'set_qty'. Take " + "precedence over @op if not None." , None, str), + 'params': ('Tuple of parameters that will be provided to the operation.', + (), tuple), 'clone_node': ('If True the dmaker will always return a copy ' \ 'of the node. (For stateless disruptors dealing with ' \ @@ -1272,11 +1273,28 @@ def disrupt_data(self, dm, target, prev_data): ok = False prev_content = prev_data.content if not isinstance(prev_content, Node): - prev_data.add_info('INVALID INPUT') + prev_data.add_info('UNSUPPORTED INPUT') return prev_data - if self.path: - l = prev_content.get_reachable_nodes(path_regexp=self.path) + if self.op_ref is not None: + if self.op_ref == 'unfreeze': + self.op = Node.unfreeze + elif self.op_ref == 'freeze': + self.op = Node.freeze + elif self.op_ref == 'walk': + self.op = Node.walk + elif self.op_ref == 'set_qty': + self.op = NodeInternals_NonTerm.set_subnode_default_qty + n = prev_content.get_first_node_by_path(path_regexp=self.path) + self.path = self.path[:self.path.rfind('/')] + '$' + self.params = (n, *self.params) + else: + prev_data.add_info('Unsupported operation') + return prev_data + + sem_crit = None if self.sem is None else NSC(optionalbut1_criteria=self.sem) + if self.path or sem_crit: + l = prev_content.get_reachable_nodes(path_regexp=self.path, semantics_criteria=sem_crit) if not l: prev_data.add_info('INVALID INPUT') return prev_data @@ -1320,15 +1338,21 @@ def _add_info(self, prev_data, n): @disruptor(tactics, dtype="MOD", weight=4, args={'path': ('Graph path regexp to select nodes on which ' \ 'the disruptor should apply.', None, str), - 'value': ('The new value to inject within the data.', '', str), + 'sem': ('Semantics to select nodes on which' \ + ' the disruptor should apply.', None, (str, list)), + 'value': ('The new value to inject within the data.', b'', bytes), 'constraints': ('Constraints for the absorption of the new value.', AbsNoCsts(), AbsCsts), - 'multi_mod': ('Dictionary of : pairs to change multiple nodes with ' - 'diferent values. can be either only the new or a ' - 'tuple (,) if new constraint for absorption is ' - 'needed', None, dict), + 'multi_mod': ('Dictionary of : pairs or ' + ': pairs or ' + ': pairs to change multiple nodes with ' + 'different values. can be either only the new or a ' + 'tuple (,) if new constraint for absorption is ' + 'needed', None, dict), + 'unfold': ('Resolve all the generator nodes within the input before performing ' + 'the @path/@sem research', False, bool), 'clone_node': ('If True the dmaker will always return a copy ' \ 'of the node. (For stateless disruptors dealing with ' \ - 'big data it can be useful to it to False.)', False, bool)}) + 'big data it can be useful to set it to False.)', False, bool)}) class d_modify_nodes(Disruptor): """ Perform modifications on the provided data. Two ways are possible: @@ -1346,15 +1370,16 @@ def setup(self, dm, user_input): def disrupt_data(self, dm, target, prev_data): prev_content = prev_data.content if not isinstance(prev_content, Node): - prev_data.add_info('INVALID INPUT') + prev_data.add_info('UNSUPPORTED INPUT') return prev_data if self.multi_mod: change_dict = self.multi_mod else: - change_dict = {self.path: (self.value, self.constraints)} + sem = None if self.sem is None else NSC(optionalbut1_criteria=self.sem) + change_dict = {self.path if sem is None else sem: (self.value, self.constraints)} - for path, item in change_dict.items(): + for selector, item in change_dict.items(): if isinstance(item, (tuple, list)): assert len(item) == 2 new_value, new_csts = item @@ -1362,18 +1387,29 @@ def disrupt_data(self, dm, target, prev_data): new_value = item new_csts = AbsNoCsts() - if path: - l = prev_content.get_reachable_nodes(path_regexp=path) + if selector: + if isinstance(selector, str): + l = prev_content.get_reachable_nodes(path_regexp=selector, + resolve_generator=self.unfold) + elif isinstance(selector, NSC): + l = prev_content.get_reachable_nodes(semantics_criteria=selector, + resolve_generator=self.unfold) + elif isinstance(selector, NIC): + l = prev_content.get_reachable_nodes(internals_criteria=selector, + resolve_generator=self.unfold) + else: + raise ValueError('Unsupported selector') + if not l: - prev_data.add_info('INVALID INPUT') + prev_data.add_info('No node found with current criteria') return prev_data for n in l: status, off, size, name = n.absorb(new_value, constraints=new_csts) - self._add_info(prev_data, n, status, size) + self._add_info(prev_data, n, new_value, status, size) else: status, off, size, name = prev_content.absorb(new_value, constraints=new_csts) - self._add_info(prev_data, prev_content, status, size) + self._add_info(prev_data, prev_content, new_value, status, size) prev_content.freeze() @@ -1384,11 +1420,11 @@ def disrupt_data(self, dm, target, prev_data): prev_data.altered = True return prev_data - def _add_info(self, prev_data, n, status, size): - val_len = len(self.value) + def _add_info(self, prev_data, n, new_value, status, size): + val_len = len(new_value) prev_data.add_info("changed node: {!s}".format(n.name)) prev_data.add_info("absorption status: {!s}".format(status)) - prev_data.add_info("value provided: {!s}".format(truncate_info(self.value))) + prev_data.add_info("value provided: {!s}".format(truncate_info(new_value))) prev_data.add_info("__ length: {:d}".format(val_len)) if status != AbsorbStatus.FullyAbsorbed: prev_data.add_info("absorbed size: {:d}".format(size)) @@ -1403,7 +1439,7 @@ def _add_info(self, prev_data, n, status, size): args={'func': ('The function that will be called with a node as its first parameter, ' 'and provided optionnaly with addtionnal parameters if @params is set.', lambda x:x, - (types.MethodType, types.FunctionType)), # python3, python2 + types.MethodType), 'params': ('Tuple of parameters that will be provided to the function.', None, tuple) }) class d_call_function(Disruptor): @@ -1426,7 +1462,8 @@ def disrupt_data(self, dm, target, prev_data): new_data = self.func(prev_data) except: new_data = prev_data - new_data.add_info("an error occurred while executing the user function '{!r}'".format(self.func)) + new_data.add_info("An error occurred while executing the user function '{!r}':".format(self.func)) + new_data.add_info(traceback.format_exc()) else: new_data.add_info("called function: {!r}".format(self.func)) if self.params: @@ -1449,7 +1486,7 @@ def setup(self, dm, user_input): def disrupt_data(self, dm, target, prev_data): prev_content = prev_data.content if not isinstance(prev_content, Node): - prev_data.add_info('INVALID INPUT') + prev_data.add_info('UNSUPPORTED INPUT') return prev_data prev_data.add_info('shallow copy of input data has been done') @@ -1459,3 +1496,270 @@ def disrupt_data(self, dm, target, prev_data): return prev_data +@disruptor(tactics, dtype="ADD", weight=4, + args={'path': ('Graph path to select the node on which ' \ + 'the disruptor should apply.', None, str), + 'after': ('If True, the addition will be done after the selected node. Otherwise, ' + 'it will be done before.', + True, bool), + 'atom': ('Name of the atom to add within the retrieved input. It is mutually ' + 'exclusive with @raw', + None, str), + 'raw': ('Raw value to add within the retrieved input. It is mutually ' + 'exclusive with @atom.', + b'', (bytes,str)), + 'name': ('If provided, the added node will have this name.', + None, str) + }) +class d_add_data(Disruptor): + """ + Add some data within the retrieved input. + """ + def setup(self, dm, user_input): + if self.atom and self.raw: + return False + return True + + def disrupt_data(self, dm, target, prev_data): + prev_content = prev_data.content + if isinstance(prev_content, bytes): + prev_content = Node('wrapper', subnodes=[Node('raw', values=[prev_content])]) + prev_content.set_env(Env()) + prev_content.freeze() + elif isinstance(prev_content, Node) and prev_content.is_term(): + prev_content = Node('wrapper', subnodes=[prev_content]) + prev_content.set_env(Env()) + prev_content.freeze() + + assert isinstance(prev_content, Node) + + if self.atom is not None: + try: + obj = dm.get_atom(self.atom) + except: + prev_data.add_info("An error occurred while retrieving the atom named '{:s}'".format(self.atom)) + return prev_data + else: + obj = Node('raw{}'.format(uuid.uuid1()), values=[self.raw]) + + if self.name is not None: + obj.name = self.name + + if self.path: + nt_node_path = self.path[:self.path.rfind('/')] + try: + nt_node = prev_content[nt_node_path][0] + pivot = prev_content[self.path][0] + except: + prev_data.add_info('An error occurred while handling @path') + return prev_data + + if self.after: + nt_node.add(obj, after=pivot) + else: + nt_node.add(obj, before=pivot) + else: + prev_content.add(obj) + prev_data.update_from(prev_content) + # prev_content.show() + + return prev_data + + +@disruptor(tactics, dtype="tWALKcsp", weight=1, modelwalker_user=False, + args={'init': ('Make the operator ignore all the steps until the provided one', 1, int), + 'clone_node': ('If True, this operator will always return a copy ' + 'of the node. (for stateless diruptors dealing with ' + 'big data it can be usefull to set it to False)', True, bool), + 'notify_exhaustion': ('When all the solutions of the CSP have been walked ' + 'through, the disruptor will notify it if this parameter ' + 'is set to True.', True, bool), + }) +class sd_walk_csp_solutions(StatefulDisruptor): + """ + + When the CSP (Constraint Satisfiability Problem) backend are used in the data description. + This operator walk through the solutions of the CSP. + + """ + + def setup(self, dm, user_input): + self._first_call_performed = False + self._count = 1 + self._step_size = self.init-1 if self.init > 1 else 1 + return True + + def set_seed(self, prev_data): + prev_content = prev_data.content + if not isinstance(prev_content, Node): + prev_data.add_info('UNSUPPORTED INPUT') + return prev_data + + self.csp = prev_content.get_csp() + if not self.csp: + prev_data.add_info('CSP BACKEND NOT USED BY THIS ATOM') + return prev_data + + self.seed = prev_content + self.seed.freeze(resolve_csp=True) + + def disrupt_data(self, dm, target, data): + + if self._first_call_performed or self.init > 1: + self.seed.unfreeze(recursive=False, dont_change_state=True, walk_csp=True, + walk_csp_step_size=self._step_size) + + self._step_size = 1 + + if self.seed.no_more_solution_for_csp and self.notify_exhaustion: + data.make_unusable() + self.handover() + + else: + if self._first_call_performed or self.init > 1: + self.seed.freeze(resolve_csp=True) + self._count += 1 + else: + self._first_call_performed = True + + data.add_info('csp solution index: {:d}'.format(self._count)) + data.add_info(' |_ variables assignment:') + solution = self.csp.get_solution() + for var, value in solution.items(): + data.add_info(f' --> {var}: {value}') + + if self.clone_node: + exported_node = Node(self.seed.name, base_node=self.seed, new_env=True) + data.update_from(exported_node) + else: + data.update_from(self.seed) + + return data + + +@disruptor(tactics, dtype="tCONST", weight=1, modelwalker_user=False, + args={'const_idx': ('Index of the constraint to begin with (first index is 1)', 1, int), + 'sample_idx': ('Index of the sample for the selected constraint to begin with (' + 'first index is 1)', 1, int), + 'clone_node': ('If True, this operator will always return a copy ' + 'of the node. (for stateless diruptors dealing with ' + 'big data it can be usefull to set it to False)', True, bool), + 'samples_per_cst': ('Maximum number of samples to output for each negated ' + 'constraint (-1 means until the end)', + -1, int), + }) +class sd_constraint_fuzz(StatefulDisruptor): + """ + + When the CSP (Constraint Satisfiability Problem) backend are used in the node description. + This operator negates the constraint one-by-one and output 1 or more samples for each negated + constraint. + + """ + + def setup(self, dm, user_input): + + assert self.const_idx > 0 + assert self.sample_idx > 0 + + self._first_call = True + self._count = 0 + self._constraint_negated = False + self._current_constraint_idx = self.const_idx-1 + self._sample_count = 0 + self._step_size = self.sample_idx + + return True + + def set_seed(self, prev_data): + prev_content = prev_data.content + if not isinstance(prev_content, Node): + prev_data.add_info('UNSUPPORTED INPUT') + return prev_data + + self.csp = prev_content.get_csp() + if not self.csp: + prev_data.add_info('CSP BACKEND NOT USED BY THIS ATOM') + return prev_data + + self.seed = prev_content + + self.seed.freeze(resolve_csp=True) + self.valid_solution = self.csp.get_solution() + self.csp_constraints = self.csp.get_all_constraints() + self.csp_variables = {v for c in self.csp_constraints for v in c.vars} + + def _update_csp(self): + current_constraint = self.csp.get_constraint(self._current_constraint_idx) + variables = self.csp_variables - set(current_constraint.vars) + for v in variables: + self.csp.set_var_domain(v, [self.valid_solution[v]]) + + def _process_next_constraint(self): + self.csp.restore_var_domains() + self.csp.reset_constraint(self._current_constraint_idx) + self._constraint_negated = False + if self._current_constraint_idx < self.csp.nb_constraints - 1: + self._current_constraint_idx += 1 + self.csp.negate_constraint(self._current_constraint_idx) + self._constraint_negated = True + self._update_csp() + self._sample_count = 1 + return True + + else: + return False + + def disrupt_data(self, dm, target, data): + + if not self._constraint_negated: + self.csp.negate_constraint(self._current_constraint_idx) + self._constraint_negated = True + self._update_csp() + self.seed.freeze(resolve_csp=True) + + if self._sample_count < self.samples_per_cst or self.samples_per_cst == -1: + if self._first_call: + self._sample_count = self._step_size + else: + self._sample_count += 1 + else: + if not self._process_next_constraint(): # no more constraint to deal with + data.make_unusable() + self.handover() + return data + + if self._first_call: + self.seed.unfreeze(recursive=False, dont_change_state=True, walk_csp=True, + walk_csp_step_size=self._step_size) + self._first_call = False + else: + self.seed.unfreeze(recursive=False, dont_change_state=True, walk_csp=True, + walk_csp_step_size=1) + + if self.seed.no_more_solution_for_csp: # Node.unfreeze() will trigger it if no new solution + if not self._process_next_constraint(): + data.make_unusable() + self.handover() + return data + + self.seed.freeze(resolve_csp=True) + self._count += 1 + + data.add_info(f'constraint fuzzing test case index: {self._count}') + data.add_info(f' |_ constraint number: {self._current_constraint_idx+1}/{self.csp.nb_constraints}') + data.add_info(f' |_ sample index: {self._sample_count}/{self.samples_per_cst}') + data.add_info(' |_ variables assignment:') + solution = self.csp.get_solution() + for var, value in solution.items(): + data.add_info(f' --> {var}: {value}') + + if self.clone_node: + exported_node = Node(self.seed.name, base_node=self.seed, new_env=True) + data.update_from(exported_node) + else: + data.update_from(self.seed) + + data.altered = True + + return data diff --git a/framework/global_resources.py b/framework/global_resources.py index 4d0480f..07b5852 100644 --- a/framework/global_resources.py +++ b/framework/global_resources.py @@ -27,14 +27,28 @@ import inspect from enum import Enum -import framework -from libs.utils import ensure_dir, ensure_file +xdg_mod_error = False +try: + from xdg.BaseDirectory import xdg_data_home, xdg_config_home +except ModuleNotFoundError: + xdg_mod_error = True + print('WARNING [FMK]: python3-xdg module is not installed!') -fuddly_version = '0.27.2' +# TODO: Taken out of libs.utils, is this the best place for them? +def ensure_dir(f): + d = os.path.dirname(f) + if not os.path.exists(d): + os.makedirs(d) + +def ensure_file(f): + if not os.path.isfile(f): + open(f, 'a').close() + + +fuddly_version = '0.30' framework_folder = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) -# framework_folder = os.path.dirname(framework.__file__) framework_folder = '.' if framework_folder == '' else framework_folder app_folder = os.path.dirname(framework_folder) @@ -43,8 +57,14 @@ data_models_folder = app_folder + os.sep + 'data_models' + os.sep fuddly_data_folder = os.path.expanduser('~' + os.sep + 'fuddly_data' + os.sep) -if not os.path.exists(fuddly_data_folder): - new_fuddly_data_folder = True +if not xdg_mod_error and not os.path.exists(fuddly_data_folder): + use_xdg = True + fuddly_data_folder = xdg_data_home + os.sep + 'fuddly' + os.sep + if not os.path.exists(fuddly_data_folder): + new_fuddly_data_folder = True +else: + use_xdg = False + ensure_dir(fuddly_data_folder) exported_data_folder = fuddly_data_folder + 'exported_data' + os.sep @@ -59,7 +79,12 @@ ensure_dir(external_libs_folder) external_tools_folder = fuddly_data_folder + 'external_tools' + os.sep ensure_dir(external_tools_folder) -config_folder = os.path.join(fuddly_data_folder, 'config') + os.sep + +if not use_xdg: + config_folder = os.path.join(fuddly_data_folder, 'config') + os.sep +else: + xdg_fuddly_config_folder = xdg_config_home + os.sep + 'fuddly' + os.sep + config_folder = xdg_fuddly_config_folder ensure_dir(config_folder) user_projects_folder = fuddly_data_folder + 'user_projects' + os.sep @@ -69,6 +94,10 @@ ensure_dir(user_data_models_folder) ensure_file(user_data_models_folder + os.sep + '__init__.py') +user_info_folder = fuddly_data_folder + 'user_info' + os.sep +ensure_dir(user_info_folder) +ensure_file(user_info_folder + os.sep + '__init__.py') + user_targets_folder = fuddly_data_folder + 'user_targets' + os.sep ensure_dir(user_targets_folder) ensure_file(user_targets_folder + os.sep + '__init__.py') @@ -87,26 +116,19 @@ def convert_to_internal_repr(val): new_v = convert_to_internal_repr(v) new_val.append(new_v) val = new_val - elif sys.version_info[0] > 2: - if isinstance(val, str): - val = val.encode(internal_repr_codec) - elif isinstance(val, unicode): # only for python2 + elif isinstance(val, str): val = val.encode(internal_repr_codec) - elif isinstance(val, str): # only for python2 - pass else: - raise ValueError + assert isinstance(val, bytes) + return val def unconvert_from_internal_repr(val): - if sys.version_info[0] == 2 and isinstance(val, buffer): - # This case occurs when reading from the FmkDB - val = str(val) - else: - try: - val = val.decode(internal_repr_codec, 'strict') - except: - val = val.decode('latin-1') + try: + val = val.decode(internal_repr_codec, 'strict') + except: + val = val.decode('latin-1') + return val def is_string_compatible(val): @@ -116,12 +138,11 @@ def is_string_compatible(val): return False else: return True - elif sys.version_info[0] > 2: - return isinstance(val, (str, bytes)) - elif isinstance(val, (unicode, str)): # only for python2 - return True else: - return False + return isinstance(val, (str, bytes)) + +def get_user_input(msg): + return input(msg) # Generic container for user inputs @@ -134,11 +155,6 @@ def __init__(self, **kwargs): for k, v in kwargs.items(): self._inputs[k] = v - # for python2 compatibility - def __nonzero__(self): - return bool(self._inputs) - - # for python3 compatibility def __bool__(self): return bool(self._inputs) @@ -155,6 +171,9 @@ def set_user_inputs(self, user_inputs): assert isinstance(user_inputs, dict) self._inputs = user_inputs + def merge_with(self, user_inputs): + self._inputs.update(user_inputs._inputs) + def check_conformity(self, valid_args): for arg in self._inputs: if arg not in valid_args: @@ -176,6 +195,8 @@ def __str__(self): else: return '[ ]' + __repr__ = __str__ + def __copy__(self): new_ui = type(self)() new_ui.__dict__.update(self.__dict__) @@ -197,13 +218,15 @@ class AbsCsts(object): Contents = 2 Regexp = 3 Structure = 4 + SimilarContent = 5 - def __init__(self, size=True, contents=True, regexp=True, struct=True): + def __init__(self, size=True, content=True, regexp=True, struct=True, similar_content=False): self.constraints = { AbsCsts.Size: size, - AbsCsts.Contents: contents, + AbsCsts.Contents: content, AbsCsts.Regexp: regexp, - AbsCsts.Structure: struct + AbsCsts.Structure: struct, + AbsCsts.SimilarContent: similar_content # for String-type nodes it means "case sensitive" } def __bool__(self): @@ -240,8 +263,9 @@ def __repr__(self): class AbsNoCsts(AbsCsts): - def __init__(self, size=False, contents=False, regexp=False, struct=False): - AbsCsts.__init__(self, size=size, contents=contents, regexp=regexp, struct=struct) + def __init__(self, size=False, content=False, regexp=False, struct=False, similar_content=False): + AbsCsts.__init__(self, size=size, content=content, regexp=regexp, struct=struct, + similar_content=similar_content) def __repr__(self): return 'AbsNoCsts()' @@ -249,8 +273,9 @@ def __repr__(self): class AbsFullCsts(AbsCsts): - def __init__(self, size=True, contents=True, regexp=True, struct=True): - AbsCsts.__init__(self, size=size, contents=contents, regexp=regexp, struct=struct) + def __init__(self, size=True, content=True, regexp=True, struct=True, similar_content=True): + AbsCsts.__init__(self, size=size, content=content, regexp=regexp, struct=struct, + similar_content=similar_content) def __repr__(self): return 'AbsFullCsts()' @@ -269,7 +294,7 @@ class Error(object): FmkWarning = -6 OperationCancelled = -7 - # FmkPlumbing.get_data() error code + # FmkPlumbing.process_data() error code CloneError = -10 InvalidDmaker = -11 HandOver = -12 @@ -348,3 +373,4 @@ class HOOK(Enum): before_sending_step2 = 3 after_sending = 4 after_fbk = 5 + final = 6 diff --git a/framework/knowledge/feedback_collector.py b/framework/knowledge/feedback_collector.py index 30064ea..04df433 100644 --- a/framework/knowledge/feedback_collector.py +++ b/framework/knowledge/feedback_collector.py @@ -28,12 +28,14 @@ class FeedbackSource(object): - def __init__(self, src, subref=None, reliability=None, related_tg=None): + def __init__(self, src, subref=None, reliability=None, related_tg=None, + display_feedback=True): self._subref = subref self._name = str(src) if subref is None else str(src) + ' - ' + str(subref) self._obj = src self._reliability = reliability self._related_tg = related_tg + self._display_feedback = display_feedback def __str__(self): return self._name @@ -52,6 +54,9 @@ def obj(self): def related_tg(self): return self._related_tg + @property + def display_feedback(self): + return self._display_feedback class FeedbackCollector(object): fbk_lock = threading.Lock() diff --git a/framework/knowledge/feedback_handler.py b/framework/knowledge/feedback_handler.py index 5dc3209..93a457d 100644 --- a/framework/knowledge/feedback_handler.py +++ b/framework/knowledge/feedback_handler.py @@ -64,19 +64,18 @@ class FeedbackHandler(object): A feedback handler extract information from binary data. """ - def __init__(self, new_window=False, new_window_title=None, xterm_prg_name='x-terminal-emulator'): + def __init__(self, new_window=False, new_window_title=None): """ Args: new_window: If `True`, a new terminal emulator is created, enabling the decoder to use it for display via the methods `print()` and `print_nl()` - xterm_prg_name: name of the terminal emulator program to be started """ self._new_window = new_window self._new_window_title = new_window_title - self._xterm_prg_name = xterm_prg_name self._s = None self.term = None + self.fmkops = None def notify_data_sending(self, current_dm, data_list, timestamp, target): """ @@ -89,7 +88,7 @@ def notify_data_sending(self, current_dm, data_list, timestamp, target): current_dm (:class:`framework.data_model.DataModel`): current loaded DataModel data_list (list): list of :class:`framework.data.Data` that were sent timestamp (datetime): date when data was sent - target (Target): target to which data was sent + target (:class:`framework.target_helpers.Target`): target to which data was sent """ pass @@ -122,19 +121,28 @@ def estimate_last_data_impact_uniqueness(self): """ return UNIQUE - def _start(self): + def start(self, current_dm): + pass + + def stop(self): + pass + + def _start(self, current_dm): self._s = '' if self._new_window: nm = self.__class__.__name__ if self._new_window_title is None else self._new_window_title - self.term = Term(name=nm, xterm_prg_name=self._xterm_prg_name, - keepterm=True) + self.term = Term(title=nm, keepterm=True) self.term.start() + self.start(current_dm) + def _stop(self): self._s = None if self._new_window and self.term is not None: self.term.stop() + self.stop() + def print(self, msg): if self._new_window: self.term.print(msg) diff --git a/framework/knowledge/information.py b/framework/knowledge/information.py index afe17e8..8d3e9f6 100644 --- a/framework/knowledge/information.py +++ b/framework/knowledge/information.py @@ -94,13 +94,24 @@ class Hardware(Info): class Language(Info): C = auto() + Ada = auto() Pascal = auto() Unknown = auto() class InputHandling(Info): Ctrl_Char_Set = auto() + Printable_Char_Set = auto() Unknown = auto() +class Test(Info): + Cursory = auto() + Medium = auto() + Deep = auto() + +class OperationMode(Info): + Determinist = auto() + Random = auto() + class InformationCollector(object): @@ -110,6 +121,8 @@ def __init__(self): def add_information(self, info, initial_trust_value=0): assert info is not None + if isinstance(info, Info): + info = [info] try: for i in info: @@ -143,11 +156,6 @@ def __str__(self): return desc - # for python2 compatibility - def __nonzero__(self): - return bool(self._collector) - - # for python3 compatibility def __bool__(self): return bool(self._collector) diff --git a/framework/logger.py b/framework/logger.py index ec3590e..8b04a53 100644 --- a/framework/logger.py +++ b/framework/logger.py @@ -22,33 +22,43 @@ ################################################################################ import os +import time + import sys import datetime import threading import itertools +from typing import List, Tuple + from libs.external_modules import * from libs.utils import get_caller_object from framework.data import Data from framework.global_resources import * from framework.database import Database from framework.knowledge.feedback_collector import FeedbackSource -from libs.utils import ensure_dir +from libs.utils import ExternalDisplay, Accumulator import framework.global_resources as gr class Logger(object): - ''' + """ The Logger is used for keeping the history of the communication with the Target. The methods are used by the framework, but can also be leveraged by an Operator. - ''' + """ fmkDB = None + FLUSH_API = 1 + WRITE_API = 2 + PRETTY_PRINT_API = 3 + PRINT_CONSOLE_API = 4 + + def __init__(self, name=None, prefix='', record_data=False, explicit_data_recording=False, - export_orig=True, export_raw_data=True, console_display_limit=800, - enable_file_logging=False): - ''' + export_raw_data=True, term_display_limit=800, enable_term_display=True, + enable_file_logging=False, highlight_marked_nodes=False): + """ Args: name (str): Name to be used in the log filenames. If not specified, the name of the project in which the logger is embedded will be used. @@ -59,20 +69,23 @@ def __init__(self, name=None, prefix='', record_data=False, explicit_data_record Such notification is possible when the framework call its method :meth:`framework.operator_helpers.Operator.do_after_all()`, where the Operator can take its decision after the observation of the target feedback and/or probes outputs. - export_orig (bool): If True, will also log the original data on which disruptors have been called. export_raw_data (bool): If True, will log the data as it is, without trying to interpret it as human readable text. - console_display_limit (int): maximum amount of characters to display on the console at once. + term_display_limit (int): maximum amount of characters to display on the terminal at once. If this threshold is overrun, the message to print on the console will be truncated. + enable_term_display (bool): If True, information will be displayed on the terminal prefix (str): prefix to use for printing on the console. enable_file_logging (bool): If True, file logging will be enabled. - ''' + highlight_marked_nodes (bool): If True, alteration performed by compatible disruptors + will be highlighted. Only possible if `export_raw_data` is False, as this option forces + data interpretation. + """ + self.name = name self.p = prefix self.__record_data = record_data self.__explicit_data_recording = explicit_data_recording - self.__export_orig = export_orig - self._console_display_limit = console_display_limit + self._term_display_limit = term_display_limit now = datetime.datetime.now() self.__prev_export_date = now.strftime("%Y%m%d_%H%M%S") @@ -82,37 +95,78 @@ def __init__(self, name=None, prefix='', record_data=False, explicit_data_record self._enable_file_logging = enable_file_logging self._fd = None + if export_raw_data and highlight_marked_nodes: + raise ValueError('When @highlight_marked_nodes is True, @export_raw_data should be False') + self._hl_marked_nodes = highlight_marked_nodes + self._tg_fbk = [] self._tg_fbk_lck = threading.Lock() + self.display_on_term = enable_term_display + + self._log_handler_thread = None + self._log_handler_stop_event = threading.Event() + + self._thread_initialized = threading.Event() + self._log_entry_submitted_cond = threading.Condition() + self._log_entry_list = [] + # self._log_displayed = threading.Event() + self._sync_lock = threading.Lock() + + self._ext_disp = ExternalDisplay() + def init_logfn(x, nl_before=True, nl_after=False, rgb=None, style=None, verbose=False, do_record=True): + if not self.display_on_term: + return + + no_format_mode = False if issubclass(x.__class__, Data): data = self._handle_binary_content(x.to_bytes(), raw=self.export_raw_data) + colored_data = x.to_formatted_str() if self._hl_marked_nodes else data rgb = None style = None + no_format_mode = self._hl_marked_nodes elif isinstance(x, str): - data = x + colored_data = data = x else: - data = self._handle_binary_content(x, raw=self.export_raw_data) - self.print_console(data, nl_before=nl_before, nl_after=nl_after, rgb=rgb, style=style) + colored_data = data = self._handle_binary_content(x, raw=self.export_raw_data) + self.print_console(colored_data, nl_before=nl_before, nl_after=nl_after, + rgb=rgb, style=style, no_format_mode=no_format_mode) if verbose and issubclass(x.__class__, Data): - x.show() + self.pretty_print_data(x) return data self.log_fn = init_logfn + def flush(self): + with self._sync_lock: + with self._log_entry_submitted_cond: + self._log_entry_list.append((Logger.FLUSH_API, None)) + self._log_entry_submitted_cond.notify() + + def write(self, data: str): + with self._sync_lock: + with self._log_entry_submitted_cond: + self._log_entry_list.append((Logger.WRITE_API, data)) + self._log_entry_submitted_cond.notify() + + def pretty_print_data(self, data: Data, fd = None, raw_limit: int = None): + with self._sync_lock: + with self._log_entry_submitted_cond: + self._log_entry_list.append((Logger.PRETTY_PRINT_API, (data, fd, raw_limit))) + self._log_entry_submitted_cond.notify() + + def set_external_display(self, disp): + self._ext_disp = disp + def __str__(self): return 'Logger' - def _handle_binary_content(self, content, raw=False): content = gr.unconvert_from_internal_repr(content) - if sys.version_info[0] > 2: - content = content if not raw else '{!a}'.format(content) - else: - content = content if not raw else repr(content) + content = content if not raw else '{!a}'.format(content) return content @@ -130,7 +184,7 @@ def start(self): self._tg_fbk = [] if self.name is None: - self.log_fn = lambda x: x + raise ValueError("Logger's name shall be provided (either by the framework or at init)") elif self._enable_file_logging: self.now = datetime.datetime.now() @@ -141,22 +195,25 @@ def start(self): def intern_func(x, nl_before=True, nl_after=False, rgb=None, style=None, verbose=False, do_record=True): + no_format_mode = False if issubclass(x.__class__, Data): data = self._handle_binary_content(x.to_bytes(), raw=self.export_raw_data) + colored_data = x.to_formatted_str() if self._hl_marked_nodes else data rgb = None style = None + no_format_mode = self._hl_marked_nodes elif isinstance(x, str): - data = x + colored_data = data = x else: - data = self._handle_binary_content(x, raw=self.export_raw_data) - self.print_console(data, nl_before=nl_before, nl_after=nl_after, rgb=rgb, style=style) + colored_data = data = self._handle_binary_content(x, raw=self.export_raw_data) + self.print_console(colored_data, nl_before=nl_before, nl_after=nl_after, + rgb=rgb, style=style, no_format_mode=no_format_mode) if not do_record: return data try: - self._fd.write(data) - self._fd.write('\n') + self._fd.write(data+'\n') if verbose and issubclass(x.__class__, Data): - x.show(log_func=self._fd.write) + self.pretty_print_data(x, fd=self._fd) self._fd.flush() except ValueError: self.print_console('\n*** ERROR: The log file has been closed.' \ @@ -171,8 +228,77 @@ def intern_func(x, nl_before=True, nl_after=False, rgb=None, style=None, verbose # No file logging pass + if self._log_handler_thread is not None: + return + + self._log_handler_thread = threading.Thread(None, self._log_handler, 'log_handler') + self._log_handler_thread.start() + while not self._thread_initialized.is_set(): + self._thread_initialized.wait(0.1) + self.print_console('*** Logger is started ***\n', nl_before=False, rgb=Color.COMPONENT_START) + def _stop_log_handler(self): + with self._sync_lock: + self._log_handler_stop_event.set() + self._log_handler_thread.join() + + def _log_handler(self): + self._thread_initialized.set() + + accu = Accumulator() + while True: + # self._log_displayed.clear() + with self._log_entry_submitted_cond: + if self._log_handler_stop_event.is_set() and not self._log_entry_list: + break + self._log_entry_submitted_cond.wait(0.001) + + if self._log_entry_list: + log_entries = self._log_entry_list + self._log_entry_list = [] + else: + continue + + for log_e in log_entries: + api, params = log_e + + if api == Logger.FLUSH_API: + if not self._ext_disp.is_enabled: + sys.stdout.flush() + elif api == Logger.WRITE_API: + if self._ext_disp.is_enabled: + self._ext_disp.disp.print(params) + else: + sys.stdout.write(params) + elif api == Logger.PRETTY_PRINT_API: + data, fd, raw_limit = params + if fd is None: + data.show(log_func=accu.accumulate, raw_limit=raw_limit) + if self._ext_disp.is_enabled: + self._ext_disp.disp.print(accu.content) + else: + sys.stdout.write(accu.content) + accu.clear() + else: + data.show(log_func=fd.write, raw_limit=raw_limit) + fd.flush() + elif api == Logger.PRINT_CONSOLE_API: + self._print_console(*params) + else: + self._print_console('*** ERROR[Logger]: Unknown API ***', rgb=Color.ERROR) + + + # self._log_displayed.set() + + self._log_handler_stop_event.wait(0.001) + + def wait_for_sync(self): + # while not self._log_displayed.is_set(): + # self._log_displayed.wait(0.01) + # self._log_displayed.clear() + time.sleep(0.1) + def stop(self): if self._fd: @@ -183,13 +309,14 @@ def stop(self): self._last_data_IDs = {} self.last_data_recordable = None + self._stop_log_handler() + self.print_console('*** Logger is stopped ***\n', nl_before=False, rgb=Color.COMPONENT_STOP) def reset_current_state(self): self._current_data = None self._current_group_id = None - self._current_orig_data_id = None self._current_size = None self._current_ack_dates = None self._current_dmaker_list= [] @@ -208,12 +335,12 @@ def commit_data_table_entry(self, group_id, prj_name): last_data_id = None for tg_ref, ack_date in self._current_ack_dates.items(): last_data_id = self.fmkDB.insert_data(init_dmaker, dm_name, - self._current_data.to_bytes(), - self._current_size, - self._current_sent_date, - ack_date, - tg_ref, prj_name, - group_id=group_id) + self._current_data.to_bytes(), + self._current_size, + self._current_sent_date, + ack_date, + tg_ref, prj_name, + group_id=group_id) # assert isinstance(tg_ref, FeedbackSource) self._last_data_IDs[tg_ref.obj] = last_data_id @@ -224,13 +351,7 @@ def commit_data_table_entry(self, group_id, prj_name): self._current_data.set_data_id(last_data_id) - if self._current_orig_data_id is not None: - self.fmkDB.insert_steps(last_data_id, 1, None, None, - self._current_orig_data_id, - None, None) - step_id_start = 2 - else: - step_id_start = 1 + step_id_start = 1 for step_id, dmaker in enumerate(self._current_dmaker_list, start=step_id_start): dmaker_type, dmaker_name, user_input = dmaker @@ -252,27 +373,51 @@ def commit_data_table_entry(self, group_id, prj_name): return None + def log_async_data(self, data_list: Data | List[Data] | Tuple[Data], sent_date, target_ref, prj_name, + current_data_id): + + if isinstance(data_list, Data): + data_list = (data_list,) + + for data in data_list: + raw_data = data.to_bytes() + data_sz = len(raw_data) + init_dmaker = data.get_initial_dmaker() + dtype = Database.DEFAULT_GTYPE_NAME if init_dmaker is None else init_dmaker[0] + dm = data.get_data_model() + dm_name = Database.DEFAULT_DM_NAME if dm is None else dm.name + self.fmkDB.insert_async_data(dtype=dtype, dm_name=dm_name, + raw_data=raw_data, + sz=data_sz, sent_date=sent_date, + target_ref=target_ref, prj_name=prj_name, + current_data_id=current_data_id) + + def log_fmk_info(self, info, nl_before=False, nl_after=False, rgb=Color.FMKINFO, - data_id=None, do_record=True, delay_recording=False): + data_id=None, do_show=True, do_record=True, delay_recording=False): now = datetime.datetime.now() p = '\n' if nl_before else '' s = '\n' if nl_after else '' msg = "{prefix:s}*** [ {message:s} ] ***{suffix:s}".format(prefix=p, suffix=s, message=info) - self.log_fn(msg, rgb=rgb) + if do_show: + self.log_fn(msg, rgb=rgb) if do_record: if not delay_recording: if data_id is None: - for d_id in self._last_data_IDs.values(): - self.fmkDB.insert_fmk_info(d_id, info, now) + if self._last_data_IDs: + for d_id in self._last_data_IDs.values(): + self.fmkDB.insert_fmk_info(d_id, info, now) + else: + self.fmkDB.insert_fmk_info(None, info, now) else: self.fmkDB.insert_fmk_info(data_id, info, now) else: self._current_fmk_info.append((info, now)) - def collect_feedback(self, content, status_code=None): + def collect_feedback(self, content, status_code=None, subref=None, fbk_src=None): """ Used within the scope of the Logger feedback-collector infrastructure. If your target implement the interface :meth:`Target.get_feedback`, no need to @@ -283,12 +428,14 @@ def collect_feedback(self, content, status_code=None): Args: content: feedback record status_code (int): should be negative for error + subref (str): specific reference to distinguish internal log sources within the same caller + fbk_src: [optional] source object of the feedback """ now = datetime.datetime.now() - fbk_src = get_caller_object() + fbk_src = get_caller_object() if fbk_src is None else fbk_src with self._tg_fbk_lck: - self._tg_fbk.append((now, FeedbackSource(fbk_src), content, status_code)) + self._tg_fbk.append((now, FeedbackSource(fbk_src, subref=subref), content, status_code)) def shall_record(self): if self.last_data_recordable or not self.__explicit_data_recording: @@ -303,17 +450,33 @@ def _log_feedback(self, source, content, status_code, timestamp, record=True): fbk_cond = status_code is not None and status_code < 0 hdr_color = Color.FEEDBACK_ERR if fbk_cond else Color.FEEDBACK body_color = Color.FEEDBACK_HLIGHT if fbk_cond else None + # now = timestamp.strftime("%d/%m/%Y - %H:%M:%S.%f") + if isinstance(timestamp, datetime.datetime) or timestamp is None: + ts_msg = f"received at {timestamp}" + elif isinstance(timestamp, list): + if len(timestamp) == 1: + ts_msg = f"received at {timestamp[0]}" + else: + ts_msg = f"received from {timestamp[0]} to {timestamp[-1]}" + else: + raise ValueError(f'Wrong format for timestamp [{type(timestamp)}]') + if not processed_feedback: - msg_hdr = "### Status from '{!s}': {!s}".format(source, status_code) + msg_hdr = "### Status from '{!s}': {!s} - {:s}".format( + source, status_code, ts_msg) else: - msg_hdr = "### Feedback from '{!s}' (status={!s}):".format(source, status_code) + msg_hdr = "### Feedback from '{!s}' (status={!s}) - {:s}:".format( + source, status_code, ts_msg) self.log_fn(msg_hdr, rgb=hdr_color, do_record=record) if processed_feedback: - if isinstance(processed_feedback, list): - for dfbk in processed_feedback: - self.log_fn(dfbk, rgb=body_color, do_record=record) + if source.display_feedback: + if isinstance(processed_feedback, list): + for dfbk in processed_feedback: + self.log_fn(dfbk, rgb=body_color, do_record=record) + else: + self.log_fn(processed_feedback, rgb=body_color, do_record=record) else: - self.log_fn(processed_feedback, rgb=body_color, do_record=record) + self.log_fn('Feedback not displayed', rgb=Color.WARNING, do_record=record) if record: assert isinstance(source, FeedbackSource) @@ -321,8 +484,10 @@ def _log_feedback(self, source, content, status_code, timestamp, record=True): try: data_id = self._last_data_IDs[source.related_tg] except KeyError: - print('\nWarning: The feedback source is related to a target to which nothing has been sent.' - ' Retrieved feedback will not be attached to any data ID.') + self.print_console( + '*** Warning: The feedback source is related to a target to which nothing has been sent.' + ' Retrieved feedback will not be attached to any data ID.', + nl_before=True, rgb=Color.WARNING) data_id = None else: ids = self._last_data_IDs.values() @@ -355,7 +520,7 @@ def log_collected_feedback(self, preamble=None, epilogue=None): bool: True if target feedback has been collected through logger infrastructure :meth:`Logger.collect_feedback`, False otherwise. """ - error_detected = {} + collected_status = {} with self._tg_fbk_lck: fbk_list = self._tg_fbk @@ -373,17 +538,12 @@ def log_collected_feedback(self, preamble=None, epilogue=None): for idx, fbk_record in enumerate(fbk_list): timestamp, fbk_src, fbk, status = fbk_record self._log_feedback(fbk_src, fbk, status, timestamp, record=record) - - if status is not None and status < 0: - error_detected[fbk_src.obj] = True - else: - error_detected[fbk_src.obj] = False + collected_status[fbk_src.obj] = status if epilogue is not None: self.log_fn(epilogue, do_record=record, rgb=Color.FMKINFO) - return error_detected - + return collected_status def log_target_feedback_from(self, source, content, status_code, timestamp, preamble=None, epilogue=None): @@ -409,16 +569,21 @@ def log_probe_feedback(self, probe, content, status_code, timestamp, related_tg= def _process_target_feedback(self, feedback): if feedback is None: - return feedback + return None if isinstance(feedback, list): new_fbk = [] for f in feedback: + if f is None: + continue new_f = f.strip() if isinstance(new_f, bytes): new_f = self._handle_binary_content(new_f, raw=self.export_raw_data) new_fbk.append(new_f) - if not list(filter(lambda x: x != b'', new_fbk)): + + if not new_fbk: + new_fbk = None + elif not list(filter(lambda x: x != b'', new_fbk)): new_fbk = None else: new_fbk = feedback.strip() @@ -436,7 +601,7 @@ def _encode_target_feedback(self, feedback): def start_new_log_entry(self, preamble=''): self.__idx += 1 self._current_sent_date = datetime.datetime.now() - now = self._current_sent_date.strftime("%d/%m/%Y - %H:%M:%S") + now = self._current_sent_date.strftime("%d/%m/%Y - %H:%M:%S.%f") msg = "====[ {:d} ]==[ {:s} ]====".format(self.__idx, now) msg += '='*(max(80-len(msg),0)) self.log_fn(msg, rgb=Color.NEWLOGENTRY, style=FontStyle.BOLD) @@ -479,10 +644,10 @@ def log_data_info(self, data_info, dmaker_type, data_maker_name): self.log_fn(" |- data info:", rgb=Color.DATAINFO) for msg in data_info: - if len(msg) > 400: - msg = msg[:400] + ' ...' + if len(msg) > self._term_display_limit: + msg = msg[:self._term_display_limit] + ' ...' - self.log_fn(' |_ ' + msg, rgb=Color.DATAINFO) + self.log_fn(' | ' + msg, rgb=Color.DATAINFO) def log_info(self, info): msg = "### Info: {:s}".format(info) @@ -500,47 +665,6 @@ def set_target_ack_date(self, tg_ref, date): else: self._current_ack_dates[tg_ref] = date - def log_orig_data(self, data): - - exportable = False if data is None else data.is_recordable() - - if self.__explicit_data_recording and not exportable: - return False - - if data is not None: - self._current_orig_data_id = data.get_data_id() - - if self.__export_orig and not self.__record_data: - if data is None: - msgs = ("### No Original Data",) - else: - msgs = ("### Original Data:", data) - - for msg in msgs: - self.log_fn(msg, rgb=Color.LOGSECTION) - - ret = True - - elif self.__export_orig: - - if data is None: - ret = False - else: - ffn = self._export_data_func(data) - if ffn: - self.log_fn("### Original data is stored in the file:", rgb=Color.DATAINFO) - self.log_fn(ffn) - ret = True - else: - self.print_console("ERROR: saving data in an extenal file has failed!", - nl_before=True, rgb=Color.ERROR) - ret = False - - else: - ret = False - - return ret - def log_data(self, data, verbose=False): self.log_fn("### Data size: ", rgb=Color.LOGSECTION, nl_after=False) @@ -599,7 +723,7 @@ def _export_data_func(self, data, suffix=''): export_full_fn = os.path.join(base_dir, dm_name, export_fname) - ensure_dir(export_full_fn) + gr.ensure_dir(export_full_fn) fd = open(export_full_fn, 'wb') fd.write(data.to_bytes()) @@ -625,33 +749,57 @@ def log_error(self, err_msg): self.fmkDB.insert_fmk_info(data_id, msg, now, error=True) def print_console(self, msg, nl_before=True, nl_after=False, rgb=None, style=None, - raw_limit=None, limit_output=True): + raw_limit=None, limit_output=True, no_format_mode=False): + + with self._sync_lock: + with self._log_entry_submitted_cond: + params = (msg,nl_before,nl_after,rgb,style,raw_limit,limit_output,no_format_mode) + self._log_entry_list.append((Logger.PRINT_CONSOLE_API, params)) + self._log_entry_submitted_cond.notify() + + + def _print_console(self, msg, nl_before=True, nl_after=False, rgb=None, style=None, + raw_limit=None, limit_output=True, no_format_mode=False): + + if not self.display_on_term: + return if raw_limit is None: - raw_limit = self._console_display_limit + raw_limit = self._term_display_limit p = '\n' if nl_before else '' s = '\n' if nl_after else '' prefix = p + self.p - if isinstance(msg, Data): - msg = repr(msg) + if no_format_mode: + if self._ext_disp.is_enabled: + self._ext_disp.disp.print(prefix + msg) + else: + sys.stdout.write(prefix + msg) + sys.stdout.flush() - suffix = '' - if limit_output and len(msg) > raw_limit: - msg = msg[:raw_limit] - suffix = ' ...' + # print(f'{msg}') - suffix += s + else: + if isinstance(msg, Data): + msg = repr(msg) + + suffix = '' + if limit_output and len(msg) > raw_limit: + msg = msg[:raw_limit] + suffix = ' ...' - if rgb is not None: - msg = colorize(msg, rgb=rgb) + suffix += s - if style is None: - style = '' + if rgb is not None: + msg = colorize(msg, rgb=rgb) - sys.stdout.write(style + prefix) - sys.stdout.write(msg) - sys.stdout.write(suffix + FontStyle.END) - sys.stdout.flush() + if style is None: + style = '' + + if self._ext_disp.is_enabled: + self._ext_disp.disp.print(style+prefix+msg+suffix+FontStyle.END) + else: + sys.stdout.write(style+prefix+msg+suffix+FontStyle.END) + sys.stdout.flush() diff --git a/framework/monitor.py b/framework/monitor.py index 1b900ae..50790c2 100644 --- a/framework/monitor.py +++ b/framework/monitor.py @@ -27,23 +27,23 @@ import time import traceback import re -import subprocess -import select +from framework.comm_backends import BackendError from libs.external_modules import * from framework.global_resources import * -import framework.error_handling as eh class ProbeUser(object): timeout = 5.0 - probe_init_timeout = 10.0 + probe_init_timeout = 15.0 def __init__(self, probe): self._probe = probe self._thread = None self._started_event = threading.Event() self._stop_event = threading.Event() + self._args = None + self._kwargs = None @property def probe(self): @@ -53,6 +53,9 @@ def start(self, *args, **kwargs): if self.is_alive(): raise RuntimeError self._clear() + self._args = args + self._kwargs = kwargs + # print('\n*** DBG start:', self._args, self._kwargs) self._thread = threading.Thread(target=self._run, name=self._probe.__class__.__name__, args=args, kwargs=kwargs) self._thread.daemon = True @@ -60,6 +63,12 @@ def start(self, *args, **kwargs): def stop(self): self._stop_event.set() + try: + self._probe._stop(*self._args, **self._kwargs) + except: + self._handle_exception('during stop()') + finally: + self._thread = None def join(self, timeout=None): if self.is_alive(): @@ -379,12 +388,11 @@ def configure_probe(self, probe, *args): def start_probe(self, probe, related_tg=None): probe_ref = self._get_probe_ref(probe) + self._related_tg = related_tg if probe_ref in self.probe_users: try: - if related_tg is None: - self.probe_users[probe_ref].start(self._dm, self._targets, self._logger) - else: - self.probe_users[probe_ref].start(self._dm, related_tg, self._logger) + tgs = self._targets if self._related_tg is None else self._related_tg + self.probe_users[probe_ref].start(self._dm, tgs, self._logger) except: self.fmk_ops.set_error("Exception raised in probe '{:s}' start".format(probe_ref), code=Error.UserCodeError) @@ -406,7 +414,8 @@ def stop_probe(self, probe): def stop_all_probes(self): for _, probe_user in self.probe_users.items(): - probe_user.stop() + if probe_user.is_alive(): + probe_user.stop() self._tg_from_probe = {} self._wait_for_specific_probes(ProbeUser, ProbeUser.join) @@ -541,6 +550,7 @@ class Probe(object): def __init__(self, delay=1.0): self._status = ProbeStatus(0) self._delay = delay + self._started = False def __str__(self): return "Probe - {:s}".format(self.__class__.__name__) @@ -563,12 +573,19 @@ def delay(self, delay): self._delay = delay def _start(self, dm, target, logger): + if self._started: + return logger.print_console("__ probe '{:s}' is starting __".format(self.__class__.__name__), nl_before=True, nl_after=True) - return self.start(dm, target, logger) + self._started = True # even if .start() fail, .stop() should be called to provide a chance for cleanup + ret = self.start(dm, target, logger) + return ret def _stop(self, dm, target, logger): + if not self._started: + return logger.print_console("__ probe '{:s}' is stopping __".format(self.__class__.__name__), nl_before=True, nl_after=True) self.stop(dm, target, logger) + self._started = False def start(self, dm, target, logger): """ @@ -671,264 +688,6 @@ def get_timestamp(self): return self._now -class Backend(object): - - def __init__(self, codec='latin_1'): - """ - Args: - codec (str): codec used by the monitored system to answer. - """ - self._started = False - self.codec = codec - self._sync_lock = threading.Lock() - - def start(self): - with self._sync_lock: - if not self._started: - self._started = True - self._start() - - def stop(self): - with self._sync_lock: - if self._started: - self._started = False - self._stop() - - def exec_command(self, cmd): - with self._sync_lock: - return self._exec_command(cmd) - - def _exec_command(self, cmd): - raise NotImplementedError - - def _start(self): - pass - - def _stop(self): - pass - - -class SSH_Backend(Backend): - """ - Backend to execute command through a serial line. - """ - def __init__(self, username, password, sshd_ip, sshd_port=22, codec='latin_1'): - """ - Args: - sshd_ip (str): IP of the SSH server. - sshd_port (int): port of the SSH server. - username (str): username to connect with. - password (str): password related to the username. - codec (str): codec used by the monitored system to answer. - """ - Backend.__init__(self, codec=codec) - if not ssh_module: - raise eh.UnavailablePythonModule('Python module for SSH is not available!') - self.sshd_ip = sshd_ip - self.sshd_port = sshd_port - self.username = username - self.password = password - self.client = None - - def _start(self): - self.client = ssh.SSHClient() - self.client.set_missing_host_key_policy(ssh.AutoAddPolicy()) - self.client.connect(self.sshd_ip, port=self.sshd_port, - username=self.username, - password=self.password) - - def _stop(self): - self.client.close() - - def _exec_command(self, cmd): - ssh_in, ssh_out, ssh_err = \ - self.client.exec_command(cmd) - - if ssh_err.read(): - # the command does not exist on the system - raise BackendError('The command does not exist on the host') - else: - return ssh_out.read() - - -class Serial_Backend(Backend): - """ - Backend to execute command through a serial line. - """ - def __init__(self, serial_port, baudrate=115200, bytesize=8, parity='N', stopbits=1, - xonxoff=False, rtscts=False, dsrdtr=False, - username=None, password=None, slowness_factor=5, - cmd_notfound=b'command not found', codec='latin_1'): - """ - Args: - serial_port (str): path to the tty device file. (e.g., '/dev/ttyUSB0') - baudrate (int): baud rate of the serial line. - bytesize (int): number of data bits. (5, 6, 7, or 8) - parity (str): parity checking. ('N', 'O, 'E', 'M', or 'S') - stopbits (int): number of stop bits. (1, 1.5 or 2) - xonxoff (bool): enable software flow control. - rtscts (bool): enable hardware (RTS/CTS) flow control. - dsrdtr (bool): enable hardware (DSR/DTR) flow control. - username (str): username to connect with. If None, no authentication step will be attempted. - password (str): password related to the username. - slowness_factor (int): characterize the slowness of the monitored system. The scale goes from - 1 (fastest) to 10 (slowest). This factor is a base metric to compute the time to wait - for the authentication step to terminate (if `username` and `password` parameter are provided) - and other operations involving to wait for the monitored system. - cmd_notfound (bytes): pattern used to detect if the command does not exist on the - monitored system. - codec (str): codec used to send/receive information through the serial line - """ - Backend.__init__(self, codec=codec) - if not serial_module: - raise eh.UnavailablePythonModule('Python module for Serial is not available!') - - self.serial_port = serial_port - self.baudrate = baudrate - self.bytesize = bytesize - self.parity = parity - self.stopbits= stopbits - self.xonxoff = xonxoff - self.rtscts = rtscts - self.dsrdtr = dsrdtr - self.slowness_factor = slowness_factor - self.cmd_notfound = cmd_notfound - if sys.version_info[0] > 2: - self.username = bytes(username, self.codec) - self.password = bytes(password, self.codec) - else: - self.username = username - self.password = password - - self.client = None - - def _start(self): - self.ser = serial.Serial(self.serial_port, self.baudrate, bytesize=self.bytesize, - parity=self.parity, stopbits=self.stopbits, - xonxoff=self.xonxoff, dsrdtr=self.dsrdtr, rtscts=self.rtscts, - timeout=self.slowness_factor*0.1) - if self.username is not None: - assert self.password is not None - self.ser.flushInput() - self.ser.write(self.username+b'\r\n') - time.sleep(0.1) - self.ser.readline() # we read login echo - pass_prompt = self.ser.readline() - retry = 0 - eot_sent = False - while pass_prompt.lower().find(b'password') == -1: - retry += 1 - if retry > 3 and eot_sent: - self.stop() - raise BackendError('Unable to establish a connection with the serial line.') - elif retry > 3: - # we send an EOT if ever the console was not in its initial state - # (already logged, or with the password prompt, ...) when we first write on - # the serial line. - self.ser.write(b'\x04\r\n') - time.sleep(self.slowness_factor*0.8) - self.ser.flushInput() - self.ser.write(self.username+b'\r\n') - time.sleep(0.1) - self.ser.readline() # we consume the login echo - pass_prompt = self.ser.readline() - retry = 0 - eot_sent = True - else: - chunks = self._read_serial(duration=self.slowness_factor*0.2) - pass_prompt = b''.join(chunks) - time.sleep(0.1) - self.ser.write(self.password+b'\r\n') - time.sleep(self.slowness_factor*0.7) - - def _stop(self): - self.ser.write(b'\x04\r\n') # we send an EOT (Ctrl+D) - self.ser.close() - - def _exec_command(self, cmd): - if not self.ser.is_open: - raise BackendError('Serial port not open') - - if sys.version_info[0] > 2: - cmd = bytes(cmd, self.codec) - cmd += b'\r\n' - self.ser.flushInput() - self.ser.write(cmd) - time.sleep(0.1) - self.ser.readline() # we consume the 'writing echo' from the input - try: - result = self._read_serial(duration=self.slowness_factor*0.8) - except serial.SerialException: - raise BackendError('Exception while reading serial line') - else: - # We have to remove the new prompt line at the end. - # But in our testing environment, the two last entries had to be removed, namely - # 'prompt_line \r\n' and 'prompt_line ' !? - # print('\n*** DBG: ', result) - result = result[:-2] - ret = b''.join(result) - if ret.find(self.cmd_notfound) != -1: - raise BackendError('The command does not exist on the host') - else: - return ret - - def _read_serial(self, duration): - result = [] - t0 = datetime.datetime.now() - delta = -1 - while delta < duration: - now = datetime.datetime.now() - delta = (now - t0).total_seconds() - res = self.ser.readline() - if res == b'': - break - result.append(res) - return result - - -class Shell_Backend(Backend): - """ - Backend to execute shell commands locally - """ - def __init__(self, timeout=None, codec='latin_1'): - """ - Args: - timeout (float): timeout in seconds for reading the result of the command - codec (str): codec used by the monitored system to answer. - """ - Backend.__init__(self, codec=codec) - self._timeout = timeout - self._app = None - - def _start(self): - pass - - def _stop(self): - pass - - def _exec_command(self, cmd): - self._app = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - ready_to_read, ready_to_write, in_error = \ - select.select([self._app.stdout, self._app.stderr], [], [], self._timeout) - - if in_error: - # the command does not exist on the system - raise BackendError('Issue with file descriptors') - elif ready_to_read: - if len(ready_to_read) == 2: - err = ready_to_read[1].read() - if err.strip(): - raise BackendError('ERROR: {!s}'.format(ready_to_read[1].read())) - if ready_to_read[0]: - return ready_to_read[0].read() - else: - raise BackendError('BUG') - else: - return b'' - - -class BackendError(Exception): pass - class ProbePID(Probe): """ Generic probe that enables you to monitor a process PID. @@ -937,7 +696,7 @@ class ProbePID(Probe): :class:`Serial_Backend`). Attributes: - backend (Backend): backend to be used (e.g., :class:`SSH_Backend`). + backend (framework.comm_backends.Backend): backend to be used (e.g., :class:`SSH_Backend`). process_name (str): name of the process to monitor. max_attempts (int): maximum number of attempts for getting the process ID. @@ -961,12 +720,13 @@ def __init__(self): def _get_pid(self, logger): try: - res = self.backend.exec_command(self.command_pattern.format(self.process_name)) + chan_desc = self.backend.exec_command(self.command_pattern.format(self.process_name)) + res = self.backend.read_output(chan_desc) except BackendError: fallback_cmd = 'ps a -opid,comm | grep {0:s}'.format(self.process_name) - res = self.backend.exec_command(fallback_cmd) - if sys.version_info[0] > 2: - res = res.decode(self.backend.codec) + chan_desc = self.backend.exec_command(fallback_cmd) + res = self.backend.read_output(chan_desc) + res = res.decode(self.backend.codec) pid_list = res.split('\n') for entry in pid_list: if entry.find(self.process_name) >= 0: @@ -979,8 +739,7 @@ def _get_pid(self, logger): # process not found pid = -1 else: - if sys.version_info[0] > 2: - res = res.decode(self.backend.codec) + res = res.decode(self.backend.codec) l = res.split() if len(l) > 1: logger.print_console("*** ERROR: more than one PID detected for process name '{:s}'" @@ -1053,7 +812,7 @@ class ProbeMem(Probe): :class:`Serial_Backend`). Attributes: - backend (Backend): backend to be used (e.g., :class:`SSH_Backend`). + backend (framework.comm_backends.Backend): backend to be used (e.g., :class:`SSH_Backend`). process_name (str): name of the process to monitor. threshold (int): memory (RSS) threshold that the monitored process should not exceed. (dimension should be the same as what is provided by the `ps` command of the system @@ -1079,10 +838,10 @@ def __init__(self): Probe.__init__(self) def _get_mem(self): - res = self.backend.exec_command(self.command_pattern.format(self.process_name)) + chan_desc = self.backend.exec_command(self.command_pattern.format(self.process_name)) + res = self.backend.read_output(chan_desc) - if sys.version_info[0] > 2: - res = res.decode(self.backend.codec) + res = res.decode(self.backend.codec) proc_list = res.split('\n') for entry in proc_list: if entry.find(self.process_name) >= 0: @@ -1160,6 +919,53 @@ def reset(self): self._saved_mem = self._max_mem self._max_mem = self._saved_mem + +class ProbeCmd(Probe): + """ + Generic probe that enables you to execute shell commands and retrieve the output. + + The monitoring can be done through different backend (e.g., :class:`SSH_Backend`, + :class:`Serial_Backend`). + + Attributes: + backend (framework.comm_backends.Backend): backend to be used (e.g., :class:`SSH_Backend`). + init_command (str): ssh command to execute at init + recurrent_command (str): ssh command to execute at each probing + """ + backend = None + init_command = None + recurrent_command = None + + def __init__(self): + assert self.backend != None + self.chan_desc = None + Probe.__init__(self) + + def start(self, dm, target, logger): + self.backend.start() + if self.init_command is not None: + try: + self.chan_desc = self.backend.exec_command(self.init_command) + data = self.backend.read_output(self.chan_desc) + except BackendError as err: + return ProbeStatus(-1, info=str(err)) + + return ProbeStatus(0, info=data) + + def stop(self, dm, target, logger): + self.backend.stop() + + def main(self, dm, target, logger): + try: + if self.recurrent_command is not None: + self.chan_desc = self.backend.exec_command(self.recurrent_command) + data = self.backend.read_output(self.chan_desc) + except BackendError as err: + return ProbeStatus(-1, info=str(err)) + + return ProbeStatus(0, info=data) + + def probe(project): def internal_func(probe_cls): project.monitor.add_probe(probe_cls(), blocking=False) diff --git a/framework/node.py b/framework/node.py index dfb6dee..31a449c 100644 --- a/framework/node.py +++ b/framework/node.py @@ -36,6 +36,9 @@ import uuid import struct import math +import time + +from pprint import pprint as pp from enum import Enum from random import shuffle @@ -46,12 +49,15 @@ from libs.external_modules import * from framework.global_resources import * from framework.error_handling import * +from framework.constraint_helpers import CSP, ConstraintError + import framework.value_types as fvt import libs.debug_facility as dbg +from libs.utils import chunk_lines DEBUG = dbg.DM_DEBUG - +DEBUG_PRINT = dbg.DEBUG_PRINT def split_with(predicate, iterable): l = [] @@ -269,7 +275,7 @@ def check(self): def _condition_satisfied(self, node, condition): exist = node.env.node_exists(id(node)) - crit_1 = True if exist else False + crit_1 = exist crit_2 = True if exist and condition is not None: try: @@ -319,13 +325,15 @@ def check(self, node): class RawCondition(NodeCondition): - def __init__(self, val=None, neg_val=None, cond_func=None): + def __init__(self, val=None, neg_val=None, cond_func=None, case_sensitive=True): """ Args: val (bytes/:obj:`list` of bytes): value(s) that satisfies the condition neg_val (bytes/:obj:`list` of bytes): value(s) that does NOT satisfy the condition (AND clause) cond_func: function that takes the node value and return a boolean + case_sensitive: if False, ignore case for performing comparison """ + self.case_sensitive = case_sensitive self.val = self._handle_cond(val) if val is not None else None self.neg_val = self._handle_cond(neg_val) if neg_val is not None else None self.cond_func = cond_func @@ -334,16 +342,22 @@ def _handle_cond(self, val): if isinstance(val, (tuple, list)): normed_val = [] for v in val: - normed_val.append(convert_to_internal_repr(v)) + normed_v = convert_to_internal_repr(v) if self.case_sensitive else convert_to_internal_repr(v).lower() + normed_val.append(normed_v) else: normed_val = convert_to_internal_repr(val) + normed_val = normed_val if self.case_sensitive else normed_val.lower() + return normed_val def check(self, node): node_val = node._tobytes() + if Node.DEFAULT_DISABLED_VALUE: node_val = node_val.replace(Node.DEFAULT_DISABLED_VALUE, b'') + if not self.case_sensitive: + node_val = node_val.lower() result = self._check_inclusion(node_val, val=self.val, neg_val=self.neg_val) if self.cond_func: result = result and self.cond_func(node_val) @@ -464,21 +478,9 @@ def __init__(self, items_to_set=None, items_to_clear=None, transform_func=None): self._transform_func = transform_func self._custo_items = copy.copy(self._custo_items) if items_to_set is not None: - if isinstance(items_to_set, int): - assert(items_to_set in self._custo_items) - self._custo_items[items_to_set] = True - elif isinstance(items_to_set, list): - for item in items_to_set: - assert(item in self._custo_items) - self._custo_items[item] = True + self.set_items(items_to_set) if items_to_clear is not None: - if isinstance(items_to_clear, int): - assert(items_to_clear in self._custo_items) - self._custo_items[items_to_clear] = False - elif isinstance(items_to_clear, list): - for item in items_to_clear: - assert(item in self._custo_items) - self._custo_items[item] = False + self.clear_items(items_to_clear) def __getitem__(self, key): if key in self._custo_items: @@ -486,10 +488,40 @@ def __getitem__(self, key): else: return None + def set_items(self, items_to_set): + if isinstance(items_to_set, int): + assert(items_to_set in self._custo_items) + self._custo_items[items_to_set] = True + elif isinstance(items_to_set, list): + for item in items_to_set: + assert(item in self._custo_items) + self._custo_items[item] = True + else: + raise ValueError + + def clear_items(self, items_to_clear): + if isinstance(items_to_clear, int): + assert(items_to_clear in self._custo_items) + self._custo_items[items_to_clear] = False + elif isinstance(items_to_clear, list): + for item in items_to_clear: + assert(item in self._custo_items) + self._custo_items[item] = False + else: + raise ValueError + + def copy_from(self, node_custo): + self._custo_items = copy.copy(node_custo._custo_items) + @property def transform_func(self): return self._transform_func + @transform_func.setter + def transform_func(self, func): + self._transform_func = func + + def __copy__(self): new_custo = type(self)() new_custo.__dict__.update(self.__dict__) @@ -502,19 +534,32 @@ class NonTermCusto(NodeCustomization): To be provided to :meth:`NodeInternals.customize` """ MutableClone = 1 - FrozenCopy = 2 - CollapsePadding = 3 + CycleClone = 2 + FrozenCopy = 3 + CollapsePadding = 4 + DelayCollapsing = 5 + + FullCombinatory = 6 + StickToDefault = 7 _custo_items = { MutableClone: True, + CycleClone:False, FrozenCopy: True, - CollapsePadding: False + CollapsePadding: False, + DelayCollapsing: False, + FullCombinatory: False, + StickToDefault: False } @property def mutable_clone_mode(self): return self._custo_items[self.MutableClone] + @property + def cycle_clone_mode(self): + return self._custo_items[self.CycleClone] + @property def frozen_copy_mode(self): return self._custo_items[self.FrozenCopy] @@ -523,6 +568,22 @@ def frozen_copy_mode(self): def collapse_padding_mode(self): return self._custo_items[self.CollapsePadding] + @property + def delay_collapsing(self): + return self._custo_items[self.DelayCollapsing] + + @property + def full_combinatory_mode(self): + return self._custo_items[self.FullCombinatory] + + @full_combinatory_mode.setter + def full_combinatory_mode(self, val: bool): + self._custo_items[self.FullCombinatory] = val + + @property + def stick_to_default_mode(self): + return self._custo_items[self.StickToDefault] + class GenFuncCusto(NodeCustomization): """ @@ -591,14 +652,19 @@ class NodeInternals(object): Abs_Postpone = 6 Separator = 15 + AutoSeparator = 16 - DEBUG = 30 + Highlight = 30 + DEBUG = 40 LOCKED = 50 DISABLED = 100 default_custo = None + def __hash__(self): + return id(self) + def __init__(self, arg=None): # if new attributes are added, set_contents_from() have to be updated self.private = None @@ -617,6 +683,9 @@ def __init__(self, arg=None): NodeInternals.Abs_Postpone: False, # Used to distinguish separator NodeInternals.Separator: False, + NodeInternals.AutoSeparator: False, + # Used to display visual effect when the node is printed on the console + NodeInternals.Highlight: False, # Used for debugging purpose NodeInternals.DEBUG: False, # Used to express that someone (a disruptor for instance) is @@ -660,7 +729,7 @@ def set_attrs_from(self, all_attrs): def _init_specific(self, arg): pass - def _get_value(self, conf=None, recursive=True, return_node_internals=False): + def _get_value(self, conf=None, recursive=True, return_node_internals=False, restrict_csp=False): raise NotImplementedError def get_raw_value(self, **kwargs): @@ -690,7 +759,7 @@ def set_node_sync(self, scope, node=None, param=None, sync_obj=None): assert node is None and param is None self._sync_with[scope] = sync_obj else: - assert node is not None + # assert node is not None self._sync_with[scope] = (node, param) def get_node_sync(self, scope): @@ -802,6 +871,27 @@ def is_attr_set(self, name): raise ValueError return self.__attrs[name] + def set_child_attr(self, name, conf=None, all_conf=False, recursive=False): + pass + + def clear_child_attr(self, name, conf=None, all_conf=False, recursive=False): + pass + + @property + def highlight(self): + return self.is_attr_set(NodeInternals.Highlight) + + @highlight.setter + def highlight(self, val): + if val: + self.set_attr(NodeInternals.Highlight) + else: + self.clear_attr(NodeInternals.Highlight) + + @property + def debug(self): + return self.is_attr_set(NodeInternals.DEBUG) + def _make_specific(self, name): return name not in [NodeInternals.Determinist, NodeInternals.Finite] @@ -1000,24 +1090,26 @@ def __init__(self, mandatory_attrs=None, negative_attrs=None, node_kinds=None, mandatory_custo=None, negative_custo=None, required_csts=None, negative_csts=None): - self.mandatory_attrs = mandatory_attrs - self.negative_attrs = negative_attrs - self.mandatory_custo = mandatory_custo - self.negative_custo = negative_custo - self.node_kinds = node_kinds - self.negative_node_kinds = negative_node_kinds - self.node_subkinds = node_subkinds - self.negative_node_subkinds = negative_node_subkinds + self.mandatory_attrs = self._handle_user_input(mandatory_attrs) + self.negative_attrs = self._handle_user_input(negative_attrs) + self.mandatory_custo = self._handle_user_input(mandatory_custo) + self.negative_custo = self._handle_user_input(negative_custo) + self.node_kinds = self._handle_user_input(node_kinds) + self.negative_node_kinds = self._handle_user_input(negative_node_kinds) + self.node_subkinds = self._handle_user_input(node_subkinds) + self.negative_node_subkinds = self._handle_user_input(negative_node_subkinds) self._node_constraints = None if required_csts is not None: - assert(isinstance(required_csts, list)) - for cst in required_csts: + req_csts = self._handle_user_input(required_csts) + for cst in req_csts: self.set_node_constraint(cst, True) if negative_csts is not None: - assert(isinstance(negative_csts, list)) - for cst in negative_csts: + neg_csts = self._handle_user_input(negative_csts) + for cst in neg_csts: self.set_node_constraint(cst, False) + def _handle_user_input(self, crit): + return crit if crit is None or isinstance(crit, (list,tuple)) else [crit] def extend(self, ic): crit = ic.mandatory_attrs @@ -1101,7 +1193,9 @@ def has_node_constraints(self): class DynNode_Helpers(object): - + + determinist = True + def __init__(self): self.reset_graph_info() @@ -1201,7 +1295,7 @@ def _get_graph_info(self): class NodeInternals_Empty(NodeInternals): - def _get_value(self, conf=None, recursive=True, return_node_internals=False): + def _get_value(self, conf=None, recursive=True, return_node_internals=False, restrict_csp=False): if return_node_internals: return (Node.DEFAULT_DISABLED_NODEINT, True) else: @@ -1213,7 +1307,12 @@ def get_raw_value(self, **kwargs): def set_child_env(self, env): self.env = env print('\n*** Empty Node: {!s}'.format(hex(id(self)))) - raise AttributeError + # raise AttributeError + + def get_child_nodes_by_attr(self, internals_criteria, semantics_criteria, owned_conf, conf, path_regexp, + exclude_self, respect_order, relative_depth, top_node, ignore_fstate, + resolve_generator=False): + return None class NodeInternals_GenFunc(NodeInternals): @@ -1249,14 +1348,18 @@ def _make_specific(self, name): # because these attributes are used to change the behaviour of # the GenFunc. if name in [NodeInternals.Determinist, NodeInternals.Finite, NodeInternals.Abs_Postpone, - NodeInternals.Separator]: + NodeInternals.Separator, NodeInternals.Highlight]: + if name == NodeInternals.Determinist: + self._node_helpers.determinist = True if self._generated_node is not None: self.generated_node.set_attr(name, recursive=True) return True def _unmake_specific(self, name): if name in [NodeInternals.Determinist, NodeInternals.Finite, NodeInternals.Abs_Postpone, - NodeInternals.Separator]: + NodeInternals.Separator, NodeInternals.Highlight]: + if name == NodeInternals.Determinist: + self._node_helpers.determinist = False if self._generated_node is not None: self.generated_node.clear_attr(name, recursive=True) return True @@ -1402,6 +1505,8 @@ def generated_node(self): self._generated_node.make_determinist(all_conf=True, recursive=True) if self.is_attr_set(NodeInternals.Finite): self._generated_node.make_finite(all_conf=True, recursive=True) + if self.is_attr_set(NodeInternals.Highlight): + self._generated_node.set_attr(NodeInternals.Highlight, recursive=True) return self._generated_node @@ -1431,12 +1536,12 @@ def set_generator_func_arg(self, generator_node_arg=None, generator_arg=None): self.generator_arg = generator_arg - def _get_value(self, conf=None, recursive=True, return_node_internals=False): + def _get_value(self, conf=None, recursive=True, return_node_internals=False, restrict_csp=False): if self.custo.trigger_last_mode and not self._trigger_registered: assert(self.env is not None) self._trigger_registered = True self.env.register_basic_djob(self._get_delayed_value, - args=[conf, recursive], + args=[conf, recursive, restrict_csp], prio=Node.DJOBS_PRIO_genfunc) if return_node_internals: @@ -1448,12 +1553,13 @@ def _get_value(self, conf=None, recursive=True, return_node_internals=False): self.reset_generator() ret = self.generated_node._get_value(conf=conf, recursive=recursive, - return_node_internals=return_node_internals) + return_node_internals=return_node_internals, + restrict_csp=restrict_csp) return (ret, False) - def _get_delayed_value(self, conf=None, recursive=True): + def _get_delayed_value(self, conf=None, recursive=True, restrict_csp=False): self.reset_generator() - ret = self.generated_node._get_value(conf=conf, recursive=recursive) + ret = self.generated_node._get_value(conf=conf, recursive=recursive, restrict_csp=restrict_csp) return (ret, False) def get_raw_value(self, **kwargs): @@ -1492,8 +1598,11 @@ def reset_state(self, recursive=False, exclude_self=False, conf=None, ignore_ent self._trigger_registered = False self.reset_generator() else: - self.generated_node.reset_state(recursive, exclude_self=exclude_self, conf=conf, - ignore_entanglement=ignore_entanglement) + if self._generated_node is not None: + self.generated_node.reset_state(recursive, exclude_self=exclude_self, conf=conf, + ignore_entanglement=ignore_entanglement) + else: + pass def is_exhausted(self): if self.is_attr_set(NodeInternals.Mutable) and self.is_attr_set(NodeInternals.Finite): @@ -1504,13 +1613,13 @@ def is_exhausted(self): elif self.is_attr_set(NodeInternals.Mutable) and not self.is_attr_set(NodeInternals.Finite): return False else: - return self.generated_node.is_exhausted() + return False if self._generated_node is None else self.generated_node.is_exhausted() def is_frozen(self): if self.is_attr_set(NodeInternals.Mutable): return self._generated_node is not None else: - return self.generated_node.is_frozen() + return None if self._generated_node is None else self.generated_node.is_frozen() def unfreeze(self, conf=None, recursive=True, dont_change_state=False, ignore_entanglement=False, only_generators=False, reevaluate_constraints=False): @@ -1523,9 +1632,12 @@ def unfreeze(self, conf=None, recursive=True, dont_change_state=False, ignore_en self._trigger_registered = False self.reset_generator() else: - self.generated_node.unfreeze(conf, recursive=recursive, dont_change_state=dont_change_state, - ignore_entanglement=ignore_entanglement, only_generators=only_generators, - reevaluate_constraints=reevaluate_constraints) + if self._generated_node is not None: + self.generated_node.unfreeze(conf, recursive=recursive, dont_change_state=dont_change_state, + ignore_entanglement=ignore_entanglement, only_generators=only_generators, + reevaluate_constraints=reevaluate_constraints) + else: + pass def unfreeze_all(self, recursive=True, ignore_entanglement=False): # if self.is_attr_set(NodeInternals.Mutable): @@ -1533,7 +1645,10 @@ def unfreeze_all(self, recursive=True, ignore_entanglement=False): self._trigger_registered = False self.reset_generator() else: - self.generated_node.unfreeze_all(recursive=recursive, ignore_entanglement=ignore_entanglement) + if self._generated_node is not None: + self.generated_node.unfreeze_all(recursive=recursive, ignore_entanglement=ignore_entanglement) + else: + pass def reset_fuzz_weight(self, recursive): if recursive: @@ -1565,12 +1680,14 @@ def reset_depth_specific(self, depth): if self._generated_node is not None: self._generated_node._reset_depth(parent_depth=self.pdepth) - def get_child_nodes_by_attr(self, internals_criteria, semantics_criteria, owned_conf, conf, path_regexp, - exclude_self, respect_order, relative_depth, top_node, ignore_fstate): + def get_child_nodes_by_attr(self, internals_criteria, semantics_criteria, owned_conf, conf, path_regexp, + exclude_self, respect_order, relative_depth, top_node, ignore_fstate, + resolve_generator=False): return self.generated_node.get_reachable_nodes(internals_criteria, semantics_criteria, owned_conf, conf, path_regexp=path_regexp, exclude_self=False, respect_order=respect_order, relative_depth=relative_depth, - top_node=top_node, ignore_fstate=ignore_fstate) + top_node=top_node, ignore_fstate=ignore_fstate, + resolve_generator=resolve_generator) def set_child_current_conf(self, node, conf, reverse, ignore_entanglement): if self.custo.forward_conf_change_mode: @@ -1579,9 +1696,10 @@ def set_child_current_conf(self, node, conf, reverse, ignore_entanglement): conf, reverse, ignore_entanglement=ignore_entanglement) - def get_child_all_path(self, name, htable, conf, recursive): + def get_child_all_path(self, name, htable, conf, recursive, resolve_generator=False): if self.env is not None: - self.generated_node._get_all_paths_rec(name, htable, conf, recursive=recursive, first=False) + self.generated_node._get_all_paths_rec(name, htable, conf, recursive=recursive, + first=False, resolve_generator=resolve_generator) else: # If self.env is None, that means that a node graph is not fully constructed # thus we avoid a freeze side-effect (by resolving 'generated_node') of the @@ -1633,16 +1751,35 @@ def _make_private_term_specific(self, ignore_frozen_state, accept_external_entan def _set_frozen_value(self, val): self.frozen_node = val - def _get_value(self, conf=None, recursive=True, return_node_internals=False): + def _set_default_value(self, val): + self.frozen_node = None + self._set_default_value_specific(val) + + def _set_default_value_specific(self, val): + raise NotImplementedError + + def _get_value(self, conf=None, recursive=True, return_node_internals=False, restrict_csp=False): + + def format_val(val): + fval = FontStyle.BOLD + colorize(val.decode('latin-1'), rgb=Color.ND_HLIGHT) + FontStyle.END + fval = fval.encode('latin-1') + return fval if self.frozen_node is not None: - return (self, False) if return_node_internals else (self.frozen_node, False) + if self.env and self.env.color_enabled and self.highlight: + fval = format_val(self.frozen_node) + else: + fval = self.frozen_node + return (self, False) if return_node_internals else (fval, False) val = self._get_value_specific(conf, recursive) if self.is_attr_set(NodeInternals.Freezable): self.frozen_node = val + if self.highlight: + val = format_val(val) + return (self, True) if return_node_internals else (val, True) def _get_value_specific(self, conf, recursive): @@ -1735,29 +1872,30 @@ def _unfreeze_reevaluate_constraints(self, current_val): def unfreeze_all(self, recursive=True, ignore_entanglement=False): self.frozen_node = None + def update_value(self, value): + self.frozen_node = self._update_value_specific(value) + + def _update_value_specific(self, value): + pass + def reset_fuzz_weight(self, recursive): pass def set_child_env(self, env): self.env = env - def set_child_attr(self, name, conf=None, all_conf=False, recursive=False): - pass - - def clear_child_attr(self, name, conf=None, all_conf=False, recursive=False): - pass - def reset_depth_specific(self, depth): pass def get_child_nodes_by_attr(self, internals_criteria, semantics_criteria, owned_conf, conf, path_regexp, - exclude_self, respect_order, relative_depth, top_node, ignore_fstate): + exclude_self, respect_order, relative_depth, top_node, ignore_fstate, + resolve_generator=False): return None def set_child_current_conf(self, node, conf, reverse, ignore_entanglement): pass - def get_child_all_path(self, name, htable, conf, recursive): + def get_child_all_path(self, name, htable, conf, recursive, resolve_generator=False): pass @@ -1778,6 +1916,9 @@ def _unmake_specific(self, name): self.value_type.make_random() return True + def _set_default_value_specific(self, val): + self.value_type.set_default_value(val) + def import_value_type(self, value_type): self.value_type = value_type # if self.env is not None: @@ -1787,12 +1928,6 @@ def import_value_type(self, value_type): else: self.value_type.make_random() - # @NodeInternals.env.setter - # def env(self, env): - # NodeInternals.env.fset(self, env) - # if self.value_type is not None and self.env is not None: - # self.value_type.knowledge_source = self.env.knowledge_source - def has_subkinds(self): return True @@ -1842,6 +1977,16 @@ def do_revert_absorb(self): def do_cleanup_absorb(self): self.value_type.do_cleanup_absorb() + def _update_value_specific(self, value): + if isinstance(value, int): + assert isinstance(self.value_type, (fvt.INT, fvt.BitField)), f'{self.value_type}' + self.value_type.update_raw_value(value) + else: + val, off, size = self.value_type.do_absorb(convert_to_internal_repr(value), + constraints=AbsNoCsts()) + + return self.value_type.get_current_value() + def _unfreeze_without_state_change(self, current_val): self.value_type.rewind() @@ -2155,7 +2300,7 @@ class NodeSeparator(object): make_private (function): used for full copy ''' - def __init__(self, node, prefix=True, suffix=True, unique=False): + def __init__(self, node, prefix=True, suffix=True, unique=False, always=False): ''' Args: node (Node): node to be used for separation. @@ -2163,12 +2308,16 @@ def __init__(self, node, prefix=True, suffix=True, unique=False): suffix (bool): if `True`, a serapator will also be placed at the end. unique (bool): if `False`, the same node will be used for each separation, otherwise a new node will be generated. + always (bool): if `True`, the separator will be always generated even if the + subnodes it separates are not generated because their evaluated quantity is 0. ''' self.node = node self.node.set_attr(NodeInternals.Separator) + self.node.set_attr(NodeInternals.AutoSeparator) self.prefix = prefix self.suffix = suffix self.unique = unique + self.always = always def make_private(self, node_dico, ignore_frozen_state): if self.node in node_dico: @@ -2180,9 +2329,121 @@ def make_private(self, node_dico, ignore_frozen_state): class NodeInternals_NonTerm(NodeInternals): - '''It is a kind of node internals that enable to structure the graph + """ + It is a kind of node internals that enable to structure the graph through a specific grammar... - ''' + """ + + class NodeAttrs(object): + _default_qty = None + _min = None + _max = None + + exhausted_seq = False + _qty_sequence = None + _current_qty = None + _previous_qty = None + _planned_reset = False + _previous_current_qty_was_none = False + + @property + def qty(self): + return self._min, self._max + + @qty.setter + def qty(self, val): + self._min, self._max = val + if self.default_qty is None: + if self._max != -1: # infinity case + self.default_qty = math.ceil((self._min + self._max) / 2) + elif self._min >= 0: + self.default_qty = self._min + 1 + else: + self.default_qty = -1 + + @property + def default_qty(self): + return self._default_qty + + @default_qty.setter + def default_qty(self, val): + if val is not None and self._min is not None and self._max is not None: + if self._max != -1: + assert self._min <= val <= self._max + elif self._min >= 0: + assert self._min <= val + else: + assert val == -1 + + self._default_qty = val + + @property + def qty_sequence(self): + if self._qty_sequence is None: + self._qty_sequence = [self.default_qty] + if self._max != self.default_qty: + self._qty_sequence.insert(0,self._max) + # default_qty is 'guaranteed' to be between mini and maxi + # which makes the condition sufficient (and avoid mini != maxi) + if self._min != self.default_qty: + self._qty_sequence.insert(0,self._min) + + return self._qty_sequence + + def next_qty(self): + qty = self.qty_sequence.pop(-1) + self._previous_qty = self._current_qty + self._current_qty = qty + if len(self.qty_sequence) == 0: + self.exhausted_seq = True + return qty + + @property + def current_qty(self): + if self._current_qty is None: + self.next_qty() + return self._current_qty + + def perform_planned_reset(self): + if self._planned_reset: + self._planned_reset = False + self._current_qty = None + self._previous_qty = None + self._qty_sequence = None + self.exhausted_seq = False + + def plan_reset(self): + self._planned_reset = True + + def unplan_reset(self): + self._planned_reset = False + + def reset(self): + self._planned_reset = True + self.perform_planned_reset() + + def unroll(self): + # print('\n*** DBG unrolling') + if self.exhausted_seq: + self._qty_sequence = None + self._qty_sequence = [self.qty_sequence[0]] + self.exhausted_seq = False + else: + if self._current_qty is not None: + self._qty_sequence.append(self._current_qty) + else: + # we have been copied without being used first + pass + + self._current_qty = self._previous_qty + self.unplan_reset() + + def __copy__(self): + node_attrs = type(self)() + node_attrs.__dict__.update(self.__dict__) + node_attrs._qty_sequence = copy.copy(self._qty_sequence) + return node_attrs + INFINITY_LIMIT = 30 # Used to limit the number of created nodes # when the max quantity is specified to be @@ -2196,6 +2457,16 @@ def _init_specific(self, arg): self.subnodes_set = None self.subnodes_order = None self.subnodes_attrs = None + self._reevaluation_pending = None + + self.current_flattened_nodelist = None + self.cursor_min = None + self.cursor_maj = None + self.previous_cursor_min = None + self.previous_cursor_maj = None + self.current_pick_section = None + self.current_picked_node_idx = None + self.reset() def reset(self, nodes_drawn_qty=None, custo=None, exhaust_info=None, preserve_node=False): @@ -2204,6 +2475,7 @@ def reset(self, nodes_drawn_qty=None, custo=None, exhaust_info=None, preserve_no self.subnodes_order_total_weight = 0 self.subnodes_attrs = {} self.separator = None + self._reevaluation_pending = False if self.encoder: self.encoder.reset() @@ -2213,6 +2485,8 @@ def reset(self, nodes_drawn_qty=None, custo=None, exhaust_info=None, preserve_no if preserve_node: pass elif custo is None: + # if self.debug: + # print('\n*** DEBUG: set default custo') self.customize(self.default_custo) else: self.customize(custo) @@ -2231,6 +2505,53 @@ def __iter_csts_verbose(self, node_list): yield idx, delim, sublist idx += 1 + + def flatten_node_list(self, node_list): + """ + Return a list of the form: [subnode1, subnode2, subnode3, ....] + In case of Pick-type sections within the parent node, sublists are included within + the previous one and include the alternative subnodes, so that the list looks like: + [subnode1, [snode21, snode22, ...], subnode3, ....] + + Args: + node_list: + + Returns: + + """ + + flatten_list = [] + pick_section_amount = 0 + for idx, delim, sublist in self.__iter_csts_verbose(node_list): + if delim[1] == '>' or delim[1:3] == '=.': + for i, node in enumerate(sublist): + flatten_list.append(node) + + elif delim[1:3] == '=+': + node_sublist = [] + if sublist[0] > -1: # it means weights exist + for _, comp in split_with(lambda x: isinstance(x, int), sublist[1]): + # sublist[1] is already ordered correctly (by weight) + node = comp[0] + shall_exist = self._existence_from_node(node) + if shall_exist is None or shall_exist: + node_sublist.append(node) + + else: + for node in sublist[1]: + shall_exist = self._existence_from_node(node) + if shall_exist is None or shall_exist: + node_sublist.append(node) + + pick_section_amount += 1 + flatten_list.append(node_sublist) + + else: + raise ValueError + + return flatten_list, pick_section_amount + + def import_subnodes_basic(self, node_list, separator=None, preserve_node=False): self.reset(preserve_node=preserve_node) @@ -2241,7 +2562,8 @@ def import_subnodes_basic(self, node_list, separator=None, preserve_node=False): for node in node_list: self.subnodes_set.add(node) - self.subnodes_attrs[node] = (1, 1) + self.subnodes_attrs[node] = NodeInternals_NonTerm.NodeAttrs() + self.subnodes_attrs[node].qty = [1, 1] def import_subnodes_with_csts(self, wlnode_list, separator=None, preserve_node=False): self.reset(preserve_node=preserve_node) @@ -2257,15 +2579,18 @@ def import_subnodes_with_csts(self, wlnode_list, separator=None, preserve_node=F new_sublist = [] for n in sublist: - node, mini, maxi = self._get_info_from_subnode_description(n) + node, mini, maxi, default_qty = self._get_info_from_subnode_description(n) self.subnodes_set.add(node) if node in self.subnodes_attrs: - prev_min, prev_max = self.subnodes_attrs[node] + prev_min, prev_max = self.subnodes_attrs[node].qty if prev_min != mini or prev_max != maxi: raise DataModelDefinitionError('Node "{:s}" is used twice in the same ' \ 'non-terminal node with ' \ 'different min/max values!'.format(node.name)) - self.subnodes_attrs[node] = (mini, maxi) + else: + self.subnodes_attrs[node] = NodeInternals_NonTerm.NodeAttrs() + self.subnodes_attrs[node].default_qty = default_qty + self.subnodes_attrs[node].qty = [mini, maxi] new_sublist.append(node) sublist = new_sublist @@ -2285,15 +2610,18 @@ def import_subnodes_with_csts(self, wlnode_list, separator=None, preserve_node=F weight_total += w if weight_l: - new_l = [] if len(weight_l) != len(sublist): raise DataModelDefinitionError('Wrong number of relative weights ({:d})!' ' Expected: {:d}'.format(len(weight_l), len(sublist))) - for w, etp in zip(weight_l, sublist): - new_l.append(w) - new_l.append(etp) - sublist = new_l + new_l = [(w, node_desc) for w, node_desc in zip(weight_l, sublist)] + new_l.sort(key=lambda x: -x[0]) + new_l2 = [] + for w, node_desc in new_l: + new_l2.append(w) + new_l2.append(node_desc) + + sublist = new_l2 else: weight_total = -1 @@ -2309,7 +2637,7 @@ def import_subnodes_with_csts(self, wlnode_list, separator=None, preserve_node=F self.subnodes_order.append(subnode_list) def import_subnodes_full_format(self, subnodes_order=None, subnodes_attrs=None, - frozen_node_list=None, internals=None, + frozen_node_list=None, current_flat_nodelist=None, internals=None, nodes_drawn_qty=None, custo=None, exhaust_info=None, separator=None): if internals is not None: @@ -2317,6 +2645,7 @@ def import_subnodes_full_format(self, subnodes_order=None, subnodes_attrs=None, self.subnodes_order = internals.subnodes_order self.subnodes_attrs = internals.subnodes_attrs + self.current_flattened_nodelist = internals.current_flattened_nodelist self.frozen_node_list = internals.frozen_node_list self.separator = internals.separator self.subnodes_set = internals.subnodes_set @@ -2331,6 +2660,7 @@ def import_subnodes_full_format(self, subnodes_order=None, subnodes_attrs=None, self.subnodes_order = subnodes_order self.subnodes_attrs = subnodes_attrs + self.current_flattened_nodelist = current_flat_nodelist self.frozen_node_list = frozen_node_list if separator is not None: self.separator = separator @@ -2341,11 +2671,11 @@ def import_subnodes_full_format(self, subnodes_order=None, subnodes_attrs=None, if delim[:3] == 'u=+' or delim[:3] == 's=+': for w, etp in split_with(lambda x: isinstance(x, int), sublist[1]): for n in etp: - node, mini, maxi = self._get_node_and_attrs_from(n) + node, mini, maxi = self._get_node_and_minmax_from(n) self.subnodes_set.add(node) else: for n in sublist: - node, mini, maxi = self._get_node_and_attrs_from(n) + node, mini, maxi = self._get_node_and_minmax_from(n) self.subnodes_set.add(node) else: @@ -2403,9 +2733,14 @@ def make_private_subnodes(self, node_dico, func_nodes, env, ignore_frozen_state, new_nodes_drawn_qty = None new_exhaust_info = None else: - new_exhaust_info = [self.exhausted, copy.copy(self.excluded_components), - self.subcomp_exhausted, self.expanded_nodelist_sz, self.expanded_nodelist_origsz, - self.component_seed, self._perform_first_step] + # new_exhaust_info = [self.exhausted, copy.copy(self.excluded_components), + # self.shape_exhausted, self.current_nodelist_sz, self.expanded_nodelist_origsz, + # self.component_seed, self._perform_first_step] + + new_exhaust_info = [self.exhausted_shapes, copy.copy(self.excluded_components), + self.combinatory_complete, self.component_seed, + self.exhausted_pick_cases] + new_nodes_drawn_qty = copy.copy(self._nodes_drawn_qty) new_fl = [] for e in self.frozen_node_list: @@ -2418,8 +2753,13 @@ def make_private_subnodes(self, node_dico, func_nodes, env, ignore_frozen_state, node_dico[e] = new_e new_fl.append(node_dico[e]) + if self.current_flattened_nodelist is None or ignore_frozen_state: + new_current_fnlist = None + else: + new_current_fnlist = [node_dico[nd] for nd in self.current_flattened_nodelist] + self.import_subnodes_full_format(subnodes_order=subnodes_order, subnodes_attrs=subnodes_attrs, - frozen_node_list=new_fl, + frozen_node_list=new_fl, current_flat_nodelist=new_current_fnlist, nodes_drawn_qty=new_nodes_drawn_qty, custo=self.custo, exhaust_info=new_exhaust_info, separator=new_separator) @@ -2561,7 +2901,7 @@ def get_drawn_node_qty(self, node_ref): def get_subnode_minmax(self, node): if node in self.subnodes_attrs: - return self.subnodes_attrs[node] + return self.subnodes_attrs[node].qty else: return None @@ -2570,18 +2910,32 @@ def set_subnode_minmax(self, node, min=None, max=None): if min is not None and max is None: assert min > -2 - self.subnodes_attrs[node][0] = min + self.subnodes_attrs[node].qty[0] = min elif max is not None and min is None: assert max > -2 - self.subnodes_attrs[node][1] = max + self.subnodes_attrs[node].qty[1] = max elif min is not None and max is not None: assert min > -2 and max > -2 and (max >= min or max == -1) - self.subnodes_attrs[node] = (min, max) + self.subnodes_attrs[node].qty = [min, max] else: raise ValueError('No values are provided!') self.reset_state(recursive=False, exclude_self=False) + def get_subnode_default_qty(self, node): + if node in self.subnodes_attrs: + return self.subnodes_attrs[node].default_qty + else: + return None + + def set_subnode_default_qty(self, node, default_qty=None): + assert node in self.subnodes_attrs + mini, maxi = self.subnodes_attrs[node].qty + assert mini <= default_qty <= maxi + self.subnodes_attrs[node].default_qty = default_qty + + self.reset_state(recursive=False, exclude_self=False) + def _get_random_component(self, comp_list, total_weight, check_existence=False): r = random.uniform(0, total_weight) s = 0 @@ -2612,7 +2966,7 @@ def _get_heavier_component(self, comp_list, check_existence=False): return current_comp @staticmethod - def _get_next_heavier_component(comp_list, excluded_idx=[]): + def _get_next_heavier_component(comp_list, excluded_idx): current_weight = -1 for idx, weight, comp in split_verbose_with(lambda x: isinstance(x, int), comp_list): if idx in excluded_idx: @@ -2628,7 +2982,7 @@ def _get_next_heavier_component(comp_list, excluded_idx=[]): return current_comp, current_idx @staticmethod - def _get_next_random_component(comp_list, excluded_idx=[], seed=None): + def _get_next_random_component(comp_list, excluded_idx, seed=None): total_weight = 0 for idx, weight, comp in split_verbose_with(lambda x: isinstance(x, int), comp_list): if idx in excluded_idx: @@ -2654,28 +3008,18 @@ def _get_next_random_component(comp_list, excluded_idx=[], seed=None): def structure_will_change(self): - ''' + """ To be used only in Finite mode. Return True if the structure will change the next time _get_value() will be called. Returns: bool - ''' - - crit1 = (len(self.subnodes_order) // 2) > 1 + """ - # assert(self.expanded_nodelist is not None) # only possible during a Node copy - # \-> this assert is a priori not needed because we force recomputation of - # the list in this case - if self.expanded_nodelist_origsz is None: - # in this case we have never been frozen + if self.current_flattened_nodelist is None: + # In this case we have never been frozen self.get_subnodes_with_csts() - crit2 = self.expanded_nodelist_origsz > 1 - # If crit2 is False while crit1 is also False, the structure - # will not change. If crit2 is False at some time, crit1 is - # then True, thus we always return the correct answer - - return crit1 or crit2 + return not self.is_exhausted() def _get_node_from(self, node_desc): @@ -2684,10 +3028,13 @@ def _get_node_from(self, node_desc): else: # node_desc is either (Node, min, max) or (Node, qty) return node_desc[0] - def _get_node_and_attrs_from(self, node_desc): + def _get_node_and_minmax_from(self, node_desc): if isinstance(node_desc, Node): + # This case exists for two situations: + # - import_subnodes_full_format() + # - _construct_subnodes() when the non-terminal node is in random mode node = node_desc - mini, maxi = self.subnodes_attrs[node_desc] + mini, maxi = self.subnodes_attrs[node_desc].qty if maxi == -1 and mini >= 0: # infinite case # for generation we limit to min+INFINITY_LIMIT maxi = mini + NodeInternals_NonTerm.INFINITY_LIMIT @@ -2713,15 +3060,22 @@ def _get_node_and_attrs_from(self, node_desc): return node, mini, maxi def _get_info_from_subnode_description(self, node_desc): - if len(node_desc) == 3: + nd_len = len(node_desc) + if nd_len == 3 or nd_len == 4: mini = node_desc[1] maxi = node_desc[2] assert mini > -2 and maxi > -2 and (maxi >= mini or maxi == -1) + if nd_len == 4: + default = node_desc[3] + assert mini <= default <= maxi, f'guilty node --> "{node_desc[0].name}"' + else: + default = None else: assert node_desc[1] > -2 mini = maxi = node_desc[1] + default = None - return node_desc[0], mini, maxi + return node_desc[0], mini, maxi, default def _copy_nodelist(self, node_list): new_list = [] @@ -2732,54 +3086,6 @@ def _copy_nodelist(self, node_list): new_list.append([delim, [sublist[0], copy.copy(sublist[1])]]) return new_list - def _generate_expanded_nodelist(self, node_list, determinist=True): - - expanded_node_list = [] - for idx, delim, sublist in self.__iter_csts_verbose(node_list): - if delim[1] == '>' or delim[1:3] == '=.': - for i, node_desc in enumerate(sublist): - node, mini, maxi = self._get_node_and_attrs_from(node_desc) - if mini < maxi: - new_nlist = self._copy_nodelist(node_list) - new_nlist[idx][1][i] = [node, mini] - expanded_node_list.insert(0, new_nlist) - new_nlist = self._copy_nodelist(node_list) - new_nlist[idx][1][i] = [node, maxi] - expanded_node_list.insert(0, new_nlist) - if mini+1 < maxi: - new_nlist = self._copy_nodelist(node_list) - new_nlist[idx][1][i] = [node, mini+1, maxi-1] - expanded_node_list.insert(0, new_nlist) - elif delim[1:3] == '=+': - new_delim = delim[0] + '>' - if sublist[0] > -1: - for weight, comp in split_with(lambda x: isinstance(x, int), sublist[1]): - node, mini, maxi = self._get_node_and_attrs_from(comp[0]) - shall_exist = self._existence_from_node(node) - if shall_exist is not None and not shall_exist: - continue - new_nlist = self._copy_nodelist(node_list) - new_nlist[idx] = [new_delim, [[node, mini, maxi]]] - expanded_node_list.insert(0, new_nlist) - else: - for node_desc in sublist[1]: - node, mini, maxi = self._get_node_and_attrs_from(node_desc) - shall_exist = self._existence_from_node(node) - if shall_exist is not None and not shall_exist: - continue - new_nlist = self._copy_nodelist(node_list) - new_nlist[idx] = [new_delim, [[node, mini, maxi]]] - expanded_node_list.insert(0, new_nlist) - else: - raise ValueError - - if not expanded_node_list: - expanded_node_list.append(node_list) - - if not determinist: - shuffle(expanded_node_list) - - return expanded_node_list def _construct_subnodes(self, node_desc, subnode_list, mode, ignore_sep_fstate, ignore_separator=False, lazy_mode=True): @@ -2788,7 +3094,9 @@ def _sync_size_handling(node): if obj is not None: obj.synchronize_nodes(node) - node, mini, maxi = self._get_node_and_attrs_from(node_desc) + node, mini, maxi = self._get_node_and_minmax_from(node_desc) + # if self.debug: + # print('\n*** construct:', node.name, mini, maxi, node_desc) shall_exist = self._existence_from_node(node) if shall_exist is not None: @@ -2799,12 +3107,19 @@ def _sync_size_handling(node): subnode_list.append(node) return - mini, maxi = self.nodeqty_corrupt_hook(node, mini, maxi) + mini, maxi, corrupted = self.nodeqty_corrupt_hook(node, mini, maxi) - if self.is_attr_set(NodeInternals.Determinist): - nb = (mini + maxi) // 2 + if corrupted or self.is_attr_set(NodeInternals.Determinist): + nb = math.ceil((mini + maxi) / 2) else: - nb = random.randint(mini, maxi) + if self._reevaluation_pending: + nb = self._nodes_drawn_qty.get(node.name) + if nb is None: + print(f'\n*** Warning[_construct_subnodes]: {node.name} node has no reference ' + f'in self._nodes_drawn_qty, thus fallback to random qty') + nb = random.randint(mini, maxi) + else: + nb = random.randint(mini, maxi) qty = self._qty_from_node(node) if qty is not None: @@ -2815,8 +3130,13 @@ def _sync_size_handling(node): base_node = node external_entangled_nodes = [] if base_node.entangled_nodes is None else list(base_node.entangled_nodes) - new_node = None + if nb == 0: + if self.separator is not None and self.separator.always and not ignore_separator: + new_sep = self._clone_separator(self.separator.node, unique=self.separator.unique, + ignore_frozen_state=ignore_sep_fstate) + subnode_list.append(new_sep) + new_node = None transformed_node = None for i in range(nb): # 'unique' mode @@ -2855,6 +3175,11 @@ def _sync_size_handling(node): print('\n*** ERROR: User-provided NodeCustomization.transform_func()' ' should return a Node. Thus we ignore its production.') + elif self.custo.cycle_clone_mode: + new_node.freeze() + new_node.walk(steps_num=1) + transformed_node = new_node + new_node._set_clone_info((base_node.tmp_ref_count-1, nb), base_node) _sync_size_handling(new_node) @@ -2889,11 +3214,78 @@ def _sync_size_handling(node): def get_subnodes_with_csts(self): - '''Generate the structure of the non terminal node. - ''' + """ + Generate the structure of the non terminal node. + """ + + def compute_next_shape(determinist, finite): + + if not self.exhausted_pick_cases: + self.excluded_components.pop(-1) + + if determinist: + node_list, idx = self._get_next_heavier_component(self.subnodes_order, + excluded_idx=self.excluded_components) + self.excluded_components.append(idx) + # 'len(self.subnodes_order)' is always even + exhausted_shape = len(self.excluded_components) == len(self.subnodes_order) // 2 + # Note that self.excluded_components is reset in a lazy way (within unfreeze) + + else: + if finite: + node_list, idx, self.component_seed = self._get_next_random_component(self.subnodes_order, + excluded_idx=self.excluded_components) + self.excluded_components.append(idx) + exhausted_shape = len(self.excluded_components) == len(self.subnodes_order) // 2 + + else: + node_list = self._get_random_component(self.subnodes_order, + self.subnodes_order_total_weight) + exhausted_shape = False + + + if determinist: + node_list, pick_section_amount = self.flatten_node_list(node_list) + if pick_section_amount > 0: + self.exhausted_pick_cases = False + new_node_list = [] + pick_section = 0 + next_pick_section = None + self.exhausted_pick_cases = False + for obj in node_list: + if isinstance(obj, Node): + new_node_list.append(obj) + else: + if pick_section == self.current_pick_section: + new_node_list.append(obj[self.current_picked_node_idx]) + self.current_picked_node_idx += 1 + if len(obj) <= self.current_picked_node_idx: + # we need to move on to the next pick section and if there is no + # more then we are exhausted. + next_pick_section = self.current_pick_section + 1 + self.current_picked_node_idx = 1 + if pick_section_amount == next_pick_section: + self.exhausted_pick_cases = True + else: + new_node_list.append(obj[0]) + pick_section += 1 + + if self.exhausted_pick_cases: + self.current_pick_section = 0 + self.current_picked_node_idx = 0 + elif next_pick_section is not None: + self.current_pick_section = next_pick_section + + node_list = new_node_list + + self.exhausted_shapes = exhausted_shape and self.exhausted_pick_cases + + return node_list + # In this case we return directly the frozen state if self.frozen_node_list is not None: + self._reevaluation_pending = False return (self.frozen_node_list, False) if self.separator is not None: @@ -2910,200 +3302,286 @@ def get_subnodes_with_csts(self): self.frozen_node_list = [] determinist = self.is_attr_set(NodeInternals.Determinist) + finite = self.is_attr_set(NodeInternals.Finite) + + if determinist: + + if self.combinatory_complete or self.custo.stick_to_default_mode: + self.current_flattened_nodelist = node_list = compute_next_shape(determinist, finite) + self.cursor_min = 0 + self.cursor_maj = 0 + self.previous_cursor_min = 0 + self.previous_cursor_maj = 0 + self.combinatory_complete = False + # self.current_flattened_nodelist = self.flatten_node_list(node_list) + + elif self.current_flattened_nodelist is None: + # This case happens when we have been cloned with 'ignore_frozen_state' + # and not frozen since then, or cloned from a node that has never been frozen. + # The parameters are already initialized by the cloning procedure. + self.current_flattened_nodelist = node_list = compute_next_shape(determinist, finite) + # self.current_flattened_nodelist = self.flatten_node_list(node_list) + + for nd in self.current_flattened_nodelist: + self.subnodes_attrs[nd].perform_planned_reset() + + next_cursor_min = None + next_cursor_maj = None + first_pass_done = False + perform_reset = False + + current_nodelist_sz = len(self.current_flattened_nodelist) + for idx, nd in enumerate(self.current_flattened_nodelist): + expanded_nodes = [] + nd_attrs = self.subnodes_attrs[nd] + + if self.custo.full_combinatory_mode: + + if idx == self.cursor_min: + if nd_attrs._current_qty is None: + first_pass_done = True + qty = nd_attrs.current_qty + + if nd_attrs.exhausted_seq: + # print('\n ## case 1 ***') + self.cursor_min = self.cursor_min + 1 + first_pass_done = False + if self.cursor_min == self.cursor_maj: + for idx in range(self.cursor_maj, current_nodelist_sz): + next_nd = self.current_flattened_nodelist[idx] + next_nd_attrs = self.subnodes_attrs[next_nd] + if not next_nd_attrs.exhausted_seq: + self.cursor_maj += 1 + break + else: + self.combinatory_complete = True + elif self.cursor_min > self.cursor_maj: + if self.cursor_maj+1 >= current_nodelist_sz: + self.combinatory_complete = True + else: + self.cursor_maj += 1 + else: + pass - if self._perform_first_step: - if determinist: + else: + if not first_pass_done: + qty = nd_attrs.next_qty() + else: + first_pass_done = False + if nd_attrs.exhausted_seq: + if self.cursor_maj - self.cursor_min > 0: + for idx in range(0, self.cursor_maj): + next_nd = self.current_flattened_nodelist[idx] + next_nd_attrs = self.subnodes_attrs[next_nd] + if not next_nd_attrs.exhausted_seq: + next_cursor_min = idx + perform_reset = True + break + else: + for idx in range(self.cursor_maj, current_nodelist_sz): + next_nd = self.current_flattened_nodelist[idx] + next_nd_attrs = self.subnodes_attrs[next_nd] + if not next_nd_attrs.exhausted_seq: + next_cursor_min = idx + next_cursor_maj = idx + perform_reset = True + break + else: + self.combinatory_complete = True - if self.subcomp_exhausted: - self.subcomp_exhausted = False + elif self.cursor_min == self.cursor_maj: + if self.cursor_maj == current_nodelist_sz-1: + next_cursor_min = 0 + else: + next_cursor_maj = self.cursor_maj + 1 + if self.cursor_min > 0: + next_cursor_min = 0 + else: + next_cursor_min = self.cursor_min + 1 + perform_reset = True + else: + # We should never reach this case + raise NotImplementedError + else: + next_cursor_min = 0 - node_list, idx = self._get_next_heavier_component(self.subnodes_order, - excluded_idx=self.excluded_components) - self.excluded_components.append(idx) - # 'len(self.subnodes_order)' is always even - if len(self.excluded_components) == len(self.subnodes_order) // 2: - # in this case we have exhausted all components - # note that self.excluded_components is reset in a lazy way (within unfreeze) - self.exhausted = True else: - self.exhausted = False - - else: - if self.is_attr_set(NodeInternals.Finite): - if self.subcomp_exhausted: - self.subcomp_exhausted = False + qty = nd_attrs.current_qty - node_list, idx, self.component_seed = self._get_next_random_component(self.subnodes_order, - excluded_idx=self.excluded_components) - self.excluded_components.append(idx) - self.exhausted = len(self.excluded_components) == len(self.subnodes_order) // 2 else: - node_list = self._get_random_component(self.subnodes_order, - self.subnodes_order_total_weight) - if not self._perform_first_step: - self._perform_first_step = True + if idx == self.cursor_min: + if nd_attrs._current_qty is None: + first_pass_done = True + qty = nd_attrs.current_qty + + if nd_attrs.exhausted_seq: + # print('\n ## case 1 ***') + self.cursor_min = self.cursor_min + 1 + first_pass_done = False + if self.cursor_min == current_nodelist_sz: + self.combinatory_complete = True + else: + if not first_pass_done: + # print('\n ## case 2.a ***') + qty = nd_attrs.next_qty() + else: + # print('\n ## case 2.b ***') + first_pass_done = False + if nd_attrs.exhausted_seq: + for idx in range(self.cursor_min, current_nodelist_sz): + next_nd = self.current_flattened_nodelist[idx] + next_nd_attrs = self.subnodes_attrs[next_nd] + if not next_nd_attrs.exhausted_seq: + next_cursor_min = idx + perform_reset = True + break + else: + self.combinatory_complete = True + else: + pass + else: + qty = nd_attrs.current_qty - if self.is_attr_set(NodeInternals.Finite) or determinist: - if self.expanded_nodelist is None: - # This case occurs when we are a copy of a node and - # keeping the state of the original node was - # requested. But the state is kept to the minimum to - # avoid memory waste, thus we need to reconstruct - # dynamically some part of the state - if determinist: - node_list, idx = self._get_next_heavier_component(self.subnodes_order, - excluded_idx=self.excluded_components) - else: - node_list, idx, self.component_seed = self._get_next_random_component(self.subnodes_order, - excluded_idx=self.excluded_components, - seed=self.component_seed) - # If the shape is Pick (=+), the shape is reduced to a singleton - self.expanded_nodelist = self._generate_expanded_nodelist(node_list, determinist=determinist) - - self.expanded_nodelist_origsz = len(self.expanded_nodelist) - if self.expanded_nodelist_sz > 0: - self.expanded_nodelist = self.expanded_nodelist[:self.expanded_nodelist_sz] - else: - self.expanded_nodelist = self.expanded_nodelist[:1] - elif not self.expanded_nodelist: # that is == [] - self.expanded_nodelist = self._generate_expanded_nodelist(node_list, determinist=determinist) - self.expanded_nodelist_origsz = len(self.expanded_nodelist) + # if self.debug: + # print(f'*** {qty} :{nd.name} {self.subnodes_attrs[nd].qty_sequence}') - node_list = self.expanded_nodelist.pop(-1) - self.expanded_nodelist_sz = len(self.expanded_nodelist) - if not self.expanded_nodelist: - self.subcomp_exhausted = True + self._construct_subnodes((nd, qty), expanded_nodes, mode='u', ignore_sep_fstate=ignore_sep_fstate) - for delim, sublist in self.__iter_csts(node_list): + self.frozen_node_list += expanded_nodes - sublist_tmp = [] + # When self.cursor_min is updated within the walking algorithms, this does not change + # the resulting data, it is just a shortcut in the algorithm. This is generally safe + # to use self.previous_cursor_m* to restore previous state. But, there is a corner + # case when the shape is exhausted, as self.cursor_m* will be incremented and will be + # longer than current_nodelist_sz. + if self.cursor_min >= current_nodelist_sz: + self.previous_cursor_min = current_nodelist_sz-1 + else: + self.previous_cursor_min = self.cursor_min + + if self.cursor_maj >= current_nodelist_sz: + self.previous_cursor_maj = current_nodelist_sz-1 + else: + self.previous_cursor_maj = self.cursor_maj + + if next_cursor_maj is not None: + self.cursor_maj = next_cursor_maj + if next_cursor_min is not None: + self.cursor_min = next_cursor_min + + if self.combinatory_complete or self.custo.stick_to_default_mode: + for nd in self.current_flattened_nodelist: + self.subnodes_attrs[nd].plan_reset() + + elif perform_reset: + for nd in self.current_flattened_nodelist[:self.cursor_min]: + self.subnodes_attrs[nd].plan_reset() + + else: # random + node_list = compute_next_shape(determinist, finite) + + for delim, sublist in self.__iter_csts(node_list): + + sublist_tmp = [] - if determinist: if delim[1] == '>': for i, node in enumerate(sublist): self._construct_subnodes(node, sublist_tmp, delim[0], ignore_sep_fstate) + elif delim[1] == '=': - if delim[2] == '+': - # This code seems never reached because, in determinist mode, we already choose - # the node during self.expanded_nodelist creation, and change the sublist - # delimiter to '>' (ordered) to avoid useless further handling. - # TODO: Check if this code can be safely removed. + + if delim[2] == '.': + l = copy.copy(sublist) + lg = len(l) + + # unfold the Nodes one after another + if delim[2:] == '..': + for i in range(lg): + node = random.choice(l) + l.remove(node) + self._construct_subnodes(node, sublist_tmp, delim[0], ignore_sep_fstate) + + # unfold all the Node and then choose randomly + else: + # In this case, NodeSeparator(always=True) have no meaning and + # thus 'always' is always considered to be False. + list_unfold = [] + for i in range(lg): + node = random.choice(l) + l.remove(node) + self._construct_subnodes(node, list_unfold, delim[0], ignore_sep_fstate, ignore_separator=True) + + lg = len(list_unfold) + for i in range(lg): + node = random.choice(list_unfold) + list_unfold.remove(node) + sublist_tmp.append(node) + if self.separator is not None: + new_sep = self._clone_separator(self.separator.node, unique=self.separator.unique, + ignore_frozen_state=ignore_sep_fstate) + sublist_tmp.append(new_sep) + + # choice of only one component within a list + elif delim[2] == '+': if sublist[0] > -1: - node = self._get_heavier_component(sublist[1], check_existence=True) + node = self._get_random_component(comp_list=sublist[1], total_weight=sublist[0], + check_existence=True) else: + ndesc_list = [] for ndesc in sublist[1]: - node, _, _ = self._get_node_and_attrs_from(ndesc) - shall_exist = self._existence_from_node(node) + n, _, _ = self._get_node_and_minmax_from(ndesc) + shall_exist = self._existence_from_node(n) if shall_exist is None or shall_exist: - node = ndesc - break - else: - node = None + ndesc_list.append(ndesc) + node = random.choice(ndesc_list) if ndesc_list else None if node is None: continue else: self._construct_subnodes(node, sublist_tmp, delim[0], ignore_sep_fstate) - else: - for i, node in enumerate(sublist): - self._construct_subnodes(node, sublist_tmp, delim[0], ignore_sep_fstate) - - else: - raise ValueError - elif delim[1] == '>': - for i, node in enumerate(sublist): - self._construct_subnodes(node, sublist_tmp, delim[0], ignore_sep_fstate) - - elif delim[1] == '=': - - if delim[2] == '.': - l = copy.copy(sublist) - lg = len(l) - - # unfold the Nodes one after another - if delim[2:] == '..': - for i in range(lg): - node = random.choice(l) - l.remove(node) - self._construct_subnodes(node, sublist_tmp, delim[0], ignore_sep_fstate) - - # unfold all the Node and then choose randomly - else: - list_unfold = [] - for i in range(lg): - node = random.choice(l) - l.remove(node) - self._construct_subnodes(node, list_unfold, delim[0], ignore_sep_fstate, ignore_separator=True) - - lg = len(list_unfold) - for i in range(lg): - node = random.choice(list_unfold) - list_unfold.remove(node) - sublist_tmp.append(node) - if self.separator is not None: - new_sep = self._clone_separator(self.separator.node, unique=self.separator.unique, - ignore_frozen_state=ignore_sep_fstate) - sublist_tmp.append(new_sep) - - # choice of only one component within a list - elif delim[2] == '+': - if sublist[0] > -1: - node = self._get_random_component(comp_list=sublist[1], total_weight=sublist[0], - check_existence=True) else: - ndesc_list = [] - for ndesc in sublist[1]: - n, _, _ = self._get_node_and_attrs_from(ndesc) - shall_exist = self._existence_from_node(n) - if shall_exist is None or shall_exist: - ndesc_list.append(ndesc) - node = random.choice(ndesc_list) if ndesc_list else None - if node is None: - continue - else: - self._construct_subnodes(node, sublist_tmp, delim[0], ignore_sep_fstate) - + raise ValueError(f"delim: '{delim}'") else: - raise ValueError("delim: '%s'"%delim) - else: - raise ValueError("delim: '%s'"%delim) + raise ValueError(f"delim: '{delim}'") - self.frozen_node_list += sublist_tmp + self.frozen_node_list += sublist_tmp for e in self.subnodes_set: e.tmp_ref_count = 1 - if self.separator is not None and self.frozen_node_list and self.frozen_node_list[-1].is_attr_set(NodeInternals.Separator): - if not self.separator.suffix: + if self.separator is not None: + if not self.separator.suffix and self.frozen_node_list \ + and self.frozen_node_list[-1].is_attr_set(NodeInternals.AutoSeparator): self.frozen_node_list.pop(-1) self._clone_separator_cleanup() + self._reevaluation_pending = False return (self.frozen_node_list, True) def _get_value(self, conf=None, recursive=True, after_encoding=True, - return_node_internals=False): + return_node_internals=False, restrict_csp=False): - ''' + """ The parameter return_node_internals is not used for non terminal nodes, only for terminal nodes. However, keeping it also for non terminal nodes avoid additional checks in the code. - ''' + """ def tobytes_helper(node_internals): if isinstance(node_internals, bytes): return node_internals else: return node_internals._get_value(conf=conf, recursive=recursive, - return_node_internals=False)[0] + return_node_internals=False, restrict_csp=restrict_csp)[0] def handle_encoding(list_to_enc): - if self.custo.collapse_padding_mode: + if self.custo.collapse_padding_mode and not self.custo.delay_collapsing: list_to_enc = list(flatten(list_to_enc)) if list_to_enc and isinstance(list_to_enc[0], bytes): return list_to_enc @@ -3163,7 +3641,9 @@ def handle_encoding(list_to_enc): node_list, was_not_frozen = self.get_subnodes_with_csts() djob_group_created = False - for n in node_list: + disabled_node = False + removed_cpt = 0 + for idx, n in enumerate(node_list): if n.is_attr_set(NodeInternals.DISABLED): val = Node.DEFAULT_DISABLED_NODEINT if not n.env.is_djob_registered(key=id(n), prio=Node.DJOBS_PRIO_nterm_existence): @@ -3176,13 +3656,33 @@ def handle_encoding(list_to_enc): cleanup = NodeInternals_NonTerm._cleanup_delayed_nodes, args=[n, node_list, len(l), conf, recursive], prio=Node.DJOBS_PRIO_nterm_existence) + disabled_node = True else: val = n._get_value(conf=conf, recursive=recursive, - return_node_internals=True) + return_node_internals=True, restrict_csp=restrict_csp) + + if self.separator is not None and isinstance(val, NodeInternals) and val.is_attr_set(NodeInternals.AutoSeparator) \ + and disabled_node and not self.separator.suffix and not self.separator.always: + + # TODO: The case "suffix False and always True" should be already handled at + # self.get_subnodes_with_csts() + # --> TBC + disabled_node = False + self.frozen_node_list.pop(idx-removed_cpt) + removed_cpt += 1 + continue + + disabled_node = False # 'val' is always a NodeInternals except if non-term encoding has been carried out l.append(val) + if self.separator is not None and l and isinstance(l[-1], NodeInternals) \ + and l[-1].is_attr_set(NodeInternals.AutoSeparator): + if not self.separator.suffix and not self.separator.always: + l.pop(-1) + self.frozen_node_list.pop(-1) + if node_list: node_env = node_list[0].env else: @@ -3251,7 +3751,7 @@ def _expand_delayed_nodes(node, node_list, idx, conf, rec): ignore_separator, lazy_mode=False) if expand_list: if node_internals.separator is not None and len(node_list) == idx - 1: - if not node_internals.separator.suffix and expand_list[-1].is_attr_set(NodeInternals.Separator): + if not node_internals.separator.suffix and expand_list[-1].is_attr_set(NodeInternals.AutoSeparator): expand_list.pop(-1) node_internals._clone_separator_cleanup() @@ -3268,7 +3768,7 @@ def _cleanup_delayed_nodes(node, node_list, idx, conf, rec): if idx < len(node_list): node_list.pop(idx) - def set_separator_node(self, sep_node, prefix=True, suffix=True, unique=False): + def set_separator_node(self, sep_node, prefix=True, suffix=True, unique=False, always=False): check_err = set() for n in self.subnodes_set: check_err.add(n.name) @@ -3276,7 +3776,7 @@ def set_separator_node(self, sep_node, prefix=True, suffix=True, unique=False): print("\n*** The separator node name shall not be used by a subnode " + \ "of this non-terminal node") raise ValueError - self.separator = NodeSeparator(sep_node, prefix=prefix, suffix=suffix, unique=unique) + self.separator = NodeSeparator(sep_node, prefix=prefix, suffix=suffix, unique=unique, always=always) def get_separator_node(self): if self.separator is not None: @@ -3329,8 +3829,105 @@ def replace_subnode(self, old, new): if n is old: sublist[idx] = new + def add(self, node, min=1, max=1, default_qty=None, after=None, before=None, idx=None): + """ + This method add a new node to this non-terminal. The location and the quantity can be configured + through the parameters. + + Args: + node (Node): The node to add + min: The minimum number of repetition of this node within the non-terminal node + max: The maximum number of repetition of this node within the non-terminal node + default_qty: the default number of repetition of this node within the non-terminal node + after: If not None, it should be the node (within the non-terminal) *after* which + the new node will be inserted. + before: If not None, it should be the node (within the non-terminal) *before* which + the new node will be inserted. + idx: If not None, it should provide the position in the list of subnodes where the new + node will be inserted. + + """ + + assert (after is not None and before is None and idx is None) or \ + (before is not None and after is None and idx is None) or \ + (idx is not None and before is None and after is None) or \ + (after is None and before is None and idx is None) + + self.subnodes_set.add(node) + self.subnodes_attrs[node] = NodeInternals_NonTerm.NodeAttrs() + self.subnodes_attrs[node].default_qty = default_qty + self.subnodes_attrs[node].qty = [min, max] + + if after is not None: + pivot = after + elif before is not None: + pivot = before + else: + pivot = None + + insert_before = after is None + + def add_to_node_list(new_node, node_list, index): + node_cpt = 0 + for delim, sublist in self.__iter_csts(node_list): + if delim[:3] == 'u=+' or delim[:3] == 's=+': + for w, etp in split_with(lambda x: isinstance(x, int), sublist[1]): + if pivot is None and node_cpt == index: + etp.insert(index, new_node) + else: + for i, ndesc in enumerate(etp): + n, _, _ = self._get_node_and_minmax_from(ndesc) + if n is pivot: + index = i if insert_before else i + 1 + ndesc = new_node + etp.insert(index, ndesc) + break + node_cpt += 1 + else: + if pivot is None and node_cpt == index: + sublist.insert(index, new_node) + else: + for i, ndesc in enumerate(sublist): + n, _, _ = self._get_node_and_minmax_from(ndesc) + if n is pivot: + index = i if insert_before else i + 1 + ndesc = new_node + sublist.insert(index, ndesc) + break + node_cpt += 1 + + for weight, lnode_list in split_with(lambda x: isinstance(x, int), self.subnodes_order): + if pivot is None and idx is None: + lnode_list[0].append(['u>', [node]]) + continue + add_to_node_list(node, lnode_list[0], idx) + + if self.current_flattened_nodelist is not None: + if pivot is not None: + cf_idx = self.current_flattened_nodelist.index(pivot) + cf_idx = cf_idx if insert_before else cf_idx+1 + elif idx is None: + cf_idx = len(self.current_flattened_nodelist) + else: + cf_idx = idx + + self.current_flattened_nodelist.insert(cf_idx, node) + + if self.frozen_node_list: + if pivot is not None: + f_idx = self.frozen_node_list.index(pivot) + f_idx = f_idx if insert_before else f_idx+1 + elif idx is None: + f_idx = len(self.frozen_node_list) + else: + f_idx = idx + + for _ in range(default_qty if default_qty is not None else min): + self.frozen_node_list.insert(f_idx, node) + + def _parse_node_desc(self, node_desc): - mini, maxi = self.subnodes_attrs[node_desc] + mini, maxi = self.subnodes_attrs[node_desc].qty return node_desc, mini, maxi def _clone_node(self, base_node, node_no, force_clone=False, ignore_frozen_state=True): @@ -3422,32 +4019,46 @@ def _qty_from_node(node): @staticmethod def _existence_from_node(node): obj = node.synchronized_with(SyncScope.Existence) - if obj is not None: if isinstance(obj, SyncExistenceObj): correct_reply = obj.check() else: sync_node, condition = obj - exist = node.env.node_exists(id(sync_node)) - crit_1 = exist - crit_2 = True - if exist and condition is not None: - try: - crit_2 = condition.check(sync_node) - except Exception as e: - print("\n*** ERROR: existence condition is not verifiable " \ - "for node '{:s}' (id: {:d})!\n" \ - "*** The condition checker raise an exception!".format(node.name, id(node))) - raise - correct_reply = crit_1 and crit_2 + if sync_node is None: + correct_reply = bool(condition) + else: + exist = node.env.node_exists(id(sync_node)) + crit_1 = exist + crit_2 = True + + if DEBUG: + print(f'\n*** [Existence Check requested by "{node.name}"]\n' + f' --> Does the node "{sync_node.name}" exist? {exist}') + if not exist: + print(f' --> The node "{sync_node.name}" is either really not existing or not\n' + f' registered in node.env.drawn_node_attrs because of a bug...') + + print(f'\n --> condition existing? {bool(condition)}') + + if exist and condition is not None: + try: + crit_2 = condition.check(sync_node) + + if DEBUG: + print(f'\n --> condition satisfied? {crit_2}') + except Exception as e: + print("\n*** ERROR: existence condition is not verifiable " \ + "for node '{:s}' (id: {:d})!\n" \ + "*** The condition checker raise an exception!".format(node.name, id(node))) + raise + correct_reply = crit_1 and crit_2 return NodeInternals_NonTerm.existence_corrupt_hook(node, correct_reply) obj = node.synchronized_with(SyncScope.Inexistence) if obj is not None: sync_node, _ = obj # condition is not checked for this scope - inexist = not node.env.node_exists(id(sync_node)) - correct_reply = True if inexist else False + correct_reply = not node.env.node_exists(id(sync_node)) return NodeInternals_NonTerm.existence_corrupt_hook(node, correct_reply) return None @@ -3484,11 +4095,11 @@ def nodeqty_corrupt_hook(node, mini, maxi): if node.env and node in node.env.nodes_to_corrupt: corrupt_type, corrupt_op = node.env.nodes_to_corrupt[node] if corrupt_type == Node.CORRUPT_NODE_QTY or corrupt_type is None: - return corrupt_op(mini, maxi) + return *corrupt_op(mini, maxi), True else: - return mini, maxi + return mini, maxi, False else: - return mini, maxi + return mini, maxi, False @staticmethod def sizesync_corrupt_hook(node, length): @@ -3532,6 +4143,8 @@ def _try_separator_absorption_with(blob, consumed_size): # We try to absorb the separator st, off, sz, name = new_sep.absorb(blob, constraints, conf=conf) + # if DEBUG: + # print(f'SEPARATOR absorb attempt, st:{st}, off:{off}, sz:{sz}, blob:{blob[:4]} ...') if st == AbsorbStatus.Reject: if DEBUG: @@ -3602,6 +4215,10 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, prepend_postponed = None postponed_appended = None + base_node_len = None + + reject_with_min_null = False + node_no = 1 while node_no <= max_node or max_node < 0: # max_node < 0 means infinity node = self._clone_node(base_node, node_no-1, force_clone) @@ -3614,8 +4231,11 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, if st == AbsorbStatus.Reject: nb_absorbed = node_no-1 if DEBUG: - print('REJECT: %s, size: %d, blob: %r ...' % (node.name, len(blob), blob[:4])) + print('\nREJECT: %s, size: %d, blob: %r ...' % (node.name, len(blob), blob[:4])) if min_node == 0: + # if DEBUG: + # print(' --> min node == 0 (No abort)') + reject_with_min_null = True # abort = False break if node_no <= min_node: @@ -3625,9 +4245,9 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, break elif st == AbsorbStatus.Absorbed or st == AbsorbStatus.FullyAbsorbed: if DEBUG: - print('\nABSORBED: %s, abort: %r, off: %d, consumed_sz: %d, blob: %r ...' \ - % (node.name, abort, off, sz, blob[off:sz][:100])) - print('\nPostpone Node: %r' % postponed) + print('\nABSORBED: %s, abort: %r, off: %d, consumed_sz: %d, blob: %r...' \ + % (node.name, abort, off, sz, blob[off:off+sz][:100])) + print(f'\nPostpone Node: {postponed.name if postponed else "N/A"} ({postponed!r})') nb_absorbed = node_no sz2 = 0 @@ -3641,6 +4261,13 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, postponed = None elif postponed is not None: + # we first set metadata related to the successful absorption as the absorbed + # node could be leveraged while trying to absorb the postponed node + # (e.g., node existence verification). + + base_node_len = len(base_node._tobytes()) + # this call is necessary for base_node existence check to work + self._set_drawn_node_attrs(base_node, nb=nb_absorbed, sz=base_node_len) # we only support one postponed node between two nodes st2, off2, sz2, name2 = \ @@ -3671,9 +4298,11 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, # We need to reject this absorption as # accepting it could prevent finding a # good non-terminal shape. + nb_absorbed = node_no-1 if node_no == 1 and min_node == 0: # this case is OK # abort = False + reject_with_min_null = True break elif node_no <= min_node: # reject in this case if DEBUG: @@ -3711,6 +4340,17 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, # considering a postpone node desc from a parent node only in the first loop first_pass = False + if reject_with_min_null and self.separator is not None and self.separator.always: + # if DEBUG: + # print(f'\n Try absorb separator\n - {blob}\n - {consumed_size}') + + abort, blob, consumed_size, new_sep = _try_separator_absorption_with(blob, consumed_size) + # if DEBUG: + # print(f'\n Try absorb separator, success={not abort}, cons_sz={consumed_size}, blob={blob}') + if not abort: + tmp_list.append(new_sep) + abort = False + if abort: blob = orig_blob consumed_size = orig_consumed_size @@ -3727,7 +4367,9 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, pending_postponed_to_send_back = None self._clear_drawn_node_attrs(base_node) else: - self._set_drawn_node_attrs(base_node, nb=nb_absorbed, sz=len(base_node._tobytes())) + if base_node_len is None: + base_node_len = len(base_node._tobytes()) + self._set_drawn_node_attrs(base_node, nb=nb_absorbed, sz=base_node_len) idx = 0 for n in tmp_list: if postponed_appended is not None and n is postponed_appended: @@ -3773,23 +4415,27 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, first_pass = True if self.custo.collapse_padding_mode: - consumed_bits = 0 - byte_aligned = None + if hasattr(self, '_private_collapse_mode'): # TODO: remove ugliness + consumed_bits, byte_aligned = self._private_collapse_mode + else: + consumed_bits = 0 + byte_aligned = None # Iterate over all sub-components of the component node_list for delim, sublist in self.__iter_csts(node_list): + blob_update_pending = False # reserved for collapse_padding_mode usage + if delim[1] == '>': for idx, node_desc in enumerate(sublist): abort = False base_node, min_node, max_node = self._parse_node_desc(node_desc) - if self.custo.collapse_padding_mode: + vt = base_node.get_value_type() if base_node.is_typed_value() else None + if self.custo.collapse_padding_mode and isinstance(vt, fvt.BitField): - vt = base_node.get_value_type() - if not isinstance(vt, fvt.BitField) \ - or min_node != 1 \ + if min_node != 1 \ or max_node != 1 \ or self.separator is not None \ or postponed_node_desc: @@ -3837,10 +4483,7 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, bval = result // (1 << i*8) result = result % (1 << i*8) # remainder l.append(bval) - if sys.version_info[0] > 2: - partial_blob = struct.pack('{:d}s'.format(nb_bytes), bytes(l)) - else: - partial_blob = struct.pack('{:d}s'.format(nb_bytes), str(bytearray(l))) + partial_blob = struct.pack('{:d}s'.format(nb_bytes), bytes(l)) else: partial_blob = blob[consumed_size:last_idx] last_byte = blob[last_idx:last_idx+1] @@ -3873,10 +4516,14 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, consumed_bits = 0 if consumed_bits == 8 else consumed_bits%8 # if vt is byte-aligned, then the consumed_size is correct + # otherwise we decrease it if vt.padding_size != 0 and consumed_bits > 0: consumed_size -= 1 + blob_update_pending = True + # if we reach the end we should update the blob if idx == len(sublist) - 1: + blob_update_pending = False blob = blob[consumed_size:] elif base_node.is_attr_set(NodeInternals.Abs_Postpone): @@ -3888,12 +4535,32 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, continue else: + if self.custo.collapse_padding_mode: + bnode_to_be_cleaned = False + aligned = consumed_bits%8 == 0 + if blob_update_pending and aligned: + # When some Bitfield were collapsed in the NT sublist + # but in this sublist other nodes are not Bitfield. + # Thus, we need to update the blob and reinit + # the "collapse_mode state" for a potential future collapse + blob_update_pending = False + blob = blob[consumed_size:] + consumed_bits = 0 + byte_aligned = None + elif not aligned: + bnode_to_be_cleaned = True + conf = base_node._check_conf(conf) + base_node.c[conf]._private_collapse_mode = (consumed_bits, byte_aligned) + # pending_upper_postpone = pending_postpone_desc abort, blob, consumed_size, consumed_nb, postponed_sent_back = \ _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, postponed_node_desc, pending_upper_postpone=pending_postpone_desc) + if self.custo.collapse_padding_mode and bnode_to_be_cleaned: + bnode_to_be_cleaned = False + del base_node.c[conf]._private_collapse_mode # In this case max_node is 0 if abort is None: @@ -4081,8 +4748,9 @@ def _try_absorption_with(base_node, min_node, max_node, blob, consumed_size, if abort: break - if self.separator is not None and self.frozen_node_list and self.frozen_node_list[-1].is_attr_set(NodeInternals.Separator): - if not self.separator.suffix: + if self.separator is not None and self.frozen_node_list and self.frozen_node_list[-1].is_attr_set(NodeInternals.AutoSeparator): + if not self.separator.suffix and not self.separator.always: + # TODO: check self.separator.always is maybe not always enough sep = self.frozen_node_list.pop(-1) data = sep._tobytes() consumed_size = consumed_size - len(data) @@ -4121,8 +4789,12 @@ def confirm_absorb(self): n.confirm_absorb() def is_exhausted(self): - if self.is_attr_set(NodeInternals.Finite): - return self.exhausted and self.subcomp_exhausted + if not self.is_attr_set(NodeInternals.Mutable): + return True + elif self.custo.stick_to_default_mode: + return self.exhausted_shapes + elif self.is_attr_set(NodeInternals.Finite): + return self.exhausted_shapes and self.combinatory_complete else: return False @@ -4130,9 +4802,26 @@ def is_frozen(self): return self.frozen_node_list is not None def _make_specific(self, name): + if name == NodeInternals.Highlight: + for node in self.subnodes_set: + node.set_attr(name, recursive=True) + if self.frozen_node_list: + for node in self.frozen_node_list: + node.set_attr(name, recursive=True) + return True def _unmake_specific(self, name): + if name == NodeInternals.Determinist: + self.current_flattened_nodelist = None + + elif name == NodeInternals.Highlight: + for node in self.subnodes_set: + node.clear_attr(name, recursive=True) + if self.frozen_node_list: + for node in self.frozen_node_list: + node.clear_attr(name, recursive=True) + return True def _cleanup_entangled_nodes(self): @@ -4162,63 +4851,35 @@ def _cleanup_entangled_nodes_from(self, node): def unfreeze(self, conf=None, recursive=True, dont_change_state=False, ignore_entanglement=False, only_generators=False, reevaluate_constraints=False): + mutable = self.is_attr_set(NodeInternals.Mutable) + # mutable = True if recursive: - if reevaluate_constraints: + if reevaluate_constraints and mutable: # In order to re-evaluate existence condition of # child node we have to recompute the previous state, # which is the purpose of the following code. We also # re-evaluate generator and function. - # - # SIDE NOTE: the previous state cannot be saved during - # a node copy in an efficient manner (as memory and - # cpu will be necessarily wasted if - # 'reevaluate_constraints' is not used), that's why we - # recompute it. iterable = self.frozen_node_list determinist = self.is_attr_set(NodeInternals.Determinist) finite = self.is_attr_set(NodeInternals.Finite) - if finite or determinist: - if self.excluded_components: - self.excluded_components.pop(-1) - self._perform_first_step = False - if determinist: - node_list, idx = self._get_next_heavier_component(self.subnodes_order, - excluded_idx=self.excluded_components) - else: - # In this case we don't recover the previous - # seed as the node is random and recovering - # the seed would make little sense because of - # the related overhead - node_list, idx, self.component_seed = self._get_next_random_component(self.subnodes_order, - excluded_idx=self.excluded_components, - seed=self.component_seed) - - fresh_expanded_nodelist = self._generate_expanded_nodelist(node_list, determinist=determinist) - if self.expanded_nodelist is None: - self.expanded_nodelist_origsz = len(fresh_expanded_nodelist) - if self.expanded_nodelist_sz is not None: - # In this case we need to go back to the previous state, thus +1 - self.expanded_nodelist_sz += 1 - self.expanded_nodelist = fresh_expanded_nodelist[:self.expanded_nodelist_sz] - else: - # This case should never trigger - raise ValueError - # self.expanded_nodelist_sz = len(fresh_expanded_nodelist) - # self.expanded_nodelist = fresh_expanded_nodelist + self._reevaluation_pending = True + + if finite or determinist: + if self.current_flattened_nodelist is None: + # This case happens when we have been cloned with 'ignore_frozen_state' + # and not frozen since then, or cloned from a node that has never been frozen. + # Thus nothing to do, the parameters are correctly initialized. + pass else: - # assert self.expanded_nodelist_origsz > self.expanded_nodelist_sz - if self.expanded_nodelist_sz is None: - # This means that nothing has been computed yet. This is the case after - # a call to reset() which is either due to a node copy after an absorption - # or at node initialization. - # In such a case, self.expanded_nodelist should be equal to [] - assert self.expanded_nodelist == [] - self.expanded_nodelist = fresh_expanded_nodelist - self.expanded_nodelist_sz = len(fresh_expanded_nodelist) - else: - self.expanded_nodelist.append(fresh_expanded_nodelist[self.expanded_nodelist_sz]) - self.expanded_nodelist_sz += 1 + self.combinatory_complete = False + self.cursor_maj = self.previous_cursor_maj + self.cursor_min = self.previous_cursor_min + nd = self.current_flattened_nodelist[self.previous_cursor_min] + self.subnodes_attrs[nd].unroll() + for nd in self.current_flattened_nodelist: + self.subnodes_attrs[nd].unplan_reset() + else: # In this case the states are random, thus we # don't bother trying to recover the previous one @@ -4236,7 +4897,7 @@ def unfreeze(self, conf=None, recursive=True, dont_change_state=False, ignore_en for n in self.subnodes_set: n.clear_clone_info_since(n) - elif dont_change_state or only_generators: + elif (dont_change_state or only_generators) and mutable: iterable = self.frozen_node_list else: iterable = copy.copy(self.subnodes_set) @@ -4249,7 +4910,7 @@ def unfreeze(self, conf=None, recursive=True, dont_change_state=False, ignore_en ignore_entanglement=ignore_entanglement, only_generators=only_generators, reevaluate_constraints=reevaluate_constraints) - if not dont_change_state and not only_generators and not reevaluate_constraints: + if not dont_change_state and not only_generators and not reevaluate_constraints and mutable: self._cleanup_entangled_nodes() self.frozen_node_list = None self._nodes_drawn_qty = {} @@ -4257,7 +4918,7 @@ def unfreeze(self, conf=None, recursive=True, dont_change_state=False, ignore_en self._clear_drawn_node_attrs(n) n.clear_clone_info_since(n) - if self.exhausted: + if self.exhausted_shapes and mutable: self.excluded_components = [] @@ -4278,7 +4939,7 @@ def unfreeze_all(self, recursive=True, ignore_entanglement=False): self._clear_drawn_node_attrs(n) n.clear_clone_info_since(n) - if self.exhausted: + if self.exhausted_shapes: self.excluded_components = [] @@ -4300,29 +4961,26 @@ def _reset_state_info(self, new_info=None, nodes_drawn_qty=None): self.frozen_node_list = None if new_info is None: - self.exhausted = False + self.exhausted_shapes = False self.excluded_components = [] - self.subcomp_exhausted = True - self.expanded_nodelist_sz = None - self.expanded_nodelist_origsz = None - self.expanded_nodelist = [] + self.combinatory_complete = True + self.exhausted_pick_cases = True self.component_seed = None - self._perform_first_step = True + self.current_pick_section = 0 + self.current_picked_node_idx = 0 + self.cursor_maj = 0 + self.cursor_min = 0 + self.previous_cursor_maj = 0 + self.previous_cursor_min = 0 + for nd in self.subnodes_attrs: + self.subnodes_attrs[nd].reset() + else: - self.exhausted = new_info[0] + self.exhausted_shapes = new_info[0] self.excluded_components = new_info[1] - self.subcomp_exhausted = new_info[2] - self.expanded_nodelist_sz = new_info[3] - self.expanded_nodelist_origsz = new_info[4] - if self.expanded_nodelist_sz is None: - # this case may exist if a node has been created (sz/origsz == None) and copied - # without being frozen first. (e.g., node absorption during a data model construction) - assert self.expanded_nodelist_origsz is None - self.expanded_nodelist = [] - else: - self.expanded_nodelist = None - self.component_seed = new_info[5] - self._perform_first_step = new_info[6] + self.combinatory_complete = new_info[2] + self.component_seed = new_info[3] + self.exhausted_pick_cases = new_info[4] if nodes_drawn_qty is None: self._nodes_drawn_qty = {} @@ -4408,7 +5066,8 @@ def reset_depth_specific(self, depth): e._reset_depth(depth) def get_child_nodes_by_attr(self, internals_criteria, semantics_criteria, owned_conf, conf, path_regexp, - exclude_self, respect_order, relative_depth, top_node, ignore_fstate): + exclude_self, respect_order, relative_depth, top_node, ignore_fstate, + resolve_generator=False): if self.frozen_node_list is not None and not ignore_fstate: iterable = self.frozen_node_list @@ -4430,7 +5089,7 @@ def get_child_nodes_by_attr(self, internals_criteria, semantics_criteria, owned_ path_regexp=path_regexp, exclude_self=False, respect_order=respect_order, relative_depth=relative_depth, top_node=top_node, - ignore_fstate=ignore_fstate) + ignore_fstate=ignore_fstate, resolve_generator=resolve_generator) if respect_order: for e in nlist: @@ -4450,7 +5109,7 @@ def set_child_current_conf(self, node, conf, reverse, ignore_entanglement): node._set_subtrees_current_conf(e, conf, reverse, ignore_entanglement=ignore_entanglement) - def get_child_all_path(self, name, htable, conf, recursive): + def get_child_all_path(self, name, htable, conf, recursive, resolve_generator=False): if self.frozen_node_list is not None: iterable = self.frozen_node_list else: @@ -4458,9 +5117,9 @@ def get_child_all_path(self, name, htable, conf, recursive): if self.separator is not None: iterable.add(self.separator.node) - for idx, e in enumerate(iterable): - e._get_all_paths_rec(name, htable, conf, recursive=recursive, first=False, - clone_idx=idx) + for idx, node in enumerate(iterable): + node._get_all_paths_rec(name, htable, conf, recursive=recursive, first=False, + resolve_generator=resolve_generator, clone_idx=idx) def set_size_from_constraints(self, size, encoded_size): # not supported @@ -4504,8 +5163,11 @@ class NodeSemantics(object): To be used while defining a data model as a means to associate semantics to an Node. ''' - def __init__(self, attrs=[]): - self.__attrs = attrs + def __init__(self, attrs=None): + self.__attrs = attrs if isinstance(attrs, (list, tuple)) else [attrs] + + def __str__(self): + return ' '.join(self.__attrs) def add_attributes(self, attrs): self.__attrs += attrs @@ -4606,6 +5268,10 @@ def __init__(self, optionalbut1_criteria=None, mandatory_criteria=None, exclusiv self.set_exclusive_criteria(exclusive_criteria) self.set_negative_criteria(negative_criteria) + def __bool__(self): + return bool(self.__optionalbut1) or bool(self.__mandatory) \ + or bool(self.__exclusive) or bool(self.__negative) + def extend(self, sc): crit = sc.get_exclusive_criteria() if crit: @@ -4628,17 +5294,20 @@ def extend(self, sc): self.__negative = [] self.__negative.extend(crit) + def _handle_user_input(self, crit): + return crit if crit is None or isinstance(crit, (list,tuple)) else [crit] + def set_exclusive_criteria(self, criteria): - self.__exclusive = criteria + self.__exclusive = self._handle_user_input(criteria) def set_mandatory_criteria(self, criteria): - self.__mandatory = criteria + self.__mandatory = self._handle_user_input(criteria) def set_optionalbut1_criteria(self, criteria): - self.__optionalbut1 = criteria + self.__optionalbut1 = self._handle_user_input(criteria) def set_negative_criteria(self, criteria): - self.__negative = criteria + self.__negative = self._handle_user_input(criteria) def get_exclusive_criteria(self): return self.__exclusive @@ -4764,8 +5433,9 @@ class Node(object): def __init__(self, name, base_node=None, copy_dico=None, ignore_frozen_state=False, accept_external_entanglement=False, acceptance_set=None, - subnodes=None, values=None, value_type=None, vt=None, new_env=False): - ''' + subnodes=None, values=None, value_type=None, vt=None, new_env=False, + description=None): + """ Args: name (str): Name of the node. Every children node of a node shall have a unique name. Useful to look for specific nodes within a graph. @@ -4791,14 +5461,18 @@ def __init__(self, name, base_node=None, copy_dico=None, ignore_frozen_state=Fal new_env (bool): [If `base_node` provided] If True, the `base_node` attached :class:`Env()` will be copied. Otherwise, the same will be used. If `ignore_frozen_state` is True, a new :class:`Env()` will be used. - ''' + description (str): textual description of the node + """ assert '/' not in name # '/' is a reserved character self.internals = {} self.name = name + self.description = description self.env = None + self._paths_htable = None + self.entangled_nodes = None self.semantics = None @@ -4821,9 +5495,8 @@ def __init__(self, name, base_node=None, copy_dico=None, ignore_frozen_state=Fal else: if new_env: self.env = Env() if ignore_frozen_state else copy.copy(base_node.env) - # we always keep a reference to objects coming from other part of the framework, - # namely: knowledge_source - # self.env.knowledge_source = base_node.env.knowledge_source + if ignore_frozen_state: + self.env.csp = copy.copy(base_node.env.csp) else: self.env = base_node.env @@ -4863,24 +5536,30 @@ def __init__(self, name, base_node=None, copy_dico=None, ignore_frozen_state=Fal else: self.make_empty() - def get_clone(self, name=None, ignore_frozen_state=False, new_env=True): - '''Create a new node. To be used wihtin a graph-based data model. - + def get_clone(self, name=None, ignore_frozen_state=False, accept_external_entanglement=False, + acceptance_set=None, new_env=True): + """Create a new node. To be used within a graph-based data model. + Args: name (str): name of the new Node instance. If ``None`` the current name will be used. ignore_frozen_state (bool): if set to False, the clone function will produce - a Node with the same state as the duplicated Node. Otherwise, the only the state won't be kept. + a Node with the same state as the duplicated Node. Otherwise, only the state won't be kept. + accept_external_entanglement (bool): refer to the corresponding Node parameter + acceptance_set (set): refer to the corresponding Node parameter new_env (bool): If True, the current :class:`Env()` will be copied. Otherwise, the same will be used. Returns: Node: duplicated Node object - ''' + """ if name is None: name = self.name - return Node(name, base_node=self, ignore_frozen_state=ignore_frozen_state, new_env=new_env) + return Node(name, base_node=self, ignore_frozen_state=ignore_frozen_state, + accept_external_entanglement=accept_external_entanglement, + acceptance_set=acceptance_set, + new_env=new_env) def __copy__(self): @@ -4900,12 +5579,13 @@ def set_contents(self, base_node, copy_dico=None, ignore_frozen_state=False, accept_external_entanglement=False, acceptance_set=None, preserve_node=True): - '''Set the contents of the node based on the one provided within + """ + Set the contents of the node based on the one provided within `base_node`. This method performs a deep copy of `base_node`, but some parameters can change the behavior of the copy. - .. note:: python deepcopy() is not used for perfomance reason - (10 to 20 times slower). + .. note:: python deepcopy() is not used for performance reason + (10 to 20 times slower) and as it does not work for all cases. Args: base_node (Node): (Optional) Used as a template to create the new node. @@ -4923,10 +5603,11 @@ def set_contents(self, base_node, Returns: dict: For each subnodes of `base_node` (keys), reference the corresponding subnodes within the new node. - ''' + """ + self.description = base_node.description self._post_freeze_handler = base_node._post_freeze_handler - + if self.internals: self.internals = {} if self.entangled_nodes: @@ -4999,6 +5680,21 @@ def set_contents(self, base_node, for n in delayed_node_internals: n._update_node_refs(node_dico, debug=n) + if self.env is not None and self.env.csp is not None: + for v in self.env.csp.iter_vars(): + old_nd = self.env.csp.var_mapping[v] + new_node = node_dico.get(old_nd, None) + if new_node is not None: + self.env.csp.map_var_to_node(v, new_node) + # print(f'\n*** DBG set_content for node "{old_nd.name}" (called from {self.name}): {v}' + # f'\n --> old node: {old_nd}' + # f'\n --> new node: {new_node}') + else: + # It means we are called from a subnode which are not linked to the + # variables thus we can break, as it exists at least one node from which some + # paths exist to all the variables (and thus we will do the update from there). + break + if base_node.entangled_nodes is not None and ((not ignore_frozen_state) or accept_external_entanglement): entangled_set.add(base_node) @@ -5017,26 +5713,26 @@ def set_contents(self, base_node, elif acceptance_set is not None and e in acceptance_set: intrics.add(e) else: - DEBUG = True - # TOFIX: the dynamically created subnodes by a - # Non terminal node, may have in their - # entangled list some mirror subnodes from - # another Non terminal node containing copies - # of these subnodes, or maybe subnodes (not - # removed) of the node in previous frozen - # state - if DEBUG: - print("\n*** WARNING: detection of entangled node outside the current graph, " \ - "whereas 'accept_external_entanglement' parameter is set to False!") - print("[ accept_external_entanglement = %r, ignore_frozen_state = %r, current copied node: %s ]" \ - % (accept_external_entanglement, ignore_frozen_state, self.name)) - print(' --> Node: ', node.name, repr(node)) - print(' --> Entangled with external node: ', e.name, repr(e)) - print(" --> Entangled nodes of node '%s':" % node.name) - for e in node.entangled_nodes: - print(' -', e.name, repr(e), - " [in node_dico.keys(): %r / .values(): %r]" % (e in node_dico, e in node_dico.values())) - # raise ValueError + pass + # Note: If base_node has entangled nodes, chances are these entangled nodes are + # not part of the current node-graph. Especially, if base node is a child node + # of an NT-node with a `qty` attribute > 1, the siblings nodes are entangled + # nodes outside of the current graph. Thus they just have to be ignored. + # In other cases (not described here) where the entangled nodes outside of + # the current node-graph needs to be cloned, the user should explicitly state it + # through the usage of the parameter `accept_external_entanglement` + # or `acceptance_set`. + # + # print("\n*** WARNING: detection of entangled node outside the current graph, " \ + # "whereas 'accept_external_entanglement' parameter is set to False!") + # print("[ accept_external_entanglement = %r, ignore_frozen_state = %r, current copied node: %s ]" \ + # % (accept_external_entanglement, ignore_frozen_state, self.name)) + # print(' --> Node: ', node.name, repr(node)) + # print(' --> Entangled with external node: ', e.name, repr(e)) + # print(" --> Entangled nodes of node '%s':" % node.name) + # for e in node.entangled_nodes: + # print(' -', e.name, repr(e), + # " [in node_dico.keys(): %r / .values(): %r]" % (e in node_dico, e in node_dico.values())) if node is base_node: self.entangled_nodes = intrics @@ -5198,7 +5894,7 @@ def __get_internals(self): '''Property linked to `self.internals` (read only)''' def conf(self, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) return self.internals[conf] def get_internals_backup(self): @@ -5217,7 +5913,7 @@ def set_internals(self, backup): self.entangled_nodes = backup.entangled_nodes self._delayed_jobs_called = backup._delayed_jobs_called - def __check_conf(self, conf): + def _check_conf(self, conf): if conf is None: conf = self.current_conf elif not self.is_conf_existing(conf): @@ -5225,15 +5921,15 @@ def __check_conf(self, conf): return conf def is_genfunc(self, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) return isinstance(self.internals[conf], NodeInternals_GenFunc) def is_func(self, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) return isinstance(self.internals[conf], NodeInternals_Func) def is_typed_value(self, conf=None, subkind=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) resp = isinstance(self.internals[conf], NodeInternals_TypedValue) if resp and subkind is not None: resp = (self.internals[conf].get_current_subkind() == subkind) or \ @@ -5241,16 +5937,16 @@ def is_typed_value(self, conf=None, subkind=None): return resp def is_nonterm(self, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) return isinstance(self.internals[conf], NodeInternals_NonTerm) def is_term(self, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) return issubclass(self.internals[conf].__class__, NodeInternals_Term) def compliant_with(self, internals_criteria=None, semantics_criteria=None, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) if internals_criteria: cond1 = self.internals[conf].match(internals_criteria) @@ -5314,7 +6010,7 @@ def _finalize_nonterm_node(self, conf, depth=None): def set_subnodes_basic(self, node_list, conf=None, ignore_entanglement=False, separator=None, preserve_node=True): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) new_internals = NodeInternals_NonTerm() if preserve_node: @@ -5331,7 +6027,7 @@ def set_subnodes_basic(self, node_list, conf=None, ignore_entanglement=False, se def set_subnodes_with_csts(self, wlnode_list, conf=None, ignore_entanglement=False, separator=None, preserve_node=True): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) new_internals = NodeInternals_NonTerm() if preserve_node: @@ -5346,7 +6042,7 @@ def set_subnodes_with_csts(self, wlnode_list, conf=None, ignore_entanglement=Fal def set_subnodes_full_format(self, subnodes_order, subnodes_attrs, conf=None, separator=None, preserve_node=True): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) new_internals = NodeInternals_NonTerm() if preserve_node: @@ -5360,7 +6056,7 @@ def set_subnodes_full_format(self, subnodes_order, subnodes_attrs, conf=None, se def set_values(self, values=None, value_type=None, conf=None, ignore_entanglement=False, preserve_node=True): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) new_internals = NodeInternals_TypedValue() if preserve_node: @@ -5387,7 +6083,7 @@ def set_values(self, values=None, value_type=None, conf=None, ignore_entanglemen def set_func(self, func, func_node_arg=None, func_arg=None, conf=None, ignore_entanglement=False, provide_helpers=False, preserve_node=True): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) new_internals = NodeInternals_Func() if preserve_node: @@ -5407,7 +6103,7 @@ def set_func(self, func, func_node_arg=None, func_arg=None, def set_generator_func(self, gen_func, func_node_arg=None, func_arg=None, conf=None, ignore_entanglement=False, provide_helpers=False, preserve_node=True): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) new_internals = NodeInternals_GenFunc() if preserve_node: @@ -5423,11 +6119,11 @@ def set_generator_func(self, gen_func, func_node_arg=None, def make_empty(self, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) self.internals[conf] = NodeInternals_Empty() def is_empty(self, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) return isinstance(self.internals[conf], NodeInternals_Empty) def absorb(self, blob, constraints=AbsCsts(), conf=None, pending_postpone_desc=None): @@ -5444,15 +6140,15 @@ def absorb(self, blob, constraints=AbsCsts(), conf=None, pending_postpone_desc=N return status, off, sz, self.name def set_absorb_helper(self, helper, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) self.internals[conf].set_absorb_helper(helper) def enforce_absorb_constraints(self, csts, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) self.internals[conf].enforce_absorb_constraints(csts) def set_size_from_constraints(self, size=None, encoded_size=None, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) self.internals[conf].set_size_from_constraints(size=size, encoded_size=encoded_size) # Does not affect function/generator Nodes @@ -5490,11 +6186,11 @@ def _set_clone_info(self, info, node): self.internals[c].set_clone_info(info, node) def make_synchronized_with(self, scope, node=None, param=None, sync_obj=None, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) self.internals[conf].set_node_sync(scope=scope, node=node, param=param, sync_obj=sync_obj) def synchronized_with(self, scope, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) val = self.internals[conf].get_node_sync(scope) return val @@ -5521,22 +6217,29 @@ def clear_attr(self, name, conf=None, all_conf=False, recursive=False): self.internals[conf].clear_child_attr(name, conf=next_conf, recursive=recursive) def is_attr_set(self, name, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) return self.internals[conf].is_attr_set(name) + @property + def debug(self): + for nd_internal in self.internals.values(): + if nd_internal.is_attr_set(NodeInternals.DEBUG): + return True + return False + def set_private(self, val, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) self.internals[conf].set_private(val) def get_private(self, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) return self.internals[conf].get_private() def set_semantics(self, sem): if isinstance(sem, NodeSemantics): self.semantics = sem else: - assert(isinstance(sem, list)) + assert isinstance(sem, (list, str)) self.semantics = NodeSemantics(sem) def get_semantics(self): @@ -5545,9 +6248,36 @@ def get_semantics(self): def get_reachable_nodes(self, internals_criteria=None, semantics_criteria=None, owned_conf=None, conf=None, path_regexp=None, exclude_self=False, - respect_order=False, relative_depth=-1, top_node=None, ignore_fstate=False): - - def __compliant(node, config, top_node): + respect_order=False, top_node=None, ignore_fstate=False, + resolve_generator=False, + relative_depth=-1): + """ + + Args: + internals_criteria: + semantics_criteria: + owned_conf: + conf: + path_regexp: + exclude_self: + respect_order: + top_node: + ignore_fstate: + resolve_generator: if `True`, the generator nodes will be resolved in order to perform + the search within. But there will be side-effects on the graph, because + some parts of the graph could end up frozen if they are used as generator parameters. + If `False`, generator nodes won't be resolved, but they could already be in a + resolved state before this method is called on them. It means that no side effects + could result from the call of this method. And thus for this latter case, + the method works as if `resolve_generator` is set to `True`. + relative_depth: For internal use only + + Returns: + + """ + + + def __compliant(node, config, top_node, side_effect_risk): if node is top_node and exclude_self: return False @@ -5565,7 +6295,8 @@ def __compliant(node, config, top_node): cond2 = True if path_regexp is not None: - paths = node.get_all_paths_from(top_node) + paths = node.get_all_paths_from(top_node, flush_cache=False, + resolve_generator=resolve_generator) for p in paths: if re.search(path_regexp, p): cond3 = True @@ -5580,10 +6311,6 @@ def __compliant(node, config, top_node): def get_reachable_nodes_rec(node, config, rdepth, top_node): s = [] - # if respect_order: - # s = [] - # else: - # s = set() if config == None: config = self.current_conf @@ -5595,14 +6322,14 @@ def get_reachable_nodes_rec(node, config, rdepth, top_node): config = node.current_conf internal = node.internals[config] + side_effect_risk = isinstance(internal, NodeInternals_GenFunc) and not internal.is_frozen() - if node.is_conf_existing(owned_conf) or (owned_conf == None): - if __compliant(node, config, top_node): + if (owned_conf == None) or node.is_conf_existing(owned_conf): + if __compliant(node, config, top_node, side_effect_risk=side_effect_risk): s.append(node) - # if respect_order: - # s.append(node) - # else: - # s.add(node) + + if not resolve_generator and side_effect_risk: + return s if rdepth <= -1 or rdepth > 0: s2 = internal.get_child_nodes_by_attr(internals_criteria=internals_criteria, @@ -5612,26 +6339,24 @@ def get_reachable_nodes_rec(node, config, rdepth, top_node): exclude_self=False, respect_order=respect_order, relative_depth = rdepth - 1, - top_node=top_node, ignore_fstate=ignore_fstate) + top_node=top_node, ignore_fstate=ignore_fstate, + resolve_generator=resolve_generator) if s2: for e in s2: if e not in s: s.append(e) - # if respect_order: - # for e in s2: - # if e not in s: - # s.append(e) - # else: - # s = s.union(s2) return s - if top_node is None: - nodes = get_reachable_nodes_rec(node=self, config=conf, rdepth=relative_depth, - top_node=self) - else: - nodes = get_reachable_nodes_rec(node=self, config=conf, rdepth=relative_depth, - top_node=top_node) + top_node = self if top_node is None else top_node + if relative_depth == -1: + top_node._paths_htable = None + + nodes = get_reachable_nodes_rec(node=self, config=conf, rdepth=relative_depth, + top_node=top_node) + + if relative_depth == -1: + top_node._paths_htable = None if respect_order: return nodes @@ -5667,32 +6392,69 @@ def filter_out_entangled_nodes(node_list): return ret - def get_node_by_path(self, path_regexp=None, path=None, conf=None): - ''' - The set of nodes that is used to perform the search include + def iter_nodes_by_path(self, path_regexp, conf=None, flush_cache=True, resolve_generator=False): + """ + iterate over all the nodes that match the `path_regexp` parameter. + + Note: the set of nodes that is used to perform the search include the node itself and all the subnodes behind it. - ''' - if path is None: - assert(path_regexp is not None) - # Find *one* Node whose path match the regexp - for n, e in self.iter_paths(conf=conf): - if re.search(path_regexp, n): - ret = e - break - else: - ret = None - else: - htable = self.get_all_paths(conf=conf) - # Find the Node through exact path - try: - ret = htable[path] - except KeyError: - ret = None - return ret + Args: + path_regexp (str): path regexp of the requested nodes + conf (str): Node configuration to use for the search + flush_cache (bool): If False, and a previous search has been performed, the outcomes will + be used for this one, which will improve the performance. + + Returns: + generator of the nodes that match the path regexp + + """ + for p, node in self.iter_paths(conf=conf, flush_cache=flush_cache, resolve_generator=resolve_generator): + if re.search(path_regexp, p): + yield node + + def get_first_node_by_path(self, path_regexp, conf=None, flush_cache=True, resolve_generator=False): + """ + Return the first Node that match the `path_regexp` parameter. - def _get_all_paths_rec(self, pname, htable, conf, recursive, first=True, clone_idx=0): + Args: + path_regexp (str): path regexp of the requested nodes + conf (str): Node configuration to use for the search + flush_cache (bool): If False, and a previous search has been performed, the outcomes will + be used for this one, which will improve the performance. + + Returns: + Node: the first Node that match the path regexp + """ + try: + node = next(self.iter_nodes_by_path(path_regexp=path_regexp, conf=conf, flush_cache=flush_cache, + resolve_generator=resolve_generator)) + except StopIteration: + node = None + + return node + + def get_nodes_by_paths(self, path_list): + """ + Provide a dictionnary of the nodes referenced by the paths provided in @path_list. + Keys of the dict are the paths provided in @path_list. + + Args: + path_list: list of paths referencing nodes of interest + + Returns: + dict: dictionary mapping path to nodes + """ + node_dict = {} + for p in path_list: + node_dict[p] = self.get_first_node_by_path(path_regexp=p, flush_cache=False) + + return node_dict + + + def _get_all_paths_rec(self, pname, htable, conf, recursive, first=True, resolve_generator=False, + clone_idx=0): next_conf = conf if recursive else None @@ -5707,55 +6469,86 @@ def _get_all_paths_rec(self, pname, htable, conf, recursive, first=True, clone_i else: htable[name] = self - internal.get_child_all_path(name, htable, conf=next_conf, recursive=recursive) + side_effect_risk = isinstance(internal, NodeInternals_GenFunc) and not internal.is_frozen() + if resolve_generator or not side_effect_risk: + internal.get_child_all_path(name, htable, conf=next_conf, recursive=recursive, + resolve_generator=resolve_generator) - def get_all_paths(self, conf=None, recursive=True, depth_min=None, depth_max=None): + def get_all_paths(self, conf=None, recursive=True, depth_min=None, depth_max=None, + resolve_generator=False, flush_cache=True): """ + Args: + resolve_generator: if `True`, the generator nodes will be resolved in order to perform + the search within. But there could be side-effects on the graph, because + some parts of the graph could end up frozen if they are used as generator parameters. + If `False`, generator nodes won't be resolved, but they could already be in a + resolved state before this method is called on them. It means that no side effects + could result from the call of this method. And thus for this latter case, + the method works as if `resolve_generator` is set to `True`. + Returns: dict: the keys are either a 'path' or a tuple ('path', int) when the path already exists (case of the same node used more than once within the same non-terminal) """ - htable = collections.OrderedDict() - self._get_all_paths_rec('', htable, conf, recursive=recursive) + + if flush_cache or self._paths_htable is None: + self._paths_htable = collections.OrderedDict() + self._get_all_paths_rec('', self._paths_htable, conf, recursive=recursive, + resolve_generator=resolve_generator) if depth_min is not None or depth_max is not None: depth_min = int(depth_min) if depth_min is not None else 0 depth_max = int(depth_max) if depth_max is not None else -1 - paths = copy.copy(htable) - for k in paths: + paths = copy.copy(self._paths_htable) + for k in self._paths_htable.keys(): depth = len(k.split('/')) if depth < depth_min: - del htable[k] + del paths[k] elif depth_max != -1 and depth > depth_max: - del htable[k] - - return htable + del paths[k] + else: + paths = self._paths_htable + + return paths + + def iter_paths(self, conf=None, recursive=True, depth_min=None, depth_max=None, only_paths=False, + resolve_generator=False, flush_cache=True): - def iter_paths(self, conf=None, recursive=True, depth_min=None, depth_max=None, only_paths=False): htable = self.get_all_paths(conf=conf, recursive=recursive, depth_min=depth_min, - depth_max=depth_max) + depth_max=depth_max, resolve_generator=resolve_generator, + flush_cache=flush_cache) for path, node in htable.items(): if isinstance(path, tuple): yield path[0] if only_paths else (path[0], node) else: yield path if only_paths else (path, node) - def get_path_from(self, node, conf=None): - for n, e in node.iter_paths(conf=conf): - if e == self: - return n + def get_path_from(self, node, conf=None, flush_cache=True, resolve_generator=False): + for path, nd in node.iter_paths(conf=conf, flush_cache=flush_cache, + resolve_generator=resolve_generator): + if nd == self: + return path else: return None - def get_all_paths_from(self, node, conf=None): + def get_all_paths_from(self, node, conf=None, flush_cache=True, resolve_generator=False): l = [] - for n, e in node.iter_paths(conf=conf): - if e == self: - l.append(n) + for path, nd in node.iter_paths(conf=conf, flush_cache=flush_cache, + resolve_generator=resolve_generator): + if nd == self: + l.append(path) return l + def is_path_valid(self, path, resolve_generator=False): + htable = self.get_all_paths(resolve_generator=resolve_generator, flush_cache=True) + for p in htable.keys(): + if re.match(path, p): + return True + else: + return False + def set_env(self, env): self.env = env for c in self.internals: @@ -5764,15 +6557,52 @@ def set_env(self, env): def get_env(self): return self.env - def freeze(self, conf=None, recursive=True, return_node_internals=False): - + def set_csp(self, csp: CSP): + self.env.csp = copy.copy(csp) + + def get_csp(self): + return self.env.csp + + @property + def no_more_solution_for_csp(self): + return self.env.csp.exhausted_solutions if self.env.csp is not None else True + + def walk(self, conf=None, recursive=True, steps_num=1): + for _ in range(steps_num): + self.unfreeze(conf=conf, recursive=recursive) + self.freeze(conf=conf, recursive=recursive) + + + def freeze(self, conf=None, recursive=True, return_node_internals=False, restrict_csp=False, resolve_csp=False): + """ + + Args: + conf: + recursive: + return_node_internals: + + restrict_csp: Only effective when a CSP is part of the data description. When + set to True, if the node on which this method is called is a variable of the CSP, then + its domain will be shrunk to its current value. Thus, the node won't change when + the CSP will be resolved. + + resolve_csp: Only effective when a CSP is part of the data description. When set to True, + the CSP will be resolved and the data generated will comply with the solution. + + Returns: + + """ + + + # Step 1 - get value ret = self._get_value(conf=conf, recursive=recursive, - return_node_internals=return_node_internals) + return_node_internals=return_node_internals, restrict_csp=restrict_csp) if self.env is None: print('Warning: freeze() is called on a node that does not have an Env()\n' ' --> node name: {!s}'.format(self.name)) + # Step 2 - DJobs resolution if self.env is not None and self.env.delayed_jobs_enabled and \ (not self._delayed_jobs_called or self.env.delayed_jobs_pending): self._delayed_jobs_called = True @@ -5784,13 +6614,43 @@ def freeze(self, conf=None, recursive=True, return_node_internals=False): self.env.execute_basic_djobs(Node.DJOBS_PRIO_genfunc) ret = self._get_value(conf=conf, recursive=recursive, - return_node_internals=return_node_internals) + return_node_internals=return_node_internals, restrict_csp=restrict_csp) + + # Step 3 - CSP resolution + if resolve_csp and self.env.csp is not None and not self.env.csp.is_current_solution_queried: + try: + solution = self.env.csp.get_solution() + except ConstraintError: + if not self.env.csp.var_domain_updated: + # in this case we let the caller handle this, as we are not responsible + raise + else: + print(f"Warning: no solution found for the current CSP, the generated data will be invalid!\n" + f" --> likely culprit: node '{self.name}' with value {self.get_raw_value()}") + else: + if solution is not None: # no more solution + for var, value in solution.items(): + nd = self.env.csp.var_mapping[var] + if self.env.csp.highlight_variables: + nd.set_attr(NodeInternals.Highlight, conf=conf) + if nd == self: + continue + # nd.set_default_value(value) + # Note: .set_default() does disruptive stuff, like re-ordering + # INT.values list, and could disturb a model walker like tWALK + nd.update_value(value) + else: + pass + # print(f'\n***DBG freeze: No more solution - exhausted: {self.env.csp.exhausted_solutions}') + finally: + if self.env.csp.var_domain_updated: + self.env.csp.restore_var_domains() return ret get_value = freeze - def _get_value(self, conf=None, recursive=True, return_node_internals=False): + def _get_value(self, conf=None, recursive=True, return_node_internals=False, restrict_csp=False): next_conf = conf if recursive else None conf2 = conf if self.is_conf_existing(conf) else self.current_conf @@ -5806,10 +6666,28 @@ def _get_value(self, conf=None, recursive=True, return_node_internals=False): raise ValueError ret, was_not_frozen = internal._get_value(conf=next_conf, recursive=recursive, - return_node_internals=return_node_internals) + return_node_internals=return_node_internals, + restrict_csp=restrict_csp) + + + if restrict_csp and self.env.csp is not None and self.is_typed_value(conf=conf): + + for v, n in self.env.csp.var_mapping.items(): + if self == n: + # print(f'\n***DBG _get_value: "{self.name}" {self.get_raw_value()}\n' + # f' --> {self.cc.value_type.values}') + self.env.csp.set_var_domain(v, [self.get_raw_value()]) + self.env.csp.reset() + break + if was_not_frozen: - self._post_freeze(internal, self) + pf_ret = self._post_freeze(internal, self, next_conf=next_conf, recursive=recursive, + return_node_internals=return_node_internals) + # post_freeze handler can perform some change on the nodes. Thus, we have to update the + # ret value accordingly. + if pf_ret is not None: + ret, _ = pf_ret # We need to test self.env because an Node can be freezed # before being registered in the data model. It triggers # for instance when a generator Node is freezed @@ -5821,19 +6699,36 @@ def _get_value(self, conf=None, recursive=True, return_node_internals=False): return ret - def _post_freeze(self, node_internals, wrapping_node): + def _post_freeze(self, node_internals, wrapping_node, + next_conf, recursive, return_node_internals): if self._post_freeze_handler is not None: self._post_freeze_handler(node_internals, wrapping_node) + # We need to call again _get_value(), so that we are sure to provide to the freeze() + # caller the updated nodes further to the execution of post_freeze. + # But even in the case there are no modifications, some bad side-effects could happen, + # that will be resolved thanks to the new call to _get_value(). + # Indeed, in the case the post_freeze handler call Node.freeze() on its associated node, there + # could be some side-effects linked to DJobs. If some DJobs are registered (to deal with + # node existence for instance in non-terminal nodes), then the post_freeze handler + # will trigger them as it will reenter Node.freeze() while step 1 of Node.freeze() is being executed. + # It means that when this step 1 has finished, step 2 won't be executed again as it will have + # already been executed though post_freeze. Thus, DJobs won't have a chance to be called again and a wrong + # value will be provided to the top caller. Thus, we need in this case to call _get_value() + # again to force potential Djobs to be registered again. + return node_internals._get_value(conf=next_conf, recursive=recursive, + return_node_internals=return_node_internals) + else: + return None def register_post_freeze_handler(self, func): self._post_freeze_handler = func def is_exhausted(self, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) return self.internals[conf].is_exhausted() def is_frozen(self, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) return self.internals[conf].is_frozen() def reset_state(self, recursive=False, exclude_self=False, conf=None, ignore_entanglement=False): @@ -5861,8 +6756,8 @@ def tobytes_helper(node_internals): if isinstance(node_internals_list, list): node_internals_list = list(flatten(node_internals_list)) if node_internals_list: - if issubclass(node_internals_list[0].__class__, NodeInternals): - node_internals_list = list(map(tobytes_helper, node_internals_list)) + # if issubclass(node_internals_list[0].__class__, NodeInternals): + node_internals_list = list(map(tobytes_helper, node_internals_list)) val = b''.join(node_internals_list) else: val = b'' @@ -5875,13 +6770,16 @@ def to_str(self, conf=None, recursive=True): val = self.to_bytes(conf=conf, recursive=recursive) return unconvert_from_internal_repr(val) + def to_formatted_str(self, conf=None, recursive=True): + self.enable_color() + val = self.to_bytes(conf=conf, recursive=recursive) + self.disable_color() + return unconvert_from_internal_repr(val) + def to_ascii(self, conf=None, recursive=True): val = self.to_str(conf=conf, recursive=recursive) try: - if sys.version_info[0] > 2: - val = eval('{!a}'.format(val)) - else: - val = str(val) + val = eval('{!a}'.format(val)) except: val = repr(val) finally: @@ -5911,7 +6809,7 @@ def tobytes_helper(node_internals): return val def set_frozen_value(self, value, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) if self.is_term(conf): value = convert_to_internal_repr(value) @@ -5919,13 +6817,21 @@ def set_frozen_value(self, value, conf=None): else: raise ValueError + def set_default_value(self, value, conf=None): + conf = self._check_conf(conf) + + if self.is_term(conf): + self.internals[conf]._set_default_value(value) + else: + raise ValueError + def fix_synchronized_nodes(self, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) self.internals[conf].synchronize_nodes(self) def unfreeze(self, conf=None, recursive=True, dont_change_state=False, ignore_entanglement=False, only_generators=False, - reevaluate_constraints=False): + reevaluate_constraints=False, walk_csp=False, walk_csp_step_size=1): self._delayed_jobs_called = False next_conf = conf @@ -5933,6 +6839,18 @@ def unfreeze(self, conf=None, recursive=True, dont_change_state=False, if not self.is_conf_existing(conf): conf = self.current_conf + # if reevaluate_constraints: + # if self.env.csp is not None: + # print(f'\n***DBG {self.env.csp._is_solution_queried}') + # self.env.csp._is_solution_queried = False + + if walk_csp: + if self.env.csp is not None and self.env.csp.is_current_solution_queried: + for i in range(walk_csp_step_size): + self.env.csp.next_solution() + if self.env.csp.exhausted_solutions: + break + # if self.is_frozen(conf): self.internals[conf].unfreeze(next_conf, recursive=recursive, dont_change_state=dont_change_state, ignore_entanglement=ignore_entanglement, only_generators=only_generators, @@ -5944,7 +6862,6 @@ def unfreeze(self, conf=None, recursive=True, dont_change_state=False, ignore_entanglement=True, only_generators=only_generators, reevaluate_constraints=reevaluate_constraints) - def unfreeze_all(self, recursive=True, ignore_entanglement=False): self._delayed_jobs_called = False @@ -5957,27 +6874,26 @@ def unfreeze_all(self, recursive=True, ignore_entanglement=False): e.unfreeze_all(recursive=recursive, ignore_entanglement=True) - def pretty_print(self, max_size=None, conf=None): - conf = self.__check_conf(conf) + conf = self._check_conf(conf) return self.internals[conf].pretty_print(max_size=max_size) - def get_nodes_names(self, conf=None, verbose=False, terminal_only=False): + def get_nodes_names(self, conf=None, verbose=False, terminal_only=False, flush_cache=True): l = [] - for n, e in self.iter_paths(conf=conf): + for path, node in self.iter_paths(conf=conf, flush_cache=flush_cache): if terminal_only: - conf = e.__check_conf(conf) - if not e.is_term(conf): + conf = node._check_conf(conf) + if not node.is_term(conf): continue if verbose: - l.append((n, e.depth, e._tobytes())) + l.append((path, node.depth, node._tobytes())) else: - l.append((n, e.depth)) + l.append((path, node.depth)) - if e.env is None: - print(n + ' (' + str(e.depth) + ')' + ' ' + str(e.env)) - print('Node value: ', e._tobytes()) + if node.env is None: + print(path + ' (' + str(node.depth) + ')' + ' ' + str(node.env)) + print('Node value: ', node._tobytes()) print("The 'env' attr of this Node is NONE") raise ValueError @@ -6048,7 +6964,7 @@ def get_args(node, conf): args += str(n.get_path_from(self, conf=conf)) else: args += ', ' + str(n.get_path_from(self, conf=conf)) - if args is '': + if args == '': args = 'None' return args @@ -6090,7 +7006,7 @@ def get_all_smaller_depth(nodes_nb, i, depth, conf): self.freeze() l = [] - for n, e in self.iter_paths(conf=conf): + for n, e in self.iter_paths(conf=conf, flush_cache=True, resolve_generator=True): l.append((n, e)) if alpha_order: @@ -6110,7 +7026,7 @@ def get_all_smaller_depth(nodes_nb, i, depth, conf): for n, i in zip(l, range(nodes_nb)): name, node = n - conf_tmp = node.__check_conf(conf) + conf_tmp = node._check_conf(conf) if isinstance(node.c[conf_tmp], NodeInternals_TypedValue): node_type = node.c[conf_tmp].get_value_type().__class__.__name__ else: @@ -6140,6 +7056,8 @@ def is_node_used_more_than_once(name): # depth always >=1 depth -= 1 + junction_sym = ' \__' + junction_sym_len = len(junction_sym) if depth == 0: indent_nonterm = '' indent_spc = '' @@ -6158,8 +7076,8 @@ def is_node_used_more_than_once(name): prefix = ' | ' + prefix else: prefix = ' ' + prefix - indent_nonterm = prefix + ' \__' - indent_term = prefix + ' \__' + indent_nonterm = prefix + junction_sym + indent_term = prefix + junction_sym # l[i+1][1].depth is not reliable in case the node is used at different level if i == nodes_nb-1 or depth != l[i+1][0].count('/'): @@ -6173,6 +7091,12 @@ def is_node_used_more_than_once(name): unindent_generated_node = False depth += 1 + node_desc_lines = chunk_lines(node.description, length=80, prefix=': ') if node.description else None + if node.semantics: + if not node_desc_lines: + node_desc_lines = [] + node_desc_lines.append(': semantics = '+str(node.semantics)) + if node.is_term(conf_tmp): raw = node._tobytes() raw_len = len(raw) @@ -6194,6 +7118,15 @@ def is_node_used_more_than_once(name): log_func=log_func, pretty_print=pretty_print) self._print(graph_deco, rgb=Color.ND_DUPLICATED, style=FontStyle.BOLD, log_func=log_func, pretty_print=pretty_print) + + if node_desc_lines: + indent_desc = indent_nonterm[:-junction_sym_len] + ' | ' + \ + ' '*len('({:d}) '.format(depth)) + for d in node_desc_lines: + print_nonterm_func("{:s}".format(indent_desc), nl=False, log_func=log_func, pretty_print=pretty_print) + self._print(d, rgb=Color.SUBINFO, style=FontStyle.BOLD, + log_func=log_func, pretty_print=pretty_print) + if val is not None: print_nonterm_func("{:s} ".format(indent_spc), nl=False, log_func=log_func, pretty_print=pretty_print) print_contents_func("\_ {:s}".format(val), log_func=log_func, pretty_print=pretty_print) @@ -6231,6 +7164,17 @@ def is_node_used_more_than_once(name): self._print(graph_deco, rgb=Color.ND_DUPLICATED, style=FontStyle.BOLD, log_func=log_func, pretty_print=pretty_print) + if node.description: + if depth == 0: + indent_desc = indent_nonterm +' | ' + else: + indent_desc = indent_nonterm[:-junction_sym_len]+' '*junction_sym_len+' | ' + for d in node_desc_lines: + print_nonterm_func("{:s}".format(indent_desc), nl=False, log_func=log_func, pretty_print=pretty_print) + self._print(d, rgb=Color.SUBINFO, style=FontStyle.BOLD, + log_func=log_func, pretty_print=pretty_print) + + else: for name, node in l: print_name_func("{:s} [{:d}]".format(name, node.depth), log_func=log_func, pretty_print=pretty_print) @@ -6239,31 +7183,32 @@ def is_node_used_more_than_once(name): def __lt__(self, other): return self.depth < other.depth - def __hash__(self): return id(self) def __str__(self): # NEVER return something with self._tobytes() as side - # effects are not welcomed + # effects are not welcome return repr(self) def __getitem__(self, key): - # self._get_value() if isinstance(key, str): - return self.get_node_by_path(key) + node_list = list(self.iter_nodes_by_path(key)) + return node_list if node_list else None elif isinstance(key, NodeInternalsCriteria): - return self.get_reachable_nodes(internals_criteria=key) + node_list = self.get_reachable_nodes(internals_criteria=key) + return node_list if node_list else None elif isinstance(key, NodeSemanticsCriteria): - return self.get_reachable_nodes(semantics_criteria=key) + node_list = self.get_reachable_nodes(semantics_criteria=key) + return node_list if node_list else None else: raise ValueError def __setitem__(self, key, val): nodes = self[key] if not nodes: - raise ValueError + raise ValueError(f'Nodes not found with the key: "{key}"') if isinstance(val, Node): if isinstance(nodes, Node): @@ -6280,10 +7225,10 @@ def __setitem__(self, key, val): elif isinstance(val, int): if isinstance(nodes, Node): # Method defined by INT object (within TypedValue nodes) - nodes.update_raw_value(val) + nodes.update_value(val) else: for n in nodes: - n.update_raw_value(val) + n.update_value(val) else: if isinstance(nodes, Node): status, off, size, name = nodes.absorb(convert_to_internal_repr(val), @@ -6297,6 +7242,29 @@ def __setitem__(self, key, val): if status != AbsorbStatus.FullyAbsorbed: raise ValueError + def update(self, node_update_dict, stop_on_error=True): + for node_ref, new_value in node_update_dict.items(): + if new_value is None: + continue + try: + self[node_ref] = new_value + except ValueError as err: + if stop_on_error: + print(f'\n\n*** Node update raised an error: "{err}"') + raise + + def enable_color(self): + assert self.env is not None + self.env._color_enabled = True + + def disable_color(self): + assert self.env is not None + self.env._color_enabled = False + + @property + def color_enabled(self): + return self.env._color_enabled + def __getattr__(self, name): internals = self.__getattribute__('internals')[self.current_conf] if hasattr(internals, name): @@ -6369,12 +7337,23 @@ def __init__(self): self._dm = None self.id_list = None self._reentrancy_cpt = 0 - # self._knowledge_source = None + self._color_enabled = False + self.csp: CSP = None @property def delayed_jobs_pending(self): return bool(self._sorted_jobs) + def enable_color(self): + self._color_enabled = True + + def disable_color(self): + self._color_enabled = False + + @property + def color_enabled(self): + return self._color_enabled + def __getattr__(self, name): if hasattr(self.env4NT, name): return self.env4NT.__getattribute__(name) @@ -6577,6 +7556,7 @@ def __copy__(self): new_env.nodes_to_corrupt = copy.copy(self.nodes_to_corrupt) new_env.env4NT = copy.copy(self.env4NT) new_env._dm = copy.copy(self._dm) + new_env.csp = copy.copy(self.csp) # DJobs are ignored in the Env copy, because they only matters # in the context of one node graph (Nodes + 1 unique Env) for performing delayed jobs diff --git a/framework/node_builder.py b/framework/node_builder.py index e96f062..0be0fd1 100644 --- a/framework/node_builder.py +++ b/framework/node_builder.py @@ -24,19 +24,27 @@ import inspect import string import sys -import six +import math +from pprint import pprint as pp +import copy from framework.dmhelpers.generic import MH from framework.error_handling import DataModelDefinitionError, CharsetError, \ InitialStateNotFoundError, QuantificationError, StructureError, InconvertibilityError, \ EscapeError, InvalidRangeError, EmptyAlphabetError -from framework.node import Node, NodeInternals_Empty, GenFuncCusto, NonTermCusto, FuncCusto, \ - NodeSemantics, SyncScope, SyncQtyFromObj, SyncSizeObj, NodeCondition, SyncExistenceObj, Env +from framework.node import Node,\ + NodeInternals_Empty, NodeInternals_NonTerm, NodeInternals_GenFunc, NodeInternals_Func, \ + GenFuncCusto, NonTermCusto, FuncCusto, \ + NodeSemantics, SyncScope, SyncQtyFromObj, SyncSizeObj, NodeCondition, SyncExistenceObj, Env, \ + NodeInternalsCriteria, NodeInternals +from framework.constraint_helpers import CSP import framework.value_types as fvt class NodeBuilder(object): + RootNS = 1 + HIGH_PRIO = 1 MEDIUM_PRIO = 2 LOW_PRIO = 3 @@ -45,15 +53,17 @@ class NodeBuilder(object): valid_keys = [ # generic description keys 'name', 'contents', 'qty', 'clone', 'type', 'alt', 'conf', - 'custo_set', 'custo_clear', 'evolution_func', + 'custo_set', 'custo_clear', 'evolution_func', 'description', + 'default_qty', 'namespace', 'from_namespace', 'highlight', + 'constraints', 'constraints_highlight', # NonTerminal description keys 'weight', 'shape_type', 'section_type', 'duplicate_mode', 'weights', - 'separator', 'prefix', 'suffix', 'unique', + 'separator', 'prefix', 'suffix', 'unique', 'always', 'encoder', # Generator/Function description keys 'node_args', 'other_args', 'provide_helpers', 'trigger_last', # Typed-node description keys - 'specific_fuzzy_vals', + 'specific_fuzzy_vals', 'default', # Import description keys 'import_from', 'data_id', # node properties description keys @@ -70,7 +80,8 @@ class NodeBuilder(object): 'debug' ] - def __init__(self, dm=None, delayed_jobs=True, add_env=True): + def __init__(self, dm=None, delayed_jobs=True, add_env=True, + default_gen_custo=None, default_nonterm_custo=None): """ Help the process of data description. This class is able to construct a :class:`framework.data_model.Node` object from a JSON-like description. @@ -89,7 +100,14 @@ def __init__(self, dm=None, delayed_jobs=True, add_env=True): SHALL have only one ``Env()`` shared between all the nodes and an Env() shall not be shared between independent graph (otherwise it could lead to unexpected results). + default_gen_custo: override default Generator node customization + default_nonterm_custo: override default NonTerminal node customization """ + self.default_gen_custo = default_gen_custo + self.default_nonterm_custo = default_nonterm_custo + self.default_gen_custo_orig = NodeInternals_GenFunc.default_custo + self.default_nonterm_custo_orig = NodeInternals_NonTerm.default_custo + self.dm = dm self.delayed_jobs = delayed_jobs self._add_env_to_the_node = add_env @@ -103,39 +121,58 @@ def _verify_keys_conformity(self, desc): def create_graph_from_desc(self, desc): self.sorted_todo = {} self.node_dico = {} - self.empty_node = Node('EMPTY') + # self.empty_node = Node('EMPTY') + + if self.default_gen_custo: + NodeInternals_GenFunc.default_custo = self.default_gen_custo + if self.default_nonterm_custo: + NodeInternals_NonTerm.default_custo = self.default_nonterm_custo n = self._create_graph_from_desc(desc, None) if self._add_env_to_the_node: - self._register_todo(n, self._set_env, prio=self.VERYLOW_PRIO) + self._register_todo(n, self._set_env, prio=self.VERYLOW_PRIO, last_position=False) todo = self._create_todo_list() + loop = 0 while todo: + if loop > 50: + print('\n*** Recursion Error ***\n') + pp(self.node_dico) + raise ValueError for node, func, args, unpack_args in todo: if isinstance(args, tuple) and unpack_args: func(node, *args) else: func(node, args) + # func could register new tasks to do todo = self._create_todo_list() + loop += 1 + + if self.default_gen_custo: + NodeInternals_GenFunc.default_custo = self.default_gen_custo_orig + if self.default_nonterm_custo: + NodeInternals_NonTerm.default_custo = self.default_nonterm_custo_orig return n - def _handle_name(self, name_desc): + def _handle_name(self, name_desc, namespace=None): if isinstance(name_desc, (tuple, list)): assert(len(name_desc) == 2) name = name_desc[0] - ident = name_desc[1] + ns = name_desc[1] elif isinstance(name_desc, str): name = name_desc - ident = 1 + ns = NodeBuilder.RootNS if namespace is None else namespace else: raise ValueError("Name is not recognized: '%s'!" % name_desc) - return name, ident + assert isinstance(name, str) + + return name, ns - def _create_graph_from_desc(self, desc, parent_node): + def _create_graph_from_desc(self, desc, parent_node, namespace=None): def _get_type(top_desc, contents): pre_ntype = top_desc.get('type', None) @@ -145,7 +182,7 @@ def _get_type(top_desc, contents): ntype = MH.RawNode elif hasattr(contents, '__call__') and pre_ntype in [None, MH.Generator]: ntype = MH.Generator - elif isinstance(contents, six.string_types) and pre_ntype in [None, MH.Regex]: + elif isinstance(contents, str) and pre_ntype in [None, MH.Regex]: ntype = MH.Regex else: ntype = MH.Leaf @@ -161,13 +198,13 @@ def _get_type(top_desc, contents): MH.RawNode: self._update_provided_node} if contents is None: - nd = self.__handle_clone(desc, parent_node) + nd = self.__handle_clone(desc, parent_node, namespace=namespace) else: # Non-terminal are recognized via its contents (avoiding # the user to always provide a 'type' field) ntype = _get_type(desc, contents) - nd = dispatcher.get(ntype)(desc) - self.__post_handling(desc, nd) + nd = dispatcher.get(ntype)(desc, namespace=namespace) + self.__post_handling(desc, nd, namespace=namespace) alt_confs = desc.get('alt', None) if alt_confs is not None: @@ -179,15 +216,15 @@ def _get_type(top_desc, contents): " into an alternate configuration is not supported") ntype = _get_type(alt, cts) # dispatcher.get(ntype)(alt, None, node=nd) - dispatcher.get(ntype)(alt, node=nd) + dispatcher.get(ntype)(alt, node=nd, namespace=namespace) return nd - def __handle_clone(self, desc, parent_node): + def __handle_clone(self, desc, parent_node, namespace=None): if isinstance(desc.get('contents'), Node): name, ident = self._handle_name(desc['contents'].name) else: - name, ident = self._handle_name(desc['name']) + name, ident = self._handle_name(desc['name'], namespace=namespace) exp = desc.get('import_from', None) if exp is not None: @@ -201,9 +238,11 @@ def __handle_clone(self, desc, parent_node): nd = Node(name) clone_ref = desc.get('clone', None) + from_ns = desc.get('from_namespace', None) + current_ns = namespace if from_ns is None else from_ns if clone_ref is not None: - ref = self._handle_name(clone_ref) - self._register_todo(nd, self._clone_from_dict, args=(ref, desc), + ref = self._handle_name(clone_ref, namespace=current_ns) + self._register_todo(nd, self._clone_from_dict, args=(ref, desc, current_ns), prio=self.LOW_PRIO) self.node_dico[(name, ident)] = nd else: @@ -217,7 +256,7 @@ def __handle_clone(self, desc, parent_node): return nd - def __pre_handling(self, desc, node): + def __pre_handling(self, desc, node, namespace=None): if node is not None: if isinstance(node.cc, NodeInternals_Empty): raise ValueError("Error: alternative configuration"\ @@ -230,30 +269,30 @@ def __pre_handling(self, desc, node): conf = None else: conf = None - ref = self._handle_name(desc['name']) + ref = self._handle_name(desc['name'], namespace=namespace) if ref in self.node_dico: raise ValueError("name {!r} is already used!".format(ref)) n = Node(ref[0]) return n, conf - def __post_handling(self, desc, node): + def __post_handling(self, desc, node, namespace=None): if not isinstance(node.cc, NodeInternals_Empty): if isinstance(desc.get('contents'), Node): ref = self._handle_name(desc['contents'].name) else: - ref = self._handle_name(desc['name']) + ref = self._handle_name(desc['name'], namespace=namespace) self.node_dico[ref] = node - def _update_provided_node(self, desc, node=None): - n, conf = self.__pre_handling(desc, node) + def _update_provided_node(self, desc, node=None, namespace=None): + n, conf = self.__pre_handling(desc, node, namespace=namespace) self._handle_custo(n, desc, conf) - self._handle_common_attr(n, desc, conf) + self._handle_common_attr(n, desc, conf, current_ns=namespace) return n - def _create_generator_node(self, desc, node=None): + def _create_generator_node(self, desc, node=None, namespace=None): - n, conf = self.__pre_handling(desc, node) + n, conf = self.__pre_handling(desc, node, namespace=namespace) contents = desc.get('contents') @@ -264,6 +303,7 @@ def _create_generator_node(self, desc, node=None): else: provide_helpers = desc.get('provide_helpers', False) node_args = desc.get('node_args', None) + from_ns = desc.get('from_namespace', None) n.set_generator_func(contents, func_arg=other_args, provide_helpers=provide_helpers, conf=conf) @@ -282,24 +322,26 @@ def _create_generator_node(self, desc, node=None): if node_args is not None: # node_args interpretation is postponed after all nodes has been created if isinstance(node_args, dict): - self._register_todo(n, self._complete_generator_from_desc, args=(node_args, conf), unpack_args=True, + self._register_todo(n, self._complete_generator_from_desc, + args=(node_args, conf), unpack_args=True, prio=self.HIGH_PRIO) else: - self._register_todo(n, self._complete_generator, args=(node_args, conf), unpack_args=True, + self._register_todo(n, self._complete_generator, + args=(node_args, conf, from_ns, namespace), unpack_args=True, prio=self.HIGH_PRIO) else: raise ValueError("*** ERROR: {:s} is an invalid contents!".format(repr(contents))) self._handle_custo(n, desc, conf) - self._handle_common_attr(n, desc, conf) + self._handle_common_attr(n, desc, conf, current_ns=namespace) return n - def _create_non_terminal_node_from_regex(self, desc, node=None): + def _create_non_terminal_node_from_regex(self, desc, node=None, namespace=None): - n, conf = self.__pre_handling(desc, node) + n, conf = self.__pre_handling(desc, node, namespace=namespace) name = desc.get('name') if desc.get('name') is not None else node.name if isinstance(name, tuple): @@ -325,22 +367,26 @@ def _create_non_terminal_node_from_regex(self, desc, node=None): prefix = sep_desc.get('prefix', True) suffix = sep_desc.get('suffix', True) unique = sep_desc.get('unique', False) - n.set_separator_node(sep_node, prefix=prefix, suffix=suffix, unique=unique) + always = sep_desc.get('always', False) + n.set_separator_node(sep_node, prefix=prefix, suffix=suffix, unique=unique, always=always) - self._handle_common_attr(n, desc, conf) + self._handle_common_attr(n, desc, conf, current_ns=namespace) return n - def _create_non_terminal_node(self, desc, node=None): + def _create_non_terminal_node(self, desc, node=None, namespace=None): - n, conf = self.__pre_handling(desc, node) + n, conf = self.__pre_handling(desc, node, namespace=namespace) shapes = [] cts = desc.get('contents') if not cts: raise ValueError + ns = desc.get('namespace') + ns = namespace if ns is None else ns + if isinstance(cts[0], (list,tuple)): # thus contains at least something that is not a # node_desc, that is directly a node. Thus, only one @@ -359,7 +405,7 @@ def _create_non_terminal_node(self, desc, node=None): shtype = s.get('shape_type', MH.Ordered) dupmode = s.get('duplicate_mode', MH.Copy) shape = self._create_nodes_from_shape(subnodes, n, shape_type=shtype, - dup_mode=dupmode) + dup_mode=dupmode, namespace=ns) shapes.append(weight) shapes.append(shape) else: @@ -367,7 +413,7 @@ def _create_non_terminal_node(self, desc, node=None): shtype = desc.get('shape_type', MH.Ordered) dupmode = desc.get('duplicate_mode', MH.Copy) shape = self._create_nodes_from_shape(cts, n, shape_type=shtype, - dup_mode=dupmode) + dup_mode=dupmode, namespace=ns) shapes.append(1) shapes.append(shape) @@ -379,22 +425,29 @@ def _create_non_terminal_node(self, desc, node=None): if sep_desc is not None: sep_node_desc = sep_desc.get('contents', None) assert(sep_node_desc is not None) - sep_node = self._create_graph_from_desc(sep_node_desc, n) + sep_node = self._create_graph_from_desc(sep_node_desc, n, namespace=ns) prefix = sep_desc.get('prefix', True) suffix = sep_desc.get('suffix', True) unique = sep_desc.get('unique', False) - n.conf(conf).set_separator_node(sep_node, prefix=prefix, suffix=suffix, unique=unique) + always = sep_desc.get('always', False) + n.conf(conf).set_separator_node(sep_node, prefix=prefix, suffix=suffix, unique=unique, always=always) - self._handle_common_attr(n, desc, conf) + self._handle_common_attr(n, desc, conf, current_ns=namespace) + + constraints = desc.get('constraints', None) + c_hlight = desc.get('constraints_highlight', None) + if constraints is not None: + self._register_todo(n, self._setup_constraints, args=(constraints, ns, c_hlight), prio=self.VERYLOW_PRIO) return n - def _create_nodes_from_shape(self, shapes, parent_node, shape_type=MH.Ordered, dup_mode=MH.Copy): + def _create_nodes_from_shape(self, shapes, parent_node, shape_type=MH.Ordered, dup_mode=MH.Copy, + namespace=None): def _handle_section(nodes_desc, sh): for n in nodes_desc: - if isinstance(n, (list,tuple)) and (len(n) == 2 or len(n) == 3): + if isinstance(n, (list,tuple)) and (2 <= len(n) <= 4): sh.append(list(n)) elif isinstance(n, dict): qty = n.get('qty', 1) @@ -406,8 +459,9 @@ def _handle_section(nodes_desc, sh): maxi = qty else: raise ValueError - l = [mini, maxi] - node = self._create_graph_from_desc(n, parent_node) + default_qty = n.get('default_qty', None) + l = [mini, maxi] if default_qty is None else [mini, maxi, default_qty] + node = self._create_graph_from_desc(n, parent_node, namespace=namespace) l.insert(0, node) sh.append(l) else: @@ -450,9 +504,9 @@ def _handle_section(nodes_desc, sh): return sh - def _create_leaf_node(self, desc, node=None): + def _create_leaf_node(self, desc, node=None, namespace=None): - n, conf = self.__pre_handling(desc, node) + n, conf = self.__pre_handling(desc, node, namespace=namespace) contents = desc.get('contents') @@ -464,18 +518,20 @@ def _create_leaf_node(self, desc, node=None): other_args = desc.get('other_args', None) provide_helpers = desc.get('provide_helpers', False) node_args = desc.get('node_args', None) + from_ns = desc.get('from_namespace', None) n.set_func(contents, func_arg=other_args, provide_helpers=provide_helpers, conf=conf) # node_args interpretation is postponed after all nodes has been created - self._register_todo(n, self._complete_func, args=(node_args, conf), unpack_args=True, + self._register_todo(n, self._complete_func, + args=(node_args, conf, from_ns, namespace), unpack_args=True, prio=self.HIGH_PRIO) else: raise ValueError("ERROR: {:s} is an invalid contents!".format(repr(contents))) self._handle_custo(n, desc, conf) - self._handle_common_attr(n, desc, conf) + self._handle_common_attr(n, desc, conf, current_ns=namespace) return n @@ -484,8 +540,9 @@ def _handle_custo(self, node, desc, conf): custo_clear = desc.get('custo_clear', None) transform_func = desc.get('evolution_func', None) + custo = None + if node.is_genfunc(conf=conf): - Custo = GenFuncCusto trig_last = desc.get('trigger_last', None) if trig_last is not None: if trig_last: @@ -501,33 +558,61 @@ def _handle_custo(self, node, desc, conf): custo_clear = [custo_clear] custo_clear.append(MH.Custo.Gen.TriggerLast) + if custo_set or custo_clear or transform_func: + custo = GenFuncCusto() if self.default_gen_custo is None else self.default_gen_custo + elif node.is_nonterm(conf=conf): - Custo = NonTermCusto + if custo_set or custo_clear or transform_func: + custo = NonTermCusto() if self.default_nonterm_custo is None else self.default_nonterm_custo elif node.is_func(conf=conf): - Custo = FuncCusto + if custo_set or custo_clear or transform_func: + custo = FuncCusto() else: - if custo_set or custo_clear: + if custo_set or custo_clear or transform_func: raise DataModelDefinitionError('Customization is not compatible with this ' 'node kind! [Guilty Node: {:s}]'.format(node.name)) else: return if custo_set or custo_clear or transform_func: - custo = Custo(items_to_set=custo_set, items_to_clear=custo_clear, - transform_func=transform_func) + if custo_set: + custo.set_items(custo_set) + if custo_clear: + custo.clear_items(custo_clear) + if transform_func: + custo.transform_func = transform_func + internals = node.conf(conf) internals.customize(custo) - def _handle_common_attr(self, node, desc, conf): + def _handle_common_attr(self, node, desc, conf, current_ns=None): + from_ns = desc.get('from_namespace', None) + ns = current_ns if from_ns is None else from_ns + + hl = desc.get('highlight') + if hl is not None: + if hl: + node.set_attr(MH.Attr.Highlight, conf=conf) + else: + node.clear_attr(MH.Attr.Highlight, conf=conf) + param = desc.get('description', None) + if param is not None: + node.description = param vals = desc.get('specific_fuzzy_vals', None) if vals is not None: if not node.is_typed_value(conf=conf): raise DataModelDefinitionError("'specific_fuzzy_vals' is only usable with Typed-nodes." " [guilty node: '{:s}']".format(node.name)) node.conf(conf).set_specific_fuzzy_values(vals) + def_val = desc.get('default', None) + if def_val is not None: + if not node.is_typed_value(conf=conf): + raise DataModelDefinitionError("'default' is only usable with Typed-nodes." + " [guilty node: '{:s}']".format(node.name)) + node.set_default_value(def_val, conf=conf) param = desc.get('mutable', None) if param is not None: if param: @@ -584,12 +669,12 @@ def _handle_common_attr(self, node, desc, conf): ref = desc.get('sync_qty_with', None) if ref is not None: self._register_todo(node, self._set_sync_node, - args=(ref, SyncScope.Qty, conf, None), + args=(ref, SyncScope.Qty, conf, None, ns), unpack_args=True) qty_from = desc.get('qty_from', None) if qty_from is not None: self._register_todo(node, self._set_sync_node, - args=(qty_from, SyncScope.QtyFrom, conf, None), + args=(qty_from, SyncScope.QtyFrom, conf, None, ns), unpack_args=True) sync_size_with = desc.get('sync_size_with', None) @@ -597,31 +682,31 @@ def _handle_common_attr(self, node, desc, conf): assert sync_size_with is None or sync_enc_size_with is None if sync_size_with is not None: self._register_todo(node, self._set_sync_node, - args=(sync_size_with, SyncScope.Size, conf, False), + args=(sync_size_with, SyncScope.Size, conf, False, ns), unpack_args=True) if sync_enc_size_with is not None: self._register_todo(node, self._set_sync_node, - args=(sync_enc_size_with, SyncScope.Size, conf, True), + args=(sync_enc_size_with, SyncScope.Size, conf, True, ns), unpack_args=True) condition = desc.get('exists_if', None) if condition is not None: self._register_todo(node, self._set_sync_node, - args=(condition, SyncScope.Existence, conf, None), + args=(condition, SyncScope.Existence, conf, None, ns), unpack_args=True) condition = desc.get('exists_if/and', None) if condition is not None: self._register_todo(node, self._set_sync_node, - args=(condition, SyncScope.Existence, conf, 'and'), + args=(condition, SyncScope.Existence, conf, 'and', ns), unpack_args=True) condition = desc.get('exists_if/or', None) if condition is not None: self._register_todo(node, self._set_sync_node, - args=(condition, SyncScope.Existence, conf, 'or'), + args=(condition, SyncScope.Existence, conf, 'or', ns), unpack_args=True) condition = desc.get('exists_if_not', None) if condition is not None: self._register_todo(node, self._set_sync_node, - args=(condition, SyncScope.Inexistence, conf, None), + args=(condition, SyncScope.Inexistence, conf, None, ns), unpack_args=True) fw = desc.get('fuzz_weight', None) if fw is not None: @@ -633,10 +718,14 @@ def _handle_common_attr(self, node, desc, conf): if encoder is not None: node.set_encoder(encoder) - def _register_todo(self, node, func, args=None, unpack_args=True, prio=MEDIUM_PRIO): + def _register_todo(self, node, func, args=None, unpack_args=True, + prio=MEDIUM_PRIO, last_position=False): if self.sorted_todo.get(prio, None) is None: self.sorted_todo[prio] = [] - self.sorted_todo[prio].insert(0, (node, func, args, unpack_args)) + if last_position: + self.sorted_todo[prio].append((node, func, args, unpack_args)) + else: + self.sorted_todo[prio].insert(0, (node, func, args, unpack_args)) def _create_todo_list(self): todo = [] @@ -646,23 +735,33 @@ def _create_todo_list(self): todo += sub_tdl return todo + ic = NodeInternalsCriteria(node_kinds=[NodeInternals_Empty]) # Should be called at the last time to avoid side effects (e.g., # when creating generator/function nodes, the node arguments are # provided at a later time. If set_contents()---which copy nodes---is called # in-between, node arguments risk to not be copied) - def _clone_from_dict(self, node, ref, desc): + def _clone_from_dict(self, node, ref, desc, current_ns): if ref not in self.node_dico: raise ValueError("arguments refer to an inexistent node ({:s}, {!s})!".format(ref[0], ref[1])) - node.set_contents(self.node_dico[ref], preserve_node=False) - self._handle_custo(node, desc, conf=None) - self._handle_common_attr(node, desc, conf=None) + + ref_nd = self.node_dico[ref] + empty_nodes = ref_nd.get_reachable_nodes(internals_criteria=self.ic) + if empty_nodes: + # We can't use it as a reference. This node needs to be resolved first, meaning its + # empty nodes have to be replaced + self._register_todo(node, self._clone_from_dict, args=(ref, desc, current_ns), + prio=self.LOW_PRIO) + else: + node.set_contents(self.node_dico[ref], preserve_node=False) + self._handle_custo(node, desc, conf=None) + self._handle_common_attr(node, desc, conf=None, current_ns=current_ns) def _get_from_dict(self, node, ref, parent_node): if ref not in self.node_dico: raise ValueError("arguments refer to an inexistent node ({:s}, {!s})!".format(ref[0], ref[1])) parent_node.replace_subnode(node, self.node_dico[ref]) - def _set_sync_node(self, node, comp, scope, conf, private): + def _set_sync_node(self, node, comp, scope, conf, private, from_ns): sync_obj = None if scope == SyncScope.QtyFrom: @@ -670,7 +769,7 @@ def _set_sync_node(self, node, comp, scope, conf, private): node_ref, base_qty = comp else: node_ref, base_qty = comp, 0 - sync_with = self.__get_node_from_db(node_ref) + sync_with = self.__get_node_from_db(node_ref, namespace=from_ns) sync_obj = SyncQtyFromObj(sync_with, base_qty=base_qty) elif scope == SyncScope.Size: @@ -678,56 +777,62 @@ def _set_sync_node(self, node, comp, scope, conf, private): node_ref, base_size = comp else: node_ref, base_size = comp, 0 - sync_with = self.__get_node_from_db(node_ref) + sync_with = self.__get_node_from_db(node_ref, namespace=from_ns) sync_obj = SyncSizeObj(sync_with, base_size=base_size, apply_to_enc_size=private) else: if isinstance(comp, (tuple,list)): if issubclass(comp[0].__class__, NodeCondition): param = comp[0] - sync_with = self.__get_node_from_db(comp[1]) + sync_with = self.__get_node_from_db(comp[1], namespace=from_ns) elif issubclass(comp[0].__class__, (tuple,list)): assert private in ['and', 'or'] sync_list = [] for subcomp in comp: assert isinstance(subcomp, (tuple,list)) and len(subcomp) == 2 param = subcomp[0] - sync_with = self.__get_node_from_db(subcomp[1]) + sync_with = self.__get_node_from_db(subcomp[1], namespace=from_ns) sync_list.append((sync_with, param)) and_junction = private == 'and' sync_obj = SyncExistenceObj(sync_list, and_junction=and_junction) else: # in this case this is a node reference in the form ('node name', ID) param = None - sync_with = self.__get_node_from_db(comp) - else: + sync_with = self.__get_node_from_db(comp, namespace=from_ns) + elif isinstance(comp, str): + # comp is a ref of a node param = None - sync_with = self.__get_node_from_db(comp) + sync_with = self.__get_node_from_db(comp, namespace=from_ns) + else: + # comp is considered to be a boolean condition + param = comp + sync_with = None if sync_obj is not None: node.make_synchronized_with(scope=scope, sync_obj=sync_obj, conf=conf) else: node.make_synchronized_with(scope=scope, node=sync_with, param=param, conf=conf) - def _complete_func(self, node, args, conf): + def _complete_func(self, node, args, conf, from_ns, current_ns): + ns = current_ns if from_ns is None else from_ns if isinstance(args, str): - func_args = self.__get_node_from_db(args) + func_args = self.__get_node_from_db(args, namespace=ns) else: assert(isinstance(args, (tuple, list))) func_args = [] for name_desc in args: - func_args.append(self.__get_node_from_db(name_desc)) + func_args.append(self.__get_node_from_db(name_desc, namespace=ns)) internals = node.cc if conf is None else node.c[conf] internals.set_func_arg(node=func_args) - def _complete_generator(self, node, args, conf): - if isinstance(args, str) or \ - (isinstance(args, tuple) and isinstance(args[1], int)): - func_args = self.__get_node_from_db(args) + def _complete_generator(self, node, args, conf, from_ns, current_ns): + ns = current_ns if from_ns is None else from_ns + if isinstance(args, str) or (isinstance(args, tuple) and len(args)==2): + func_args = self.__get_node_from_db(args, namespace=ns) else: - assert(isinstance(args, (tuple, list))) + assert isinstance(args, list) func_args = [] for name_desc in args: - func_args.append(self.__get_node_from_db(name_desc)) + func_args.append(self.__get_node_from_db(name_desc, namespace=ns)) internals = node.cc if conf is None else node.c[conf] internals.set_generator_func_arg(generator_node_arg=func_args) @@ -741,8 +846,8 @@ def _set_env(self, node, args): env.delayed_jobs_enabled = self.delayed_jobs node.set_env(env) - def __get_node_from_db(self, name_desc): - ref = self._handle_name(name_desc) + def __get_node_from_db(self, name_desc, namespace=None): + ref = self._handle_name(name_desc, namespace=namespace) if ref not in self.node_dico: raise ValueError("arguments refer to an inexistent node ({:s}, {!s})!".format(ref[0], ref[1])) @@ -752,6 +857,20 @@ def __get_node_from_db(self, name_desc): return node + def _setup_constraints(self, node, constraints, root_namespace, constraint_highlight): + csp = CSP(constraints=constraints, highlight_variables=constraint_highlight) + + for v in csp.iter_vars(): + nd = self.__get_node_from_db(csp.from_var_to_varns(v), namespace=root_namespace) + csp.map_var_to_node(v, nd) + if nd.cc.value_type.values is not None: + domain = copy.copy(nd.cc.value_type.values) + else: + domain = range(nd.cc.value_type.mini_gen, nd.cc.value_type.maxi_gen + 1) + csp.set_var_domain(v, domain) + + csp.save_current_var_domains() + node.set_csp(csp) class State(object): """ @@ -1172,7 +1291,7 @@ def advance(self, ctx): elif ctx.input in ('(', '['): raise InconvertibilityError() elif ctx.input == '-': - raise InvalidRangeError() + return self.machine.BeforeRange elif ctx.input == ']': raise EmptyAlphabetError() elif ctx.input == '\\': @@ -1208,8 +1327,11 @@ def _run(self, ctx): pass def advance(self, ctx): - if ctx.input in ('?', '*', '+', '{', '}', '(', ')', '[', ']', '|', '-', None): + if ctx.input in ('?', '*', '+', '{', '}', '(', ')', '[', '|', '-', None): raise InvalidRangeError() + elif ctx.input == ']': + ctx.append_to_alphabet('-') + return self.machine.Final elif ctx.input == '\\': return self.machine.EscapeAfterRange else: @@ -1402,7 +1524,7 @@ def reset(self): def parse(self, inputs, name, charset=MH.Charset.ASCII_EXT): self._name = name self.charset = charset - self.int_to_string = chr if sys.version_info[0] == 2 and self.charset != MH.Charset.UNICODE else six.unichr + self.int_to_string = chr if self.charset == MH.Charset.ASCII: max = 0x7F @@ -1445,7 +1567,16 @@ def _create_terminal_node(self, name, type, values=None, alphabet=None, qty=None if type == fvt.String: node = Node(name=name, vt=fvt.String(values=values, codec=self.codec)) else: - node = Node(name=name, vt=fvt.INT_str(values=values)) + if values == [i for i in range(10)] and (qty[1] != 0 or qty[1] is None): + max_digit = qty[1] + min_digit = qty[0] + node = Node(name=name, vt=fvt.INT_str(min=(min_digit-1)*10, + max=10**max_digit-1 if max_digit is not None else None)) + + return [node, 1, 1] + + else: + node = Node(name=name, vt=fvt.INT_str(values=values)) return [node, qty[0], -1 if qty[1] is None else qty[1]] diff --git a/framework/plumbing.py b/framework/plumbing.py index d4f4cc3..6007bb5 100644 --- a/framework/plumbing.py +++ b/framework/plumbing.py @@ -38,8 +38,10 @@ import time import signal -from functools import wraps +from functools import wraps, partial +from typing import Sequence +from framework.data import Data, DataProcess from framework.database import FeedbackGate from framework.knowledge.feedback_collector import FeedbackSource from framework.error_handling import * @@ -51,8 +53,6 @@ from framework.scenario import * from framework.tactics_helpers import * from framework.target_helpers import * -from framework.targets.local import LocalTarget -from framework.targets.printer import PrinterTarget from framework.cosmetics import aligned_stdout from framework.config import config, config_dot_proxy from libs.utils import * @@ -65,6 +65,8 @@ from framework.global_resources import * from libs.utils import * +import io + sys.path.insert(0, fuddly_data_folder) sys.path.insert(0, external_libs_folder) @@ -82,12 +84,43 @@ def is_python_file(fname): return r_pyfile.match(fname) +class Printer(io.StringIO): + + def __init__(self, fmk): + io.StringIO.__init__(self) + self.fmk = fmk + + def start(self): + pass + + def stop(self): + pass + + def write(self, data): + if False and self.fmk.lg: + self.fmk.lg.write(data) + else: + sys.__stdout__.write(data) + + def flush(self): + if self.fmk.lg: + self.fmk.lg.flush() + else: + sys.__stdout__.flush() + + def wait_for_sync(self): + if self.fmk.lg: + self.fmk.lg.wait_for_sync() + + def print(self, msg): + self.write(msg+'\n') + self.flush() class ExportableFMKOps(object): def __init__(self, fmk): - self.set_fuzz_delay = fmk.set_fuzz_delay - self.set_fuzz_burst = fmk.set_fuzz_burst + self.set_sending_delay = fmk.set_sending_delay + self.set_sending_burst_counter = fmk.set_sending_burst_counter self.set_health_check_timeout = fmk.set_health_check_timeout self.cleanup_all_dmakers = fmk.cleanup_all_dmakers self.cleanup_dmaker = fmk.cleanup_dmaker @@ -96,8 +129,9 @@ def __init__(self, fmk): self.load_data_model = fmk.load_data_model self.load_multiple_data_model = fmk.load_multiple_data_model self.reload_all = fmk.reload_all - self.get_data = fmk.get_data + self.process_data = fmk.process_data self.unregister_task = fmk._unregister_task + self.handle_data_desc = fmk.handle_data_desc class FmkFeedback(object): @@ -148,7 +182,7 @@ class EnforceOrder(object): current_state = None - def __init__(self, accepted_states=None, final_state=None, + def __init__(self, accepted_states=None, final_state=None, reset_sm=False, initial_func=False, always_callable=False, transition=None): accepted_states = [] if accepted_states is None else accepted_states if initial_func: @@ -158,6 +192,7 @@ def __init__(self, accepted_states=None, final_state=None, self.final_state = final_state self.always_callable = always_callable self.transition = transition + self.reset_sm = reset_sm def __call__(self, func): @wraps(func) @@ -168,6 +203,8 @@ def wrapped_func(*args, **kargs): rgb=Color.ERROR)) return False ok = func(*args, **kargs) + if (ok or ok is None) and self.reset_sm: + EnforceOrder.current_state = None if (ok or ok is None) and self.final_state is not None: EnforceOrder.current_state = self.final_state @@ -191,8 +228,16 @@ def __init__(self, name, func, arg, period=None, self._stop = threading.Event() self._error_func = error_func self._cleanup_func=cleanup_func + if isinstance(func, Task): + func.stop_event = self._stop def run(self): + if isinstance(self._func, Task): + self._func._setup() + + if isinstance(self._func, Task): + time.sleep(self._func.init_delay) + while not self._stop.is_set(): try: # print("\n*** Function '{!s}' executed by Task '{!s}' ***".format(self._func, self._name)) @@ -207,26 +252,43 @@ def run(self): self._error_func("Task '{!s}' has crashed!".format(self._name)) break if self._period is not None: - self._stop.wait(max(self._period,0.01)) + self._stop.wait(max(self._period,0.0001)) else: self._cleanup_func() break + if isinstance(self._func, Task): + self._func._cleanup() + def stop(self): self._stop.set() class FmkPlumbing(object): - ''' + """ Defines the methods to operate every sub-systems of fuddly - ''' + """ - def __init__(self, exit_on_error=False, debug_mode=False, quiet=False): + def __init__(self, exit_on_error=False, debug_mode=False, quiet=False, + external_term=False, fmkdb_path=None): self._debug_mode = debug_mode self._exit_on_error = exit_on_error self._quiet = quiet + self._fmkdb_path = fmkdb_path + self.external_display = ExternalDisplay() + if external_term: + self.external_display.start_term(title='Fuddly log', keepterm=True) + + self.printer = Printer(self) + self.print = self.printer.print + + self._reset_main_objects() + def __str__(self): + return 'Fuddly FmK' + + def _reset_main_objects(self): self.prj_list = [] self.dm_list = [] @@ -266,6 +328,11 @@ def __init__(self, exit_on_error=False, debug_mode=False, quiet=False): self._hc_timeout = {} # health-check tiemout, further initialized as a dict (tg -> hc_timeout) self._hc_timeout_max = None + # self._hc_timeout_increment = 0.01 + + self._fbk_timeout_max = 0 + self._fbk_timeout_default = 0.005 + self._last_sending_date = None self._current_sent_date = None @@ -279,41 +346,44 @@ def __init__(self, exit_on_error=False, debug_mode=False, quiet=False): self._dm_to_be_reloaded = False self._generic_tactics = framework.generic_data_makers.tactics - self._generic_tactics.set_exportable_fmk_ops(self._exportable_fmk_ops) + self._generic_tactics.set_additional_info(self._exportable_fmk_ops) self._tactics = None - - def __str__(self): - return 'Fuddly FmK' + self.last_data_id = None + self.next_data_id = None @EnforceOrder(initial_func=True) def start(self): - self.import_text_reg = re.compile('(.*?)(#####)', re.S) - self.check_clone_re = re.compile('(.*)#(\w{1,20})') + self.printer.start() + + self.check_clone_re = re.compile('(.*)#(\w{1,30})') self.config = config(self, path=[config_folder]) - def save_config(): - filename=os.path.join( - config_folder, - self.config.config_name + '.ini') - with open(filename, 'w') as cfile: - self.config.write(cfile) - atexit.register(save_config) - self.fmkDB = Database() + external_term = self.config.terminal.external_term + if external_term and not self.external_display.is_enabled: + self.switch_term() + + self.fmkDB = Database(fmkdb_path=self._fmkdb_path) ok = self.fmkDB.start() if not ok: raise InvalidFmkDB("The database {:s} is invalid!".format(self.fmkDB.fmk_db_path)) - self.feedback_gate = FeedbackGate(self.fmkDB) - Project.feedback_gate = self.feedback_gate + + self.last_data_id = None + self.next_data_id = None + + # Fbk Gate to be used for sequencial tasks (scenario steps callbacks, etc.) + self.last_feedback_gate = FeedbackGate(self.fmkDB, only_last_entries=True) + + # Fbk Gate to be used for out-of-fmk-order tasks (e.g., Tasks). This gate is agnostic to + # fmkdb.flush_feedback(). Thus feedback entries last longer there. + self.feedback_gate = FeedbackGate(self.fmkDB, only_last_entries=False) self._fmkDB_insert_dm_and_dmakers('generic', self._generic_tactics) self.group_id = 0 self._recovered_tgs = None # used by self._recover_target() - self.enable_wkspace() - self.import_successfull = True self.get_data_models() if self._exit_on_error and not self.import_successfull: @@ -326,25 +396,38 @@ def save_config(): raise ProjectDefinitionError('Error with some Project imports') if not self._quiet: - print(colorize(FontStyle.BOLD + '='*44 + '[ Fuddly Data Folder Information ]==\n', + self.print(colorize(FontStyle.BOLD + '='*44 + '[ Fuddly Home Information ]==\n', rgb=Color.FMKINFOGROUP)) if not self._quiet and hasattr(gr, 'new_fuddly_data_folder'): - print(colorize(FontStyle.BOLD + \ - ' *** New Fuddly Data Folder Has Been Created ***\n', + self.print(colorize(FontStyle.BOLD + \ + ' *** New Fuddly Home Has Been Created ***\n', rgb=Color.FMKINFO_HLIGHT)) if not self._quiet: - print(colorize(' --> path: {:s}'.format(gr.fuddly_data_folder), - rgb=Color.FMKINFO)) - print(colorize(' --> contains: - fmkDB.db, logs, imported/exported data, ...\n' - ' - user projects and user data models', - rgb=Color.FMKSUBINFO)) + self.print(colorize(' --> data folder: {:s}'.format(gr.fuddly_data_folder), + rgb=Color.FMKINFO)) + self.print(colorize(' --> contains: - fmkDB.db, logs, imported/exported data, ...\n' + ' - user projects and user data models, ...', + rgb=Color.FMKSUBINFO)) + self.print(colorize(' --> config folder: {:s}'.format(gr.config_folder), + rgb=Color.FMKINFO)) + + + def switch_term(self): + if not self.external_display.is_enabled: + self.external_display.start_term(title='Fuddly log', keepterm=False) + else: + self.external_display.stop() - @EnforceOrder(accepted_states=['20_load_prj','25_load_dm','S1','S2']) + @EnforceOrder(accepted_states=['20_load_prj','25_load_dm','S1','S2'], reset_sm=True) def stop(self): self._stop_fmk_plumbing() self.fmkDB.stop() + self._reset_main_objects() + self.printer.stop() + if self.external_display.is_enabled: + self.external_display.stop() @property def prj(self): @@ -373,6 +456,11 @@ def is_not_ok(self): def is_ok(self): return not self.error + def show_and_flush_errors(self): + err_list = self.get_error() + for e in err_list: + self.print(colorize(" (_ [#{err!s:s}]: {msg:s} _)".format(err=e, msg=e.msg), rgb=e.color)) + def flush_errors(self): self.error = False self.fmk_error = [] @@ -381,26 +469,32 @@ def _reset_fmk_internals(self, reset_existing_seed=True): self.cleanup_all_dmakers(reset_existing_seed) # Warning: fuzz delay is not set to 0 by default in order to have a time frame # where SIGINT is accepted from user - self.set_fuzz_delay(self.config.defvalues.fuzz.delay) - self.set_fuzz_burst(self.config.defvalues.fuzz.burst) - for tg in self.targets.values(): - self._recompute_health_check_timeout(tg.feedback_timeout, tg.sending_delay, target=tg) + delay = self.config.misc.fuzz.delay if self.prj.default_sending_delay is None else self.prj.default_sending_delay + burst = self.config.misc.fuzz.burst if self.prj.default_burst_value is None else self.prj.default_burst_value + self.set_sending_delay(delay) + self.set_sending_burst_counter(burst) - def _recompute_health_check_timeout(self, base_timeout, sending_delay, target=None, do_show=True): - if base_timeout is not None: - if base_timeout != 0: - if 0 < base_timeout < 1: - hc_timeout = base_timeout + sending_delay + 0.5 - else: - hc_timeout = base_timeout + sending_delay + 2.0 - self.set_health_check_timeout(hc_timeout, target=target, do_show=do_show) + if self.prj.default_fbk_timeout is not None: + self.set_feedback_timeout(self.prj.default_fbk_timeout) + if self.prj.default_fbk_mode is not None: + self.set_feedback_mode(self.prj.default_fbk_mode) + + for tg in self.targets.values(): + self._recompute_generic_timeouts(tg.feedback_timeout, tg.sending_delay, target=tg) + + def _recompute_generic_timeouts(self, fbk_timeout, sending_delay, target=None, do_show=True): + # if target is None or target in self._currently_used_targets: + self._fbk_timeout_max = max(0 if fbk_timeout is None else fbk_timeout, self._fbk_timeout_max) + if fbk_timeout is not None: + if fbk_timeout != 0: + self.set_health_check_timeout(fbk_timeout + sending_delay, target=target, do_show=do_show) else: - # base_timeout comes from feedback_timeout, if it is equal to 0 + # if fbk_timeout is equal to 0 # this is a special meaning used internally to collect residual feedback. # Thus, we don't change the current health_check timeout. return else: - self.set_health_check_timeout(max(10,sending_delay), target=target, do_show=do_show) + self.set_health_check_timeout(sending_delay, target=target, do_show=do_show) def _handle_user_code_exception(self, msg='', context=None): self.set_error(msg, code=Error.UserCodeError, context=context) @@ -408,10 +502,10 @@ def _handle_user_code_exception(self, msg='', context=None): self.lg.log_error("Exception in user code detected! Outcomes " \ "of this log entry has to be considered with caution.\n" \ " (_ cause: '%s' _)" % msg) - print("Exception in user code:") - print('-'*60) - traceback.print_exc(file=sys.stdout) - print('-'*60) + self.print("Exception in user code:") + self.print('-'*60) + traceback.print_exc(file=self.printer) + self.print('-'*60) def _handle_fmk_exception(self, cause=''): self.set_error(cause, code=Error.UserCodeError) @@ -419,10 +513,10 @@ def _handle_fmk_exception(self, cause=''): self.lg.log_error("Not handled exception detected! Outcomes " \ "of this log entry has to be considered with caution.\n" \ " (_ cause: '%s' _)" % cause) - print("Call trace:") - print('-'*60) - traceback.print_exc(file=sys.stdout) - print('-'*60) + self.print("Call trace:") + self.print('-'*60) + traceback.print_exc(file=self.printer) + self.print('-'*60) def _is_data_valid(self, data): def is_valid(d): @@ -554,6 +648,10 @@ def _reload_all(self, tg_ids=None): self._add_project(prj_params['project'], prj_params['target'], prj_params['logger'], prj_params['prj_rld_args'], reload_prj=True) + prj_params['project'].share_knowlegde_source() + self._show_knowledge(knowledge_src=prj_params['project'].knowledge_source, + do_record=True) + if dm_prefix is None: # it is ok to call reload_dm() here because it is a # composed DM, and it won't call the methods used within @@ -688,11 +786,11 @@ def populate_data_models(path): rexp_strategy = re.compile("(.*)_strategy\.py$") if not self._quiet: - print(colorize(FontStyle.BOLD + "="*63+"[ Data Models ]==", rgb=Color.FMKINFOGROUP)) + self.print(colorize(FontStyle.BOLD + "="*63+"[ Data Models ]==", rgb=Color.FMKINFOGROUP)) for dname, file_list in data_models.items(): if not self._quiet: - print(colorize(">>> Look for Data Models within '%s' directory" % dname, + self.print(colorize(">>> Look for Data Models within '%s' directory" % dname, rgb=Color.FMKINFOSUBGROUP)) prefix = dname.replace(os.sep, '.') + '.' for f in file_list: @@ -722,13 +820,9 @@ def _import_dm(self, prefix, name, reload_dm=False): try: if reload_dm: - if sys.version_info[0] == 2: - eval('reload(' + prefix + name + ')') - eval('reload(' + prefix + name + '_strategy' + ')') - else: - exec('import importlib') - eval('importlib.reload(' + prefix + name + ')') - eval('importlib.reload(' + prefix + name + '_strategy' + ')') + exec('import importlib') + eval('importlib.reload(' + prefix + name + ')') + eval('importlib.reload(' + prefix + name + '_strategy' + ')') else: exec('import ' + prefix + name) exec('import ' + prefix + name + '_strategy') @@ -737,12 +831,12 @@ def _import_dm(self, prefix, name, reload_dm=False): return None if reload_dm: - print(colorize("*** Problem during reload of '%s.py' and/or '%s_strategy.py' ***" % (name, name), rgb=Color.ERROR)) + self.print(colorize("*** Problem during reload of '%s.py' and/or '%s_strategy.py' ***" % (name, name), rgb=Color.ERROR)) else: - print(colorize("*** Problem during import of '%s.py' and/or '%s_strategy.py' ***" % (name, name), rgb=Color.ERROR)) - print('-'*60) - traceback.print_exc(file=sys.stdout) - print('-'*60) + self.print(colorize("*** Problem during import of '%s.py' and/or '%s_strategy.py' ***" % (name, name), rgb=Color.ERROR)) + self.print('-'*60) + traceback.print_exc(file=self.printer) + self.print('-'*60) return None @@ -755,16 +849,17 @@ def _import_dm(self, prefix, name, reload_dm=False): dm_params['dm'] = eval(prefix + name + '.data_model') except: if not self._quiet: - print(colorize("*** ERROR: '%s.py' shall contain a global variable 'data_model' ***" % (name), rgb=Color.ERROR)) + self.print(colorize("*** ERROR: '%s.py' shall contain a global variable 'data_model' ***" % (name), rgb=Color.ERROR)) return None try: dm_params['tactics'] = eval(prefix + name + '_strategy' + '.tactics') except: if not self._quiet: - print(colorize("*** ERROR: '%s_strategy.py' shall contain a global variable 'tactics' ***" % (name), rgb=Color.ERROR)) + self.print(colorize("*** ERROR: '%s_strategy.py' shall contain a global variable 'tactics' ***" % (name), rgb=Color.ERROR)) return None - dm_params['tactics'].set_exportable_fmk_ops(self._exportable_fmk_ops) + dm_params['tactics'].set_additional_info(fmkops=self._exportable_fmk_ops, + related_dm=dm_params['dm']) if dm_params['dm'].name is None: dm_params['dm'].name = name @@ -772,9 +867,9 @@ def _import_dm(self, prefix, name, reload_dm=False): if not self._quiet: if reload_dm: - print(colorize("*** Data Model '%s' updated ***" % dm_params['dm'].name, rgb=Color.DATA_MODEL_LOADED)) + self.print(colorize("*** Data Model '%s' updated ***" % dm_params['dm'].name, rgb=Color.DATA_MODEL_LOADED)) else: - print(colorize("*** Found Data Model: '%s' ***" % dm_params['dm'].name, rgb=Color.FMKSUBINFO)) + self.print(colorize("*** Found Data Model: '%s' ***" % dm_params['dm'].name, rgb=Color.FMKSUBINFO)) return dm_params @@ -838,11 +933,11 @@ def populate_projects(path): rexp_proj = re.compile("(.*)_proj\.py$") if not self._quiet: - print(colorize(FontStyle.BOLD + "="*66+"[ Projects ]==", rgb=Color.FMKINFOGROUP)) + self.print(colorize(FontStyle.BOLD + "="*66+"[ Projects ]==", rgb=Color.FMKINFOGROUP)) for dname, file_list in projects.items(): if not self._quiet: - print(colorize(">>> Look for Projects within '%s' Directory" % dname, + self.print(colorize(">>> Look for Projects within '%s' Directory" % dname, rgb=Color.FMKINFOSUBGROUP)) prefix = dname.replace(os.sep, '.') + '.' for f in file_list: @@ -865,11 +960,8 @@ def _import_project(self, prefix, name, reload_prj=False): try: if reload_prj: - if sys.version_info[0] == 2: - eval('reload(' + prefix + name + '_proj' + ')') - else: - exec('import importlib') - eval('importlib.reload(' + prefix + name + '_proj' + ')') + exec('import importlib') + eval('importlib.reload(' + prefix + name + '_proj' + ')') else: exec('import ' + prefix + name + '_proj') except: @@ -877,12 +969,12 @@ def _import_project(self, prefix, name, reload_prj=False): return None if reload_prj: - print(colorize("*** Problem during reload of '%s_proj.py' ***" % (name), rgb=Color.ERROR)) + self.print(colorize("*** Problem during reload of '%s_proj.py' ***" % (name), rgb=Color.ERROR)) else: - print(colorize("*** Problem during import of '%s_proj.py' ***" % (name), rgb=Color.ERROR)) - print('-'*60) - traceback.print_exc(file=sys.stdout) - print('-'*60) + self.print(colorize("*** Problem during import of '%s_proj.py' ***" % (name), rgb=Color.ERROR)) + self.print('-'*60) + traceback.print_exc(file=self.printer) + self.print('-'*60) return None @@ -895,7 +987,7 @@ def _import_project(self, prefix, name, reload_prj=False): prj_params['project'] = eval(prefix + name + '_proj' + '.project') except: if not self._quiet: - print(colorize("*** ERROR: '%s_proj.py' shall contain a global variable 'project' ***" % (name), rgb=Color.ERROR)) + self.print(colorize("*** ERROR: '%s_proj.py' shall contain a global variable 'project' ***" % (name), rgb=Color.ERROR)) return None else: prj = prj_params['project'] @@ -910,33 +1002,37 @@ def _import_project(self, prefix, name, reload_prj=False): except: logger = Logger(name) logger.fmkDB = self.fmkDB + logger.set_external_display(self.external_display) if logger.name is None: logger.name = name prj_params['logger'] = logger try: targets = eval(prefix + name + '_proj' + '.targets') - targets.insert(0, EmptyTarget()) + targets.insert(0, EmptyTarget(verbose=self.config.targets.empty_tg.verbose)) except: - targets = [EmptyTarget()] + tg = EmptyTarget(verbose=self.config.targets.empty_tg.verbose) + tg.set_project(prj) + targets = [tg] else: new_targets = [] for obj in targets: if isinstance(obj, (tuple, list)): tg = obj[0] obj = obj[1:] - tg.remove_probes() + tg.del_extensions() for p in obj: - tg.add_probe(p) + tg.add_extensions(p) else: assert issubclass(obj.__class__, Target), 'project: {!s}'.format(name) tg = obj - tg.remove_probes() + tg.del_extensions() + tg.set_project(prj) new_targets.append(tg) targets = new_targets for idx, tg_id in enumerate(self._tg_ids): if tg_id >= len(targets): - print(colorize("*** Incorrect Target ID detected: {:d} --> replace with 0 ***".format(tg_id), + self.print(colorize("*** Incorrect Target ID detected: {:d} --> replace with 0 ***".format(tg_id), rgb=Color.WARNING)) self._tg_ids[idx] = 0 @@ -948,9 +1044,9 @@ def _import_project(self, prefix, name, reload_prj=False): if not self._quiet: if reload_prj: - print(colorize("*** Project '%s' updated ***" % prj_params['project'].name, rgb=Color.FMKSUBINFO)) + self.print(colorize("*** Project '%s' updated ***" % prj_params['project'].name, rgb=Color.FMKSUBINFO)) else: - print(colorize("*** Found Project: '%s' ***" % prj_params['project'].name, rgb=Color.FMKSUBINFO)) + self.print(colorize("*** Found Project: '%s' ***" % prj_params['project'].name, rgb=Color.FMKSUBINFO)) return prj_params @@ -1044,11 +1140,25 @@ def _load_data_model(self): self.__dynamic_generator_ids[self.dm].append(dmaker_type) self.fmkDB.insert_dmaker(self.dm.name, dmaker_type, gen_cls_name, True, True) - print(colorize("*** Data Model '%s' loaded ***" % self.dm.name, rgb=Color.DATA_MODEL_LOADED)) + self.print(colorize("*** Data Model '%s' loaded ***" % self.dm.name, rgb=Color.DATA_MODEL_LOADED)) self._dm_to_be_reloaded = False return True + def _create_fmktask_from_task(self, task_obj, task_ref, period): + task_obj.fmkops = self._exportable_fmk_ops + task_obj.feedback_gate = self.feedback_gate + task_obj.targets = self.targets + task_obj.dm = self.dm + task_obj.prj = self.prj + + fmktask = FmkTask(task_ref, func=task_obj, arg=None, period=period, + error_func=self._handle_user_code_exception, + cleanup_func=partial(self._unregister_task, task_ref)) + self._register_task(task_ref, fmktask) + if self.is_ok(): + self.lg.log_fmk_info('A task has been registered (Task ID #{!s})'.format(task_ref)) + def _start_fmk_plumbing(self): if not self._is_started(): signal.signal(signal.SIGINT, signal.SIG_IGN) @@ -1079,20 +1189,31 @@ def _start_fmk_plumbing(self): need_monitoring = False for tg in self.targets.values(): - if tg.probes: - need_monitoring = True - for p in tg.probes: - pobj, delay = self._extract_info_from_probe(p) - if delay is not None: - self.mon.set_probe_delay(pobj, delay) - self.mon.start_probe(pobj, related_tg=tg) + for ext in tg.extensions: + ext_obj, period = self._extract_info_from_tg_extensions(ext) + if isinstance(ext_obj, type) and issubclass(ext_obj, Probe): + need_monitoring = True + if period is not None: + self.mon.set_probe_delay(ext_obj, period) + self.mon.start_probe(ext_obj, related_tg=tg) + elif isinstance(ext_obj, Task): + self._create_fmktask_from_task(ext_obj, task_ref=id(ext_obj), period=ext_obj.period) + else: + raise ValueError self.mon.wait_for_probe_initialization() self.prj.start() + if self.prj.fmkdb_enabled and not self.fmkDB.is_enabled(): + self.enable_fmkdb() + elif not self.prj.fmkdb_enabled and self.fmkDB.is_enabled(): + self.disable_fmkdb() + else: + pass if self.prj.project_scenarios: self._generic_tactics.register_scenarios(*self.prj.project_scenarios) + self._fmkDB_insert_dm_and_dmakers('generic', self._generic_tactics) if need_monitoring: time.sleep(0.5) @@ -1119,6 +1240,7 @@ def _stop_fmk_plumbing(self): self.log_target_residual_feedback() self._cleanup_tasks() + self.fmkDB.flush_feedback() if self.is_target_enabled(): self.mon.stop() @@ -1142,6 +1264,7 @@ def load_targets(self, tg_ids): return self._load_targets(tg_ids) def _load_targets(self, tg_ids): + tg_ids = [tg_ids] if isinstance(tg_ids, int) else tg_ids for tg_id in tg_ids: if tg_id >= len(self.__target_dict[self.prj]): self.set_error('The provided target number does not exist!', @@ -1157,15 +1280,15 @@ def get_available_targets(self): return self.__target_dict[self.prj] - def _extract_info_from_probe(self, p): - if isinstance(p, (tuple, list)): - assert(len(p) == 2) - pobj = p[0] - delay = p[1] + def _extract_info_from_tg_extensions(self, ext): + if isinstance(ext, (tuple, list)): + assert(len(ext) == 2) + ext_obj = ext[0] + delay = ext[1] else: - pobj = p + ext_obj = ext delay = None - return pobj, delay + return ext_obj, delay def _get_detailed_target_desc(self, tg): @@ -1176,30 +1299,30 @@ def _get_detailed_target_desc(self, tg): @EnforceOrder(accepted_states=['25_load_dm','S1','S2']) def show_targets(self): - print(colorize(FontStyle.BOLD + '\n-=[ Available Targets ]=-\n', rgb=Color.INFO)) + self.print(colorize(FontStyle.BOLD + '\n-=[ Available Targets ]=-\n', rgb=Color.INFO)) idx = 0 for tg in self.get_available_targets(): name = self.available_targets_desc[tg] msg = "[{:d}] {:s}".format(idx, name) - probes = tg.probes - if probes: - msg += '\n \-- monitored by:' - for p in probes: - pobj, delay = self._extract_info_from_probe(p) - pname = pobj.__name__ + extensions = tg.extensions + if extensions: + msg += '\n \-- extensions:' + for ext in extensions: + ext_obj, delay = self._extract_info_from_tg_extensions(ext) + ext_name = ext_obj.__name__ if isinstance(ext_obj, type) else ext_obj.__class__.__name__ if delay: - msg += " {:s}(refresh={:.2f}s),".format(pname, delay) + msg += " {:s}(refresh={:.2f}s),".format(ext_name, delay) else: - msg += " {:s},".format(pname) + msg += " {:s},".format(ext_name) msg = msg[:-1] if idx in self._tg_ids: msg = colorize(FontStyle.BOLD + msg, rgb=Color.SELECTED) else: msg = colorize(msg, rgb=Color.SUBINFO) - print(msg) + self.print(msg) idx += 1 @@ -1211,13 +1334,15 @@ def dynamic_generator_ids(self): @EnforceOrder(accepted_states=['S2']) def show_fmk_internals(self): - print(colorize(FontStyle.BOLD + '\n-=[ FMK Internals ]=-\n', rgb=Color.INFO)) - print(colorize(' [ General Information ]', rgb=Color.INFO)) - print(colorize(' FmkDB enabled: ', rgb=Color.SUBINFO) + repr(self.fmkDB.enabled)) - print(colorize(' Workspace enabled: ', rgb=Color.SUBINFO) + repr(self._wkspace_enabled)) - print(colorize(' Fuzz delay: ', rgb=Color.SUBINFO) + str(self._delay)) - print(colorize(' Number of data sent in burst: ', rgb=Color.SUBINFO) + str(self._burst)) - print(colorize(' Target(s) health-check timeout: ', rgb=Color.SUBINFO) + str(self._hc_timeout_max)) + delay_str = str(self._delay) if self._delay >=0 else 'Step by step mode' + + self.print(colorize(FontStyle.BOLD + '\n-=[ FMK Internals ]=-\n', rgb=Color.INFO)) + self.print(colorize(' [ General Information ]', rgb=Color.INFO)) + self.print(colorize(' FmkDB enabled: ', rgb=Color.SUBINFO) + repr(self.fmkDB.enabled)) + self.print(colorize(' Workspace enabled: ', rgb=Color.SUBINFO) + repr(self.prj.wkspace_enabled)) + self.print(colorize(' Sending delay: ', rgb=Color.SUBINFO) + delay_str) + self.print(colorize(' Number of data sent in burst: ', rgb=Color.SUBINFO) + str(self._burst)) + self.print(colorize(' Target(s) health-check timeout: ', rgb=Color.SUBINFO) + str(self._hc_timeout_max)) for tg_id, tg in self.targets.items(): if not tg.supported_feedback_mode: @@ -1229,19 +1354,25 @@ def show_fmk_internals(self): fbk_timeout = str(tg.feedback_timeout) tg_name = self.available_targets_desc[tg] - print(colorize('\n [ Target Specific Information - ({:d}) {!s} ]'.format(tg_id, tg_name), rgb=Color.INFO)) - print(colorize(' Feedback timeout: ', rgb=Color.SUBINFO) + fbk_timeout) - print(colorize(' Feedback mode: ', rgb=Color.SUBINFO) + fbk_mode) + self.print(colorize('\n [ Target Specific Information - ({:d}) {!s} ]'.format(tg_id, tg_name), rgb=Color.INFO)) + self.print(colorize(' Feedback timeout: ', rgb=Color.SUBINFO) + fbk_timeout) + self.print(colorize(' Feedback mode: ', rgb=Color.SUBINFO) + fbk_mode) @EnforceOrder(accepted_states=['S2']) def show_knowledge(self): - k = self.prj.knowledge_source - print(colorize(FontStyle.BOLD + '\n-=[ Status of Knowledge ]=-\n', rgb=Color.INFO)) + self._show_knowledge(do_record=False) + + def _show_knowledge(self, knowledge_src=None, do_record=False): + k = self.prj.knowledge_source if knowledge_src is None else knowledge_src + self.print(colorize(FontStyle.BOLD + '\n-=[ Status of Knowledge ]=-\n', rgb=Color.INFO)) if k: - print(colorize(str(k), rgb=Color.SUBINFO)) + self.print(colorize(str(k), rgb=Color.SUBINFO)) else: - print(colorize('No knowledge', rgb=Color.SUBINFO)) + self.print(colorize('No knowledge', rgb=Color.SUBINFO)) + if do_record: + msg = ('Status of Knowledge:\n' + str(k)) if k else 'No knowledge' + self.lg.log_fmk_info(msg, do_record=True, do_show=False) @EnforceOrder(accepted_states=['20_load_prj','25_load_dm','S1','S2']) def projects(self): @@ -1254,10 +1385,10 @@ def _projects(self): @EnforceOrder(accepted_states=['20_load_prj','25_load_dm','S1','S2']) def show_projects(self): - print(colorize(FontStyle.BOLD + '\n-=[ Projects ]=-\n', rgb=Color.INFO)) + self.print(colorize(FontStyle.BOLD + '\n-=[ Projects ]=-\n', rgb=Color.INFO)) idx = 0 for prj in self._projects(): - print(colorize('[%d] ' % idx + prj.name, rgb=Color.SUBINFO)) + self.print(colorize('[%d] ' % idx + prj.name, rgb=Color.SUBINFO)) idx += 1 @@ -1272,13 +1403,13 @@ def _iter_data_models(self): @EnforceOrder(accepted_states=['20_load_prj','25_load_dm','S1','S2']) def show_data_models(self): - print(colorize(FontStyle.BOLD + '\n-=[ Data Models ]=-\n', rgb=Color.INFO)) + self.print(colorize(FontStyle.BOLD + '\n-=[ Data Models ]=-\n', rgb=Color.INFO)) idx = 0 for dm in self._iter_data_models(): if dm is self.dm: - print(colorize(FontStyle.BOLD + '[{:d}] {!s}'.format(idx, dm.name), rgb=Color.SELECTED)) + self.print(colorize(FontStyle.BOLD + '[{:d}] {!s}'.format(idx, dm.name), rgb=Color.SELECTED)) else: - print(colorize('[{:d}] {!s}'.format(idx, dm.name), rgb=Color.SUBINFO)) + self.print(colorize('[{:d}] {!s}'.format(idx, dm.name), rgb=Color.SUBINFO)) idx += 1 def _init_fmk_internals_step1(self, prj, dm): @@ -1527,6 +1658,8 @@ def load_project(self, prj=None, name=None): self._update_targets_desc(prj) + self.prj.share_knowlegde_source() + return True @@ -1548,6 +1681,8 @@ def _launch(self): self._stop_fmk_plumbing() return False + self._show_knowledge(do_record=True) + self._init_fmk_internals_step2(self.prj, self.dm) return True @@ -1564,24 +1699,24 @@ def _disable_target(self): @EnforceOrder(always_callable=True) def enable_wkspace(self): - self._wkspace_enabled = True + self.prj.wkspace_enabled = True @EnforceOrder(always_callable=True) def disable_wkspace(self): - self._wkspace_enabled = False + self.prj.wkspace_enabled = False @EnforceOrder(accepted_states=['S1','S2']) - def set_fuzz_delay(self, delay, do_record=False): + def set_sending_delay(self, delay, do_record=True): if delay >= 0 or delay == -1: self._delay = delay - self.lg.log_fmk_info('Fuzz delay = {:.2f}s'.format(self._delay), do_record=do_record) + self.lg.log_fmk_info('Sending delay = {:.2f}s'.format(self._delay), do_record=do_record) return True else: self.lg.log_fmk_info('Wrong delay value!', do_record=False) return False @EnforceOrder(accepted_states=['S1','S2']) - def set_fuzz_burst(self, val, do_record=False): + def set_sending_burst_counter(self, val, do_record=False): if val >= 1: self._burst = int(val) self._burst_countdown = self._burst @@ -1593,7 +1728,7 @@ def set_fuzz_burst(self, val, do_record=False): return False @EnforceOrder(accepted_states=['S1','S2']) - def set_health_check_timeout(self, timeout, target=None, do_record=False, do_show=True): + def set_health_check_timeout(self, timeout, target=None, do_record=True, do_show=True): if timeout >= 0: if target is None: self._hc_timeout = {} @@ -1617,7 +1752,8 @@ def set_health_check_timeout(self, timeout, target=None, do_record=False, do_sho return False @EnforceOrder(accepted_states=['S1','S2']) - def set_feedback_timeout(self, timeout, tg_id=None, do_record=False, do_show=True): + def set_feedback_timeout(self, timeout, tg_id=None, do_record=True, do_show=True): + self._fbk_timeout_max = 0 if tg_id is None: max_sending_delay = 0 @@ -1630,24 +1766,24 @@ def set_feedback_timeout(self, timeout, tg_id=None, do_record=False, do_show=Tru if tg_id is None: for tg in self.targets.values(): tg.set_feedback_timeout(None) - self._recompute_health_check_timeout(timeout, max_sending_delay, do_show=do_show) + self._recompute_generic_timeouts(timeout, max_sending_delay, do_show=do_show) else: tg = self.targets[tg_id] tg.set_feedback_timeout(None) - self._recompute_health_check_timeout(timeout, tg.sending_delay, target=tg, do_show=do_show) + self._recompute_generic_timeouts(timeout, tg.sending_delay, target=tg, do_show=do_show) elif timeout >= 0: if tg_id is None: for tg in self.targets.values(): tg.set_feedback_timeout(timeout) - self._recompute_health_check_timeout(timeout, max_sending_delay, do_show=do_show) + self._recompute_generic_timeouts(timeout, max_sending_delay, do_show=do_show) if do_show or do_record: self.lg.log_fmk_info('Target(s) feedback timeout = {:.1f}s'.format(timeout), do_record=do_record) else: tg = self.targets[tg_id] tg.set_feedback_timeout(timeout) - self._recompute_health_check_timeout(timeout, tg.sending_delay, target=tg, do_show=do_show) + self._recompute_generic_timeouts(timeout, tg.sending_delay, target=tg, do_show=do_show) if do_show or do_record: tg_desc = self._get_detailed_target_desc(tg) self.lg.log_fmk_info('Target {!s} feedback timeout = {:.1f}s'.format(tg_desc, timeout), @@ -1663,7 +1799,9 @@ def set_feedback_mode(self, mode, tg_id=None, do_record=False, do_show=True): def _set_fbk_mode(tg): ok = tg.set_feedback_mode(mode) if not ok: - self.set_error('The target does not support this feedback Mode', code=Error.CommandError) + mode_desc = tg.get_fbk_mode_desc(mode, short=True) + self.set_error(f'The target does not support the requested feedback mode ({mode_desc})', + code=Error.CommandError) elif do_show or do_record: if tg.fbk_wait_full_time_slot_mode: msg = 'Feedback Mode = ' + tg.fbk_wait_full_time_slot_msg @@ -1691,7 +1829,7 @@ def switch_feedback_mode(self, tg_id, do_record=False, do_show=True): self.set_feedback_mode(Target.FBK_WAIT_FULL_TIME, tg_id=tg_id, do_record=do_record, do_show=do_show) # Used to introduce some delay after sending data - def _delay_fuzzing(self): + def _delay_sending(self): ''' return False if the user want to stop fuzzing (action possible if delay is set to -1) @@ -1704,10 +1842,8 @@ def _delay_fuzzing(self): if self._delay == -1.0: try: signal.signal(signal.SIGINT, sig_int_handler) - if sys.version_info[0] == 2: - cont = raw_input("\n*** Press [ENTER] to continue ('q' to exit) ***\n") - else: - cont = input("\n*** Press [ENTER] to continue ('q' to exit) ***\n") + cont = get_user_input(colorize("\n*** Press [ENTER] to continue ('q' to exit) ***\n", + rgb=Color.PROMPT)) if cont == 'q': ret = False except KeyboardInterrupt: @@ -1737,7 +1873,7 @@ def _delay_fuzzing(self): return ret - def _do_before_sending_data(self, data_list): + def _do_before_sending_data(self, data_list: Sequence[Data]) -> Sequence[Data]: # Monitor hook function before sending self.mon.notify_imminent_data_sending() # Callbacks that triggers before sending a data are executed here @@ -1758,12 +1894,15 @@ def _do_before_sending_data(self, data_list): def _do_after_sending_data(self, data_list): self._recovered_tgs = None + # _last_sending_date is not as precise as the one stored in Target object. But it is + # sufficient for its purpose within the plumbing. + self._last_sending_date = datetime.datetime.now() self._handle_data_callbacks(data_list, hook=HOOK.after_sending) - self.prj.notify_data_sending(data_list, self._current_sent_date, self.targets) def _do_sending_and_logging_init(self, data_list): for d in data_list: - mapping = self.prj.scenario_target_mapping.get(d.scenario_dependence, None) + if d.sending_delay is not None: + self.set_sending_delay(d.sending_delay) if d.feedback_timeout is not None: tg_ids = self._vtg_to_tg(d) @@ -1779,12 +1918,19 @@ def _do_sending_and_logging_init(self, data_list): data_list = list(filter(lambda x: not x.is_blocked(), data_list)) if self._burst_countdown == self._burst: - user_interrupt, go_on = self._collect_residual_feedback(force_mode=False) + try: + max_fbk_timeout = max([tg.feedback_timeout for tg in self._currently_used_targets + if tg.feedback_timeout is not None]) + except ValueError: + # empty list + max_fbk_timeout = self._fbk_timeout_default + user_interrupt, go_on = self._collect_residual_feedback(timeout=max_fbk_timeout) else: user_interrupt, go_on = False, True if blocked_data: self._handle_data_callbacks(blocked_data, hook=HOOK.after_fbk) + self._handle_data_callbacks(blocked_data, hook=HOOK.final) self.fmkDB.flush_current_feedback() if user_interrupt: @@ -1821,15 +1967,15 @@ def _collect_residual_feedback(self, force_mode=False, timeout=0): collected = False for tg in targets_to_retrieve_fbk.values(): - if tg.collect_pending_feedback(timeout=timeout): + if tg.collect_unsolicited_feedback(timeout=timeout): self._recovered_tgs = None collected = True if collected: # We have to make sure the targets are ready for sending data after # collecting feedback. - ftimeout = None if timeout == 0 else timeout + 0.1 - ret = self.check_target_readiness(forced_timeout=ftimeout) + ftimeout = None if timeout == 0 else timeout + ret = self.wait_for_target_readiness(forced_feedback_timeout=ftimeout) user_interrupt = ret == -2 tg_ready = ret >= 0 @@ -1847,18 +1993,29 @@ def _collect_residual_feedback(self, force_mode=False, timeout=0): def _do_after_feedback_retrieval(self, data_list): self._handle_data_callbacks(data_list, hook=HOOK.after_fbk) + self._handle_data_callbacks(data_list, hook=HOOK.final) self.fmkDB.flush_current_feedback() def _do_after_dmaker_data_retrieval(self, data): self._handle_data_callbacks([data], hook=HOOK.after_dmaker_production) - def _handle_data_desc(self, data_desc, resolve_dataprocess=True, original_data=None): + def handle_data_desc(self, data_desc, resolve_dataprocess=True, original_data=None, + save_generator_seed=False, reset_dmakers=False): if isinstance(data_desc, Data): data = data_desc - data.generate_info_from_content(original_data=original_data) + data.generate_info_from_content(data=original_data) elif isinstance(data_desc, DataProcess): + if data_desc.dp_completed: + # It means we already resolve the dataprocess (as @dp_completed is only set + # in these code path) and we are called a second time + # while @dp_completed has not been cleared. + # Typical of Replace_Data cbkops used in Scenario that could get called twice in + # HOOK.before_sending_step1 and HOOK.before_sending_step2. + data_desc.dp_completed = False + return None + if isinstance(data_desc.seed, str): try: seed_node = self.dm.get_atom(data_desc.seed) @@ -1869,44 +2026,56 @@ def _handle_data_desc(self, data_desc, resolve_dataprocess=True, original_data=N return None else: seed = Data(seed_node) - seed.generate_info_from_content(original_data=original_data) + seed.generate_info_from_content(data=original_data) elif isinstance(data_desc.seed, Data): seed = data_desc.seed - seed.generate_info_from_content(original_data=original_data) + seed.generate_info_from_content(data=original_data) elif data_desc.seed is None: seed = None else: - self.set_error(msg='DataProcess object contains an unrecognized seed type!', - code=Error.UserCodeError) + self.set_error(msg='DataProcess object contains an unrecognized seed' + ' type {!r}'.format(data_desc.seed), + code=Error.UserCodeError) return None if resolve_dataprocess or data_desc.outcomes is None: - data = self.get_data(data_desc.process, data_orig=seed) - if data is None and data_desc.auto_regen: - data_desc.auto_regen_cpt += 1 - if data is None: - other_process = data_desc.next_process() - if other_process or data_desc.auto_regen: - data = self.get_data(data_desc.process, data_orig=seed) - if data is None and data_desc.process_qty > 1: - for i in range(data_desc.process_qty-1): - data_desc.next_process() - data = self.get_data(data_desc.process, data_orig=seed) - if data is not None: - break + if data_desc.process is None: + data = seed + else: + data = self.process_data(data_desc.process, seed=seed, + save_gen_seed=save_generator_seed, + reset_dmakers=reset_dmakers) + if data is None: + if data_desc.auto_regen: + data_desc.auto_regen_cpt += 1 + + while data_desc.next_process() or data_desc.auto_regen: + data = self.process_data(data_desc.process, seed=seed, + save_gen_seed=save_generator_seed, + reset_dmakers=reset_dmakers) + if data is not None: + break + + if data is not None: + data.generate_info_from_content(data=original_data) data_desc.outcomes = data + if data is not None and data.tg_ids is None: + data.tg_ids = data_desc.tg_ids + else: data = data_desc.outcomes + # we do not update data with data_desc information because when we reached this case + # ("else" branch case) we have already done it when .outcomes was computed (which is + # in the "if" branch case). if data is None: + data_desc.dp_completed = True self.set_error(msg='Data creation process has yielded!', code=Error.DPHandOver) return None - data.tg_ids = data_desc.vtg_ids - elif isinstance(data_desc, str): try: node = self.dm.get_atom(data_desc) @@ -1916,7 +2085,7 @@ def _handle_data_desc(self, data_desc, resolve_dataprocess=True, original_data=N return None else: data = Data(node) - data.generate_info_from_content(original_data=original_data) + data.generate_info_from_content(data=original_data) if original_data is not None: data.tg_ids = original_data.tg_ids else: @@ -1941,7 +2110,7 @@ def _handle_data_callbacks(self, data_list, hook, resolve_dataprocess=True): try: if hook == HOOK.after_fbk: - data.run_callbacks(feedback=self.feedback_gate, hook=hook) + data.run_callbacks(feedback=self.last_feedback_gate, hook=hook) else: data.run_callbacks(feedback=None, hook=hook) except: @@ -1950,6 +2119,9 @@ def _handle_data_callbacks(self, data_list, hook, resolve_dataprocess=True): new_data_list.append(data) continue + if hook == HOOK.final: + continue + new_data = data data_tg_ids = data.tg_ids if data.tg_ids is not None else [self._tg_ids[0]] @@ -1981,15 +2153,16 @@ def _handle_data_callbacks(self, data_list, hook, resolve_dataprocess=True): new_data = [] first_step = True for d_desc, vtg_ids in zip(data_desc, vtg_ids_list): - data_tmp = self._handle_data_desc(d_desc, - resolve_dataprocess=resolve_dataprocess, - original_data=data) + data_tmp = self.handle_data_desc(d_desc, + resolve_dataprocess=resolve_dataprocess, + original_data=data, + reset_dmakers=data.attrs.is_set(DataAttr.Reset_DMakers)) + if data_tmp is not None: if first_step: first_step = False data_tmp.copy_callback_from(data) - data_tmp.tg_ids = vtg_ids - data_tmp.scenario_dependence = data.scenario_dependence + data_tmp.tg_ids = data.tg_ids if vtg_ids is None else vtg_ids new_data.append(data_tmp) else: # We mark the data unusable in order to make sending methods @@ -1997,8 +2170,10 @@ def _handle_data_callbacks(self, data_list, hook, resolve_dataprocess=True): # In this case it is either the normal end of a scenario or an error # within a scenario step. newd = Data() - newd.tg_ids = vtg_ids - newd.scenario_dependence = data.scenario_dependence + newd.tg_ids = data.tg_ids if vtg_ids is None else vtg_ids + newd.set_basic_attributes(from_data=data) + if not data.on_error_handover_to_user: + newd.copy_callback_from(data) newd.make_unusable() new_data = [newd] break @@ -2006,29 +2181,32 @@ def _handle_data_callbacks(self, data_list, hook, resolve_dataprocess=True): for idx in op[CallBackOps.Del_PeriodicData]: self._unregister_task(idx) + for idx in op[CallBackOps.Stop_Task]: + self._unregister_task(idx) + final_data_tg_ids = self._vtg_to_tg(data) for idx, obj in op[CallBackOps.Add_PeriodicData].items(): - periodic_obj, period = obj - data_desc = periodic_obj.data - if periodic_obj.vtg_ids_list: - final_data_tg_ids = self._vtg_to_tg(data, vtg_ids_list=periodic_obj.vtg_ids_list) + task_obj, period = obj + data_desc = task_obj.data + if task_obj.vtg_ids_list: + final_data_tg_ids = self._vtg_to_tg(data, vtg_ids_list=task_obj.vtg_ids_list) if isinstance(data_desc, DataProcess): # In this case each time we send the periodic we walk through the process # (thus, sending a new data each time) periodic_data = data_desc - func = functools.partial(self._send_periodic, final_data_tg_ids) + func = partial(self._send_periodic, final_data_tg_ids) else: - periodic_data = self._handle_data_desc(data_desc, - resolve_dataprocess=resolve_dataprocess, - original_data=data) + periodic_data = self.handle_data_desc(data_desc, + resolve_dataprocess=resolve_dataprocess, + original_data=data) targets = [self.targets[x] for x in final_data_tg_ids] - func = [tg.send_data_sync for tg in targets] + func = [partial(tg.send_data_sync, from_fmk=False) for tg in targets] if periodic_data is not None: task = FmkTask(idx, func, periodic_data, period=period, error_func=self._handle_user_code_exception, - cleanup_func=functools.partial(self._unregister_task, idx)) + cleanup_func=partial(self._unregister_task, idx)) self._register_task(idx, task) if self.is_ok(): self.lg.log_fmk_info('A periodic data sending has been registered (Task ID #{!s})'.format(idx)) @@ -2036,6 +2214,10 @@ def _handle_data_callbacks(self, data_list, hook, resolve_dataprocess=True): self.set_error(msg='Data descriptor is incorrect!', code=Error.UserCodeError) + for idx, obj in op[CallBackOps.Start_Task].items(): + task_obj, period = obj + self._create_fmktask_from_task(task_obj, task_ref=idx, period=period) + if isinstance(new_data, list): for newd in new_data: new_data_list.append(newd) @@ -2077,7 +2259,7 @@ def _vtg_to_tg(self, data, vtg_ids_list=None): return valid_tg_ids def _send_periodic(self, tg_ids, data_desc): - data = self._handle_data_desc(data_desc) + data = self.handle_data_desc(data_desc) if data is not None: for tg in [self.targets[tg_id] for tg_id in tg_ids]: tg.send_data_sync(data, from_fmk=False) @@ -2090,7 +2272,7 @@ def _unregister_task(self, id, ign_error=False): if id in self._task_list: self._task_list[id].stop() del self._task_list[id] - self.lg.log_fmk_info('Removal of a periodic data sending ' + self.lg.log_fmk_info('Removal of a Task ' '(Task ID #{!s})'.format(id)) elif not ign_error: self.set_error('ERROR: Task ID #{!s} does not exist. ' @@ -2115,24 +2297,115 @@ def stop_all_tasks(self): self._cleanup_tasks() @EnforceOrder(accepted_states=['S2']) - def send_data_and_log(self, data_list, original_data=None, verbose=False): + def process_data_and_send(self, data_desc=None, id_from_fmkdb=None, id_from_db=None, + max_loop=1, tg_ids=None, + verbose=False, console_display=True, + save_generator_seed=False): + """ + Send data to the selected targets. These data can follow a specific processing before + being emitted. The latter depends on what is provided in `data_desc`. + + + Args: + data_desc: Can be either a :class:`framework.data.DataProcess`, a :class:`framework.data.Data`, + the name (str) of an atom of the loaded data models, or a list of the previous types. + id_from_fmkdb: Data can be fetched from the FmkDB and send directly to the targets or be + used as the seed of a DataProcess if such object is provided in `data_desc`. + id_from_db: Data can be fetched from the Data Bank and send directly to the targets or be + used as the seed of a DataProcess if such object is provided in `data_desc`. + max_loop: Maximum number of iteration. -1 one means "infinite" or until some criteria occurs + (e.g., a disruptor has exhausted, the end-user issued Ctrl-C, ...) + tg_ids: Target ID or list of the Target IDs on which data will be sent. If provided + it will supersede the `tg_ids` parameter of any DataProcess provided in `data_desc` + verbose: Pretty print sent data + console_display: If `False`, nothing will be displayed on the screen (that could cause latency) + save_generator_seed: If random Generators are used, the generated data will be internally saved + and will be reused next time this generator will be called, until + FmkPlumbing.cleanup_dmaker(... reset_existing_seed=True) is called on this Generator. + + Returns: + The list of data that have been sent. `None` if nothing was sent due to some error. + + """ + + assert data_desc is not None or id_from_fmkdb is not None or id_from_db is not None + assert id_from_fmkdb is None or id_from_db is None + + if id_from_fmkdb is not None: + data = self.fmkdb_fetch_data(start_id=id_from_fmkdb, end_id=id_from_fmkdb) + if data is None: + return None + + elif id_from_db is not None: + data = self.get_from_data_bank(id_from_db) + if data is None: + return None + + if data_desc is not None: + if id_from_fmkdb is not None: + assert isinstance(data_desc, DataProcess) + data_desc.seed = data + elif id_from_db is not None: + assert isinstance(data_desc, DataProcess) + data_desc.seed = data + + data_desc = data_desc if isinstance(data_desc, list) else [data_desc] + cpt = 0 + data_to_send = [] + while cpt < max_loop or max_loop == -1: + cpt += 1 + data_list = [] + for d_desc in data_desc: + data = self.handle_data_desc(d_desc, resolve_dataprocess=True, + save_generator_seed=save_generator_seed) + if data is None: + data = Data() + data.make_unusable() + if tg_ids: + data.tg_ids = tg_ids + data_list.append(data) + + data_to_send += data_list + go_on = self.send_data_and_log(data_list, verbose=verbose, + console_display=console_display) + if not go_on: + break + + else: + cpt = 0 + data_to_send = [] + while cpt < max_loop or max_loop == -1: + cpt += 1 + data_to_send.append(data) + go_on = self.send_data_and_log(data, verbose=verbose, + console_display=console_display) + if not go_on: + break + + return data_to_send + + + @EnforceOrder(accepted_states=['S2']) + def send_data_and_log(self, data_list, verbose=False, console_display=True): - orig_data_provided = original_data is not None + if not console_display: + lg_display_on_term_save = self.lg.display_on_term + self.lg.display_on_term = False if isinstance(data_list, Data): data_list = [data_list] - if orig_data_provided: - original_data = [original_data] - elif isinstance(data_list, list): - assert original_data is None or isinstance(original_data, (list, tuple)) - else: - raise ValueError try: data_list = self._do_sending_and_logging_init(data_list) except (TargetFeedbackError, UserInterruption): return False + # we delay sending after calling self._do_sending_and_logging_init(data_list) + # as this delay can be changed while data is handled by this method. + go_on = self._delay_sending() + if not go_on: + return False + if not data_list: return True @@ -2142,9 +2415,13 @@ def send_data_and_log(self, data_list, original_data=None, verbose=False): return False if data_list is None: - # In this case, some data callbacks have triggered to block the emission of - # what was in data_list. We go on because this is a normal behavior (especially in the - # context of Scenario() execution). + # In this case: + # - either some data callbacks have triggered to block the emission of + # what was in data_list and we go on because this is a normal behavior (especially + # in the context of Scenario() execution). + # - or the framework is requested to go on even if there are no data to send (e.g., a + # DataProcess of a Scenario has completed and it exists a transition for that condition + # in the scenario). return True # All feedback entries that are available for relevant framework users (scenario @@ -2152,51 +2429,59 @@ def send_data_and_log(self, data_list, original_data=None, verbose=False): # means the previous feedback entries are obsolete. self.fmkDB.flush_current_feedback() + # For loaded Target()s not currently stimulated (i.e., ._send_data() not triggered on them) + # we also want to get data that may have been sent by the associated real targets. So that + # after the sending operation we get + # a full snapshot of the feedback coming from all the loaded Target()s. + # Note: Feedback retrieved from a real target has to be provided to the framework through + # the associated Target object either after Target.send_data() + # is called or when Target.collect_unsolicited_feedback() is called. if self._burst_countdown == self._burst: try: - max_fbk_timeout = max([tg.feedback_timeout for tg in self.targets.values() + # we compute the max fbk_timeout onl yon the target that have been stimulated + # as they rule the sequencing + max_fbk_timeout = max([tg.feedback_timeout for tg in self._currently_used_targets if tg.feedback_timeout is not None]) except ValueError: # empty list - max_fbk_timeout = 0 + max_fbk_timeout = self._fbk_timeout_default for tg in self.targets.values(): if tg not in self._currently_used_targets: - tg.collect_pending_feedback(timeout=max_fbk_timeout) + tg.collect_unsolicited_feedback(timeout=max_fbk_timeout) # the provided data_list can be changed after having called self._send_data() multiple_data = len(data_list) > 1 - if self._wkspace_enabled: - for idx, dt in zip(range(len(data_list)), data_list): - if orig_data_provided: - self.__current.append((original_data[idx], dt)) - else: - self.__current.append((None, dt)) - - if orig_data_provided: - for dt_orig in original_data: - if dt_orig is not None: - dt_orig.make_recordable() + if self.prj.wkspace_enabled: + if self.prj.wkspace_size == 1: + self.__current = [data_list[-1]] + else: + for dt in data_list: + self.__current.append(dt) + wkspace_len = len(self.__current) + if wkspace_len > self.prj.wkspace_size: + self.lg.log_fmk_info(f'Workspace is full (size={self.prj.wkspace_size}). Older ' + f'entries will be removed (ratio={self.prj.wkspace_free_slot_ratio_when_full*100}%)') + fslots = int(self.prj.wkspace_free_slot_ratio_when_full * self.prj.wkspace_size) + self.__current = self.__current[wkspace_len-self.prj.wkspace_size+fslots:] for dt in data_list: dt.make_recordable() # When checking target readiness, feedback timeout is taken into account indirectly - # through the call to Target.is_target_ready_for_new_data() - cont0 = self.check_target_readiness() >= 0 + # through the call to Target.is_feedback_received() + cont0 = self.wait_for_target_readiness() >= 0 if multiple_data: - self._log_data(data_list, original_data=original_data, - verbose=verbose) + self._log_data(data_list, verbose=verbose) else: - orig = original_data[0] if orig_data_provided else None - self._log_data(data_list[0], original_data=orig, verbose=verbose) + self._log_data(data_list[0], verbose=verbose) cont1 = True cont2 = True # That means this is the end of a burst if self._burst_countdown == self._burst: - cont1 = self.log_target_feedback() + cont1 = self.retrieve_and_log_target_feedback() self.mon.notify_target_feedback_retrieval() self.mon.wait_for_probe_status_retrieval() @@ -2209,14 +2494,14 @@ def send_data_and_log(self, data_list, original_data=None, verbose=False): self._do_after_feedback_retrieval(data_list) - if cont0: - cont0 = self._delay_fuzzing() + if not console_display: + self.lg.display_on_term = lg_display_on_term_save return cont0 and cont1 and cont2 @EnforceOrder(accepted_states=['S2']) - def _send_data(self, data_list): + def _send_data(self, data_list: Sequence[Data]): ''' @data_list: either a list of Data() or a Data() ''' @@ -2245,10 +2530,20 @@ def _send_data(self, data_list): self._stop_sending = False if data_list[0].is_unusable(): - self.set_error("_send_data(): A DataProcess has yielded. No more data to send.", - code=Error.NoMoreData) self.mon.notify_error() - self._stop_sending = True + if data_list[0].on_error_handover_to_user: + # We ask the framework to stop its current sending task and to give the + # control back to the end user. + self._stop_sending = True + self.set_error("_send_data(): A DataProcess has yielded. No more data to send." + " We give control back to the end user.", + code=Error.NoMoreData) + else: + self.set_error("_send_data(): A DataProcess has yielded. No more data to send." + " We are asked to go on silently with the next steps, if any.", + code=Error.NoMoreData) + self._handle_data_callbacks(data_list, hook=HOOK.final) + return None self._setup_new_sending() @@ -2256,6 +2551,8 @@ def _send_data(self, data_list): used_targets = [] for d in data_list: + self.next_data_id = self.fmkDB.get_next_data_id(prev_id=self.next_data_id) + d.estimated_data_id = self.next_data_id tg_ids = self._vtg_to_tg(d) for tg_id in tg_ids: if tg_id not in self.targets: @@ -2296,7 +2593,7 @@ def _send_data(self, data_list): @EnforceOrder(accepted_states=['S2']) - def _log_data(self, data_list, original_data=None, verbose=False): + def _log_data(self, data_list, verbose=False): if self.__tg_enabled: @@ -2310,15 +2607,8 @@ def _log_data(self, data_list, original_data=None, verbose=False): self._recovered_tgs = None gen = self.__current_gen - if original_data is None: - orig_data_provided = False - else: - orig_data_provided = True - if isinstance(data_list, Data): data_list = [data_list] - if orig_data_provided: - original_data = [original_data] multiple_data = False elif isinstance(data_list, list): multiple_data = True @@ -2350,10 +2640,6 @@ def _log_data(self, data_list, original_data=None, verbose=False): num = 0 if dt_mk_h is not None: - if orig_data_provided: - self.lg.log_orig_data(original_data[idx]) - else: - self.lg.log_orig_data(None) for dmaker_type, data_maker_name, user_input in dt_mk_h: num += 1 @@ -2411,12 +2697,12 @@ def _log_data(self, data_list, original_data=None, verbose=False): self.lg.set_target_ack_date(FeedbackSource(tg), date=ack_date) if self.fmkDB.enabled: - data_id = self.lg.commit_data_table_entry(self.group_id, self.prj.name) - if data_id is None: + self.last_data_id = self.lg.commit_data_table_entry(self.group_id, self.prj.name) + if self.last_data_id is None: self.lg.print_console('### Data not recorded in FmkDB', rgb=Color.DATAINFO, nl_after=True) else: - self.lg.print_console('### FmkDB Data ID: {!r}'.format(data_id), + self.lg.print_console('### FmkDB Data ID: {!r}'.format(self.last_data_id), rgb=Color.DATAINFO, nl_after=True) if multiple_data: @@ -2435,8 +2721,8 @@ def _setup_new_sending(self): self._current_sent_date = self.lg.start_new_log_entry(preamble=p) @EnforceOrder(accepted_states=['S2']) - def log_target_feedback(self, residual=False): - collected_err, err_detected2 = None, False + def retrieve_and_log_target_feedback(self, residual=False): + collected_status, err_detected2 = None, False ok = True if self.__tg_enabled: if residual: @@ -2446,12 +2732,17 @@ def log_target_feedback(self, residual=False): p = "::[ END BURST ]::\n" if self._burst > 1 else None e = None try: - collected_err = self.lg.log_collected_feedback(preamble=p, epilogue=e) + collected_status = self.lg.log_collected_feedback(preamble=p, epilogue=e) except NotImplementedError: pass for tg in self.targets.values(): - err_detected1 = collected_err.get(tg, False) if collected_err else False + if collected_status: + status = collected_status.get(tg, tg.STATUS_THRESHOLD_FOR_RECOVERY) + status = tg.STATUS_THRESHOLD_FOR_RECOVERY if status is None else status + err_detected1 = status < tg.STATUS_THRESHOLD_FOR_RECOVERY + else: + err_detected1 = False err_detected2 = self._log_directly_retrieved_target_feedback(tg=tg, preamble=p, epilogue=e) go_on = self._recover_target(tg) if err_detected1 or err_detected2 else True if not go_on: @@ -2461,7 +2752,7 @@ def log_target_feedback(self, residual=False): @EnforceOrder(accepted_states=['S2']) def log_target_residual_feedback(self): - return self.log_target_feedback(residual=True) + return self.retrieve_and_log_target_feedback(residual=True) def _log_directly_retrieved_target_feedback(self, tg, preamble=None, epilogue=None): """ @@ -2470,17 +2761,22 @@ def _log_directly_retrieved_target_feedback(self, tg, preamble=None, epilogue=No access the feedback from Target directly """ err_detected = False + + if not tg.is_feedback_received(): + return err_detected + tg_fbk = tg.get_feedback() if tg_fbk is not None: err_code = tg_fbk.get_error_code() - if err_code is not None and err_code < 0: + if err_code is not None and err_code < tg.STATUS_THRESHOLD_FOR_RECOVERY: err_detected = True if tg_fbk.has_fbk_collector(): for ref, fbk, status, tstamp in tg_fbk.iter_and_cleanup_collector(): - if status < 0: + if status < tg.STATUS_THRESHOLD_FOR_RECOVERY: err_detected = True - self.lg.log_target_feedback_from(source=FeedbackSource(tg, subref=ref), + self.lg.log_target_feedback_from(source=FeedbackSource(tg, subref=ref, + display_feedback=tg.display_feedback), content=fbk, status_code=status, timestamp=tstamp, @@ -2489,7 +2785,7 @@ def _log_directly_retrieved_target_feedback(self, tg, preamble=None, epilogue=No raw_fbk = tg_fbk.get_bytes() if raw_fbk is not None: - self.lg.log_target_feedback_from(source=FeedbackSource(tg), + self.lg.log_target_feedback_from(source=FeedbackSource(tg, display_feedback=tg.display_feedback), content=raw_fbk, status_code=err_code, timestamp=tg_fbk.get_timestamp(), @@ -2501,20 +2797,67 @@ def _log_directly_retrieved_target_feedback(self, tg, preamble=None, epilogue=No return err_detected @EnforceOrder(accepted_states=['S2']) - def check_target_readiness(self, forced_timeout=None): + def wait_for_target_readiness(self, forced_feedback_timeout=None): + """ - if self.__tg_enabled: - t0 = datetime.datetime.now() + Args: + forced_feedback_timeout: should be an integer >= 0 if only feedback need to be checked + + Returns: + + """ + if self.__tg_enabled: + t0 = datetime.datetime.now() if self._last_sending_date is None else self._last_sending_date signal.signal(signal.SIGINT, sig_int_handler) ret = 0 - tg = None - hc_timeout = self._hc_timeout_max if forced_timeout is None else forced_timeout + if forced_feedback_timeout is not None: + fbk_timeout = forced_feedback_timeout + elif self._currently_used_targets: + fbkt_list = [tg.feedback_timeout for tg in self._currently_used_targets + if tg.feedback_timeout is not None] + fbk_timeout = max(fbkt_list) if fbkt_list else self._fbk_timeout_default + elif self._last_sending_date is None: + # This case happens when we are waiting for feedback but no data has been sent, + # It is typical when a NoDataStep() is reached in a Scenario + fbk_timeout = self._fbk_timeout_default + else: + fbk_timeout = self._fbk_timeout_max + + # self.print('\n*** DBG: wait for target - fbk timeout: {!r} - forced fbk timeout: {!r}'.format(fbk_timeout, forced_feedback_timeout)) - # Wait until the target is ready or timeout expired + tg = None try: + # Wait for potential feedback from enabled targets + for tg in self.targets.values(): + while True: + # self.print('\n*** wait for target fbk busy loop') + if fbk_timeout == 0: + # self.print('\n*** DBG exit fast path1') + break + fbk_received = tg.is_feedback_received() + if (tg.fbk_wait_until_recv_mode and fbk_received) or \ + (forced_feedback_timeout is not None and fbk_received): + # self.print('\n*** DBG exit fast path2') + break + now = datetime.datetime.now() + if (now - t0).total_seconds() > fbk_timeout: + # if not fbk_received: + # self.lg.log_target_feedback_from( + # source=FeedbackSource(self), + # content='*** No feedback received from target {!s}.' + # .format(self.available_targets_desc[tg]), + # status_code=0, + # timestamp=now + # ) + break + time.sleep(0.005) + + hc_timeout = self._hc_timeout_max + # Wait until the target is ready to send data or timeout expired for tg in self.targets.values(): while not tg.is_target_ready_for_new_data(): + # self.print('\n***DBG wait for target ready busy loop') time.sleep(0.005) now = datetime.datetime.now() if (now - t0).total_seconds() > hc_timeout: @@ -2522,16 +2865,17 @@ def check_target_readiness(self, forced_timeout=None): source=FeedbackSource(self), content='*** Timeout! The target {!s} does not seem to be ready.' .format(self.available_targets_desc[tg]), - status_code=-1, + status_code=-2, timestamp=now ) go_on = self._recover_target(tg) ret = 0 if go_on else -1 # tg.cleanup() break + except KeyboardInterrupt: - self.lg.log_comment("*** Waiting for target to become ready has been cancelled by the user!\n") - self.set_error("Waiting for target to become ready has been cancelled by the user!", + self.lg.log_comment("*** Waiting for target readiness has been cancelled by the user!\n") + self.set_error("Waiting for target readiness has been cancelled by the user!", code=Error.OperationCancelled) ret = -2 if tg: @@ -2542,6 +2886,7 @@ def check_target_readiness(self, forced_timeout=None): if tg: tg.cleanup() finally: + self._last_sending_date = None signal.signal(signal.SIGINT, signal.SIG_IGN) return ret @@ -2550,9 +2895,9 @@ def check_target_readiness(self, forced_timeout=None): return 0 @EnforceOrder(accepted_states=['S2']) - def show_data(self, data, verbose=True): + def show_data(self, data: Data, verbose=True): self.lg.print_console('-=[ Data Visualization ]=-\n', rgb=Color.INFO, style=FontStyle.BOLD) - data.show(raw_limit=400) + self.lg.pretty_print_data(data, raw_limit=400) self.lg.print_console('\n\n', nl_before=False) @EnforceOrder(accepted_states=['S2']) @@ -2595,12 +2940,16 @@ def log_comment(self, comments): self.lg.log_comment(comments) @EnforceOrder(accepted_states=['S2']) - def _register_in_data_bank(self, data_orig, data): - self.__db_idx += 1 - self.__data_bank[self.__db_idx] = (data_orig, data) + def register_in_data_bank(self, data): + if isinstance(data, Data): + data = [data] + for d in data: + self.__db_idx += 1 + self.__data_bank[self.__db_idx] = d @EnforceOrder(accepted_states=['S2']) def fmkdb_fetch_data(self, start_id=1, end_id=-1): + data_list = [] for record in self.fmkDB.fetch_data(start_id=start_id, end_id=end_id): data_id, content, dtype, dmk_name, dm_name = record data = Data(content) @@ -2610,13 +2959,20 @@ def fmkdb_fetch_data(self, start_id=1, end_id=-1): if dm_name != Database.DEFAULT_DM_NAME: dm = self.get_data_model_by_name(dm_name) data.set_data_model(dm) - self._register_in_data_bank(None, data) + data_list.append(data) + + if not data_list: + data_list = None + elif len(data_list) == 1: + data_list = data_list[0] + + return data_list def _log_fmk_info(self, msg): if self.lg: self.lg.log_fmk_info(msg, do_record=False) else: - print(colorize('*** [ {:s} ] ***'.format(msg), rgb=Color.FMKINFO)) + self.print(colorize('*** [ {:s} ] ***'.format(msg), rgb=Color.FMKINFO)) def enable_fmkdb(self): self.fmkDB.enable() @@ -2628,38 +2984,32 @@ def disable_fmkdb(self): @EnforceOrder(accepted_states=['S2']) def get_last_data(self): - if not self._wkspace_enabled: + if not self.prj.wkspace_enabled: self.set_error('Workspace is disabled!', code=Error.CommandError) - return None, None + return None if self.__current: entry = self.__current[-1] return entry else: - return None, None + return None @EnforceOrder(accepted_states=['S2']) def get_from_data_bank(self, i): try: entry = self.__data_bank[i] except KeyError: - return None, None + return None return entry @EnforceOrder(accepted_states=['S2']) def iter_data_bank(self): for i in self.__data_bank: - entry = self.__data_bank[i] - yield entry + yield self.__data_bank[i] - def _show_entry(self, data_orig, data): - if data_orig != None: - self.lg.print_console('|_ IN < ', rgb=Color.SUBINFO) - self.lg.print_console(data_orig, nl_before=False) - else: - self.lg.print_console('|_ !IN', rgb=Color.SUBINFO) + def _show_entry(self, data): gen = self.__current_gen @@ -2722,59 +3072,24 @@ def show_data_bank(self): msg = '===[ {:d} ]==='.format(idx) msg += '='*(max(80-len(msg),0)) self.lg.print_console(msg, rgb=Color.INFO) - self._show_entry(*entry) + self._show_entry(entry) self.lg.print_console('\n', nl_before=False) @EnforceOrder(accepted_states=['S2']) def show_wkspace(self): - if not self._wkspace_enabled: + if not self.prj.wkspace_enabled: self.set_error('Workspace is disabled!', code=Error.CommandError) return self.lg.print_console("-=[ Workspace ]=-\n", rgb=Color.INFO, style=FontStyle.BOLD) - for data_orig, data in self.__current: - self._show_entry(data_orig, data) + for data in self.__current: + self._show_entry(data) self.lg.print_console('\n', nl_before=False) - @EnforceOrder(accepted_states=['S2']) - def dump_db_to_file(self, f): - if f: - try: - pickle.dump(self.__data_bank, f) - except (pickle.PicklingError, TypeError): - print("*** ERROR: Can't pickle the data bank!") - print('-'*60) - traceback.print_exc(file=sys.stdout) - print('-'*60) - - @EnforceOrder(accepted_states=['S2']) - def load_db_from_file(self, f): - if f: - self.__data_bank = pickle.load(f) - self.__idx = len(self.__data_bank) - - @EnforceOrder(accepted_states=['S2']) - def load_db_from_text_file(self, f): - if f: - text = f.read() - - self.__data_bank = {} - self.__idx = 0 - - while True: - obj = self.import_text_reg.match(text) - if obj is None: - break - - data = Data(obj.group(1)[:-1]) - - self._register_in_data_bank(None, data) - text = text[len(obj.group(0))+1:] - @EnforceOrder(accepted_states=['S2']) def empty_data_bank(self): self.__data_bank = {} @@ -2782,7 +3097,7 @@ def empty_data_bank(self): @EnforceOrder(accepted_states=['S2']) def empty_workspace(self): - if not self._wkspace_enabled: + if not self.prj.wkspace_enabled: self.set_error('Workspace is disabled!', code=Error.CommandError) return @@ -2790,23 +3105,22 @@ def empty_workspace(self): @EnforceOrder(accepted_states=['S2']) def register_current_in_data_bank(self): - if not self._wkspace_enabled: + if not self.prj.wkspace_enabled: self.set_error('Workspace is disabled!', code=Error.CommandError) return if self.__current: - for data_orig, data in self.__current: - self._register_in_data_bank(data_orig, data) + for data in self.__current: + self.register_in_data_bank(data) @EnforceOrder(accepted_states=['S2']) def register_last_in_data_bank(self): - if not self._wkspace_enabled: + if not self.prj.wkspace_enabled: self.set_error('Workspace is disabled!', code=Error.CommandError) return if self.__current: - data_orig, data = self.__current[-1] - self._register_in_data_bank(data_orig, data) + self.register_in_data_bank(self.__current[-1]) @EnforceOrder(accepted_states=['S2']) def show_operators(self): @@ -2874,7 +3188,7 @@ def launch_operator(self, name, user_input=None, use_existing_seed=True, verbose self.cleanup_all_dmakers(reset_existing_seed=False) if operation.is_flag_set(Operation.Stop): - self.log_target_feedback() + self.retrieve_and_log_target_feedback() break else: retry = False @@ -2888,8 +3202,7 @@ def launch_operator(self, name, user_input=None, use_existing_seed=True, verbose if action_list is None: data = orig else: - data = self.get_data(action_list, data_orig=orig, - save_seed=use_existing_seed) + data = self.process_data(action_list, seed=orig, save_gen_seed=use_existing_seed) if data: data.tg_ids = tg_ids @@ -2909,7 +3222,7 @@ def launch_operator(self, name, user_input=None, use_existing_seed=True, verbose change_list.append((e.context, idx)) retry = True else: - self.set_error('Unrecoverable error in get_data() method!', + self.set_error('Unrecoverable error in process_data() method!', code=Error.UnrecoverableError) return False @@ -2960,14 +3273,14 @@ def launch_operator(self, name, user_input=None, use_existing_seed=True, verbose if linst.is_instruction_set(LastInstruction.RecordData): for dt in data_list: dt.make_recordable() - self._register_in_data_bank(None, dt) + self.register_in_data_bank(dt) if multiple_data: self._log_data(data_list, verbose=verbose) else: self._log_data(data_list[0], verbose=verbose) - ret = self.check_target_readiness() + ret = self.wait_for_target_readiness() # Note: the condition (ret = -1) is supposed to be managed by the operator if ret < -1: exit_operator = True @@ -2978,7 +3291,7 @@ def launch_operator(self, name, user_input=None, use_existing_seed=True, verbose # Target fbk is logged only at the end of a burst if self._burst_countdown == self._burst: - cont1 = self.log_target_feedback() + cont1 = self.retrieve_and_log_target_feedback() self.mon.notify_target_feedback_retrieval() self.mon.wait_for_probe_status_retrieval() @@ -3014,7 +3327,7 @@ def launch_operator(self, name, user_input=None, use_existing_seed=True, verbose tg.cleanup() # Delay introduced after logging data - if not self._delay_fuzzing(): + if not self._delay_sending(): exit_operator = True self.lg.log_fmk_info("Operator will shutdown because waiting has been cancelled by the user") @@ -3028,17 +3341,26 @@ def launch_operator(self, name, user_input=None, use_existing_seed=True, verbose return True + @EnforceOrder(accepted_states=['S2']) - def get_data(self, action_list, data_orig=None, valid_gen=False, save_seed=False): - ''' - @action_list shall have a format compatible with what follows: - [(action_1, UserInput_1), ..., - (action_n, UserInput_n)] + def exec_dm_tests(self): + try: + self.dm.validation_tests() + except Exception: + self._handle_user_code_exception(f'Validation tests has crashed on current data model {self.dm.name}') - [action_1, (action_2, UserInput_2), ... action_n] - where action_N can be either: dmaker_type_N or (dmaker_type_N, dmaker_name_N) - ''' + @EnforceOrder(accepted_states=['S2']) + def process_data(self, action_list, seed=None, valid_gen=False, save_gen_seed=False, reset_dmakers=False): + """ + + Args: + action_list (list): Shall have a format compatible with what follows + [(action_1, UserInput_1), ..., (action_n, UserInput_n)] + [action_1, (action_2, UserInput_2), ... action_n] + where action_N can be either: dmaker_type_N or (dmaker_type_N, dmaker_name_N) + + """ l = [] action_list = action_list[:] @@ -3060,14 +3382,11 @@ def get_data(self, action_list, data_orig=None, valid_gen=False, save_seed=False clone_dmaker = self._tactics.clone_generator clone_gen_dmaker = self._generic_tactics.clone_generator - if data_orig != None: - data = copy.copy(data_orig) - initial_generator_info = data_orig.get_initial_dmaker() - # print('\n***') - # print(data_orig.get_history(), data_orig.info_list, data_orig.info) - data.generate_info_from_content(original_data=data_orig) + if seed != None: + data = copy.copy(seed) + initial_generator_info = seed.get_initial_dmaker() + # data.generate_info_from_content(data=seed) history = data.get_history() - # print(history, data_orig.info_list, data_orig.info) if history: for h_entry in history: l.append(h_entry) @@ -3091,6 +3410,11 @@ def get_data(self, action_list, data_orig=None, valid_gen=False, save_seed=False action = full_action user_input = None + if first and action == 'NOGEN': + self.lg.log_fmk_info('We are asked to ignore the generator and to go on with disruptors resolution') + first = False + continue + if unrecoverable_error: break @@ -3169,11 +3493,15 @@ def get_data(self, action_list, data_orig=None, valid_gen=False, save_seed=False code=Error.InvalidDmaker) return None + if reset_dmakers: + dmaker_obj._cleanup() + get_name = get_generic_dmaker_name if generic else get_dmaker_name dmaker_name = get_name(dmaker_type, dmaker_obj) - if first: - if dmaker_obj in self.__initialized_dmakers and self.__initialized_dmakers[dmaker_obj][0]: + if first: # Generator() case + if not reset_dmakers and \ + (dmaker_obj in self.__initialized_dmakers and self.__initialized_dmakers[dmaker_obj][0]): ui = self.__initialized_dmakers[dmaker_obj][1] else: ui = user_input @@ -3231,7 +3559,7 @@ def get_data(self, action_list, data_orig=None, valid_gen=False, save_seed=False if dmaker_obj not in self.__initialized_dmakers: self.__initialized_dmakers[dmaker_obj] = (False, None) - if not self.__initialized_dmakers[dmaker_obj][0]: + if reset_dmakers or not self.__initialized_dmakers[dmaker_obj][0]: initialized = dmaker_obj._setup(self.dm, user_input) if not initialized: setup_err = True @@ -3256,7 +3584,11 @@ def get_data(self, action_list, data_orig=None, valid_gen=False, save_seed=False else: data = dmaker_obj.generate_data(self.dm, self.mon, self.targets) - if save_seed and dmaker_obj.produced_seed is None: + if data.scenario_dependence: + init_dmaker = data.get_initial_dmaker() + if init_dmaker[0] != Database.DEFAULT_GTYPE_NAME: + initial_generator_info = data.get_initial_dmaker() + if save_gen_seed and dmaker_obj.produced_seed is None: # Usefull to replay from the beginning a modelwalking sequence dmaker_obj.produced_seed = Data(data.get_content(do_copy=True)) invalid_data = not self._is_data_valid(data) @@ -3549,31 +3881,41 @@ def print_dmaker(dmaker_list, title): self.lg.print_console('') self.lg.print_console('===[ Generator Types ]' + '='*58, rgb=Color.FMKINFOGROUP, nl_after=True) - l1 = [] - for dt in self._tactics.generator_types: - l1.append(dt) - l1 = sorted(l1) + + dmakers = {} + for dt, related_dm in self._tactics.generators_info(): + if related_dm is None: + continue + if related_dm not in dmakers: + dmakers[related_dm] = [] + dmakers[related_dm].append(dt) + + for related_dm, dt_list in dmakers.items(): + print_dmaker(dt_list, "Data Model: {}".format(related_dm.name)) l2 = [] for dt in self._generic_tactics.generator_types: l2.append(dt) l2 = sorted(l2) - - print_dmaker(l1, 'Specific') print_dmaker(l2, 'Generic') self.lg.print_console('===[ Disruptor Types ]' + '='*58, rgb=Color.FMKINFOGROUP, nl_after=True) - l1 = [] - for dmaker_type in self._tactics.disruptor_types: - l1.append(dmaker_type) - l1 = sorted(l1) + + dmakers = {} + for dt, related_dm in self._tactics.disruptors_info(): + if related_dm is None: + continue + if related_dm not in dmakers: + dmakers[related_dm] = [] + dmakers[related_dm].append(dt) + + for related_dm, dt_list in dmakers.items(): + print_dmaker(dt_list, "Data Model: {}".format(related_dm.name)) l2 = [] for dmaker_type in self._generic_tactics.disruptor_types: l2.append(dmaker_type) l2 = sorted(l2) - - print_dmaker(l1, 'Specific') print_dmaker(l2, 'Generic') @@ -3614,6 +3956,8 @@ def _make_str(k, v): for x in arg_type: args_type_desc += x.__name__ + ', ' args_type_desc = args_type_desc[:-2] + elif arg_type is None: + args_type_desc = 'unspecified' else: args_type_desc = arg_type.__name__ msg += ' |' + ' '*prefix_len + \ @@ -3737,14 +4081,18 @@ def display_color_theme(self): class FmkShell(cmd.Cmd): - def __init__(self, title, fmk_plumbing, completekey='tab', stdin=None, stdout=None): - cmd.Cmd.__init__(self, completekey, stdin, stdout) + def __init__(self, title, fmk_plumbing, completekey='tab'): + cmd.Cmd.__init__(self, completekey, stdin=None, stdout=None) self.fz = fmk_plumbing + self.printer = fmk_plumbing.printer + self.print = fmk_plumbing.printer.print + self.intro = colorize(FontStyle.BOLD + "\n-=[ %s ]=- (with Fuddly FmK %s)\n" % (title, fuddly_version), rgb=Color.TITLE) self.__allowed_cmd = re.compile( - '^quit$|^show_projects$|^show_data_models$|^load_project|^load_data_model|^load_targets|^show_targets$|^launch$' \ - '|^run_project|^config|^display_color_theme$|^fmkdb_disable$|^fmkdb_enable$|^help' + '^quit$|^switch_term$|^show_projects$|^show_data_models$|^load_project|^load_data_model' + '|^load_targets|^show_targets$|^launch$|^run_project|^config|^display_color_theme$' + '|^disable_fmkdb$|^enable_fmkdb$|^disable_fbk_handlers$|^enable_fbk_handlers$|^help' ) self.dmaker_name_re = re.compile('^([#\-\w]+)(\(?[^\(\)]*\)?)$', re.S) @@ -3752,18 +4100,22 @@ def __init__(self, title, fmk_plumbing, completekey='tab', stdin=None, stdout=No self.input_param_values_re = re.compile('(.*)=(.*)', re.S) self.config = config(self, path=[config_folder]) - def save_config(): - filename=os.path.join( - config_folder, - self.config.config_name + '.ini') - with open(filename, 'w') as cfile: - self.config.write(cfile) - atexit.register(save_config) - self.prompt = self.config.prompt + ' ' self.available_configs = { "framework": self.fz.config, - "shell": self.config} + "shell": self.config, + "db": self.fz.fmkDB.config} + + def save_config(): + for conf in self.available_configs.values(): + filename=os.path.join( + config_folder, + conf.config_name + '.ini') + with open(filename, 'w') as cfile: + conf.write(cfile) + atexit.register(save_config) + + self.prompt = '\n' + self.config.prompt + ' ' self.__error = False self.__error_msg = '' @@ -3784,15 +4136,12 @@ def save_history(history_path=history_path): def postcmd(self, stop, line): - self.prompt = self.config.prompt + ' ' + self.prompt = '\n' + self.config.prompt + ' ' if self._quit_shell: self._quit_shell = False msg = colorize(FontStyle.BOLD + "\nReally Quit? [Y/n]", rgb=Color.WARNING) - if sys.version_info[0] == 2: - cont = raw_input(msg) - else: - cont = input(msg) + cont = get_user_input(msg) cont = cont.upper() if cont == 'Y' or cont == '': self.fz.stop() @@ -3801,29 +4150,32 @@ def postcmd(self, stop, line): return False printed_err = False - print('') + self.print('') if self.fz.is_not_ok() or self.__error: printed_err = True msg = '| ERROR / WARNING / INFO |' - print(colorize('-'*len(msg), rgb=Color.WARNING)) - print(colorize(msg, rgb=Color.WARNING)) - print(colorize('-'*len(msg), rgb=Color.WARNING)) + self.print(colorize('-'*len(msg), rgb=Color.WARNING)) + self.print(colorize(msg, rgb=Color.WARNING)) + self.print(colorize('-'*len(msg), rgb=Color.WARNING)) if self.fz.is_not_ok(): err_list = self.fz.get_error() for e in err_list: - print(colorize(" (_ FMK [#{err!s:s}]: {msg:s} _)".format(err=e, msg=e.msg), rgb=e.color)) + self.print(colorize(" (_ FMK [#{err!s:s}]: {msg:s} _)".format(err=e, msg=e.msg), rgb=e.color)) if self.__error: self.__error = False if self.__error_msg != '': - print(colorize(" (_ SHELL: {:s} _)".format(self.__error_msg), rgb=Color.WARNING)) + self.print(colorize(" (_ SHELL: {:s} _)".format(self.__error_msg), rgb=Color.WARNING)) if printed_err: - print('') + self.print('') self.__error_msg = '' self.__error_fmk = '' + + self.printer.wait_for_sync() + return stop def emptyline(self): @@ -3857,6 +4209,12 @@ def precmd(self, line): self.__error_msg = 'You shall first load a project and/or enable all the framework components!' return '' + def do_switch_term(self, line): + """Display information on another terminal""" + self.fz.switch_term() + + return False + def do_show_projects(self, line): '''Show the available Projects''' @@ -3953,7 +4311,7 @@ def complete_config(self, text, line, bgidx, endix, target=None): return comp def do_config(self, line, target=None): - '''Get and set miscellaneous options + """Get and set miscellaneous options Usage: - config @@ -3962,7 +4320,7 @@ def do_config(self, line, target=None): Get value associated with . - config [name [subname...]] value Set value associated with . - ''' + """ self.__error = True level = self.config.config.indent.level @@ -3972,10 +4330,10 @@ def do_config(self, line, target=None): args = line.split() if target is None: if len(args) == 0: - print('Available configurations:') + self.print('Available configurations:') for target in self.available_configs: - print(' - {}'.format(target)) - print('\n\t > Type "config " to display documentation.') + self.print(' - {}'.format(target)) + self.print('\n\t > Type "config " to display documentation.') self.__error = False return False else: @@ -3984,15 +4342,15 @@ def do_config(self, line, target=None): self.__error = False return self.do_config(' '.join(args[1:]), target) except KeyError as e: - print('Unknown config "{}": '.format(args[0]) + str(e)) - return True + self.print('Unknown config "{}": '.format(args[0]) + str(e)) + return False if len(args) == 0: - print(target.help(None, level, indent, middle)) + self.print(target.help(None, level, indent, middle)) self.__error = False return False elif len(args) == 1: - print(target.help(args[0], level, indent, middle)) + self.print(target.help(args[0], level, indent, middle)) self.__error = False return False @@ -4012,7 +4370,7 @@ def do_config(self, line, target=None): if isinstance(attr, config_dot_proxy): self.__error = False key = '.'.join(args) - print(target.help(key, level, indent, middle)) + self.print(target.help(key, level, indent, middle)) self.__error = False return False @@ -4022,7 +4380,7 @@ def do_config(self, line, target=None): self.__error_msg = 'config: ' + str(e) return False - print(target.help(args[0], level, indent, middle)) + self.print(target.help(args[0], level, indent, middle)) self.__error = False return False @@ -4041,7 +4399,7 @@ def do_config(self, line, target=None): self.__error_msg = 'config: ' + str(e) return False - print(target.help(key, level, indent, middle)) + self.print(target.help(key, level, indent, middle)) self.__error = False return False @@ -4049,6 +4407,14 @@ def do_config(self, line, target=None): "'{}' do not have subkeys".format(args[0])) return False + + def do_exec_dm_tests(self, line): + """ + Execute the validation tests of current Data Model + """ + self.fz.exec_dm_tests() + return False + def do_load_data_model(self, line): '''Load a Data Model by name''' self.__error = True @@ -4374,24 +4740,6 @@ def do_disable_wkspace(self, line): self.fz.disable_wkspace() return False - def do_send_valid(self, line): - ''' - Build a data in multiple step from a valid source - |_ syntax: send_valid [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] - |_ Note: generator_type shall have at least one valid generator - ''' - ret = self.do_send(line, valid_gen=True) - return ret - - def do_send_loop_valid(self, line): - ''' - Execute the 'send_valid' command in a loop - |_ syntax: send_loop_valid <#loop> [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] - |_ Note: generator_type shall have at least one valid generator - ''' - ret = self.do_send_loop(line, valid_gen=True) - return ret - def do_send_loop_keepseed(self, line): ''' Execute the 'send' command in a loop and save the seed @@ -4569,6 +4917,7 @@ def _retrieve_tg_ids(self, args): tg_ids = tg_ids[::-1] args = args[:-len(tg_ids)] + tg_ids = tg_ids if tg_ids else None return args, tg_ids def do_reload_data_model(self, line): @@ -4668,8 +5017,23 @@ def do_flush_feedback(self, line): self.fz.collect_residual_feedback(timeout=3) return False + def do_collect_feedback(self, line): + """ + Collect the residual feedback (received by the target and the probes) indefinitely. + Only stop with Ctrl+C + """ + # t0 = datetime.datetime.now() + # while (datetime.datetime.now() - t0).total_seconds() < 100: + try: + while True: + self.fz.collect_residual_feedback(timeout=0.01) + except UserInterruption: + pass + + return False - def do_send(self, line, valid_gen=False, verbose=False): + + def do_send(self, line, verbose=False): ''' Carry out multiple fuzzing steps in sequence |_ syntax: send [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] @@ -4684,20 +5048,13 @@ def do_send(self, line, valid_gen=False, verbose=False): args, tg_ids = self._retrieve_tg_ids(args) - t = self.__parse_instructions(args) - if t is None: + actions = self.__parse_instructions(args) + if actions is None: self.__error_msg = "Syntax Error!" return False - data = self.fz.get_data(t, valid_gen=valid_gen) - if data is None: - return False - - if tg_ids: - data.tg_ids = tg_ids - self.fz.send_data_and_log(data, verbose=verbose) - - self.__error = False + self.__error = self.fz.process_data_and_send(DataProcess(actions, tg_ids=tg_ids), + verbose=verbose) is None return False @@ -4710,7 +5067,7 @@ def do_send_verbose(self, line): return ret - def do_send_loop(self, line, valid_gen=False, use_existing_seed=False): + def do_send_loop(self, line, use_existing_seed=False, verbose=False): ''' Execute the 'send' command in a loop |_ syntax: send_loop <#loop> [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] @@ -4735,8 +5092,8 @@ def do_send_loop(self, line, valid_gen=False, use_existing_seed=False): except ValueError: return False - t = self.__parse_instructions(args) - if t is None: + actions = self.__parse_instructions(args) + if actions is None: self.__error_msg = "Syntax Error!" return False @@ -4750,22 +5107,23 @@ def do_send_loop(self, line, valid_gen=False, use_existing_seed=False): } with aligned_stdout(**kwargs): - # for i in range(nb): - cpt = 0 - while cpt < max_loop or max_loop == -1: - cpt += 1 - data = self.fz.get_data(t, valid_gen=valid_gen, save_seed=use_existing_seed) - if data is None: - return False - if tg_ids: - data.tg_ids = tg_ids - cont = self.fz.send_data_and_log(data) - if not cont: - break - self.__error = False + self.__error = self.fz.process_data_and_send(DataProcess(actions, tg_ids=tg_ids), + max_loop=max_loop, verbose=verbose, + save_generator_seed=use_existing_seed) is None + return False + def do_send_loop_verbose(self, line): + """ + Execute the 'send' command in a loop and save the seed + |_ syntax: send_loopv <#loop> [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] + + Notes: + - To loop indefinitely use -1 for #loop. To stop the loop use Ctrl+C + """ + ret = self.do_send_loop(line, use_existing_seed=True, verbose=True) + return ret def do_send_with(self, line): ''' @@ -4786,16 +5144,9 @@ def do_send_with(self, line): self.__error_msg = "Syntax Error!" return False - action = [((t[0], args[1]), t[1])] - data = self.fz.get_data(action) - if data is None: - return False - - if tg_ids: - data.tg_ids = tg_ids - self.fz.send_data_and_log(data) + actions = [((t[0], args[1]), t[1])] - self.__error = False + self.__error = self.fz.process_data_and_send(DataProcess(actions, tg_ids=tg_ids)) is None return False @@ -4818,12 +5169,12 @@ def do_send_loop_with(self, line): except ValueError: return False - t = self.__parse_instructions([args[1]])[0] - if t is None: + actions = self.__parse_instructions([args[1]])[0] + if actions is None: self.__error_msg = "Syntax Error!" return False - action = [((t[0], args[2]), t[1])] + action = [((actions[0], args[2]), actions[1])] conf = self.config.send_loop.aligned_options kwargs = { @@ -4835,20 +5186,12 @@ def do_send_loop_with(self, line): } with aligned_stdout(**kwargs): - for i in range(nb): - data = self.fz.get_data(action) - if data is None: - return False + self.__error = self.fz.process_data_and_send(DataProcess(actions, tg_ids=tg_ids), + max_loop=nb) is None - if tg_ids: - data.tg_ids = tg_ids - self.fz.send_data_and_log(data) - - self.__error = False return False - def do_multi_send(self, line): ''' Send several data to one or more targets. Generation instructions must be provided when @@ -4867,20 +5210,18 @@ def do_multi_send(self, line): except: loop_count = 1 - actions_list = [] + dp_list = [] idx = 0 while True: idx += 1 - msg = "*** Data generation instructions [#{:d}] (type '!' when all instructions are provided):\n".format(idx) - if sys.version_info[0] == 2: - actions_str = raw_input(msg) - else: - actions_str = input(msg) + msg = "*** Data generation instructions [#{:d}] (type '!' " \ + "when all instructions are provided):\n# ".format(idx) + actions_str = get_user_input(colorize(msg, rgb=Color.PROMPT)) if actions_str == '!': - print("*** Configuration terminated.") + self.print("*** Configuration terminated.") break l = actions_str.split() @@ -4893,59 +5234,59 @@ def do_multi_send(self, line): self.__error_msg = "Syntax Error!" return False - actions_list.append((actions, tg_ids)) - - prev_data_list = None - exhausted_data_cpt = 0 - exhausted_data = {} - nb_data = len(actions_list) - - for i in range(loop_count): - data_list = [] - - for j in range(nb_data): - if j not in exhausted_data: - exhausted_data[j] = False - - if not exhausted_data[j]: - action_seq, tg_ids = actions_list[j] - data = self.fz.get_data(action_seq) - if tg_ids and data is not None: - data.tg_ids = tg_ids - else: - if prev_data_list is not None: - data = prev_data_list[j] - else: - self.__error_msg = 'The loop has terminated too soon! (number of exhausted data: %d)' % exhausted_data_cpt - return False - - if data is None and exhausted_data_cpt < nb_data: - exhausted_data_cpt += 1 - if prev_data_list is not None: - data = prev_data_list[j] - exhausted_data[j] = True - else: - self.__error_msg = 'The loop has terminated too soon! (number of exhausted data: %d)' % exhausted_data_cpt - return False + dp_list.append(DataProcess(actions, tg_ids=tg_ids)) - if exhausted_data[j] and exhausted_data_cpt >= nb_data: - self.__error_msg = 'The loop has terminated because all data are exhausted ' \ - '(number of exhausted data: %d)' % exhausted_data_cpt - return False - - data_list.append(data) - - prev_data_list = data_list - - self.fz.send_data_and_log(data_list) - - if exhausted_data_cpt > 0: - print("\nThe loop has terminated normally, but it remains non exhausted " \ - "data (number of exhausted data: %d)" % exhausted_data_cpt) - - self.__error = False + self.__error = self.fz.process_data_and_send(dp_list, max_loop=loop_count) is None return False + # prev_data_list = None + # exhausted_data_cpt = 0 + # exhausted_data = {} + # nb_data = len(dp_list) + # + # for i in range(loop_count): + # data_list = [] + # + # for j in range(nb_data): + # if j not in exhausted_data: + # exhausted_data[j] = False + # + # if not exhausted_data[j]: + # action_seq, tg_ids = dp_list[j] + # data = self.fz.process_data(action_seq) + # if tg_ids and data is not None: + # data.tg_ids = tg_ids + # else: + # if prev_data_list is not None: + # data = prev_data_list[j] + # else: + # self.__error_msg = 'The loop has terminated too soon! (number of exhausted data: %d)' % exhausted_data_cpt + # return False + # + # if data is None and exhausted_data_cpt < nb_data: + # exhausted_data_cpt += 1 + # if prev_data_list is not None: + # data = prev_data_list[j] + # exhausted_data[j] = True + # else: + # self.__error_msg = 'The loop has terminated too soon! (number of exhausted data: %d)' % exhausted_data_cpt + # return False + # + # if exhausted_data[j] and exhausted_data_cpt >= nb_data: + # self.__error_msg = 'The loop has terminated because all data are exhausted ' \ + # '(number of exhausted data: %d)' % exhausted_data_cpt + # return False + # + # data_list.append(data) + # + # prev_data_list = data_list + # + # self.fz.send_data_and_log(data_list) + # + # if exhausted_data_cpt > 0: + # self.print("\nThe loop has terminated normally, but it remains non exhausted " \ + # "data (number of exhausted data: %d)" % exhausted_data_cpt) + def do_set_feedback_timeout(self, line): ''' Set the time duration for feedback gathering (if supported by the target) @@ -5049,7 +5390,7 @@ def do_set_delay(self, line): return False try: delay = float(args[0]) - self.fz.set_fuzz_delay(delay) + self.fz.set_sending_delay(delay) except: return False @@ -5075,7 +5416,7 @@ def do_set_burst(self, line): return False try: val = float(args[0]) - self.fz.set_fuzz_burst(val) + self.fz.set_sending_burst_counter(val) except: return False @@ -5108,10 +5449,11 @@ def do_empty_wkspace(self, line): return False def do_replay_db(self, line): - ''' - Replay data from the Data Bank and optionnaly apply new disruptors on it - |_ syntax: replay_db i [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] - ''' + """ + Replay data from the FmkDB or the Data Bank and optionnaly apply new disruptors on it + |_ syntax for FmkDB: replay_db f [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] + |_ syntax for DBank: replay_db d [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] + """ self.__error = True @@ -5123,40 +5465,41 @@ def do_replay_db(self, line): return False try: - idx = int(args.pop(0)[1:]) - except ValueError: - return False + full_id = args.pop(0) + idx = int(full_id[1:]) + if full_id[0] == 'f': + id_from_fmkdb = idx + id_from_db = None + elif full_id[0] == 'd': + id_from_fmkdb = None + id_from_db = idx + else: + raise ValueError - data_orig, data = self.fz.get_from_data_bank(idx) - if data is None: + except ValueError: return False if args_len > 1: - data_orig = data - - t = self.__parse_instructions(args) - if t is None: + actions = self.__parse_instructions(args) + if actions is None: self.__error_msg = "Syntax Error!" return False - data = self.fz.get_data(t, data_orig=data) - if data is None: - return False - - self.__error = False - - if tg_ids: - data.tg_ids = tg_ids - self.fz.send_data_and_log(data, original_data=data_orig) + else: + actions = None + self.__error = self.fz.process_data_and_send(DataProcess(actions, tg_ids=tg_ids), + id_from_fmkdb=id_from_fmkdb, + id_from_db=id_from_db) is None return False def do_replay_db_loop(self, line): - ''' + """ Loop ( Replay data from the Data Bank and optionnaly apply new disruptors on it ) - |_ syntax: replay_db_loop <#loop> i [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] - ''' + |_ syntax for FmkDB: replay_db_loop <#loop> f [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] + |_ syntax for DBank: replay_db_loop <#loop> d [disruptor_type_1 ... disruptor_type_n] [targetID1 ... targetIDN] + """ self.__error = True @@ -5170,39 +5513,32 @@ def do_replay_db_loop(self, line): try: nb = int(args.pop(0)) - idx = int(args.pop(0)[1:]) - except ValueError: - return False + full_id = args.pop(0) + idx = int(full_id[1:]) + if full_id[0] == 'f': + id_from_fmkdb = idx + id_from_db = None + elif full_id[0] == 'd': + id_from_fmkdb = None + id_from_db = idx + else: + raise ValueError - data_orig, data = self.fz.get_from_data_bank(idx) - if data is None: + except ValueError: return False if args_len > 2: - data_orig = data - - t = self.__parse_instructions(args) - if t is None: + actions = self.__parse_instructions(args) + if actions is None: self.__error_msg = "Syntax Error!" return False - - for i in range(nb): - new_data = self.fz.get_data(t, data_orig=data) - if new_data is None: - return False - - if tg_ids: - new_data.tg_ids = tg_ids - self.fz.send_data_and_log(new_data, original_data=data_orig) - else: - for i in range(nb): - if tg_ids: - data.tg_ids = tg_ids - self.fz.send_data_and_log(data, original_data=data_orig) - - self.__error = False + actions = None + self.__error = self.fz.process_data_and_send(DataProcess(actions, tg_ids=tg_ids), + id_from_fmkdb=id_from_fmkdb, + id_from_db=id_from_db, + max_loop=nb) is None return False @@ -5222,10 +5558,8 @@ def do_replay_db_all(self, line): self.__error_msg = "the Data Bank is empty" return False - for data_orig, data in self.fz.iter_data_bank(): - if tg_ids: - data.tg_ids = tg_ids - self.fz.send_data_and_log(data, original_data=data_orig) + for data in self.fz.iter_data_bank(): + self.fz.process_data_and_send(data, tg_ids=tg_ids) return False @@ -5236,7 +5570,7 @@ def do_show_data_paths(self, line): ''' self.__error = True - data_orig, data = self.fz.get_last_data() + data = self.fz.get_last_data() if data is None: return False @@ -5251,7 +5585,7 @@ def do_show_data(self, line): ''' self.__error = True - data_orig, data = self.fz.get_last_data() + data = self.fz.get_last_data() if data is None: return False @@ -5297,7 +5631,7 @@ def do_replay_last(self, line): self.__error = True - data_orig, data = self.fz.get_last_data() + data = self.fz.get_last_data() if data is None: return False @@ -5305,69 +5639,82 @@ def do_replay_last(self, line): if line: args = line.split() - data_orig = data args, tg_ids = self._retrieve_tg_ids(args) - t = self.__parse_instructions(args) - if t is None: + actions = self.__parse_instructions(args) + if actions is None: self.__error_msg = "Syntax Error!" return False + else: + actions = None - data = self.fz.get_data(t, data_orig=data) - if data is None: - return False - - self.__error = False - - if tg_ids: - data.tg_ids = tg_ids - self.fz.send_data_and_log(data, original_data=data_orig) + self.__error = self.fz.process_data_and_send(DataProcess(actions, tg_ids=tg_ids, seed=data)) is None return False def do_send_raw(self, line): - ''' + """ Send raw data - |_ syntax: send_raw - ''' + |_ syntax: send_raw [targetID1 ... targetIDN] + + A prompt will then ask you to write your input + """ self.__error_msg = "Syntax Error!" args = line.split() - args_len = len(args) - - if args_len < 1: + args, tg_ids = self._retrieve_tg_ids(args) + if args: self.__error = True return False - args, tg_ids = self._retrieve_tg_ids(args) - line = ''.join(args) - - if line: - data = Data(line) + data_str = get_user_input(colorize('*** Data to be sent without interpretation:\n# ', + rgb=Color.PROMPT)) - if tg_ids: - data.tg_ids = tg_ids - self.fz.send_data_and_log(data, None) - else: - self.__error = True + self.fz.process_data_and_send(Data(data_str), tg_ids=tg_ids) return False def do_send_eval(self, line): ''' Send python-evaluation of the parameter - |_ syntax: send_eval + |_ syntax: send_eval [targetID1 ... targetIDN] ''' self.__error_msg = "Syntax Error!" args = line.split() + + args, tg_ids = self._retrieve_tg_ids(args) + if args: + self.__error = True + return False + + data_str = get_user_input(colorize('*** Data to be sent after evaluation:\n# ', + rgb=Color.PROMPT)) + + try: + data = Data(eval(data_str)) + except: + self.__error = True + return False + + self.fz.process_data_and_send(data, tg_ids=tg_ids) + + return False + + def do_register_db(self, line): + """ + Register in the data bank the python-evaluation of the parameter + This can then be used as a seed in a disruptors chain + |_ syntax: register_db + """ + self.__error_msg = "Syntax Error!" + args = line.split() args_len = len(args) if args_len < 1: self.__error = True return False - args, tg_ids = self._retrieve_tg_ids(args) line = ''.join(args) if line: @@ -5377,15 +5724,12 @@ def do_send_eval(self, line): self.__error = True return False - if tg_ids: - data.tg_ids = tg_ids - self.fz.send_data_and_log(data, None) + self.fz.register_in_data_bank(data) else: self.__error = True return False - def do_register_wkspace(self, line): '''Register the workspace to the Data Bank''' self.fz.register_current_in_data_bank() @@ -5428,80 +5772,35 @@ def do_fmkdb_fetch_data(self, line): sid = 1 eid = -1 - self.fz.fmkdb_fetch_data(start_id=sid, end_id=eid) + data_list = self.fz.fmkdb_fetch_data(start_id=sid, end_id=eid) + self.fz.register_in_data_bank(data_list) self.__error = False return False - def do_fmkdb_enable(self, line): - '''Enable FmkDB recording''' + def do_enable_fmkdb(self, line): + """Enable FmkDB recording""" self.fz.enable_fmkdb() return False - def do_fmkdb_disable(self, line): - '''Enable FmkDB recording''' + def do_disable_fmkdb(self, line): + """Enable FmkDB recording""" self.fz.disable_fmkdb() return False - def do_dump_db_to_file(self, line): - ''' - Dump the Data Bank to a file in pickle format - |_ syntax: dump_db_to_file - ''' - - if line: - arg = line.split()[0] - - f = open(arg, 'wb') - self.fz.dump_db_to_file(f) - f.close() - else: - self.__error = True - + def do_enable_fbk_handlers(self, line): + """Enable Feedback Handlers""" + self.fz.prj.enable_feedback_handlers() return False - - def do_load_db_from_file(self, line): - ''' - Load a previous saved Data Bank from a file - |_ syntax: load_db_from_file - ''' - - if line: - arg = line.split()[0] - - f = open(arg, 'rb') - self.fz.load_db_from_file(f) - f.close() - else: - self.__error = True - + def do_disable_fbk_handlers(self, line): + """Disable Feedback Handlers""" + self.fz.prj.disable_feedback_handlers() return False - def do_load_db_from_text_file(self, line): - ''' - Load a previous saved Data Bank from a file - |_ syntax: load_db_from_text_file - ''' - - if line: - arg = line.split()[0] - - f = open(arg, 'r') - self.fz.load_db_from_text_file(f) - f.close() - else: - self.__error = True - - return False - - def do_comment(self, line): - if sys.version_info[0] == 2: - comments = raw_input("*** Write your comments:\n") - else: - comments = input("*** Write your comments:\n") - + comments = get_user_input(colorize("*** Write your comments:\n# ", + rgb=Color.PROMPT)) self.fz.log_comment(comments) return False diff --git a/framework/project.py b/framework/project.py index 017b71f..896e19a 100644 --- a/framework/project.py +++ b/framework/project.py @@ -42,14 +42,55 @@ class Project(object): name = None default_dm = None - feedback_gate = None + wkspace_enabled = None + wkspace_size = None + wkspace_free_slot_ratio_when_full = None + + def __init__(self, enable_fbk_processing=True, + wkspace_enabled=True, wkspace_size=1000, wkspace_free_slot_ratio_when_full=0.5, + fmkdb_enabled=True, + default_fbk_timeout=None, default_fbk_mode=None, + default_sending_delay=None, default_burst_value=None): + """ + + Args: + enable_fbk_processing: enable or disable the execution of feedback + handlers, if any are set in the project. + wkspace_enabled: If set to True, enable the framework workspace that store + the generated data. + wkspace_size: Maximum number of data that can be stored in the workspace. + wkspace_free_slot_ratio_when_full: when the workspace is full, provide the ratio + of the workspace size that will be used as the amount of entries to free in + the workspace. + fmkdb_enabled: If set to `True`, the fmkDB will be used. Otherwise, no DB transactions will + occur and thus the fmkDB won't be filled during the session. + default_fbk_timeout: If not None, when the project will be run, this value will be used + to initialize the feedback timeout of all the targets + default_fbk_mode: If not None, when the project will be run, this value will be used + to initialize the feedback mode of all the targets + default_sending_delay: If not None, when the project will be run, this value will be used + to initialize the delay that is applied by the framework between each data sending. + default_burst_value: If not None, when the project will be run, this value will be used + to initialize the burst value of the framework (number of data that can be sent in burst + before a delay is applied). + """ - def __init__(self, enable_fbk_processing=True): self.monitor = Monitor() self._knowledge_source = InformationCollector() self._fbk_processing_enabled = enable_fbk_processing self._feedback_processing_thread = None self._fbk_handlers = [] + self._fbk_handlers_disabled = False + + self.wkspace_enabled = wkspace_enabled + self.wkspace_size = wkspace_size + self.wkspace_free_slot_ratio_when_full = wkspace_free_slot_ratio_when_full + self.fmkdb_enabled = fmkdb_enabled + + self.default_fbk_timeout = default_fbk_timeout + self.default_fbk_mode = default_fbk_mode + self.default_sending_delay = default_sending_delay + self.default_burst_value = default_burst_value self.scenario_target_mapping = None self.reset_target_mappings() @@ -78,12 +119,21 @@ def reset_knowledge(self): def register_feedback_handler(self, fbk_handler): self._fbk_handlers.append(fbk_handler) + def disable_feedback_handlers(self): + self._fbk_handlers_disabled = True + + def enable_feedback_handlers(self): + self._fbk_handlers_disabled = False + def notify_data_sending(self, data_list, timestamp, target): + if self._fbk_handlers_disabled: + return + for fh in self._fbk_handlers: fh.notify_data_sending(self.dm, data_list, timestamp, target) def trigger_feedback_handlers(self, source, timestamp, content, status): - if not self._fbk_processing_enabled: + if not self._fbk_processing_enabled or self._fbk_handlers_disabled: return self._feedback_fifo.put((source, timestamp, content, status)) @@ -169,15 +219,17 @@ def register_probe(self, probe, blocking=False): ### Runtime Operations ### ########################## - def start(self): + def share_knowlegde_source(self): VT.knowledge_source = self.knowledge_source Env.knowledge_source = self.knowledge_source DataModel.knowledge_source = self.knowledge_source DataMaker.knowledge_source = self.knowledge_source ScenarioEnv.knowledge_source = self.knowledge_source + def start(self): for fh in self._fbk_handlers: - fh._start() + fh.fmkops = self._fmkops + fh._start(self.dm) if self._fbk_processing_enabled: self._run_fbk_handling_thread = True @@ -188,12 +240,6 @@ def start(self): def stop(self): - VT.knowledge_source = None - Env.knowledge_source = None - DataModel.knowledge_source = None - DataMaker.knowledge_source = None - ScenarioEnv.knowledge_source = None - if self._fbk_processing_enabled: self._run_fbk_handling_thread = False if self._feedback_processing_thread: @@ -203,6 +249,7 @@ def stop(self): for fh in self._fbk_handlers: fh._stop() + def get_operator(self, name): try: ret = self.operators[name] diff --git a/framework/scenario.py b/framework/scenario.py index 6ce9233..b10b995 100644 --- a/framework/scenario.py +++ b/framework/scenario.py @@ -24,129 +24,18 @@ import copy import subprocess import platform +import random + +from typing import Tuple from framework.global_resources import * -from framework.data import Data +from framework.data import Data, DataProcess, EmptyDataProcess, DataAttr, NodeBackend from framework.node import Node from framework.target_helpers import Target from libs.external_modules import * -from libs.utils import find_file, retrieve_app_handler - -class DataProcess(object): - def __init__(self, process, seed=None, auto_regen=False, vtg_ids=None): - """ - Describe a process to generate a data. - - Args: - process (list): List of disruptors (possibly complemented by parameters) to apply to - a ``seed``. However, if the list begin with a generator, the disruptor chain will apply - to the outcome of the generator. The generic form for a process is: - ``[action_1, (action_2, generic_UI_2, specific_UI_2), ... action_n]`` - where ``action_N`` can be either: ``dmaker_type_N`` or ``(dmaker_type_N, dmaker_name_N)`` - seed: (Optional) Can be a registered :class:`framework.data_model.Node` name or - a :class:`framework.data_model.Data`. Will be provided to the first disruptor in - the disruptor chain (described by the parameter ``process``) if it does not begin - with a generator. - auto_regen (boolean): If ``True``, the data process will notify the framework to - rerun the data maker chain after a disruptor has yielded (meaning it is exhausted with - the data that has been provided to it). - It will make the chain going on with new data coming either from the first - non-exhausted disruptor (preceding the exhausted one), or from the generator if - all disruptors are exhausted. If ``False``, the data process won't notify the - framework to rerun the data maker chain, thus triggering the end of the scenario - that embeds this data process. - vtg_ids (list): Virtual ID list of the targets to which the outcomes of this data process will be sent. - If ``None``, the outcomes will be sent to the first target that has been enabled. - """ - self.seed = seed - self.auto_regen = auto_regen - self.auto_regen_cpt = 0 - self.outcomes = None - self.feedback_timeout = None - self.feedback_mode = None - self.vtg_ids = vtg_ids - self._process = [process] - self._process_idx = 0 - self._blocked = False - - def append_new_process(self, process): - """ - Append a new process to the list. - """ - self._process.append(process) - - def next_process(self): - if self._process_idx == len(self._process) - 1: - self._process_idx = 0 - return False - else: - self._process_idx += 1 - return True - - def reset(self): - self.auto_regen_cpt = 0 - self.outcomes = None - self._process_idx = 0 - - @property - def process(self): - return self._process[self._process_idx] - - @process.setter - def process(self, value): - self._process[self._process_idx] = value - - @property - def process_qty(self): - return len(self._process) - - def make_blocked(self): - self._blocked = True - if self.outcomes is not None: - self.outcomes.make_blocked() - - def make_free(self): - self._blocked = False - if self.outcomes is not None: - self.outcomes.make_free() - - def formatted_str(self, oneliner=False): - desc = '' - suffix = ', proc=' if oneliner else '\n' - if isinstance(self.seed, str): - desc += "seed='" + self.seed + "'" + suffix - elif isinstance(self.seed, Data): - if isinstance(self.seed.content, Node): - seed_str = self.seed.content.name - else: - seed_str = "Data('{!s}'...)".format(self.seed.to_str()[:10]) - desc += "seed='{:s}'".format(seed_str) + suffix - else: - desc += suffix[2:] - - for proc in self._process: - for d in proc: - if isinstance(d, (list, tuple)): - desc += '{!s}/'.format(d[0]) - else: - assert isinstance(d, str) - desc += '{!s}/'.format(d) - desc = desc[:-1] - desc += ',' if oneliner else '\n' - desc = desc[:-1] # if oneliner else desc[:-1] - - return desc - - def __repr__(self): - return self.formatted_str(oneliner=True) - - def __copy__(self): - new_datap = type(self)(self.process) - new_datap.__dict__.update(self.__dict__) - new_datap._process = copy.copy(self._process) - new_datap.reset() - return new_datap +from libs.utils import find_file, retrieve_app_handler, Task +data_graph_desc_fstr = "Data('{!a}'...)" class Periodic(object): def __init__(self, data, period=None, vtg_ids=None): @@ -167,9 +56,9 @@ def __str__(self): desc += 'DP({:s})'.format(d.formatted_str(oneliner=True)) elif isinstance(d, Data): if isinstance(d.content, Node): - desc += d.content.name + desc += d.content.name.upper() else: - desc += "Data('{!s}'...)".format(d.to_str()[:10]) + desc += data_graph_desc_fstr.format(d.to_str()[:10]) if d.description is None else f'"{d.description}"' elif isinstance(d, str): desc += "{:s}".format(d.upper()) else: @@ -185,14 +74,15 @@ def __str__(self): return desc - class Step(object): def __init__(self, data_desc=None, final=False, - fbk_timeout=None, fbk_mode=None, + fbk_timeout=None, fbk_mode=None, sending_delay=None, set_periodic=None, clear_periodic=None, step_desc=None, + start_tasks=None, stop_tasks=None, do_before_data_processing=None, do_before_sending=None, - valid=True, vtg_ids=None): + valid=True, vtg_ids=None, + refresh_atoms=True): """ Step objects are the building blocks of Scenarios. @@ -211,10 +101,26 @@ def __init__(self, data_desc=None, final=False, If ``None``, the outcomes will be sent to the first target that has been enabled. If ``data_desc`` is a list, this parameter should be a list where each item is the ``vtg_ids`` of the corresponding item in the ``data_desc`` list. + transition_on_dp_complete (bool): + this attribute is set + to ``True`` by the framework. + refresh_atoms (bool): if set to `True` atoms described by names in `data_desc` will be re-instanced + each time the step is entered. + """ self.final = final self.valid = valid + + self.data_attrs = DataAttr() + + # In the context of a step hosting a DataProcess, if the latter is completed, meaning that all + # the registered processes are exhausted (data makers have yielded), then if a transition for this + # condition has been defined (this attribute will be set to True), the scenario will walk through it. + self.transition_on_dp_complete = False + + self.refresh_atoms = refresh_atoms + self._step_desc = step_desc self._transitions = [] self._do_before_data_processing = do_before_data_processing @@ -236,6 +142,7 @@ def __init__(self, data_desc=None, final=False, # need to be set after self._data_desc self.feedback_timeout = fbk_timeout self.feedback_mode = fbk_mode + self.sending_delay = sending_delay self._scenario_env = None self._periodic_data = list(set_periodic) if set_periodic else None @@ -246,6 +153,17 @@ def __init__(self, data_desc=None, final=False, else: self._periodic_data_to_remove = None + self._tasks = list(start_tasks) if start_tasks else None + if stop_tasks: + self._tasks_to_stop = [] + for t in stop_tasks: + self._tasks_to_stop.append(id(t)) + else: + self._tasks_to_stop = None + + self._stutter_cpt = None + self._stutter_max = None + def _handle_data_desc(self, data_desc): self._atom = None @@ -261,7 +179,7 @@ def _handle_data_desc(self, data_desc): self._data_desc = [data_desc] for desc in self._data_desc: - assert isinstance(desc, (str, Data, DataProcess)), '{!r}, class:{:s}'.format(desc, self.__class__.__name__) + assert isinstance(desc, (str, Data, DataProcess, EmptyDataProcess)), '{!r}, class:{:s}'.format(desc, self.__class__.__name__) if isinstance(data_desc, str): self._node_name = [data_desc] @@ -278,24 +196,40 @@ def _handle_data_desc(self, data_desc): def set_scenario_env(self, env): self._scenario_env = env - def connect_to(self, step, cbk_after_sending=None, cbk_after_fbk=None, prepend=False): + def connect_to(self, obj, dp_completed_guard=False, cbk_after_sending=None, cbk_after_fbk=None, + prepend=False, description=None): if isinstance(self, NoDataStep): assert cbk_after_sending is None - tr = Transition(step, - cbk_after_sending=cbk_after_sending, - cbk_after_fbk=cbk_after_fbk) - if prepend: - self._transitions.insert(0, tr) + + if dp_completed_guard: + assert cbk_after_sending is None and cbk_after_fbk is None + self.transition_on_dp_complete = True + self._transitions.insert(0, Transition(obj, dp_completed_guard=dp_completed_guard, + description=description)) + else: - self._transitions.append(tr) + tr = Transition(obj, + cbk_after_sending=cbk_after_sending, + cbk_after_fbk=cbk_after_fbk, + description=description) + if prepend: + self._transitions.insert(0, tr) + else: + self._transitions.append(tr) def do_before_data_processing(self): + if self.refresh_atoms: + self._atom = None + if self._do_before_data_processing is not None: self._do_before_data_processing(self._scenario_env, self) def do_before_sending(self): if self._do_before_sending is not None: self._do_before_sending(self._scenario_env, self) + return True + else: + return False def make_blocked(self): self._blocked = True @@ -309,9 +243,61 @@ def make_free(self): if isinstance(d, (Data, DataProcess)): d.make_free() + def _stutter_cbk(self, env, current_step, next_step): + if self._stutter_cpt == 1 and self._rd_count_range: + self._stutter_max = random.randint(self._rd_count_range[0], self._rd_count_range[1]) + self._stutter_cpt += 1 + if self._stutter_fbk_timeout_range: + min, max = self._stutter_fbk_timeout_range + next_step.feedback_timeout = random.random() * (max-min) + min + if self._stutter_cpt > self._stutter_max: + self._stutter_cpt = 1 + return False + else: + return True + + def make_stutter(self, count=None, rd_count_range: Tuple[int, int] = None, + fbk_timeout_range: Tuple[float, float] = None): + """ + Further to this call, a step is connected to itself with a guard enabling looping on the + step for a number of time: either @count times or a random value within @rd_count_range. + + Args: + count: number of loops. + rd_count_range: number of loops is determined randomly + within the bounds provided by this parameter. + fbk_timeout_range: feedback timeout is chosen randomly within the bounds + provided by this parameter. + + """ + assert bool(count is None) ^ bool(rd_count_range is None) + self._stutter_cpt = 1 + self._stutter_max = count + self._rd_count_range = rd_count_range + self._stutter_fbk_timeout_range = fbk_timeout_range + if count is not None: + desc_str = f'Loop {count} times' if count > 1 else f'Loop once' + else: + desc_str = f'Loop randomly between {rd_count_range[0]} and {rd_count_range[1]} times' + self.connect_to(self, cbk_after_sending=self._stutter_cbk, description=desc_str) + def is_blocked(self): return self._blocked + def set_dmaker_reset(self): + """ + Request the framework to reset the data makers involved + in the step before processing them. + Relevant only when DataProcess are in use. + """ + self.data_attrs.set(DataAttr.Reset_DMakers) + + def clear_dmaker_reset(self): + """ + Restore the state changed by .set_dmaker_reset() + """ + self.data_attrs.clear(DataAttr.Reset_DMakers) + def cleanup(self): for d in self._data_desc: if isinstance(d, DataProcess): @@ -329,6 +315,17 @@ def has_dataprocess(self): else: return False + @property + def sending_delay(self): + return self._sending_delay + + @sending_delay.setter + def sending_delay(self, delay): + self._sending_delay = delay + for d in self._data_desc: + if isinstance(d, (Data, DataProcess)): + d.sending_delay = delay + @property def feedback_timeout(self): return self._feedback_timeout @@ -387,9 +384,9 @@ def content(self): atom_list.append(None) else: atom_list.append(None) - elif isinstance(d, Data): + elif isinstance(d, Data) and self._node_name[idx] is None: atom_list.append(d.content if isinstance(d.content, Node) else None) - elif isinstance(d, Data) or self._node_name[idx] is None: + elif self._node_name[idx] is None: # that means that a data creation process has been registered and will be # carried out by the framework through a callback atom_list.append(None) @@ -400,7 +397,6 @@ def content(self): if update_node: self._atom[idx] = self._scenario_env.dm.get_atom(self._node_name[idx]) self._data_desc[idx] = Data(self._atom[idx]) - update_node = False atom_list.append(self._atom[idx]) return atom_list[0] if len(atom_list) == 1 else atom_list @@ -408,9 +404,9 @@ def content(self): @content.setter def content(self, atom_list): if isinstance(atom_list, list): - self._data_desc = atom_list + self._data_desc = [Data(a) for a in atom_list] if isinstance(atom_list, Node): - self._data_desc = [atom_list] + self._data_desc = [Data(atom_list)] else: raise ValueError @@ -426,6 +422,12 @@ def get_data(self): # in this case a data creation process is provided to the framework through the # callback HOOK.before_sending_step1 d = Data('STEP:POISON_1') + + name = node_list.name if isinstance(node_list, Node) else None + if name is not None: + gen_info = [name.upper(), 'g_' + name, None] + d.set_initial_dmaker(gen_info) + else: # In this case we have multiple data # Practically it means that the creation of these data need to be performed @@ -439,11 +441,10 @@ def get_data(self): for idx, d_desc in enumerate(self._data_desc): if isinstance(d_desc, DataProcess): - d.add_info(repr(d_desc)) - elif isinstance(d_desc, Data): + d.add_info('DP({:s})'.format(d_desc.formatted_str(oneliner=True))) + elif isinstance(d_desc, Data) and not d_desc.has_node_content(): d.add_info('User-provided Data()') else: - assert isinstance(d_desc, str) d.add_info("Data Model: '{!s}'" .format(self._scenario_env.dm.name)) d.add_info("Node Name: '{!s}'" @@ -464,6 +465,10 @@ def get_data(self): d.feedback_timeout = self._feedback_timeout if self._feedback_mode is not None: d.feedback_mode = self._feedback_mode + if self._sending_delay is not None: + d.sending_delay = self._sending_delay + + d.set_attributes_from(self.data_attrs) if self.vtg_ids_list: # Note in the case of self._data_desc contains multiple data, related @@ -496,27 +501,48 @@ def periodic_to_clear(self): for pid in self._periodic_data_to_remove: yield pid + @property + def tasks_to_start(self): + if self._tasks is None: + return + else: + for t in self._tasks: + yield t + + @property + def tasks_to_stop(self): + if self._tasks_to_stop is None: + return + else: + for tid in self._tasks_to_stop: + yield tid + def set_transitions(self, transitions): self._transitions = transitions - def __str__(self): + def get_desc(self, oneliner=True): if self._step_desc: step_desc = self._step_desc else: step_desc = '' for idx, d in enumerate(self._data_desc): if isinstance(d, DataProcess): - step_desc += 'DP({:s})'.format(d.formatted_str(oneliner=True)) + step_desc += 'DP({:s})'.format(d.formatted_str(oneliner=oneliner)) elif isinstance(d, Data): if self.__class__.__name__ != 'Step': step_desc += '[' + self.__class__.__name__ + ']' else: - step_desc += d.content.name.upper() if isinstance(d.content, Node) else "Data('{!s}'...)".format(d.to_str()[:10]) + if isinstance(d.content, Node): + step_desc += d.content.name.upper() + else: + step_desc += data_graph_desc_fstr.format(d.to_str()[:10]) if d.description is None else f'"{d.description}"' elif isinstance(d, str): step_desc += "{:s}".format(self._node_name[idx].upper()) + elif isinstance(d, EmptyDataProcess): + step_desc += 'DP(not defined yet)' else: - assert d is None + assert d is None, f'incorrect object: {d}' step_desc += '[' + self.__class__.__name__ + ']' vtgids_str = ' -(vtg)-\> {:s}'.format(str(self.vtg_ids_list[idx])) if self.vtg_ids_list is not None else '' step_desc += vtgids_str + '\n' @@ -524,9 +550,12 @@ def __str__(self): return step_desc - def get_description(self): + def __str__(self): + return self.get_desc(oneliner=True) + + def get_full_description(self, oneliner=True): # Note the provided string is dot/graphviz oriented. - step_desc = str(self).replace('\n', '\\n') # for graphviz display in 'record' boxes + step_desc = self.get_desc(oneliner=oneliner).replace('\n', '\\n') # for graphviz display in 'record' boxes if self._do_before_sending is not None or self._do_before_data_processing is not None: if self._do_before_data_processing is None: @@ -544,12 +573,21 @@ def get_description(self): step_desc = step_desc + '|{{{:s}|{:s}}}'.format(cbk_before_dataproc_str, cbk_before_sending_str) fbk_mode = None if self.feedback_mode is None else Target.get_fbk_mode_desc(self.feedback_mode, short=True) + delay = f'|sending delay: {self.sending_delay}s' if self.sending_delay is not None else '' if self.feedback_timeout is not None and self.feedback_mode is not None: - step_desc = '{{fbk timeout {!s}s|{:s}}}|{:s}'.format(self.feedback_timeout, fbk_mode, step_desc) + step_desc = f'{{fbk timeout: {self.feedback_timeout}s|fbk mode: {fbk_mode}{delay}}}|{step_desc}' elif self.feedback_timeout is not None: - step_desc = 'fbk timeout\\n{!s}s|{:s}'.format(self.feedback_timeout, step_desc) + if self.sending_delay is not None: + step_desc = f'{{fbk timeout: {self.feedback_timeout}s{delay}}}|{step_desc}' + else: + step_desc = f'fbk timeout\\n{self.feedback_timeout}s|{step_desc}' elif self.feedback_mode is not None: - step_desc = 'fbk mode\\n{:s}|{:s}'.format(fbk_mode, step_desc) + if self.sending_delay is not None: + step_desc = f'{{fbk mode: {fbk_mode}{delay}}}|{step_desc}' + else: + step_desc = f'fbk mode\\n{fbk_mode}|{step_desc}' + elif self.sending_delay is not None: + step_desc = f'sending delay\\n{self.sending_delay}s|{step_desc}' else: pass @@ -561,6 +599,12 @@ def is_periodic_set(self): def is_periodic_cleared(self): return bool(self._periodic_data_to_remove) + def has_tasks_to_start(self): + return bool(self._tasks) + + def has_tasks_to_stop(self): + return bool(self._tasks_to_stop) + def get_periodic_description(self): # Note the provided string is dot/graphviz oriented. if self.is_periodic_set() or self.is_periodic_cleared(): @@ -577,6 +621,22 @@ def get_periodic_description(self): else: return 'No periodic to set' + def get_tasks_description(self): + # Note the provided string is dot/graphviz oriented. + if self.has_tasks_to_start() or self.has_tasks_to_stop(): + desc = '{' + if self.has_tasks_to_start(): + for t in self.tasks_to_start: + desc += 'START Task [{:s}]\l [{:s}]\l|'.format(str(id(t))[-6:], str(t)) + + if self.has_tasks_to_stop(): + for t in self.tasks_to_stop: + desc += 'STOP Task [{:s}]\l|'.format(str(t)[-6:]) + desc = desc[:-1] + '}' + return desc + else: + return 'No tasks to start' + def get_periodic_ref(self): if self.is_periodic_set(): ref = id(self._periodic_data) @@ -587,6 +647,16 @@ def get_periodic_ref(self): return ref + def get_tasks_ref(self): + if self.has_tasks_to_start(): + ref = id(self._tasks) + elif self.has_tasks_to_stop(): + ref = id(self._tasks_to_stop) + else: + ref = None + + return ref + def __hash__(self): return id(self) @@ -599,37 +669,61 @@ def __copy__(self): # their IDs (memory addr) are used for registration and cancellation new_step._periodic_data = copy.copy(self._periodic_data) new_step._periodic_data_to_remove = copy.copy(self._periodic_data_to_remove) + new_step._tasks = copy.copy(self._tasks) + new_step._tasks_to_stop = copy.copy(self._tasks_to_stop) + new_step._periodic_data_to_remove = copy.copy(self._periodic_data_to_remove) new_step._scenario_env = None # we ignore the environment, a new one will be provided new_step._transitions = copy.copy(self._transitions) + new_step.data_attrs = copy.copy(self.data_attrs) return new_step class FinalStep(Step): - def __init__(self, data_desc=None, final=False, fbk_timeout=None, fbk_mode=None, + def __init__(self, data_desc=None, final=False, fbk_timeout=None, fbk_mode=None, sending_delay=None, set_periodic=None, clear_periodic=None, step_desc=None, - do_before_data_processing=None, valid=True, vtg_ids=None): + start_tasks=None, stop_tasks=None, + do_before_data_processing=None, do_before_sending=None, valid=True, vtg_ids=None): Step.__init__(self, final=True, do_before_data_processing=do_before_data_processing, + do_before_sending=do_before_sending, valid=valid, vtg_ids=vtg_ids) class NoDataStep(Step): - def __init__(self, data_desc=None, final=False, fbk_timeout=None, fbk_mode=None, + def __init__(self, data_desc=None, final=False, fbk_timeout=None, fbk_mode=None, sending_delay=None, set_periodic=None, clear_periodic=None, step_desc=None, - do_before_data_processing=None, valid=True, vtg_ids=None): + start_tasks=None, stop_tasks=None, + do_before_data_processing=None, do_before_sending=None, valid=True, vtg_ids=None): Step.__init__(self, data_desc=Data(''), final=final, - fbk_timeout=fbk_timeout, fbk_mode=fbk_mode, + fbk_timeout=fbk_timeout, fbk_mode=fbk_mode, sending_delay=sending_delay, set_periodic=set_periodic, clear_periodic=clear_periodic, + start_tasks=start_tasks, stop_tasks=stop_tasks, step_desc=step_desc, do_before_data_processing=do_before_data_processing, + do_before_sending=do_before_sending, valid=valid, vtg_ids=vtg_ids) self.make_blocked() def make_free(self): pass +class StepStub(Step): + def __init__(self, data_desc=None, final=False, fbk_timeout=None, fbk_mode=None, sending_delay=None, + set_periodic=None, clear_periodic=None, step_desc=None, + start_tasks=None, stop_tasks=None, + do_before_data_processing=None, do_before_sending=None, valid=True, vtg_ids=None): + Step.__init__(self, data_desc=EmptyDataProcess(), final=final, + fbk_timeout=fbk_timeout, fbk_mode=fbk_mode, sending_delay=sending_delay, + set_periodic=set_periodic, clear_periodic=clear_periodic, + start_tasks=start_tasks, stop_tasks=stop_tasks, + step_desc=step_desc, do_before_data_processing=do_before_data_processing, + do_before_sending=do_before_sending, + valid=valid, vtg_ids=vtg_ids) + class Transition(object): - def __init__(self, step, cbk_after_sending=None, cbk_after_fbk=None): + def __init__(self, obj, dp_completed_guard=False, cbk_after_sending=None, cbk_after_fbk=None, + description=None): self._scenario_env = None - self._step = step + self._obj = obj + self.dp_completed_guard = dp_completed_guard self._callbacks = {} if cbk_after_sending: self._callbacks[HOOK.after_sending] = cbk_after_sending @@ -637,20 +731,32 @@ def __init__(self, step, cbk_after_sending=None, cbk_after_fbk=None): self._callbacks[HOOK.after_fbk] = cbk_after_fbk self._callbacks_qty = self._callbacks_pending = len(self._callbacks) + self._description = description + self._invert_conditions = False self._crossable = True @property def step(self): - return self._step + if isinstance(self._obj, Step): + return self._obj + elif isinstance(self._obj, Scenario): + return self._obj.anchor + else: + raise NotImplementedError @step.setter def step(self, value): - self._step = value + self._obj = value - def set_scenario_env(self, env): + def set_scenario_env(self, env, merge_user_contexts: bool = True): self._scenario_env = env - self._step.set_scenario_env(env) + if isinstance(self._obj, Step): + self._obj.set_scenario_env(env) + elif isinstance(self._obj, Scenario): + self._obj.set_scenario_env(env, merge_user_contexts=merge_user_contexts) + else: + raise NotImplementedError def register_callback(self, callback, hook=HOOK.after_fbk): assert isinstance(hook, HOOK) @@ -697,10 +803,15 @@ def is_crossable(self): return self._crossable def __str__(self): - desc = '' - for k, v in self._callbacks.items(): - desc += str(k) + '\n' + v.__name__ + '()\n' - desc = desc[:-1] + if self.dp_completed_guard: + desc = 'DP completed?' + elif self._description is not None: + desc = self._description + else: + desc = '' + for k, v in self._callbacks.items(): + desc += str(k) + '\n' + v.__name__ + '()\n' + desc = desc[:-1] return desc @@ -708,7 +819,7 @@ def __hash__(self): return id(self) def __copy__(self): - new_transition = type(self)(self._step) + new_transition = type(self)(self._obj) new_transition.__dict__.update(self.__dict__) new_transition._callbacks = copy.copy(self._callbacks) new_transition._callbacks_qty = new_transition._callbacks_pending @@ -759,14 +870,6 @@ def user_context(self): def user_context(self, val): self._context = val - # @property - # def knowledge_source(self): - # return self._knowledge_source - # - # @knowledge_source.setter - # def knowledge_source(self, val): - # self._knowledge_source = val - def __copy__(self): new_env = type(self)() new_env.__dict__.update(self.__dict__) @@ -785,8 +888,21 @@ def __copy__(self): class Scenario(object): - def __init__(self, name, anchor=None, reinit_anchor=None, user_context=None): + def __init__(self, name, anchor=None, reinit_anchor=None, user_context=None, + user_args=None): + """ + Note: only at copy the ScenarioEnv are propagated to the steps and transitions + + Args: + name: + anchor: + reinit_anchor: + user_context: + user_args: + """ + self.name = name + self._user_args = user_args self._steps = None self._reinit_steps = None self._transitions = None @@ -796,6 +912,7 @@ def __init__(self, name, anchor=None, reinit_anchor=None, user_context=None): self._env.scenario = self self._env.user_context = user_context self._periodic_ids = set() + self._task_ids = set() self._current = None self._anchor = None self._reinit_anchor = None @@ -815,9 +932,17 @@ def clone(self, new_name): def reset(self): self._current = self._anchor - def set_user_context(self, user_context): + @property + def user_context(self): + return self._env.user_context + + @user_context.setter + def user_context(self, user_context): self._env.user_context = user_context + def merge_user_context_with(self, user_context): + self._env.user_context.merge_with(user_context) + def set_data_model(self, dm): self._dm = dm self._env.dm = dm @@ -825,21 +950,15 @@ def set_data_model(self, dm): def set_target(self, target): self._env.target = target - # @property - # def knowledge_source(self): - # return self._env.knowledge_source - # - # @knowledge_source.setter - # def knowledge_source(self, val): - # self._env.knowledge_source = val - def _graph_setup(self, init_step, steps, transitions): for tr in init_step.transitions: transitions.append(tr) + tr.set_scenario_env(self.env) if tr.step in steps: continue else: steps.append(tr.step) + tr.step.set_scenario_env(self.env) self._graph_setup(tr.step, steps, transitions) def _init_main_properties(self): @@ -873,6 +992,23 @@ def set_reinit_anchor(self, reinit_anchor): def env(self): return self._env + def set_scenario_env(self, env: ScenarioEnv, merge_user_contexts: bool = True): + """ + + Args: + env: + merge_user_contexts: the new env will have a user_context that is the merging of + the current one and the one provided through the new env. + In case some parameter names overlaps, the new values are kept. + + """ + if merge_user_contexts: + self._env.user_context.merge_with(env.user_context) + env.user_context = self._env.user_context + + self._env = env + self._init_main_properties() + @property def steps(self): if self._steps is None: @@ -928,6 +1064,11 @@ def periodic_to_clear(self): for pid in self._periodic_ids: yield pid + @property + def tasks_to_stop(self): + for tid in self._task_ids: + yield tid + def _view_linux(self, filepath, graph_filename): """Open filepath in the user's preferred application (linux).""" @@ -972,11 +1113,22 @@ def graph_periodic(step, node_list): id_node = str(id(step)) id_periodic = str(step.get_periodic_ref()) graph.node(id_periodic, label=step.get_periodic_description(), - shape='record', style='filled', color='black', fillcolor='palegreen', - fontcolor='black', fontsize='8') + shape='record', style='filled', color='black', fillcolor='palegreen', + fontcolor='black', fontsize='8') node_list.append(step.get_periodic_ref()) graph.edge(id_node, id_periodic, arrowhead='dot') # headport='se', tailport='nw') + def graph_tasks(step, node_list): + if (step.has_tasks_to_start() or step.has_tasks_to_stop()) \ + and step.get_tasks_ref() not in node_list: + id_node = str(id(step)) + id_tasks = str(step.get_tasks_ref()) + graph.node(id_tasks, label=step.get_tasks_description(), + shape='record', style='filled', color='black', fillcolor='palegreen', + fontcolor='black', fontsize='8') + node_list.append(step.get_tasks_ref()) + graph.edge(id_node, id_tasks, arrowhead='dot') # headport='se', tailport='nw') + step_color = current_color if init_step is current_step else 'black' if init_step.final or init_step is self._anchor: step_fillcolor = current_fillcolor if init_step is current_step else 'slategray' @@ -990,8 +1142,9 @@ def graph_periodic(step, node_list): step_fontcolor = current_fontcolor if init_step is current_step else 'black' graph.attr('node', fontcolor=step_fontcolor, shape='record', style=step_style, color=step_color, fillcolor=step_fillcolor) - graph.node(str(id(init_step)), label=init_step.get_description()) + graph.node(str(id(init_step)), label=init_step.get_full_description(oneliner=False)) graph_periodic(init_step, node_list) + graph_tasks(init_step, node_list) for idx, tr in enumerate(init_step.transitions): if tr.step not in node_list: step_color = current_color if tr.step is current_step else 'black' @@ -1006,8 +1159,9 @@ def graph_periodic(step, node_list): step_style = 'rounded,dotted' if isinstance(tr.step, NoDataStep) else 'rounded,filled' graph.attr('node', fontcolor=step_fontcolor, shape='record', style=step_style, fillcolor=step_fillcolor, color=step_color) - graph.node(str(id(tr.step)), label=tr.step.get_description()) + graph.node(str(id(tr.step)), label=tr.step.get_full_description(oneliner=False)) graph_periodic(tr.step, node_list) + graph_tasks(tr.step, node_list) if id(tr) not in edge_list: graph.edge(str(id(init_step)), str(id(tr.step)), label='[{:d}] {!s}'.format(idx+1, tr)) edge_list.append(id(tr)) @@ -1040,7 +1194,10 @@ def graph_periodic(step, node_list): uctxt_desc = '{' uinputs = self.env.user_context.get_inputs() for k, v in uinputs.items(): - uctxt_desc += '{:s} = {!r}\l|'.format(k, v) + v = f'{v!s}' + v = v.replace('{', '\{') + v = v.replace('}', '\}') + uctxt_desc += '{:s} = {:s}\l|'.format(k, v) uctxt_desc = uctxt_desc[:-1] + '}' else: uctxt_desc = str(self.env.user_context) @@ -1071,6 +1228,8 @@ def graph_copy(init_step, dico, env): init_step.set_transitions(new_transitions) for periodic in init_step.periodic_to_set: new_sc._periodic_ids.add(id(periodic)) + for task in init_step.tasks_to_start: + new_sc._task_ids.add(id(task)) for tr in init_step.transitions: tr.set_scenario_env(env) @@ -1090,6 +1249,7 @@ def graph_copy(init_step, dico, env): new_sc._env = copy.copy(self._env) new_sc._env.scenario = new_sc new_sc._periodic_ids = set() # periodic ids are gathered only during graph_copy() + new_sc._task_ids = set() # task ids are gathered only during graph_copy() if self._current is self._anchor: new_current = new_anchor = copy.copy(self._current) else: diff --git a/framework/tactics_helpers.py b/framework/tactics_helpers.py index 45bae3b..a9c670b 100644 --- a/framework/tactics_helpers.py +++ b/framework/tactics_helpers.py @@ -31,6 +31,7 @@ from framework.data import * from framework.global_resources import * import framework.scenario as sc +import framework.node as nd DEBUG = False @@ -38,6 +39,7 @@ XT_CLS_LIST_K = 2 XT_WEIGHT_K = 3 XT_VALID_CLS_LIST_K = 4 +XT_RELATED_DM = 5 class Tactics(object): @@ -47,16 +49,23 @@ def __init__(self): self.disruptor_clones = {} self.generator_clones = {} self._fmkops = None - # self._knowledge_source = None + self._related_dm = None - def set_exportable_fmk_ops(self, fmkops): + def set_additional_info(self, fmkops, related_dm=None): self._fmkops = fmkops + self._related_dm = related_dm for dtype in self.generator_types: + self.generators[dtype][XT_RELATED_DM] = related_dm for name, attrs in self.get_generators_list(dtype).items(): attrs['obj'].set_exportable_fmk_ops(fmkops) + if self._related_dm: + attrs['obj'].related_dm_name = self._related_dm.name for dtype in self.disruptor_types: + self.disruptors[dtype][XT_RELATED_DM] = related_dm for name, attrs in self.get_disruptors_list(dtype).items(): attrs['obj'].set_exportable_fmk_ops(fmkops) + if self._related_dm: + attrs['obj'].related_dm_name = self._related_dm.name @staticmethod def scenario_ref_from(scenario): @@ -78,16 +87,22 @@ def __register_new_data_maker(self, dict_var, name, obj, weight, dmaker_type, va dict_var[dmaker_type][XT_CLS_LIST_K] = {} dict_var[dmaker_type][XT_WEIGHT_K] = 0 dict_var[dmaker_type][XT_VALID_CLS_LIST_K] = {} + dict_var[dmaker_type][XT_RELATED_DM] = self._related_dm if name in dict_var[dmaker_type][XT_NAME_LIST_K]: print("\n*** /!\\ ERROR: The name '%s' is already used for the dmaker_type '%s'\n" % \ (name, dmaker_type)) raise ValueError + if self._fmkops is not None: + obj.set_exportable_fmk_ops(self._fmkops) + if self._related_dm is not None: + obj.related_dm_name = self._related_dm.name + dict_var[dmaker_type][XT_NAME_LIST_K][name] = { 'obj': obj, 'weight': weight, - 'valid': False + 'valid': False, } dict_var[dmaker_type][XT_CLS_LIST_K][obj] = name @@ -99,9 +114,6 @@ def __register_new_data_maker(self, dict_var, name, obj, weight, dmaker_type, va dict_var[dmaker_type][XT_VALID_CLS_LIST_K][name] = \ dict_var[dmaker_type][XT_NAME_LIST_K][name] - if self._fmkops is not None: - obj.set_exportable_fmk_ops(self._fmkops) - # obj.knowledge_source = self.knowledge_source def register_new_disruptor(self, name, obj, weight, dmaker_type, valid=False): self.__register_new_data_maker(self.disruptors, name, obj, @@ -197,6 +209,13 @@ def get_generators_list(self, dmaker_type): return ret + def generators_info(self): + for gen_type, attrs in self.generators.items(): + yield gen_type, attrs[XT_RELATED_DM] + + def disruptors_info(self): + for dis_type, attrs in self.disruptors.items(): + yield dis_type, attrs[XT_RELATED_DM] def get_disruptor_weight(self, dmaker_type, name): try: @@ -403,6 +422,9 @@ def _handle_user_inputs(dmaker, user_input): assert(type(ui_val) in arg_type or ui_val is None) elif isinstance(arg_type, type): assert(type(ui_val) == arg_type or issubclass(type(ui_val), arg_type) or ui_val is None) + elif arg_type is None: + # we ignore type verification + pass else: raise ValueError if ui_val is None: @@ -425,23 +447,22 @@ def _restore_dmaker_internals(dmaker): ################################ GENERIC_ARGS = { - 'init': ('make the model walker ignore all the steps until the provided one', 1, int), - 'max_steps': ('maximum number of steps (-1 means until the end)', -1, int), - 'runs_per_node': ('maximum number of test cases for a single node (-1 means until the end)', -1, int), - 'clone_node': ('if True the dmaker will always return a copy ' \ + 'init': ('Make the model walker ignore all the steps until the provided one', 1, int), + 'max_steps': ('Maximum number of steps (-1 means until the end)', -1, int), + 'min_node_tc': ('Minimum number of test cases per node (-1 means until the end)', -1, int), + 'max_node_tc': ('Maximum number of test cases per node (-1 means until the end). This value is ' + 'used for nodes with a fuzz weight strictly greater than 1.', -1, int), + 'clone_node': ('If True, this operator will always return a copy ' \ 'of the node. (for stateless diruptors dealing with ' \ - 'big data it can be usefull to it to False)', True, bool) + 'big data it can be usefull to set it to False)', True, bool) } def modelwalker_inputs_handling_helper(dmaker): - assert(dmaker.runs_per_node > 0 or dmaker.runs_per_node == -1) + assert dmaker.max_node_tc > 0 or dmaker.max_node_tc == -1 + assert dmaker.min_node_tc > 0 or dmaker.min_node_tc == -1 - if dmaker.runs_per_node == -1: - dmaker.max_runs_per_node = -1 - dmaker.min_runs_per_node = -1 - else: - dmaker.max_runs_per_node = dmaker.runs_per_node + 3 - dmaker.min_runs_per_node = max(dmaker.runs_per_node - 2, 1) + dmaker.max_runs_per_node = dmaker.max_node_tc + dmaker.min_runs_per_node = dmaker.min_node_tc ### Generator & Disruptor @@ -456,10 +477,10 @@ class DataMaker(object): knowledge_source = None _modelwalker_user = False _args_desc = None + related_dm_name = None def __init__(self): self._fmkops = None - # self._knowledge_source = None def set_exportable_fmk_ops(self, fmkops): self._fmkops = fmkops @@ -542,7 +563,6 @@ def generate_data(self, dm, monitor, target): class dyn_generator(type): data_id = '' - def __init__(cls, name, bases, attrs): attrs['_args_desc'] = DynGenerator._args_desc type.__init__(cls, name, bases, attrs) @@ -552,15 +572,21 @@ def __init__(cls, name, bases, attrs): class DynGenerator(Generator): data_id = '' _args_desc = { - 'finite': ('make the data model finite', False, bool), - 'determinist': ('make the data model determinist', False, bool), - 'random': ('make the data model random', False, bool) + 'finite': ('Make the data model finite', False, bool), + 'determinist': ("Make the data model determinist if set to 'True', random if set to " + "'False', or do nothing if set to 'None'", None, bool), + 'leaf_determinism': ("If set to 'True', all the typed nodes of the model will be " + "set to determinist mode prior to any fuzzing. If set " + "to 'False', they will be set to random mode. " + "Otherwise, if set to 'None', nothing will be done.", None, bool), + 'min_def': ("Set the default quantity of all the nodes to the defined minimum quantity if " + "this parameter is set to 'True', or maximum quantity if set to 'False'. " + "Otherwise if set to 'None', nothing is done.", None, bool), + 'freeze': ("Freeze the generated node.", False, bool), + 'resolve_csp': ("Resolve any CSP if any", True, bool) } def setup(self, dm, user_input): - if self.determinist or self.random: - assert(self.random != self.determinist) - return True def generate_data(self, dm, monitor, target): @@ -568,20 +594,49 @@ def generate_data(self, dm, monitor, target): if isinstance(atom, Node): if self.finite: atom.make_finite(all_conf=True, recursive=True) - if self.determinist: + + if self.determinist is None: + pass + elif self.determinist: atom.make_determinist(all_conf=True, recursive=True) - if self.random: + else: atom.make_random(all_conf=True, recursive=True) + if self.leaf_determinism is not None: + nic = nd.NodeInternalsCriteria(node_kinds=[nd.NodeInternals_TypedValue]) + nl = atom.get_reachable_nodes(internals_criteria=nic, ignore_fstate=True) + for n in nl: + if not n.is_attr_set(nd.NodeInternals.Mutable): + continue + if self.leaf_determinism: + n.make_determinist() + else: + n.make_random() + + if self.min_def is not None: + nic = nd.NodeInternalsCriteria(node_kinds=[nd.NodeInternals_NonTerm]) + nl = atom.get_reachable_nodes(internals_criteria=nic, ignore_fstate=True) + for node in nl: + subnodes = node.subnodes_set + for snd in subnodes: + min, max = node.get_subnode_minmax(snd) + node.set_subnode_default_qty(snd, min if self.min_def else max) + + if self.freeze: + atom.freeze(resolve_csp=self.resolve_csp) + return Data(atom) class dyn_generator_from_scenario(type): scenario = None - def __init__(cls, name, bases, attrs): - attrs['_args_desc'] = DynGeneratorFromScenario._args_desc - type.__init__(cls, name, bases, attrs) - cls.scenario = dyn_generator_from_scenario.scenario + def __new__(cls, name, bases, attrs): + attrs['_args_desc'] = copy.copy(DynGeneratorFromScenario._args_desc) + if dyn_generator_from_scenario.scenario._user_args: + attrs['_args_desc'].update(dyn_generator_from_scenario.scenario._user_args) + cls_obj = type(name, bases, attrs) + cls_obj.scenario = dyn_generator_from_scenario.scenario + return cls_obj class DynGeneratorFromScenario(Generator): scenario = None @@ -629,6 +684,8 @@ def cleanup(self, fmkops): self._cleanup_walking_attrs() for periodic_id in self.scenario.periodic_to_clear: fmkops.unregister_task(periodic_id, ign_error=True) + for task_id in self.scenario.tasks_to_stop: + fmkops.unregister_task(task_id, ign_error=True) def _cleanup_walking_attrs(self): self.tr_selected = None @@ -636,10 +693,7 @@ def _cleanup_walking_attrs(self): self.tr_selected_idx = -1 def setup(self, dm, user_input): - if not _user_input_conformity(self, user_input, self._args_desc): - return False self.__class__.scenario.set_data_model(dm) - # self.__class__.scenario.knowledge_source = self.knowledge_source self.scenario = copy.copy(self.__class__.scenario) assert (self.data_fuzz and not (self.cond_fuzz or self.ignore_timing)) or not self.data_fuzz @@ -708,12 +762,12 @@ def _alter_data_step(self): data_desc = step.data_desc if isinstance(data_desc[0], str) \ or (isinstance(data_desc[0], Data) and data_desc[0].content is not None): - dp = sc.DataProcess(process=['tTYPE#{:d}'.format(self._step_num)], seed=data_desc[0], + dp = DataProcess(process=['tTYPE#{:d}'.format(self._step_num)], seed=data_desc[0], auto_regen=True) dp.append_new_process([('tSTRUCT#{:d}'.format(self._step_num), UI(init=1, deep=True))]) data_desc[0] = dp step.data_desc = data_desc - elif isinstance(data_desc[0], sc.DataProcess): + elif isinstance(data_desc[0], DataProcess): proc = copy.copy(data_desc[0].process) proc2 = copy.copy(data_desc[0].process) proc.append('tTYPE#{:d}'.format(self._step_num)) @@ -722,7 +776,7 @@ def _alter_data_step(self): data_desc[0].append_new_process(proc2) data_desc[0].auto_regen = True elif isinstance(data_desc[0], Data): - dp = sc.DataProcess(process=['C#{:d}'.format(self._step_num)], seed=data_desc[0], + dp = DataProcess(process=['C#{:d}'.format(self._step_num)], seed=data_desc[0], auto_regen=True) data_desc[0] = dp step.data_desc = data_desc @@ -739,7 +793,7 @@ def _check_data_fuzz_completion_cbk(self, env, step): if self._prev_func is not None: self._prev_func(env, step) data_desc = step.data_desc[0] - assert isinstance(data_desc, sc.DataProcess) + assert isinstance(data_desc, DataProcess) if data_desc.auto_regen_cpt > 0: data_desc.auto_regen_cpt = 0 self._data_fuzz_change_step = True @@ -823,6 +877,11 @@ def generate_data(self, dm, monitor, target): self._alteration_just_performed = False self.scenario.set_target(target) + + if self.scenario._user_args: + for ua in self.scenario._user_args.keys(): + setattr(self.scenario.env, str(ua), getattr(self, str(ua))) + self.step = self.scenario.current_step self.step.do_before_data_processing() @@ -837,9 +896,10 @@ def generate_data(self, dm, monitor, target): else: self.need_reset() data = Data() - data.register_callback(self._callback_cleanup_periodic, hook=HOOK.after_dmaker_production) + # data.register_callback(self._callback_cleanup_periodic, hook=HOOK.after_dmaker_production) data.make_unusable() data.origin = self.scenario + data.scenario_dependence = self.scenario.name return data data = self.step.get_data() @@ -855,19 +915,13 @@ def generate_data(self, dm, monitor, target): data.register_callback(self._callback_dispatcher_before_sending_step2, hook=HOOK.before_sending_step2) data.register_callback(self._callback_dispatcher_after_sending, hook=HOOK.after_sending) data.register_callback(self._callback_dispatcher_after_fbk, hook=HOOK.after_fbk) + data.register_callback(self._callback_dispatcher_final, hook=HOOK.final) data.scenario_dependence = self.scenario.name return data - def _callback_cleanup_periodic(self): - cbkops = CallBackOps() - for periodic_id in self.scenario.periodic_to_clear: - cbkops.add_operation(CallBackOps.Del_PeriodicData, id=periodic_id) - return cbkops - - def __handle_transition_callbacks(self, hook, feedback=None): for idx, tr in self.pending_tr_eval: if tr.run_callback(self.step, feedback=feedback, hook=hook): @@ -880,7 +934,14 @@ def __handle_transition_callbacks(self, hook, feedback=None): if self.tr_selected is None: for idx, tr in enumerate(self.step.transitions): if self.tr_selected is None: - if not tr.has_callback() and tr.is_crossable(): + if tr.dp_completed_guard: + for d_desc in self.step.data_desc: + if isinstance(d_desc, DataProcess) and d_desc.dp_completed: + # d_desc.dp_completed = False + self.tr_selected = tr + self.tr_selected_idx = idx + break + elif not tr.has_callback() and tr.is_crossable(): self.tr_selected = tr self.tr_selected_idx = idx break @@ -901,23 +962,33 @@ def _callback_dispatcher_before_sending_step1(self): if self.step.has_dataprocess(): cbkops.add_operation(CallBackOps.Replace_Data, param=(self.step.data_desc, self.step.vtg_ids_list)) + if self.step.transition_on_dp_complete: + cbkops.set_flag(CallBackOps.ForceDataHandling) return cbkops def _callback_dispatcher_before_sending_step2(self): # Callback called after any data have been processed but not sent yet - self.step.do_before_sending() - # We add again the operation CallBackOps.Replace_Data, because the step contents could have changed + did_something = self.step.do_before_sending() + + self.__handle_transition_callbacks(HOOK.before_sending_step2) + cbkops = CallBackOps() - cbkops.add_operation(CallBackOps.Replace_Data, - param=(self.step.data_desc, self.step.vtg_ids_list)) + if did_something: + # We add again the operation CallBackOps.Replace_Data, because the step contents could have changed + cbkops.add_operation(CallBackOps.Replace_Data, + param=(self.step.data_desc, self.step.vtg_ids_list)) + return cbkops def _callback_dispatcher_after_sending(self): self.__handle_transition_callbacks(HOOK.after_sending) def _callback_dispatcher_after_fbk(self, fbk): - """This callback is always called by the framework""" + """ + This callback is always called by the framework + It allows for a NoDataStep to perform actions (trigger periodic data, tasks, ...) + """ self.__handle_transition_callbacks(HOOK.after_fbk, feedback=fbk) @@ -929,6 +1000,16 @@ def _callback_dispatcher_after_fbk(self, fbk): for periodic_id in self.step.periodic_to_clear: cbkops.add_operation(CallBackOps.Del_PeriodicData, id=periodic_id) + for desc in self.step.tasks_to_start: + cbkops.add_operation(CallBackOps.Start_Task, id=id(desc), + param=desc, period=desc.period) + + for task_id in self.step.tasks_to_stop: + cbkops.add_operation(CallBackOps.Stop_Task, id=task_id) + + return cbkops + + def _callback_dispatcher_final(self): if self.tr_selected is not None: self.scenario.walk_to(self.tr_selected.step) else: @@ -938,8 +1019,6 @@ def _callback_dispatcher_after_fbk(self, fbk): # In case the same Data is used again without going through self.generate_data() self._cleanup_walking_attrs() - return cbkops - class Disruptor(DataMaker): diff --git a/framework/target_helpers.py b/framework/target_helpers.py index 156f2be..7876c86 100644 --- a/framework/target_helpers.py +++ b/framework/target_helpers.py @@ -25,14 +25,23 @@ import threading from framework.data import Data +from framework.knowledge.feedback_collector import FeedbackSource from libs.external_modules import * class TargetStuck(Exception): pass +class TargetError(Exception): pass +class TargetNotReady(Exception): pass class Target(object): - ''' - Class abstracting the target we interact with. - ''' + """ + Class abstracting the real target we interact with. + + About feedback: + Feedback retrieved from a real target has to be provided to the user (i.e., the framework) through + either after Target.send_data() is called or when Target.collect_unsolicited_feedback() is called. + + """ + name = None feedback_timeout = None sending_delay = 0 @@ -44,15 +53,30 @@ class Target(object): fbk_wait_until_recv_msg = 'Wait until the target has sent something back to us' _feedback_mode = None - supported_feedback_mode = [] + supported_feedback_mode = [FBK_WAIT_FULL_TIME, FBK_WAIT_UNTIL_RECV] + + STATUS_THRESHOLD_FOR_RECOVERY = 0 # When a feedback status gathered by FmkPlumbing is + # strictly lesser than this value, .recover_target() will be called + + _started = None _logger = None - _probes = None + _extensions = None _send_data_lock = threading.Lock() _altered_data_queued = None _pending_data = None + _pending_data_id = None + + _last_sending_date = None + + display_feedback = False + + def __init__(self, name=None, display_feedback=True): + self.name = name + self.display_feedback = display_feedback + self._started = False @staticmethod def get_fbk_mode_desc(fbk_mode, short=False): @@ -67,17 +91,25 @@ def set_logger(self, logger): def set_data_model(self, dm): self.current_dm = dm + def set_project(self, prj): + self._project = prj + def _start(self, target_desc, tg_id): self._logger.print_console('*** Target initialization: ({:d}) {!s} ***\n'.format(tg_id, target_desc), nl_before=False, rgb=Color.COMPONENT_START) self._pending_data = [] - return self.start() + self._pending_data_id = None + self._started = self.start() + return self._started def _stop(self, target_desc, tg_id): self._logger.print_console('*** Target cleanup procedure for ({:d}) {!s} ***\n'.format(tg_id, target_desc), nl_before=False, rgb=Color.COMPONENT_STOP) self._pending_data = None - return self.stop() + self._pending_data_id = None + ret = self.stop() + self._started = not ret + return ret def start(self): ''' @@ -91,6 +123,9 @@ def stop(self): ''' return True + def is_started(self): + return self._started + def record_info(self, info): """ Can be used by the target to record some information during initialization or anytime @@ -136,11 +171,18 @@ def send_multiple_data(self, data_list, from_fmk=False): def is_target_ready_for_new_data(self): - ''' - The FMK busy wait on this method() before sending a new data. - This method should take into account feedback timeout (that is the maximum - time duration for gathering feedback from the target) - ''' + """ + To be overloaded if the target needs some time (for conditions to occur) before data can be sent. + Note: The FMK busy wait on this method() before sending a new data. + """ + return True + + + def is_feedback_received(self): + """ + To be overloaded if the target implements FBK_WAIT_UNTIL_RECV mode, so that + it can informs the framework about feedback reception. + """ return True def get_last_target_ack_date(self): @@ -148,7 +190,7 @@ def get_last_target_ack_date(self): If different from None the return value is used by the FMK to log the date of the target acknowledgment after a message has been sent to it. - [Note: If this method is overloaded, is_target_ready_for_new_data() should also be] + [Note: If this method is overloaded, is_feedback_received() should also be] ''' return None @@ -175,10 +217,11 @@ def get_feedback(self): ''' return None - def collect_pending_feedback(self, timeout=0): + def collect_unsolicited_feedback(self, timeout=0): """ - If overloaded, it can be used by the framework to retrieve additional feedback from the - target without sending any new data. + If overloaded, it should collect any data from the associated real target that may be sent + without solicitation (i.e. without any data sent through it) and make it available through + the method .get_feedback() Args: timeout: Maximum delay before returning from feedback collecting @@ -221,6 +264,10 @@ def set_feedback_mode(self, mode): def fbk_wait_full_time_slot_mode(self): return self._feedback_mode == Target.FBK_WAIT_FULL_TIME + @property + def fbk_wait_until_recv_mode(self): + return self._feedback_mode == Target.FBK_WAIT_UNTIL_RECV + def set_sending_delay(self, sending_delay): """ Set the sending delay. @@ -236,7 +283,8 @@ def __str__(self): return self.__class__.__name__ + ' [' + self.get_description() + ']' def get_description(self): - return 'ID: ' + str(id(self))[-6:] + prefix = '{:s} | '.format(self.name) if self.name is not None else '' + return '{:s}ID: {:s}'.format(prefix, str(id(self))[-6:]) def add_pending_data(self, data): with self._send_data_lock: @@ -257,44 +305,72 @@ def send_pending_data(self, from_fmk=False): else: raise ValueError('No pending data') - def send_data_sync(self, data, from_fmk=False): - ''' + def send_data_sync(self, data: Data, from_fmk=False): + """ Can be used in user-code to send data to the target without interfering with the framework. Use case example: The user needs to send some message to the target on a regular basis in background. For that purpose, it can quickly define a :class:`framework.monitor.Probe` that just emits the message by itself. - ''' + """ with self._send_data_lock: if data is not None: self._altered_data_queued = data.altered - self.send_data(data, from_fmk=from_fmk) + if self.is_target_ready_for_new_data(): + self._last_sending_date = datetime.datetime.now() + self.send_data(data, from_fmk=from_fmk) + self._project.notify_data_sending([data], self._last_sending_date, self) + if from_fmk: + self._pending_data_id = data.estimated_data_id + if not from_fmk: + self._logger.log_async_data(data, sent_date=self._last_sending_date, + target_ref=FeedbackSource(self), + prj_name=self._project.name, + current_data_id=self._pending_data_id) + else: + self._logger.print_console(f'*** Target {self!s} Not ready ***\n', + nl_before=False, rgb=Color.WARNING) + # raise TargetNotReady def send_multiple_data_sync(self, data_list, from_fmk=False): - ''' + """ Can be used in user-code to send data to the target without interfering with the framework. - ''' + """ with self._send_data_lock: if data_list is not None: self._altered_data_queued = data_list[0].altered - self.send_multiple_data(data_list, from_fmk=from_fmk) + if self.is_target_ready_for_new_data(): + self._last_sending_date = datetime.datetime.now() + self.send_multiple_data(data_list, from_fmk=from_fmk) + self._project.notify_data_sending(data_list, self._last_sending_date, self) + if from_fmk: + self._pending_data_id = data_list[-1].estimated_data_id + if not from_fmk: + self._logger.log_async_data(data_list, sent_date=self._last_sending_date, + target_ref=FeedbackSource(self), + prj_name=self._project.name, + current_data_id=self._pending_data_id) + else: + self._logger.print_console(f'*** Target {self!s} Not ready ***\n', + nl_before=False, rgb=Color.WARNING) + # raise TargetNotReady - def add_probe(self, probe): - if self._probes is None: - self._probes = [] - self._probes.append(probe) + def add_extensions(self, probe): + if self._extensions is None: + self._extensions = [] + self._extensions.append(probe) - def remove_probes(self): - self._probes = None + def del_extensions(self): + self._extensions = None def is_processed_data_altered(self): return self._altered_data_queued @property - def probes(self): - return self._probes if self._probes is not None else [] + def extensions(self): + return self._extensions if self._extensions is not None else [] class EmptyTarget(Target): @@ -302,24 +378,14 @@ class EmptyTarget(Target): _feedback_mode = Target.FBK_WAIT_FULL_TIME supported_feedback_mode = [Target.FBK_WAIT_FULL_TIME, Target.FBK_WAIT_UNTIL_RECV] - def __init__(self, enable_feedback=True): + def __init__(self, verbose=False): Target.__init__(self) - self._feedback_enabled = enable_feedback - self._sending_time = None + self.verbose = verbose def send_data(self, data, from_fmk=False): - if self._feedback_enabled: - self._sending_time = datetime.datetime.now() + if self.verbose: + print(f'\n*** data sent: {data.to_bytes()}') def send_multiple_data(self, data_list, from_fmk=False): - if self._feedback_enabled: - self._sending_time = datetime.datetime.now() - - def is_target_ready_for_new_data(self): - if self._feedback_enabled and self.feedback_timeout is not None and \ - self._sending_time is not None: - return (datetime.datetime.now() - self._sending_time).total_seconds() > self.feedback_timeout - else: - return True - + pass diff --git a/framework/targets/debug.py b/framework/targets/debug.py index 46dac5e..289dff5 100644 --- a/framework/targets/debug.py +++ b/framework/targets/debug.py @@ -24,51 +24,369 @@ import random import datetime import time +import queue +import threading + +import struct +from multiprocessing import shared_memory from framework.target_helpers import Target from framework.basic_primitives import rand_string +from framework.knowledge.feedback_collector import FeedbackCollector +from framework.data import Data +from libs.external_modules import Color + +class IncorrectTargetError(Exception): pass +class ShmemMappingError(Exception): pass class TestTarget(Target): - _feedback_mode = None - supported_feedback_mode = [Target.FBK_WAIT_UNTIL_RECV] + _feedback_mode = Target.FBK_WAIT_FULL_TIME + supported_feedback_mode = [Target.FBK_WAIT_UNTIL_RECV, Target.FBK_WAIT_FULL_TIME] _last_ack_date = None - def __init__(self, recover_ratio=100, fbk_samples=None): - Target.__init__(self) + # shared memory constants + dlen_start = 0 + dlen_stop = 4 + dlen_format = '>L' + producer_status_idx = 5 + consumer_start = 6 + consumer_stop = 15 + max_consumer = consumer_stop - consumer_start + 1 + data_start = meta_data_size = 16 + shmem_size = 4096 + + + def __init__(self, name=None, recover_ratio=100, fbk_samples=None, repeat_input=False, + fbk_timeout=0.05, shmem_mode=False, shmem_timeout=10): + Target.__init__(self, name) self._cpt = None self._recover_ratio = recover_ratio self._fbk_samples = fbk_samples + self._repeat_input = repeat_input + self._bound_targets = [] + if shmem_mode and not name: + raise ValueError('name parameter should be specified in shmem_mode') + self._shmem_mode= shmem_mode + self.output_shmem = None + self.input_shmem_list = None + self.fbk_sources = None + + self.controled_targets = None + self.control_delay = 0 + self.forward_queue = queue.Queue() + + self._current_consumer_idx = -1 + self._stop_event = None + self._fbk_collector_exit_event = None + self._fbk_collector_thread = None + self._map_timeout = shmem_timeout + self._target_ready = None + self._send_data_finished_event = None + + self._shared_queue = queue.Queue() + self._fbk_collector = FeedbackCollector() + self.set_feedback_timeout(fbk_timeout) def start(self): self._cpt = 0 + if self._shmem_mode: + self._target_ready = False + if self.fbk_sources is not None: + self.input_shmem_list = [] + self.output_shmem = shared_memory.SharedMemory(name=self.name, + create=True, + size=self.shmem_size) + buf = self.output_shmem.buf + for i in range(self._current_consumer_idx+1): + # every consumers are ready at start + buf[self.consumer_start+i] = 1 + # nothing has been produced + buf[self.producer_status_idx] = 0 + + self._stop_event = threading.Event() + self._stop_event.clear() + self._fbk_collector_exit_event = threading.Event() + self._fbk_collector_exit_event.set() + self._send_data_finished_event = threading.Event() + self._send_data_finished_event.set() + + if self.controled_targets is not None: + self.forward_queue = queue.Queue() + self._forward_data_thread = threading.Thread(None, self._forward_data) + self._forward_data_thread.start() + + self._fbk_collector_thread = threading.Thread(None, self._collect_fbk_loop) + self._fbk_collector_thread.start() + + return True + + def stop(self): + if self._shmem_mode: + self._stop_event.set() + self._target_ready = False + while not self._fbk_collector_exit_event.is_set(): + time.sleep(0.05) + while not self._send_data_finished_event.is_set(): + time.sleep(0.001) + if self.output_shmem: + self.output_shmem.close() + try: + self.output_shmem.unlink() + except FileNotFoundError: + pass + if self.input_shmem_list: + for shm, _ in self.input_shmem_list: + shm.close() + + def _map_input_shmem(self): + if self.fbk_sources is None: + return False + + else: + for tg, c_idx in self.fbk_sources: + if tg._shmem_mode: + try: + shm = shared_memory.SharedMemory(name=tg.name, create=False) + except FileNotFoundError: + return False + else: + self.input_shmem_list.append((shm, c_idx)) + else: + raise IncorrectTargetError() + return True + def _collect_fbk_loop(self): + self._fbk_collector_exit_event.clear() + shmem_ok = False + t0 = datetime.datetime.now() + while (datetime.datetime.now() - t0).total_seconds() < self._map_timeout: + if self._map_input_shmem(): + shmem_ok = True + break + else: + # print('\n*** DBG wait for fbk sources') + time.sleep(0.2) + + if shmem_ok: + self._target_ready = True + self._logger.print_console("*** Shared memory from feedback sources have been mapped ***", + rgb=Color.COMMENTS, + nl_before=True, nl_after=True) + else: + self._logger.print_console("*** ERROR: cannot map shared memory from feedback sources ***", + rgb=Color.ERROR, + nl_before=True, nl_after=True) + self._fbk_collector_exit_event.set() + return + + while not self._stop_event.is_set(): + feedback_items = [] + for shm, _ in self.input_shmem_list: + if shm.buf[self.producer_status_idx] == 1: + break + else: + time.sleep(0.01) + continue + + for tg_idx, obj in enumerate(self.input_shmem_list): + shm, c_idx = obj + if shm.buf[self.producer_status_idx] == 0 \ + or shm.buf[self.consumer_start+c_idx] == 1: + continue + + dlen = struct.unpack(self.dlen_format, + bytes(shm.buf[self.dlen_start:self.dlen_stop]))[0] + fbk_item = bytes(shm.buf[self.data_start:self.data_start+dlen]) + self._logger.collect_feedback(fbk_item, status_code=0, + fbk_src=self.fbk_sources[tg_idx][0]) + shm.buf[self.consumer_start+c_idx] = 1 + feedback_items.append(fbk_item) + + if self.controled_targets: + for fi in feedback_items: + # print(f'\n***DBG put {fi}') + self.forward_queue.put(fi) + + time.sleep(0.01) + + # print('\n*** DBG fbk collector exits') + self._fbk_collector_exit_event.set() + + def _forward_data(self): + while not self._stop_event.is_set(): + data_to_send = [] + while not self.forward_queue.empty(): + data_to_send.append(self.forward_queue.get_nowait()) + + # print(f'\n***DBG get {data_to_send}') + + if data_to_send: + time.sleep(self.control_delay) + for tg, fbk_filter in self.controled_targets: + for data in data_to_send: + d = fbk_filter(data, self.current_dm) + if d is not None: + tg.send_data_sync(Data(d)) + else: + time.sleep(0.1) + + # print('\n***DBG leave forward') + + def _handle_fbk(self, data): + if self._repeat_input: + fbk_content = data.to_bytes() + elif self._fbk_samples: + fbk_content = random.choice(self._fbk_samples) + else: + fbk_content = rand_string(size=10) + return fbk_content + + def get_consumer_idx(self): + # This function is OK even when linked TestTargets are run in different fuddly instance + if self._current_consumer_idx + 1 > self.max_consumer: + raise IndexError + self._current_consumer_idx += 1 + return self._current_consumer_idx + + def add_feedback_sources(self, *targets): + if self.fbk_sources is None: + self.fbk_sources = [] + for tg in targets: + self.fbk_sources.append((tg, tg.get_consumer_idx())) + + def set_control_over(self, *test_targets, feedback_filter=lambda x, y: x): + if self.controled_targets is None: + self.controled_targets = [] + for tg in test_targets: + self.controled_targets.append((tg, feedback_filter)) + + def set_control_delay(self, delay): + self.control_delay = delay + + # obsolete API + def add_binding(self, target): + if self.output_shmem: + raise ValueError('In shmem mode, binding are not possible') + self._bound_targets.append(target) + def send_data(self, data, from_fmk=False): - self._last_ack_date = datetime.datetime.now() + datetime.timedelta(microseconds=random.randint(20, 40)) - time.sleep(0.001) - fbk_content = random.choice(self._fbk_samples) if self._fbk_samples else rand_string(size=10) - self._logger.collect_feedback(content=fbk_content, status_code=random.randint(-3, 3)) + if self.output_shmem: + self._send_data_finished_event.clear() + if self._stop_event.is_set(): + self._send_data_finished_event.set() + time.sleep(0.01) + return + + d = data.to_bytes() + dlen = len(d) + if dlen > self.shmem_size - self.meta_data_size: + raise ValueError('data too long, exceeds shared memory size') + + data_consumed = False + buf = self.output_shmem.buf + t0 = datetime.datetime.now() + self._last_ack_date = None + while (datetime.datetime.now() - t0).total_seconds() < 2: + time.sleep(0.001) + for i in range(self._current_consumer_idx+1): + if buf[self.consumer_start+i] == 0: + # one consumer is not ready thus we busy wait + break + else: + # All the consumers are ready (0 everywhere) + # print('\n*** consumer are ready') + self._last_ack_date = datetime.datetime.now() + data_consumed = True + # print('\n*** DBG: data consumed on time!') + break + # print('\n*** DBG: data not consumed yet') + + + if data_consumed: + buf[self.producer_status_idx] = 0 + + for i in range(self._current_consumer_idx+1): + buf[self.consumer_start+i] = 0 + buf[self.dlen_start:self.dlen_stop] = struct.pack(self.dlen_format, dlen) + buf[self.data_start:self.data_start+dlen] = d + buf[self.producer_status_idx] = 1 + else: + print(f'\n*** Warning: previous data not consumed on time, thus ignore new sending of "{d[:10]}..." ***') + + self._send_data_finished_event.set() + else: + time.sleep(0.001) + if self._bound_targets: + for tg in self._bound_targets: + tg._shared_queue.put((data.to_bytes(), str(self))) + else: + self._logger.collect_feedback(content=self._handle_fbk(data), + status_code=random.randint(-3, 3)) + + self._last_ack_date = datetime.datetime.now() + datetime.timedelta(microseconds=random.randint(20, 40)) def send_multiple_data(self, data_list, from_fmk=False): - self._last_ack_date = datetime.datetime.now() + datetime.timedelta(microseconds=random.randint(20, 40)) - time.sleep(0.001) - fbk_content = random.choice(self._fbk_samples) if self._fbk_samples else rand_string(size=20) - self._logger.collect_feedback(content=fbk_content, status_code=random.randint(-3, 3)) + for data in data_list: + self.send_data(data, from_fmk=from_fmk) + def is_target_ready_for_new_data(self): - self._cpt += 1 - if self._cpt > 5 and random.choice([True, False]): - self._cpt = 0 + if self._shmem_mode: + return self._target_ready + else: return True + + def is_feedback_received(self): + if self._shmem_mode: + for shm, c_idx in self.input_shmem_list: + if shm.buf[self.producer_status_idx] == 1 and \ + shm.buf[self.consumer_start+c_idx] == 0: + return True + else: + return False + + elif self._bound_targets: + return not self._shared_queue.empty() + else: - return False + self._cpt += 1 + if self._cpt > 5 and random.choice([True, False]): + self._cpt = 0 + return True + else: + return False + + def get_feedback(self): + fbk = None + if self._bound_targets: + t0 = datetime.datetime.now() + timeout = 0.01 if self.feedback_timeout is None else self.feedback_timeout + while (datetime.datetime.now() - t0).total_seconds() < timeout: + try: + item = self._shared_queue.get(block=True, timeout=0.01) + except queue.Empty: + pass + else: + self._fbk_collector.add_fbk_from(f'sent by {item[1]}', item[0]) + self._shared_queue.task_done() + + fbk = self._fbk_collector - def recover_target(self): - if random.randint(1, 100) > (100 - self._recover_ratio): - return True else: + pass + + return fbk + + def recover_target(self): + if self._shmem_mode or self._bound_targets: return False + else: + if random.randint(1, 100) > (100 - self._recover_ratio): + return True + else: + return False def get_last_target_ack_date(self): return self._last_ack_date diff --git a/framework/targets/local.py b/framework/targets/local.py index 93bee19..4d57a97 100644 --- a/framework/targets/local.py +++ b/framework/targets/local.py @@ -38,8 +38,9 @@ class LocalTarget(Target): _feedback_mode = Target.FBK_WAIT_UNTIL_RECV supported_feedback_mode = [Target.FBK_WAIT_UNTIL_RECV] - def __init__(self, target_path=None, pre_args='', post_args='', - tmpfile_ext='.bin', send_via_stdin=False, send_via_cmdline=False): + def __init__(self, target_path=None, pre_args=None, post_args=None, + tmpfile_ext='.bin', send_via_stdin=False, send_via_cmdline=False, + error_samples=None, error_parsing_func=lambda x: (False, '')): Target.__init__(self) self._suffix = '{:0>12d}'.format(random.randint(2 ** 16, 2 ** 32)) self._app = None @@ -47,6 +48,8 @@ def __init__(self, target_path=None, pre_args='', post_args='', self._post_args = post_args self._send_via_stdin = send_via_stdin self._send_via_cmdline = send_via_cmdline + self._error_samples = [b'error', b'invalid'] if error_samples is None else error_samples + self._error_parsing_func = error_parsing_func self._data_sent = None self._feedback_computed = None self._feedback = FeedbackCollector() @@ -56,7 +59,10 @@ def __init__(self, target_path=None, pre_args='', post_args='', def get_description(self): pre_args = self._pre_args post_args = self._post_args - args = ', Args: ' + pre_args + post_args if pre_args or post_args else '' + if pre_args or post_args: + args = ', Args: ' + ('' if pre_args is None else pre_args) + ('' if post_args is None else post_args) + else: + args = '' return 'Program: ' + self._target_path + args def set_tmp_file_extension(self, tmpfile_ext): @@ -121,13 +127,19 @@ def send_data(self, data, from_fmk=False): f.write(data) if self._pre_args is not None and self._post_args is not None: - cmd = [self._target_path] + self._pre_args.split() + [name] + self._post_args.split() + if self._send_via_stdin: + cmd = [self._target_path] + self._pre_args.split() + self._post_args.split() + else: + cmd = [self._target_path] + self._pre_args.split() + [name] + self._post_args.split() elif self._pre_args is not None: - cmd = [self._target_path] + self._pre_args.split() + [name] + cmd = [self._target_path] + self._pre_args.split() + if not self._send_via_stdin: + cmd += [name] elif self._post_args is not None: - cmd = [self._target_path, name] + self._post_args.split() + cmd = [self._target_path] if self._send_via_stdin else [self._target_path, name] + cmd += self._post_args.split() else: - cmd = [self._target_path, name] + cmd = [self._target_path] if self._send_via_stdin else [self._target_path, name] stdin_arg = subprocess.PIPE if self._send_via_stdin else None self._app = subprocess.Popen(args=cmd, stdin=stdin_arg, stdout=subprocess.PIPE, @@ -164,40 +176,58 @@ def get_feedback(self, timeout=0.2): else: self._feedback_computed = True - err_detected = False - - if self._app is None and self._data_sent: - err_detected = True - self._feedback.add_fbk_from("LocalTarget", "Application has terminated (crash?)", - status=-3) - return self._feedback - elif self._app is None: + if self._app is None: + # application not yet started return self._feedback - exit_status = self._app.poll() - if exit_status is not None and exit_status < 0: - err_detected = True - self._feedback.add_fbk_from("Application[{:d}]".format(self._app.pid), - "Negative return status ({:d})".format(exit_status), - status=exit_status) + exit_error = False + proc_killed = False + return_code = self._app.poll() + if return_code is not None: # process has terminate + if return_code < 0: + # process terminated by a signal (python behavior for POSIX system) + proc_killed = True + self._feedback.add_fbk_from(f"Application[{self._app.pid}]", + f"Application terminated by a signal (ID: {-return_code})", + status=-3) + else: + if self.is_processed_data_altered() and return_code == 0: + exit_error = True + msg = f"Wrong exit status ({return_code}) - expecting != 0 (as altered data was provided)" + elif not self.is_processed_data_altered() and return_code != 0: + exit_error = True + msg = f"Wrong exit status ({return_code}) - expecting 0 (as valid data was provided)" + else: + msg = f'Expected exit status ({return_code})' + + self._feedback.add_fbk_from(f"Application[{self._app.pid}]", msg, status=-1 if exit_error else 0) + + console_error = False ret = select.select([self._app.stdout, self._app.stderr], [], [], timeout) if ret[0]: byte_string = b'' for fd in ret[0][:-1]: byte_string += fd.read() + b'\n\n' - if b'error' in byte_string or b'invalid' in byte_string: - err_detected = True + console_error, msg = self._error_parsing_func(byte_string) + if console_error: self._feedback.add_fbk_from("LocalTarget[stdout]", - "Application outputs errors on stdout", + f"Error detected on stdout (by provided parsing function)\n --> {msg}", status=-1) + for err_msg in self._error_samples: + if err_msg in byte_string: + console_error = True + self._feedback.add_fbk_from("LocalTarget[stdout]", + f"Error detected on stdout (from provided samples): '{err_msg.decode()}'", + status=-1) + stderr_msg = ret[0][-1].read() if stderr_msg: - err_detected = True + console_error = True self._feedback.add_fbk_from("LocalTarget[stderr]", - "Application outputs on stderr", + "Application outputs on stderr", status=-2) byte_string += stderr_msg else: @@ -206,7 +236,7 @@ def get_feedback(self, timeout=0.2): else: byte_string = b'' - if err_detected: + if proc_killed or exit_error or console_error: self._feedback.set_error_code(-1) self._feedback.set_bytes(byte_string) diff --git a/framework/targets/network.py b/framework/targets/network.py index 9f993b5..8972a25 100644 --- a/framework/targets/network.py +++ b/framework/targets/network.py @@ -59,10 +59,11 @@ class NetworkTarget(Target): - '''Generic target class for interacting with a network resource. Can + """ + Generic target class for interacting with a network resource. Can be used directly, but some methods may require to be overloaded to fit your needs. - ''' + """ General_Info_ID = 'General Information' @@ -74,10 +75,11 @@ class NetworkTarget(Target): supported_feedback_mode = [Target.FBK_WAIT_FULL_TIME, Target.FBK_WAIT_UNTIL_RECV] def __init__(self, host='localhost', port=12345, socket_type=(socket.AF_INET, socket.SOCK_STREAM), - data_semantics=UNKNOWN_SEMANTIC, server_mode=False, target_address=None, wait_for_client=True, + data_semantics=UNKNOWN_SEMANTIC, + server_mode=False, listen_on_start=True, target_address=None, wait_for_client=True, hold_connection=False, keep_first_client=True, mac_src=None, mac_dst=None, add_eth_header=False, - fbk_timeout=2, sending_delay=1, recover_timeout=0.5): + fbk_timeout=2, fbk_mode=Target.FBK_WAIT_FULL_TIME, sending_delay=1, recover_timeout=0.5): """ Args: host (str): IP address of the target to connect to, or @@ -97,6 +99,9 @@ def __init__(self, host='localhost', port=12345, socket_type=(socket.AF_INET, so server_mode (bool): If `True`, the interface will be set in server mode, which means we will wait for the real target to connect to us for sending it data. + listen_on_start (bool): If `True`, servers will be launched right after the `NetworkTarget` + starts. Otherwise, they will be launched in a lazy mode, meaning just when something is + about to be sent through the server mode interface. target_address (tuple): Used only if `server_mode` is `True` and socket type is `SOCK_DGRAM`. To be used if data has to be sent to a specific address (which is not necessarily the client). It is especially @@ -138,10 +143,9 @@ def __init__(self, host='localhost', port=12345, socket_type=(socket.AF_INET, so if not self._is_valid_socket_type(socket_type): raise ValueError("Unrecognized socket type") - if sys.platform in ['linux', 'linux2']: # python3, python2 + if sys.platform in ['linux']: def get_mac_addr(ifname): - if sys.version_info[0] > 2: - ifname = bytes(ifname, 'latin_1') + ifname = bytes(ifname, 'latin_1') s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: info = fcntl.ioctl(s.fileno(), 0x8927, struct.pack('256s', ifname[:15])) @@ -203,8 +207,10 @@ def get_mac_addr(ifname): self._raw_server_private = None self._recover_timeout = recover_timeout - self.set_timeout(fbk_timeout=fbk_timeout, sending_delay=sending_delay) + self._listen_on_start = listen_on_start + self.set_timeout(fbk_timeout=fbk_timeout, sending_delay=sending_delay) + self.set_feedback_mode(fbk_mode) def _is_valid_socket_type(self, socket_type): skt_sz = len(socket_type) @@ -276,7 +282,8 @@ def terminate(self): def add_additional_feedback_interface(self, host, port, socket_type=(socket.AF_INET, socket.SOCK_STREAM), - fbk_id=None, fbk_length=None, server_mode=False): + fbk_id=None, fbk_length=None, server_mode=False, + wait_time=None): '''Allows to register additional socket to get feedback from. Connection is attempted be when target starts, that is when :meth:`NetworkTarget.start()` is called. @@ -286,7 +293,8 @@ def add_additional_feedback_interface(self, host, port, fbk_id = 'Default Additional Feedback ID %d' % self._default_additional_fbk_id else: assert(not str(fbk_id).startswith('Default Additional Feedback ID')) - self._additional_fbk_desc[fbk_id] = (host, port, socket_type, fbk_id, fbk_length, server_mode) + self._additional_fbk_desc[fbk_id] = (host, port, socket_type, fbk_id, fbk_length, + server_mode, wait_time) self.hold_connection[(host, port)] = True self._server_mode_additional_info[(host, port)] = (None, None, None) @@ -422,9 +430,10 @@ def _connect_to_additional_feedback_sockets(self): Connection to additional feedback sockets, if any. ''' if self._additional_fbk_desc: - for host, port, socket_type, fbk_id, fbk_length, server_mode in self._additional_fbk_desc.values(): + for host, port, socket_type, fbk_id, fbk_length, server_mode, wait_time in self._additional_fbk_desc.values(): if server_mode: - self._raw_listen_to(host, port, fbk_id, socket_type, chk_size=fbk_length) + self._raw_listen_to(host, port, fbk_id, socket_type, chk_size=fbk_length, + wait_time=wait_time) else: self._raw_connect_to(host, port, fbk_id, socket_type, chk_size=fbk_length) @@ -479,10 +488,17 @@ def start(self): self._connect_to_additional_feedback_sockets() + if self._listen_on_start: + # In the case there are server mode interfaces, + # we initiate the server threads by the following method and could thus catch + # information from any connected clients trying to reach us. This could then + # be leveraged when we sent data. + self.collect_unsolicited_feedback() + for k, mac_src in self._mac_src.items(): if mac_src is not None: if mac_src: - mac_src = mac_src.hex() if sys.version_info[0] > 2 else mac_src.encode('hex') + mac_src = mac_src.hex() self.record_info('*** Detected HW address for {!s}: {!s} ***' .format(k[0], mac_src)) else: @@ -519,7 +535,7 @@ def stop(self): def recover_target(self): t0 = datetime.datetime.now() - while not self.is_target_ready_for_new_data(): + while not self.is_feedback_received(): time.sleep(0.0001) now = datetime.datetime.now() if (now - t0).total_seconds() > self._recover_timeout: @@ -554,8 +570,8 @@ def send_multiple_data(self, data_list, from_fmk=False): for data, host, port, socket_type, server_mode in sending_list: if server_mode: - if from_fmk: - self._fbk_collector_to_launch_cpt += 1 + # if from_fmk: + # self._fbk_collector_to_launch_cpt += 1 connected_client_event[(host, port)] = threading.Event() self._listen_to_target(host, port, socket_type, self._handle_target_connection, @@ -580,8 +596,8 @@ def send_multiple_data(self, data_list, from_fmk=False): # this case exist when data are only sent through 'server_mode'-configured interfaces # (because self._send_data() is called through self._handle_target_connection()) # or a connection error has occurred. - pass - # self._feedback_handled = True + if from_fmk: + self._fbk_collector_to_launch_cpt += 1 if data_list is None: return @@ -607,6 +623,10 @@ def send_multiple_data(self, data_list, from_fmk=False): err_msg = ">>> WARNING: unable to send data because the target did not connect" \ " to us [{:s}:{:d}] <<<".format(host, port) self._feedback.add_fbk_from(self._INTERNALS_ID, err_msg, status=-1) + # self._fbk_collector_to_launch_cpt -= 1 + + # else: + # self._fbk_collector_to_launch_cpt -= 1 def _get_data_semantic_key(self, data): if not isinstance(data.content, Node): @@ -639,7 +659,6 @@ def _connect_to_target(self, host, port, socket_type): fd = self._hclient_hp2sock[(host, port)].fileno() if fd == -1: # if the socket has been closed, -1 is received by python3 - # (with python2 previous instruction raise a Bad file descriptor Exception) raise OSError except Exception: print('\n*** WARNING: Current socket was closed unexpectedly! --> create new one.') @@ -762,6 +781,9 @@ def start_raw_server(serversocket, sending_event, notif_host_event): else: raise ValueError("Unrecognized socket type") + def _cleanup_state(self): + self._fbk_collector_to_launch_cpt -= 1 + # For SOCK_STREAM def _server_main(self, serversocket, host, port, func): _first_client = {} @@ -881,6 +903,8 @@ def _handle_target_connection(self, clientsocket, address, args, pre_fbk=None): with self._server_thread_lock: self._last_client_hp2sock[(host, port)] = (clientsocket, address) self._last_client_sock2hp[clientsocket] = (host, port) + # if from_fmk: + # self._fbk_collector_to_launch_cpt += 1 connected_client_event.set() self._send_data([clientsocket], {clientsocket:(data, host, port, address)}, fbk_timeout=self.feedback_timeout, from_fmk=from_fmk, @@ -1074,7 +1098,7 @@ def _send_data(self, sockets, data_refs, fbk_timeout, from_fmk, pre_fbk=None): if data_refs[sockets[0]][0] is None: # We check the data to send. If it is None, we only collect feedback from the sockets. - # This is used by self.collect_pending_feedback() + # This is used by self.collect_unsolicited_feedback() if fbk_sockets is None: assert fbk_ids is None assert fbk_lengths is None @@ -1177,12 +1201,14 @@ def _feedback_collect(self, fbk, ref, error=0): def _feedback_complete(self): self._fbk_collector_finished_cpt += 1 + # print('\n***DBG _fc: {} {}'.format(self._fbk_collector_to_launch_cpt, self._fbk_collector_finished_cpt)) def _before_sending_data(self, data_list, from_fmk): if from_fmk: self._last_ack_date = None self._first_send_data_call = True # related to additional feedback with self._fbk_handling_lock: + # print('\n***DBG _bsd: {} {}'.format(self._fbk_collector_to_launch_cpt, self._fbk_collector_finished_cpt)) assert self._fbk_collector_to_launch_cpt == self._fbk_collector_finished_cpt self._fbk_collector_finished_cpt = 0 self._fbk_collector_to_launch_cpt = 0 @@ -1235,7 +1261,7 @@ def _before_sending_data(self, data_list, from_fmk): return self._custom_data_handling_before_emission(new_data_list) - def collect_pending_feedback(self, timeout=0): + def collect_unsolicited_feedback(self, timeout=0): self._flush_feedback_delay = timeout self.send_multiple_data_sync(None, from_fmk=True) return True @@ -1243,9 +1269,13 @@ def collect_pending_feedback(self, timeout=0): def get_feedback(self): return self._feedback - def is_target_ready_for_new_data(self): + def is_feedback_received(self): + # print('\n*** DBG network is fbk received: {} {}'.format(self._fbk_collector_to_launch_cpt, self._fbk_collector_finished_cpt)) return self._fbk_collector_to_launch_cpt == self._fbk_collector_finished_cpt + def is_target_ready_for_new_data(self): + return self.is_feedback_received() + def _register_last_ack_date(self, ack_date): self._last_ack_date = ack_date diff --git a/framework/targets/printer.py b/framework/targets/printer.py index e961ec1..69cf23f 100755 --- a/framework/targets/printer.py +++ b/framework/targets/printer.py @@ -27,10 +27,7 @@ from framework.global_resources import workspace_folder from framework.target_helpers import Target from framework.knowledge.feedback_collector import FeedbackCollector -from libs.external_modules import cups_module - -if cups_module: - import cups +from libs.external_modules import cups_module, cups class PrinterTarget(Target): diff --git a/framework/targets/sim.py b/framework/targets/sim.py index b73cf0a..507563d 100644 --- a/framework/targets/sim.py +++ b/framework/targets/sim.py @@ -48,8 +48,7 @@ def __init__(self, serial_port, baudrate, pin_code, targeted_tel_num, codec='lat self.tel_num = targeted_tel_num self.pin_code = pin_code self.codec = codec - if sys.version_info[0]>2: - self.pin_code = bytes(self.pin_code, self.codec) + self.pin_code = bytes(self.pin_code, self.codec) self.set_feedback_timeout(2) def start(self): @@ -116,15 +115,14 @@ def send_data(self, data, from_fmk=False): print('\nWARNING: Data does not contain a mobile number.') pdu = b'' raw_data = data.to_bytes() + pdu_sz = len(raw_data) for c in raw_data: - if sys.version_info[0] == 2: - c = ord(c) pdu += binascii.b2a_hex(struct.pack('B', c)) pdu = pdu.upper() pdu = b'00' + pdu + b"\x1a\r\n" - - self.ser.write(b"AT+CMGS=23\r\n") # PDU mode + at_cmd = "AT+CMGS={:d}\r\n".format(pdu_sz-1).encode() + self.ser.write(at_cmd) # used for PDU mode time.sleep(self.delay_between_write) self.ser.write(pdu) diff --git a/framework/targets/ssh.py b/framework/targets/ssh.py new file mode 100644 index 0000000..ef9d9ab --- /dev/null +++ b/framework/targets/ssh.py @@ -0,0 +1,184 @@ +import datetime +import socket +import time + +from framework.target_helpers import Target, TargetError +from libs.external_modules import * +from framework.global_resources import * +from framework.comm_backends import SSH_Backend, BackendError +from framework.knowledge.feedback_collector import FeedbackCollector + +import framework.error_handling as eh + +class SSHTarget(Target): + + NO_PASSWORD = SSH_Backend.NO_PASSWORD + ASK_PASSWORD = SSH_Backend.ASK_PASSWORD + + STATUS_THRESHOLD_FOR_RECOVERY = -2 + + def __init__(self, target_addr='localhost', port=12345, bind_address=None, + username=None, password=None, pkey_path=None, pkey_password=None, + proxy_jump_addr=None, proxy_jump_bind_addr=None, proxy_jump_port=None, + proxy_jump_username=None, proxy_jump_password=None, + proxy_jump_pkey_path=None, proxy_jump_pkey_password=None, + targeted_command=None, file_parameter_path=None, + fbk_timeout=0.5, read_stdout=True, read_stderr=True, char_mapping=False, + get_pty=False, + ref=None): + """ + This generic target enables you to interact with a remote target requiring an SSH connection. + + Args: + target_addr: IP address to reach the SSH server + port: port on which the SSH server listen to. + bind_address: source address for communication. + username: username to use for the connection. + password: (optional) password related to the username. Could also be the special value + `SSHTarget.ASK_PASSWORD` that will prompt the user for the password at the time of connection. + pkey_path: (optional) path to the private key related to the username (if no password provided). + pkey_password: (optional) if the private key is encrypted, this parameter + can be either the password to decrypt it, or the special value `SSHTarget.ASK_PASSWORD` that will + prompt the user for the password at the time of connection. If the private key is + not encrypted, then this parameter should be set to `SSHTarget.NO_PASSWORD` + proxy_jump_addr: If a proxy jump has to be done before reaching the target, this parameter + should be provided with the proxy address to connect with. + proxy_jump_bind_addr: internal address of the proxy to communication with the target. + proxy_jump_port: port on which the SSH server of the proxy listen to. + proxy_jump_username: username to use for the connection with the proxy. + proxy_jump_password: (optional) password related to the username. Could also be the special value + `SSHTarget.ASK_PASSWORD` that will prompt the user for the password at the time of connection. + proxy_jump_pkey_path: (optional) path to the private key related to the username. + proxy_jump_pkey_password: (optional) if the private key is encrypted, this parameter + can be either the password to decrypt it, or the special value `SSHTarget.ASK_PASSWORD` that will + prompt the user for the password at the time of connection. If the private key is + not encrypted, then this parameter should be set to `SSHTarget.NO_PASSWORD`. + targeted_command: If not None, it should be a format string taking one argument that will + be automatically filled either with + the data to be sent or with @file_parameter_path if it is not None (meaning the data + have to be provided through a file). + file_parameter_path: If data should be provided to the targeted command through a file, + then this parameter should provide the remote path where the data to be sent will be + first copied into (otherwise it should remain equal to None). + it will be provided as a parameter of @targeted_command. + fbk_timeout: delay for the framework to wait before it requests feedback from us. + read_stdout (bool): If `True`, collect as feedback what the executed command will write + in stdout. + read_stderr (bool): If `True`, collect as feedback what the executed command will write + in stderr. + char_mapping (dict): If provided, specific characters in the payload will be + replaced based on it. + get_pty (bool): Request a pseudo-terminal from the server. + ref (str): Reference for the target. Used for description only. + """ + Target.__init__(self) + if not ssh_module: + raise eh.UnavailablePythonModule('Python module for SSH is not available!') + + self.ssh_backend = SSH_Backend(target_addr=target_addr, port=port, bind_address=bind_address, username=username, + password=password, pkey_path=pkey_path, pkey_password=pkey_password, + proxy_jump_addr=proxy_jump_addr, proxy_jump_bind_addr = proxy_jump_bind_addr, + proxy_jump_port=proxy_jump_port, + proxy_jump_username=proxy_jump_username, proxy_jump_password=proxy_jump_password, + proxy_jump_pkey_path=proxy_jump_pkey_path, + proxy_jump_pkey_password=proxy_jump_pkey_password, + get_pty=get_pty) + + self.read_stdout = read_stdout + self.read_stderr = read_stderr + self.char_mapping = char_mapping + self.file_paremeter_path = file_parameter_path + if file_parameter_path: + self.targeted_command = targeted_command.format(file_parameter_path) + else: + self.targeted_command = targeted_command + self.tg_ref = ref + self._fbk_collector = FeedbackCollector() + self._set_feedback_timeout_specific(fbk_timeout) + + def start(self): + self._fbk_received = False + self._last_ack_date = None + self.chan_desc = None + + self.ssh_backend.start() + self.sftp = self.ssh_backend.client.open_sftp() if self.file_paremeter_path else None + + return True + + def stop(self): + self.ssh_backend.stop() + self.chan_desc = None + if self.sftp is not None: + self.sftp.close() + + return True + + def recover_target(self): + self.stop() + return self.start() + + def send_data(self, data, from_fmk=False): + self._fbk_received = False + data_str = data.to_str() + if self.char_mapping: + for old_c, new_c in self.char_mapping.items(): + data_str = data_str.replace(old_c, new_c) + if not self.targeted_command: + cmd = data_str + elif self.file_paremeter_path: + input_f = self.sftp.file(self.file_paremeter_path, mode='w', bufsize=-1) + input_f.write(data_str) + input_f.flush() + cmd = self.targeted_command + else: + cmd = self.targeted_command.format(data_str) + + try: + self.chan_desc = self.ssh_backend.exec_command(cmd) + self._last_ack_date = datetime.datetime.now() + self._fbk_received = True + except BackendError as err: + self._logger.collect_feedback(content='{}'.format(err), status_code=err.status) + return + + if self.read_stdout: + try: + data = self.ssh_backend.read_stdout(self.chan_desc) + if data: + self._logger.collect_feedback(content=data, status_code=0, subref='stdout') + except BackendError as err: + self._logger.collect_feedback(content='{}'.format(err), status_code=err.status, + subref='stdout') + + if self.read_stderr: + try: + data = self.ssh_backend.read_stderr(self.chan_desc) + if data: + self._logger.collect_feedback(content=data, status_code=0, subref='stderr') + except BackendError as err: + self._logger.collect_feedback(content='{}'.format(err), status_code=err.status, + subref='stderr') + + def is_feedback_received(self): # useless currently as no-threaded send_data + return self._fbk_received + + def _set_feedback_timeout_specific(self, fbk_timeout): + self.feedback_timeout = fbk_timeout + self.ssh_backend.set_timeout(fbk_timeout) + + def get_last_target_ack_date(self): + return self._last_ack_date + + def get_description(self): + prefix = '{:s} | '.format(self.tg_ref) if self.tg_ref is not None else '' + desc = '{:s}host:{:s},port:{:d},user:{:s}'.format(prefix, self.ssh_backend.host, + self.ssh_backend.port, self.ssh_backend.username) + if self.ssh_backend.proxy_jump_addr: + desc += f" | proxy_jump:{self.ssh_backend.proxy_jump_addr}" + + if self.targeted_command: + desc += f" | cmd='{self.targeted_command}'" + + return desc + diff --git a/framework/value_types.py b/framework/value_types.py index 21f80a9..2ae70cc 100644 --- a/framework/value_types.py +++ b/framework/value_types.py @@ -36,14 +36,7 @@ import zlib import codecs -if sys.version_info[0] > 2: - # python3 - import builtins - CHR_compat = chr -else: - # python2.7 - import __builtin__ as builtins - CHR_compat = unichr +import builtins import framework.basic_primitives as bp from framework.encoders import * @@ -56,9 +49,9 @@ DEBUG = dbg.VT_DEBUG class VT(object): - ''' - Base class for value type classes accepted by value Elts - ''' + """ + Base class to implement Types that are leveraged by typed nodes + """ mini = None maxi = None knowledge_source = None @@ -75,9 +68,6 @@ class VT(object): endian = None - # def __init__(self, endian=BigEndian): - # self.endian = self.enc2struct[endian] - def make_private(self, forget_current_state): pass @@ -109,6 +99,9 @@ def get_current_value(self): def get_current_raw_val(self): return None + def set_default_value(self, val): + raise NotImplementedError + def reset_state(self): raise NotImplementedError @@ -194,50 +187,50 @@ class String(VT_Alt): """ DEFAULT_MAX_SZ = 10000 - encoded_string = False ctrl_char_set = ''.join([chr(i) for i in range(0, 0x20)])+'\x7f' printable_char_set = ''.join([chr(i) for i in range(0x20, 0x7F)]) - extended_char_set = ''.join([CHR_compat(i) for i in range(0x80, 0x100)]) + extended_char_set = ''.join([chr(i) for i in range(0x80, 0x100)]) non_ctrl_char = printable_char_set + extended_char_set - def encode(self, val): + encoded_string = False + + ### used by @from_encoder decorator ### + _encoder_cls = None + _encoder_obj = None + init_encoder = None + ### + + def subclass_specific_init(self, **kwargs): """ - To be overloaded by a subclass that deals with encoding. - (Should be stateless.) + To be overwritten by class that inherits from String if specific init is necessary, + for instance new parameters. Args: - val (bytes): the value + **kwargs: Returns: - bytes: the encoded value + """ - return val + pass - def decode(self, val): + def subclass_specific_test_cases(self, knowledge, orig_val, fuzz_magnitude): """ - To be overloaded by a subclass that deals with encoding. - (Should be stateless.) + To be overwritten by class that inherits from String if specific test cases need to be + implemented + Args: - val (bytes): the encoded value + knowledge: + orig_val: + fuzz_magnitude: Returns: - bytes: the decoded value - """ - return val + list: list of test cases or None - def init_encoding_scheme(self, arg): """ - To be optionally overloaded by a subclass that deals with encoding, - if encoding need to be initialized in some way. (called at init and - in :meth:`String.reset`) - - Args: - arg: provided through the `encoding_arg` parameter of the `String` constructor - """ - return + return None def encoding_test_cases(self, current_val, max_sz, min_sz, min_encoded__sz, max_encoded_sz): """ @@ -256,6 +249,21 @@ def encoding_test_cases(self, current_val, max_sz, min_sz, min_encoded__sz, max_ """ return None + def reset_encoder(self): + self._encoder_obj.reset() + + def encode(self, val): + """ + Exclusively overloaded by the decorator @from_encoder + """ + return val + + def decode(self, val): + """ + Exclusively overloaded by the decorator @from_encoder + """ + return val + def __repr__(self): if DEBUG: return VT_Alt.__repr__(self)[:-1] + ' contents:' + str(self.values) + '>' @@ -270,18 +278,7 @@ def _str2bytes(self, val): for v in val: b.append(self._str2bytes(v)) else: - if sys.version_info[0] > 2: - b = val if isinstance(val, bytes) else val.encode(self.codec) - else: - try: - b = val.encode(self.codec) - except: - if len(val) > 30: - val = val[:30] + ' ...' - err_msg = "\n*** WARNING: Encoding issue. With python2 'str' or 'bytes' means " \ - "ASCII, prefix the string {:s} with 'u'".format(repr(val)) - print(err_msg) - b = val + b = val if isinstance(val, bytes) else val.encode(self.codec) return b def _bytes2str(self, val): @@ -297,16 +294,17 @@ def _bytes2str(self, val): LATIN_1 = codecs.lookup('latin-1').name def __init__(self, values=None, size=None, min_sz=None, - max_sz=None, determinist=True, codec='latin-1', + max_sz=None, determinist=True, codec='latin-1', case_sensitive=True, default=None, extra_fuzzy_list=None, absorb_regexp=None, - alphabet=None, min_encoded_sz=None, max_encoded_sz=None, encoding_arg=None): + alphabet=None, min_encoded_sz=None, max_encoded_sz=None, encoding_arg=None, + values_desc=None, **kwargs): """ Initialize the String Args: values: List of the character strings that are considered valid for the node - backed by this *String object*. + backed by this *String object*. The first item of the list is the default value size: Valid character string size for the node backed by this *String object*. min_sz: Minimum valid size for the character strings for the node backed by this *String object*. If not set, this parameter will be @@ -319,6 +317,10 @@ def __init__(self, values=None, size=None, min_sz=None, determinist: If set to ``True`` generated values will be in a deterministic order, otherwise in a random order. codec: codec to use for encoding the string (e.g., 'latin-1', 'utf8') + case_sensitive: If the string is set to be case sensitive then specific additional + test cases will be generated in fuzzing mode. + default: If not None, this value will be provided by default at first and also each time + :meth:`framework.value_types.String.reset_state() is called`. extra_fuzzy_list: During data generation, if this parameter is specified with some specific values, they will be part of the test cases generated by the generic disruptor tTYPE. @@ -327,6 +329,9 @@ def __init__(self, values=None, size=None, min_sz=None, alphabet: The alphabet to use for generating data, in case no ``values`` is provided. Also use during absorption to validate the contents. It is checked if there is no ``values``. + values_desc (dict): Dictionary that maps string values to their descriptions (character + strings). Leveraged for display purpose. Even if provided, all values do not need to + be described. min_encoded_sz: Only relevant for subclasses that leverage the encoding infrastructure. Enable to provide the minimum legitimate size for an encoded string. max_encoded_sz: Only relevant for subclasses that leverage the encoding infrastructure. @@ -335,6 +340,7 @@ def __init__(self, values=None, size=None, min_sz=None, and that allow their encoding scheme to be configured. This parameter is directly provided to :meth:`String.init_encoding_scheme`. Any object that go through this parameter should support the ``__copy__`` method. + kwargs: for subclass usage """ VT_Alt.__init__(self) @@ -346,37 +352,42 @@ def __init__(self, values=None, size=None, min_sz=None, self.values_fuzzy = None self.values_save = None + self.values_desc = values_desc + if self.values_desc and not isinstance(self.values_desc, dict): + raise ValueError('@values_desc should be a dictionary') + self.is_values_provided = None self.min_sz = None self.max_sz = None - if self.__class__.encode != String.encode: + if self._encoder_cls is not None: self.encoded_string = True - if not hasattr(self, 'encoding_arg'): - self.encoding_arg = encoding_arg - self.init_encoding_scheme(self.encoding_arg) + self._encoding_arg = encoding_arg + self.init_encoder() self.set_description(values=values, size=size, min_sz=min_sz, max_sz=max_sz, determinist=determinist, codec=codec, + case_sensitive=case_sensitive, default=default, extra_fuzzy_list=extra_fuzzy_list, absorb_regexp=absorb_regexp, alphabet=alphabet, min_encoded_sz=min_encoded_sz, max_encoded_sz=max_encoded_sz) + self.subclass_specific_init(**kwargs) + def make_private(self, forget_current_state): + if self.encoded_string: + # we always forget current state of the encoder but it should not impact the current state + # of the String + self._encoding_arg = copy.copy(self._encoding_arg) + self.init_encoder() + if forget_current_state: - if self.is_values_provided: - self.values = copy.copy(self.values) - else: - self._populate_values(force_max_enc_sz=self.max_enc_sz_provided, - force_min_enc_sz=self.min_enc_sz_provided) - self._ensure_enc_sizes_consistency() + self.values = copy.copy(self.values) if self.is_values_provided else None self.reset_state() else: self.values = copy.copy(self.values) self.values_copy = copy.copy(self.values_copy) - if self.encoded_string: - self.encoding_arg = copy.copy(self.encoding_arg) def make_determinist(self): self.determinist = True @@ -462,6 +473,7 @@ def do_absorb(self, blob, constraints, off=0, size=None): self.orig_values = copy.copy(self.values) self.orig_values_copy = copy.copy(self.values_copy) self.orig_drawn_val = self.drawn_val + self.orig_default = self.default if constraints[AbsCsts.Size]: sz = size if size is not None and size < self.max_encoded_sz else self.max_encoded_sz @@ -472,26 +484,15 @@ def do_absorb(self, blob, constraints, off=0, size=None): val_enc_sz = len(self.encode(val)) # maybe different from sz if blob is smaller if val_enc_sz < self.min_encoded_sz: raise ValueError('min_encoded_sz constraint not respected!') - if not self.encoded_string: - val_sz = val_enc_sz + + val_sz = val_enc_sz if not self.encoded_string else None else: blob = blob[off:] #blob[off:size+off] if size is not None else blob[off:] val = self._read_value_from(blob, constraints) val_sz = len(val) - if constraints[AbsCsts.Contents] and self.is_values_provided: - for v in self.values: - if val.startswith(v): - val = v - val_sz = len(val) - break - else: - if self.alphabet is not None: - val, val_sz = self._check_alphabet(val, constraints) - else: - raise ValueError('contents not valid!') - elif constraints[AbsCsts.Contents] and self.alphabet is not None: - val, val_sz = self._check_alphabet(val, constraints) + if constraints[AbsCsts.Contents] or constraints[AbsCsts.SimilarContent]: + val, val_sz = self._check_contents(val, val_sz, constraints) if self.encoded_string: val_enc = self.encode(val) @@ -513,7 +514,16 @@ def do_absorb(self, blob, constraints, off=0, size=None): if self.values is None: self.values = [] - self.values.insert(0, val) + if val in self.values: + # self.values.remove(val) + idx_val = self.values.index(val) + if idx_val + 1 == len(self.values): + pass + else: + self.values = self.values[idx_val:]+self.values[:idx_val] + else: + self.values.insert(0, val) + self.default = val self.reset_state() @@ -525,9 +535,16 @@ def do_absorb(self, blob, constraints, off=0, size=None): def _check_alphabet(self, val, constraints): + if constraints[AbsCsts.SimilarContent]: + alphabet = self.alphabet.lower() + val_tmp = val.lower() + else: + alphabet = self.alphabet + val_tmp = val + i = -1 # to cover case where val is '' - for i, l in enumerate(val): - if l not in self.alphabet: + for i, l in enumerate(val_tmp): + if l not in alphabet: sz = i break else: @@ -550,6 +567,33 @@ def _check_alphabet(self, val, constraints): return val, val_sz + def _check_contents(self, val, val_sz, constraints): + + if self.is_values_provided: + if constraints[AbsCsts.Contents]: + value_list = self.values + compared_val = val + elif constraints[AbsCsts.SimilarContent]: + value_list = map(lambda x: x.lower(), self.values) + compared_val = val.lower() + else: + raise NotImplementedError('constraint is not supported') + + for v in value_list: + if compared_val.startswith(v): + val_sz = len(v) + val = val[:val_sz] + break + else: + if self.alphabet is not None: + val, val_sz = self._check_alphabet(val, constraints) + else: + raise ValueError('contents not valid!') + + elif self.alphabet is not None: + val, val_sz = self._check_alphabet(val, constraints) + + return val, val_sz def do_revert_absorb(self): ''' @@ -564,6 +608,7 @@ def do_revert_absorb(self): self.min_encoded_sz = self.orig_min_encoded_sz self.max_encoded_sz = self.orig_max_encoded_sz self.drawn_val = self.orig_drawn_val + self.default = self.orig_default def do_cleanup_absorb(self): ''' @@ -576,6 +621,7 @@ def do_cleanup_absorb(self): del self.orig_max_sz del self.orig_max_encoded_sz del self.orig_drawn_val + del self.orig_default def _read_value_from(self, blob, constraints): if self.encoded_string: @@ -590,11 +636,23 @@ def _read_value_from(self, blob, constraints): return blob def reset_state(self): + if self.values is None: + assert not self.is_values_provided + self._populate_values(force_max_enc_sz=self.max_enc_sz_provided, + force_min_enc_sz=self.min_enc_sz_provided) + self._ensure_enc_sizes_consistency() + else: + if self.default is not None and self.default != self.values[0]: + # this case may pass if a successful absorption changed the first value in the list + self.values.remove(self.default) + self.values.insert(0, self.default) + self.values_copy = copy.copy(self.values) + self.drawn_val = None + if self.encoded_string: - self.encoding_arg = copy.copy(self.encoding_arg) - self.init_encoding_scheme(self.encoding_arg) + self.reset_encoder() def rewind(self): sz_vlist_copy = len(self.values_copy) @@ -606,24 +664,22 @@ def rewind(self): self.drawn_val = None - def _check_sizes(self, values): - if values is not None: - for v in values: - sz = len(v) - if self.max_sz is not None: - assert(self.max_sz >= sz >= self.min_sz) - else: - assert(sz >= self.min_sz) + def _check_size_constraints(self, value): + sz = len(value) + if self.max_sz < sz or self.min_sz > sz: + raise DataModelDefinitionError def set_description(self, values=None, size=None, min_sz=None, max_sz=None, determinist=True, codec='latin-1', + case_sensitive=True, default=None, extra_fuzzy_list=None, absorb_regexp=None, alphabet=None, min_encoded_sz=None, max_encoded_sz=None): - ''' + """ @size take precedence over @min_sz and @max_sz - ''' + """ self.codec = codecs.lookup(codec).name # normalize + self.case_sensitive = case_sensitive self.max_encoded_sz = max_encoded_sz self.min_encoded_sz = min_encoded_sz self.max_enc_sz_provided = max_encoded_sz is not None @@ -646,27 +702,15 @@ def set_description(self, values=None, size=None, min_sz=None, self.add_specific_fuzzy_vals(extra_fuzzy_list) if values is not None: - assert isinstance(values, list) + if isinstance(values, (str, bytes)): + values = [values] + elif not isinstance(values, list): + raise DataModelDefinitionError + else: + pass self.values = self._str2bytes(values) - for val in self.values: - if not self._check_compliance(val, force_max_enc_sz=self.max_enc_sz_provided, - force_min_enc_sz=self.min_enc_sz_provided, - update_list=False): - raise DataModelDefinitionError - - if self.alphabet is not None: - for l in val: - if l not in self.alphabet: - raise ValueError("The value '%s' does not conform to the alphabet!" % val) - - self.values_copy = copy.copy(self.values) - self.is_values_provided = True # distinguish cases where - # values is provided or - # created based on size - self.user_provided_list = copy.copy(self.values) else: - self.is_values_provided = False - self.user_provided_list = None + self.values = None if size is not None: self.min_sz = size @@ -683,14 +727,10 @@ def set_description(self, values=None, size=None, min_sz=None, elif max_sz is not None: self.max_sz = max_sz self.min_sz = 0 - elif values is not None: - sz = 0 - for v in values: - length = len(v) - if length > sz: - sz = length - self.max_sz = sz - self.min_sz = 0 + elif self.values is not None: + sz_list = list(map(lambda x: len(x), self.values)) + self.max_sz = builtins.max(sz_list) + self.min_sz = builtins.min(sz_list) elif max_encoded_sz is not None: # If we reach this condition, that means no size has been provided, we thus decide # an arbitrary default value for max_sz. Regarding absorption, this arbitrary choice will @@ -701,12 +741,38 @@ def set_description(self, values=None, size=None, min_sz=None, self.min_sz = 0 self.max_sz = self.DEFAULT_MAX_SZ - self._check_sizes(values) + if self.values is not None: + for val in self.values: + if not self._check_constraints(val, force_max_enc_sz=self.max_enc_sz_provided, + force_min_enc_sz=self.min_enc_sz_provided, + update_list=False): + raise DataModelDefinitionError + + if self.alphabet is not None: + for l in val: + if l not in self.alphabet: + raise ValueError("The value '%s' does not conform to the alphabet!" % val) + + self.values_copy = copy.copy(self.values) + self.is_values_provided = True # distinguish cases where + # values is provided or + # created based on size + self.user_provided_list = copy.copy(self.values) + else: + self.is_values_provided = False + self.user_provided_list = None self.determinist = determinist self._ensure_enc_sizes_consistency() + if default is not None: + default = self._str2bytes(default) + self._check_constraints_and_update(default) + self.default = default + else: + self.default = None + def _ensure_enc_sizes_consistency(self): if not self.encoded_string: # For a non-Encoding type, the size of the string is always lesser or equal than the size @@ -720,7 +786,8 @@ def _ensure_enc_sizes_consistency(self): (not self.min_enc_sz_provided and self.min_encoded_sz > self.min_sz): self.min_encoded_sz = self.min_sz - def _check_compliance(self, value, force_max_enc_sz, force_min_enc_sz, update_list=True): + def _check_constraints(self, value, force_max_enc_sz, force_min_enc_sz, update_list=False): + self._check_size_constraints(value) if self.encoded_string: try: enc_val = self.encode(value) @@ -770,17 +837,45 @@ def _check_compliance(self, value, force_max_enc_sz, force_min_enc_sz, update_li self.values.append(value) return True + def _check_constraints_and_update(self, val): + if self._check_constraints(val, + force_max_enc_sz=self.max_enc_sz_provided, + force_min_enc_sz=self.min_enc_sz_provided, + update_list=False): + if self.values is None: + # This case is happening when only @alphabet is provided. Thus, the default value + # will be added at the time self._populate_values() is called. + pass + else: + if val not in self.values: + self.values.insert(0, val) + elif self.default != self.values[0]: + self.values.remove(val) + self.values.insert(0, val) + else: + pass + self.values_copy = None + else: + raise DataModelDefinitionError + def _populate_values(self, force_max_enc_sz=False, force_min_enc_sz=False): self.values = [] alpbt = string.printable if self.alphabet is None else self._bytes2str(self.alphabet) + if self.default is not None: + self._check_constraints(self._str2bytes(self.default), + force_max_enc_sz=force_max_enc_sz, force_min_enc_sz=force_min_enc_sz, + update_list=True) if self.min_sz < self.max_sz: - self._check_compliance(self._str2bytes(bp.rand_string(size=self.max_sz, str_set=alpbt)), - force_max_enc_sz=force_max_enc_sz, force_min_enc_sz=force_min_enc_sz) - self._check_compliance(self._str2bytes(bp.rand_string(size=self.min_sz, str_set=alpbt)), - force_max_enc_sz=force_max_enc_sz, force_min_enc_sz=force_min_enc_sz) + self._check_constraints(self._str2bytes(bp.rand_string(size=self.max_sz, str_set=alpbt)), + force_max_enc_sz=force_max_enc_sz, force_min_enc_sz=force_min_enc_sz, + update_list=True) + self._check_constraints(self._str2bytes(bp.rand_string(size=self.min_sz, str_set=alpbt)), + force_max_enc_sz=force_max_enc_sz, force_min_enc_sz=force_min_enc_sz, + update_list=True) else: - self._check_compliance(self._str2bytes(bp.rand_string(size=self.max_sz, str_set=alpbt)), - force_max_enc_sz=force_max_enc_sz, force_min_enc_sz=force_min_enc_sz) + self._check_constraints(self._str2bytes(bp.rand_string(size=self.max_sz, str_set=alpbt)), + force_max_enc_sz=force_max_enc_sz, force_min_enc_sz=force_min_enc_sz, + update_list=True) if self.min_sz+1 < self.max_sz: NB_VALS_MAX = 3 for idx in range(NB_VALS_MAX): @@ -788,8 +883,8 @@ def _populate_values(self, force_max_enc_sz=False, force_min_enc_sz=False): retry_cpt = 0 while nb_vals < NB_VALS_MAX and retry_cpt < 5: val = bp.rand_string(min=self.min_sz+1, max=self.max_sz-1, str_set=alpbt) - if self._check_compliance(self._str2bytes(val), force_max_enc_sz=force_max_enc_sz, - force_min_enc_sz=force_min_enc_sz): + if self._check_constraints(self._str2bytes(val), force_max_enc_sz=force_max_enc_sz, + force_min_enc_sz=force_min_enc_sz, update_list=True): nb_vals += 1 else: retry_cpt += 1 @@ -812,20 +907,7 @@ def add_to_fuzz_list(flist): if v not in self.values_fuzzy: self.values_fuzzy.append(v) - if self.knowledge_source is None \ - or not self.knowledge_source.is_info_class_represented(Language) \ - or self.knowledge_source.is_assumption_valid(Language.C): - C_strings_enabled = True - else: - C_strings_enabled = False - - if self.knowledge_source \ - and self.knowledge_source.is_info_class_represented(InputHandling) \ - and self.knowledge_source.is_assumption_valid(InputHandling.Ctrl_Char_Set): - CTRL_char_enabled = True - else: - CTRL_char_enabled = False - + ### Common Test Cases if self.drawn_val is not None: orig_val = self.drawn_val else: @@ -867,17 +949,29 @@ def add_to_fuzz_list(flist): unsupported_chars = base_char_set - set(self._bytes2str(self.alphabet)) if unsupported_chars: - sample = random.sample(unsupported_chars, 1)[0] + sample = random.choice(tuple(unsupported_chars))[0] test_case = orig_val[:-1] + sample.encode(self.codec) self.values_fuzzy.append(test_case) - self.values_fuzzy += String.fuzz_cases_ctrl_chars(self.knowledge_source, orig_val, sz, - self.max_sz, self.codec) - self.values_fuzzy += String.fuzz_cases_c_strings(self.knowledge_source, orig_val, sz, - fuzz_magnitude) - self.values_fuzzy.append(orig_val + b'\r\n' * int(100*fuzz_magnitude)) + ### Conditional Test Cases + ctrl_chars_tc = String.fuzz_cases_ctrl_chars(self.knowledge_source, orig_val, sz, + self.max_sz, self.codec) + if ctrl_chars_tc: + add_to_fuzz_list(ctrl_chars_tc) + + c_strings_tc = String.fuzz_cases_c_strings(self.knowledge_source, orig_val, sz, + fuzz_magnitude) + if c_strings_tc: + add_to_fuzz_list(c_strings_tc) + + if self.case_sensitive: + fc_letter_cases = String.fuzz_cases_letter_case(self.knowledge_source, orig_val) + if fc_letter_cases: + add_to_fuzz_list(fc_letter_cases) + + ### CODEC related test cases if self.codec == self.ASCII: val = bytearray(orig_val) if len(val) > 0: @@ -896,16 +990,21 @@ def add_to_fuzz_list(flist): if val not in self.values_fuzzy: self.values_fuzzy.append(val) + ### Specific test cases added by optional string encoders enc_cases = self.encoding_test_cases(orig_val, self.max_sz, self.min_sz, self.min_encoded_sz, self.max_encoded_sz) if enc_cases: self.values_fuzzy += enc_cases + ### Specific test cases added by class that inherits from String() + extended_fuzz_cases = self.subclass_specific_test_cases(self.knowledge_source, orig_val, fuzz_magnitude) + if extended_fuzz_cases is not None: + add_to_fuzz_list(extended_fuzz_cases) + + ### Specific static test cases added at String() initialization through @extra_fuzzy_list specif = self.get_specific_fuzzy_vals() if specif: add_to_fuzz_list(specif) - if hasattr(self, 'subclass_fuzzing_list'): - add_to_fuzz_list(self.subclass_fuzzing_list) self.values_save = self.values self.values = self.values_fuzzy @@ -915,6 +1014,19 @@ def add_to_fuzz_list(flist): @staticmethod def fuzz_cases_c_strings(knowledge, orig_val, sz, fuzz_magnitude): + """ + Produces test cases relevant for C strings + This method is also used by INT_str() + + Args: + knowledge: + orig_val: + sz: + fuzz_magnitude: + + Returns: + + """ if knowledge is None \ or not knowledge.is_info_class_represented(Language) \ or knowledge.is_assumption_valid(Language.C): @@ -934,19 +1046,34 @@ def fuzz_cases_c_strings(knowledge, orig_val, sz, fuzz_magnitude): fuzzy_values.append(b'%n') fuzzy_values.append(b'%s') - fuzzy_values.append(orig_val + b'%n' * int(400*fuzz_magnitude)) - fuzzy_values.append(orig_val + b'%s' * int(400*fuzz_magnitude)) - fuzzy_values.append(orig_val + b'\"%n\"' * int(400*fuzz_magnitude)) - fuzzy_values.append(orig_val + b'\"%s\"' * int(400*fuzz_magnitude)) + if knowledge is None \ + or not knowledge.is_info_class_represented(Test) \ + or not knowledge.is_assumption_valid(Test.Cursory): + fuzzy_values.append(orig_val + b'%n' * int(400*fuzz_magnitude)) + fuzzy_values.append(orig_val + b'%s' * int(400*fuzz_magnitude)) return fuzzy_values else: - return [] + return None @staticmethod def fuzz_cases_ctrl_chars(knowledge, orig_val, sz, max_sz, codec): + """ + Produces test cases relevant when control characters are interpreted by the consumer + This method is also used by INT_str() + + Args: + knowledge: + orig_val: + sz: + max_sz: + codec: + + Returns: + + """ if knowledge \ and knowledge.is_info_class_represented(InputHandling) \ and knowledge.is_assumption_valid(InputHandling.Ctrl_Char_Set): @@ -968,7 +1095,32 @@ def fuzz_cases_ctrl_chars(knowledge, orig_val, sz, max_sz, codec): return fuzzy_values else: - return [] + return None + + + @staticmethod + def fuzz_cases_letter_case(knowledge, orig_val): + """ + Produces test cases relevant if the described element is case sensitive. + + Args: + knowledge: + orig_val: + + Returns: + + """ + fuzzy_values = [] + for idx, l in enumerate(orig_val): + if ord('a') <= l <= ord('z'): + fuzzy_values.append(orig_val[:idx]+chr(l).upper().encode()+orig_val[idx+1:]) + break + for idx, l in enumerate(orig_val): + if ord('A') <= l <= ord('Z'): + fuzzy_values.append(orig_val[:idx]+chr(l).lower().encode()+orig_val[idx+1:]) + break + return fuzzy_values + def get_value(self): if not self.values: @@ -999,6 +1151,12 @@ def get_current_value(self): self.get_value() return self.encode(self.drawn_val) if self.encoded_string else self.drawn_val + def set_default_value(self, val): + val = self._str2bytes(val) + self._check_constraints_and_update(val) + self.default = val + self.reset_state() + def is_exhausted(self): if self.values_copy: return False @@ -1027,17 +1185,21 @@ def pretty_print(self, max_size=None): if self.drawn_val is None: self.get_value() + if self.values_desc: + desc = self.values_desc.get(self.drawn_val) + desc = '' if desc is None else ", desc='" + desc + "'" + else: + desc = '' + if self.encoded_string or self.codec not in [self.ASCII, self.LATIN_1]: dec = self.drawn_val sz = len(dec) if max_size is not None and sz > max_size: dec = dec[:max_size] dec = dec.decode(self.codec, 'replace') - if sys.version_info[0] == 2: - dec = dec.encode('latin-1') - return dec + ' [decoded, sz={!s}, codec={!s}]'.format(len(dec), self.codec) + return dec + ' [decoded, sz={!s}, codec={!s}{:s}]'.format(len(dec), self.codec, desc) else: - return 'codec={!s}'.format(self.codec) + return 'codec={!s}{:s}'.format(self.codec, desc) class INT(VT): @@ -1070,7 +1232,7 @@ def __init__(self, values=None, min=None, max=None, default=None, determinist=Tr self.determinist = determinist self.exhausted = False self.drawn_val = None - self.default = None + self.default = default self._specific_fuzzy_vals = None self.values_desc = values_desc @@ -1101,11 +1263,7 @@ def __init__(self, values=None, min=None, max=None, default=None, determinist=Tr raise DataModelDefinitionError("Incompatible value ({!r}) with {!s}".format(v, self.__class__)) self.values = list(values) - if default is not None: - assert default in self.values - self.values.remove(default) - self.values.insert(0, default) - self.values_copy = list(self.values) + self.values_copy = copy.copy(self.values) else: if min is not None and max is not None: @@ -1116,12 +1274,6 @@ def __init__(self, values=None, min=None, max=None, default=None, determinist=Tr # we keep min/max information as it may be valuable for fuzzing self.mini = self.mini_gen = min self.maxi = self.maxi_gen = max - if default is not None: - assert min <= default <= max - self.values.remove(default) - self.values.insert(0, default) - # Once inserted at this place, its position is preserved, especially with reset_state() - # (assuming do_absorb() is not called), so we do not save 'default' value in this case self.values_copy = copy.copy(self.values) else: @@ -1149,19 +1301,44 @@ def __init__(self, values=None, min=None, max=None, default=None, determinist=Tr else: self.maxi = self.maxi_gen = max - if default is not None: - assert self.mini_gen <= default <= self.maxi_gen - self.default = default - self.idx = default - self.mini_gen + if default is not None: + self._check_constraints_and_update(default) + self.default = default + + def _check_constraints_and_update(self, val, no_update=False): + if self.values: + if val in self.values: + if not no_update and val != self.values[0]: + self.values.remove(val) + self.values.insert(0, val) + self.values_copy = copy.copy(self.values) + else: + pass + else: + raise DataModelDefinitionError + else: + if self.mini_gen <= val <= self.maxi_gen: + if not no_update: + self.idx = val - self.mini_gen + else: + pass + else: + raise DataModelDefinitionError + def make_private(self, forget_current_state): # no need to copy self.default (that should not be modified) if forget_current_state: - self.values_copy = copy.copy(self.values) - self.idx = 0 + self.values = copy.copy(self.values) + self.values_copy = None #copy.copy(self.values) + if self.default is not None and self.values is None: + self.idx = self.default - self.mini_gen + else: + self.idx = 0 self.exhausted = False self.drawn_val = None else: + self.values = copy.copy(self.values) self.values_copy = copy.copy(self.values_copy) def copy_attrs_from(self, vt): @@ -1174,8 +1351,8 @@ def get_fuzzed_vt_list(self): val = self.get_current_raw_val() if val is not None: - # don't use a set to preserve determinism if needed - if val-1 not in supp_list: + # don't use a set to preserve the order if needed + if val+1 not in supp_list: supp_list.append(val+1) if val-1 not in supp_list: supp_list.append(val-1) @@ -1262,47 +1439,58 @@ def do_absorb(self, blob, constraints, off=0, size=None): self.orig_values = copy.copy(self.values) self.orig_values_copy = copy.copy(self.values_copy) self.orig_drawn_val = self.drawn_val + self.orig_default = self.default + self.orig_idx = self.idx blob = blob[off:] val, sz = self._read_value_from(blob, size) - orig_val = self._unconvert_value(val) + decoded_val = self._unconvert_value(val) if self.values is not None: if constraints[AbsCsts.Contents]: - if orig_val not in self.values: + if decoded_val not in self.values: raise ValueError('contents not valid!') - self.values.insert(0, orig_val) + if decoded_val in self.values: + # self.values.remove(decoded_val) + idx_val = self.values.index(decoded_val) + if idx_val + 1 == len(self.values): + pass + else: + self.values = self.values[idx_val:]+self.values[:idx_val] + else: + self.values.insert(0, decoded_val) self.values_copy = copy.copy(self.values) elif self.maxi is None and self.mini is None: # this case means 'self' is an unlimited INT (like INT_str subclass) where no constraints # have been provided to the constructor, like INT_str(). - self.values = [orig_val] - self.values_copy = [orig_val] + self.values = [decoded_val] + self.values_copy = [decoded_val] else: if constraints[AbsCsts.Contents]: - if self.maxi is not None and orig_val > self.maxi: + if self.maxi is not None and decoded_val > self.maxi: raise ValueError('contents not valid! (max limit)') - if self.mini is not None and orig_val < self.mini: + if self.mini is not None and decoded_val < self.mini: raise ValueError('contents not valid! (min limit)') else: # mini_gen and maxi_gen are always defined - if orig_val < self.mini_gen: - if self.__class__.mini is not None and orig_val < self.__class__.mini: + if decoded_val < self.mini_gen: + if self.__class__.mini is not None and decoded_val < self.__class__.mini: raise ValueError('The type {!s} is not able to represent the value {:d}' - .format(self.__class__, orig_val)) - self.mini = self.mini_gen = orig_val - if orig_val > self.maxi_gen: - if self.__class__.maxi is not None and orig_val > self.__class__.maxi: + .format(self.__class__, decoded_val)) + self.mini = self.mini_gen = decoded_val + if decoded_val > self.maxi_gen: + if self.__class__.maxi is not None and decoded_val > self.__class__.maxi: raise ValueError('The type {!s} is not able to represent the value {:d}' - .format(self.__class__, orig_val)) - self.maxi = self.maxi_gen = orig_val + .format(self.__class__, decoded_val)) + self.maxi = self.maxi_gen = decoded_val - self.idx = orig_val - self.mini_gen + self.idx = decoded_val - self.mini_gen # self.reset_state() self.exhausted = False - self.drawn_val = orig_val + self.drawn_val = decoded_val + self.default = decoded_val return val, off, sz @@ -1315,12 +1503,15 @@ def do_revert_absorb(self): self.values = self.orig_values self.values_copy = self.orig_values_copy self.drawn_val = self.orig_drawn_val + self.default = self.orig_default + self.idx = self.orig_idx def do_cleanup_absorb(self): if hasattr(self, 'orig_drawn_val'): del self.orig_values del self.orig_values_copy del self.orig_drawn_val + del self.orig_default def make_determinist(self): self.determinist = True @@ -1328,9 +1519,6 @@ def make_determinist(self): def make_random(self): self.determinist = False - def get_value_list(self): - return self.values - def get_current_raw_val(self): if self.drawn_val is None: self.get_value() @@ -1347,59 +1535,6 @@ def is_size_compatible(self, integer): else: return False - def set_value_list(self, new_list): - ret = False - if self.values: - l = list(filter(self.is_compatible, new_list)) - if l: - self.values = l - self.values_copy = copy.copy(self.values) - self.idx = 0 - ret = True - - return ret - - def extend_value_list(self, new_list): - if self.values is not None: - l = list(filter(self.is_compatible, new_list)) - if l: - values_enc = list(map(self._convert_value, self.values)) - - # We copy the list as it is a class attribute in - # Fuzzy_* classes, and we don't want to change the classes - # (as we modify the list contents and not the list itself) - self.values = list(self.values) - - # we don't use a set to preserve the order - for v in l: - # we check the converted value to avoid duplicated - # values (negative and positive value coded the - # same) --> especially usefull for the Fuzzy_INT class - if self._convert_value(v) not in values_enc: - self.values.insert(0, v) - - self.idx = 0 - self.values_copy = copy.copy(self.values) - - - def remove_value_list(self, value_list): - if self.values is not None: - l = list(filter(self.is_compatible, value_list)) - if l: - # We copy the list as it is a class attribute in - # Fuzzy_* classes, and we don't want to change the classes - # (as we modify the list contents and not the list itself) - self.values = list(self.values) - - for v in l: - try: - self.values.remove(v) - except ValueError: - pass - - self.idx = 0 - self.values_copy = copy.copy(self.values) - def get_value(self): if self.values is not None: if not self.values_copy: @@ -1446,6 +1581,11 @@ def get_current_value(self): self.get_value() return self._convert_value(self.drawn_val) + def set_default_value(self, val): + self._check_constraints_and_update(val) + self.default = val + self.reset_state() + def set_size_from_constraints(self, size=None, encoded_size=None): raise DataModelDefinitionError @@ -1505,11 +1645,15 @@ def _read_value_from(self, blob, size): return struct.pack(self.cformat, val), sz def reset_state(self): - if self.default is not None: + if self.default is not None and self.values is None: self.idx = self.default - self.mini_gen else: self.idx = 0 if self.values is not None: + if self.default is not None and self.default != self.values[0]: + # this case may pass if a successful absorption changed the first value in the list + self.values.remove(self.default) + self.values.insert(0, self.default) self.values_copy = copy.copy(self.values) self.exhausted = False self.drawn_val = None @@ -1517,11 +1661,22 @@ def reset_state(self): def update_raw_value(self, val): ok = True if isinstance(val, int): - if val > self.__class__.maxi: + if self.__class__.maxi is not None and val > self.__class__.maxi: + # self.__class__.maxi is None for INT_str which don't have any limit for maximum value + # thus this check has to be ignored with INT_str val = self.__class__.maxi ok = False if self.values is not None: - self.values.append(val) + if val in self.values: + # self.values.remove(val) + idx_val = self.values.index(val) + if idx_val + 1 == len(self.values): + pass + else: + self.values = self.values[idx_val:]+self.values[:idx_val] + else: + self.values.insert(0, val) + self.values_copy = copy.copy(self.values) else: self.idx = val - self.mini_gen @@ -1539,44 +1694,192 @@ def is_exhausted(self): class Filename(String): - @property - def subclass_fuzzing_list(self): - linux_spe = [b'../../../../../../etc/password'] - windows_spe = [b'..\\..\\..\\..\\..\\..\\Windows\\system.ini'] - c_spe = [b'file%n%n%n%nname.txt'] + linux_prefix = [ + b'../', + b'..\xc0\xaf', # incorrect UTF-8 encoding for '/' from Markus Kuhn + b'\xc0\xae\xc0\xae\xc0\xaf' # incorrect UTF-8 encoding for '../' + ] + linux_suffix = [b'etc/password'] + linux_specific_fnames = [ + b'A'*256, # most of FS filename limit is 255 chars + b'./'*2046 + b'TEST' # most FS path limit is 4096 including NULL end byte + ] + + windows_prefix = [ + b'..\x5c', + b'\xc0\xae\xc0\xae\\' + ] + windows_suffix = [b'Windows\\system.ini'] + windows_specific_fnames = [ + b'PRN', # reserved + b'NUL.txt', # not recommended fname + b'C:\\..\\..\\', # invalid reference + b'A'*256 + b'.txt' # MAX_PATH == 260 including NULL end byte + ] + + uri_prefix = [ + b'%2e%2e%2f', # hex encoding for '../' + b'%2e%2e/', # mixed encoding for '../' + b'..%2f', # mixed encoding for '../' + b'..%252f', # double hex encoding for '../' + b'.%252e/', # double hex encoding for '../' + b'%2e%2e%5c', # hex encoding for '..\' + b'..%255c', # double hex encoding for '..\' + b'..%c0%af', # incorrect UTF-8 encoding for '/' + b'%c0%ae%c0%ae%c0%af' # incorrect UTF-8 encoding for '../' + ] + uri_suffix = [b'MARKER.txt'] + + path_mode = False + + def subclass_specific_init(self, specific_suffix=None, uri_parsing=False): + """ + Specific init for Filename + + Args: + specific_suffix: List of specific suffixes that will be used for path traversal test + cases in addition to the current list. + uri_parsing: if the filename is to be consumed as an URI + + """ + if isinstance(specific_suffix, list) or specific_suffix is None: + self.__specific_suffix = specific_suffix + else: + self.__specific_suffix = list(specific_suffix) + self.__uri_parsing = uri_parsing + + def _get_path_from_value(self, value, knowledge): + """ Returned path always terminates with a separator """ + + if value[-1:] == b'/' or value[-1:] == b'\\': + path = value + elif self.path_mode: + if knowledge and knowledge.is_info_class_represented(OS): + if knowledge.is_assumption_valid(OS.Linux): + path = value + b'/' + elif knowledge.is_assumption_valid(OS.Windows): + path = value + b'\\' + else: + raise NotImplementedError + else: + if value.find(b'/') != -1: + path = value + b'/' + elif value.find(b'\\') != -1: + path = value + b'\\' + else: + raise NotImplementedError + else: + idx_linux = value.rfind(b'/') + if idx_linux != -1: + if not self.__uri_parsing and knowledge is not None: + knowledge.add_information(OS.Linux) + path = value[:idx_linux+1] + else: + idx_windows = value.rfind(b'\\') + if idx_windows != -1: + if knowledge is not None: + knowledge.add_information(OS.Windows) + path = value[:idx_windows+1] + else: + path = b'' + + return path - if self.knowledge_source is None: - flist = linux_spe+windows_spe+c_spe + def _get_path_depth(self, path): + nb_sep_linux = path.count(b'/') + if nb_sep_linux > 0: + depth = nb_sep_linux if path[:1] != b'/' else nb_sep_linux-1 else: - flist = [] - if self.knowledge_source.is_info_class_represented(OS): - if self.knowledge_source.is_assumption_valid(OS.Linux): - flist += linux_spe - if self.knowledge_source.is_assumption_valid(OS.Windows): - flist += windows_spe + nb_sep_windows = path.count(b'\\') + if nb_sep_windows > 0: + depth = nb_sep_windows if path[2:3] != b'\\' else nb_sep_windows-1 else: - flist = linux_spe+windows_spe - if self.knowledge_source.is_info_class_represented(Language): - if self.knowledge_source.is_assumption_valid(Language.C): - flist += c_spe + depth = 1 if self.path_mode else 0 + + return depth + + def subclass_specific_test_cases(self, knowledge, orig_val, fuzz_magnitude=1.0): + orig_path = self._get_path_from_value(orig_val, knowledge) + orig_depth = self._get_path_depth(orig_path) + dir_depth = int((5+orig_depth)*fuzz_magnitude) + + def gen_path_traversal_tc(tc_output, prefixes, suffixes, orig_path=b''): + for pre in prefixes: + for suf in suffixes: + tc_output.append(orig_path+pre*dir_depth+suf) + + def gen_windows_invalid_fname(tc_output, orig_fname): + tc_output += self.windows_specific_fnames + tc_output += [ + orig_fname + b'.', # incorrect position for '.' + orig_fname + b' ', # incorrect position for ' ' + orig_fname + b':' # invalid character ':' + ] + + def gen_linux_invalid_fname(tc_output, orig_fname): + tc_output += self.linux_specific_fnames + tc_output += [ + orig_fname + b'/A', # invalid character '/' + orig_fname + b'\0A', # invalid character '\0' + ] + + test_cases = [] + + if self.__uri_parsing: + gen_path_traversal_tc(test_cases, prefixes=self.uri_prefix, + suffixes=self.uri_suffix if self.__specific_suffix is None else self.__specific_suffix, + orig_path=orig_path) + else: + test_cases += [ + b'*.*' + ] + + if knowledge is None or not knowledge.is_info_class_represented(OS): + gen_linux_invalid_fname(test_cases, orig_val) + gen_windows_invalid_fname(test_cases, orig_val) + gen_path_traversal_tc(test_cases, prefixes=self.linux_prefix, + suffixes=self.linux_suffix if self.__specific_suffix is None else self.__specific_suffix, + orig_path=orig_path) + gen_path_traversal_tc(test_cases, prefixes=self.windows_prefix, + suffixes=self.windows_suffix if self.__specific_suffix is None else self.__specific_suffix, + orig_path=orig_path) else: - flist += c_spe + if knowledge.is_assumption_valid(OS.Linux): + gen_linux_invalid_fname(test_cases, orig_val) + gen_path_traversal_tc(test_cases, prefixes=self.linux_prefix, + suffixes=self.linux_suffix if self.__specific_suffix is None else self.__specific_suffix, + orig_path=orig_path) + if knowledge.is_assumption_valid(OS.Windows): + gen_windows_invalid_fname(test_cases, orig_val) + gen_path_traversal_tc(test_cases, prefixes=self.windows_prefix, + suffixes=self.windows_suffix if self.__specific_suffix is None else self.__specific_suffix, + orig_path=orig_path) + + return test_cases + +class FolderPath(Filename): + + def subclass_specific_test_cases(self, knowledge, orig_val, fuzz_magnitude=1.0): + self.path_mode = True + return Filename.subclass_specific_test_cases(self, knowledge=knowledge, orig_val=orig_val, + fuzz_magnitude=fuzz_magnitude) - return flist def from_encoder(encoder_cls, encoding_arg=None): def internal_func(string_subclass): - def new_meth(meth): - return meth if sys.version_info[0] > 2 else meth.im_func - string_subclass.encode = new_meth(encoder_cls.encode) - string_subclass.decode = new_meth(encoder_cls.decode) - string_subclass.init_encoding_scheme = new_meth(encoder_cls.init_encoding_scheme) - if encoding_arg is not None: - string_subclass.encoding_arg = encoding_arg + def init_encoder(self): + self._encoder_obj = self._encoder_cls(self._encoding_arg) + self.encode = self._encoder_obj.encode + self.decode = self._encoder_obj.decode + + string_subclass._encoder_cls = encoder_cls + string_subclass._encoder_arg = encoding_arg + string_subclass.init_encoder = init_encoder + return string_subclass - return internal_func + return internal_func @from_encoder(GZIP_Enc) class GZIP(String): pass @@ -1595,14 +1898,14 @@ class INT_str(INT): endian = VT.Native usable = True - regex_decimal = b'-?\d+' + regex_decimal = b'-?\d' - regex_upper_hex = b'-?[0123456789ABCDEF]+' - regex_lower_hex = b'-?[0123456789abcdef]+' + regex_upper_hex = b'-?[0123456789ABCDEF]' + regex_lower_hex = b'-?[0123456789abcdef]' - regex_octal = b'-?[01234567]+' + regex_octal = b'-?[01234567]' - regex_bin = b'-?[01]+' + regex_bin = b'-?[01]' fuzzy_values = [0, -1, -2**32, 2 ** 32 - 1, 2 ** 32] value_space_size = -1 # means infinite @@ -1633,11 +1936,33 @@ def copy_attrs_from(self, vt): self._regex = vt._regex def _prepare_format_str(self, min_size, base, letter_case): + + if self.maxi is not None: + max_val = self.maxi + elif self.values: + max_val = max(self.values) + else: + max_val = INT.GEN_MAX_INT + + if self.mini is not None: + min_val = self.mini + elif self.values: + min_val = min(self.values) + else: + min_val = 0 + + self.max_digit = math.floor(math.log(abs(max_val), base))+1 if max_val != 0 else 1 + self.min_digit = math.floor(math.log(abs(min_val), base))+1 if min_val != 0 else 1 if min_size is not None: + self.min_digit = max(self.min_digit, min_size) format_str = '{:0' + str(min_size) + if self.max_digit < self.min_digit: + self.max_digit = self.min_digit else: format_str = '{:' + regex_prefix = '{{{},{}}}'.format(self.min_digit,self.max_digit).encode() + if base == 10: format_str += '}' regex = self.regex_decimal @@ -1657,7 +1982,7 @@ def _prepare_format_str(self, min_size, base, letter_case): else: raise ValueError(self._base) - return (format_str, regex) + return (format_str, regex+regex_prefix) def get_fuzzed_vt_list(self): @@ -1710,10 +2035,15 @@ def handle_size(self, v): else: max_sz = sz - fuzzed_vals += String.fuzz_cases_ctrl_chars(self.knowledge_source, orig_val, sz, - max_sz, codec=String.ASCII) - fuzzed_vals += String.fuzz_cases_c_strings(self.knowledge_source, orig_val, sz, + ctrl_chars_tc = String.fuzz_cases_ctrl_chars(self.knowledge_source, orig_val, sz, + max_sz, codec=String.ASCII) + if ctrl_chars_tc: + fuzzed_vals += ctrl_chars_tc + + c_strings_tc = String.fuzz_cases_c_strings(self.knowledge_source, orig_val, sz, fuzz_magnitude=0.3) + if c_strings_tc: + fuzzed_vals += c_strings_tc fuzzed_vals.append(orig_val + b'\r\n' * 100) @@ -1767,9 +2097,9 @@ class BitField(VT_Alt): def __init__(self, subfield_limits=None, subfield_sizes=None, subfield_values=None, subfield_val_extremums=None, - padding=0, lsb_padding=True, + padding=0, lsb_padding=True, show_padding=False, endian=VT.BigEndian, determinist=True, - subfield_descs=None, defaults=None): + subfield_descs=None, subfield_value_descs=None, defaults=None): VT_Alt.__init__(self) @@ -1785,8 +2115,10 @@ def __init__(self, subfield_limits=None, subfield_sizes=None, self.endian = endian self.padding = padding self.lsb_padding = lsb_padding + self.show_padding = show_padding self.subfield_descs = None + self.subfield_value_descs = None self.subfield_limits = [] self.subfield_sizes = [] self.subfield_vals = None @@ -1794,21 +2126,25 @@ def __init__(self, subfield_limits=None, subfield_sizes=None, self.subfield_extrems = None self.subfield_extrems_save = None self.subfield_fuzzy_vals = [] - self.current_idx = None + self.current_subfield = None self.idx = None self.idx_inuse = None self.set_bitfield(sf_values=subfield_values, sf_val_extremums=subfield_val_extremums, sf_limits=subfield_limits, sf_sizes=subfield_sizes, - sf_descs=subfield_descs, sf_defaults=defaults) + sf_descs=subfield_descs, sf_val_descs=subfield_value_descs, + sf_defaults=defaults) def make_private(self, forget_current_state): - # no need to copy self.default (that should not be modified) + # subfield_defaults, subfield_descs and subfield_value_descs are not made private + # as it is not meant to be modified outside of .set_bitfield() which completely change the BitField + # object and thus will definitely remove the dependencies to the original BitField. self.subfield_limits = copy.deepcopy(self.subfield_limits) self.subfield_sizes = copy.deepcopy(self.subfield_sizes) self.subfield_vals = copy.deepcopy(self.subfield_vals) self.subfield_vals_save = copy.deepcopy(self.subfield_vals_save) self.subfield_extrems = copy.deepcopy(self.subfield_extrems) self.subfield_extrems_save = copy.deepcopy(self.subfield_extrems_save) + self.subfield_defaults = copy.deepcopy(self.subfield_defaults) if forget_current_state: self.reset_state() else: @@ -1818,10 +2154,6 @@ def make_private(self, forget_current_state): def reset_state(self): self._reset_idx() - for i, default in enumerate(self.subfield_defaults): - if default is not None: - mini, _ = self.subfield_extrems[i] - self.idx[i] = default - mini self.drawn_val = None self.__count_of_possible_values = None self.exhausted = False @@ -1832,22 +2164,46 @@ def reset_state(self): self.switch_mode() def _reset_idx(self, reset_idx_inuse=True): - self.current_idx = 0 - self.idx = [1 for i in self.subfield_limits] - if not self._fuzzy_mode: - self.idx[0] = 0 + self.current_subfield = 0 + if self._fuzzy_mode: + self.idx = [1 for i in self.subfield_limits] + else: + self.idx = [0 for i in self.subfield_limits] + for i, default in enumerate(self.subfield_defaults): + if default is not None: + if self.subfield_extrems[i] is None: + self.idx[i] = self.subfield_vals[i].index(default) + else: + mini, _ = self.subfield_extrems[i] + self.idx[i] = default - mini # initially we don't make copy, as it will be copied anyway # during .get_value() if reset_idx_inuse: self.idx_inuse = self.idx - + + + def idx_from_desc(self, sf_desc): + if self.subfield_descs is None: + raise DataModelAccessError + try: + idx = self.subfield_descs.index(sf_desc) + except ValueError: + raise DataModelAccessError(f'wrong subfield description: {sf_desc!r}') + + return idx + def set_subfield(self, idx, val): - ''' + """ Args: - idx (int): subfield index, from 0 (low significant subfield) to nb_subfields-1 + idx: Either an integer which should be the subfield index, + from 0 (low significant subfield) to nb_subfields-1 (specific index -1 is used to choose the last subfield). + Or a string which should be the description of the field. val (int): new value for the subfield - ''' + """ + + if isinstance(idx, str): + idx = self.idx_from_desc(idx) if idx == -1: idx = len(self.subfield_sizes) - 1 assert(self.is_compatible(val, self.subfield_sizes[idx])) @@ -1868,8 +2224,61 @@ def set_subfield(self, idx, val): self.current_val_update_pending = True + def change_subfield(self, idx, values=None, extremums=None): + """ + Change the constraints on a given subfield. + + Args: + idx (int): subfield index, from 0 (low significant subfield) to nb_subfields-1 + (specific index -1 is used to choose the last subfield). + values (list): new values for the subfield (remove previous value list or remove previous + extremums if no value list was used for this subfield) + extremums (list): new extremums for the subfield (remove previous extremums or remove + previous value list if no extremums were used for this subfield) + """ + if isinstance(idx, str): + idx = self.idx_from_desc(idx) + if idx == -1: + idx = len(self.subfield_sizes) - 1 + + if values is not None: + for v in values: + assert self.is_compatible(v, self.subfield_sizes[idx]) + self.subfield_extrems[idx] = None + if self.subfield_extrems_save is not None: + self.subfield_extrems_save[idx] = None + self.subfield_vals[idx] = copy.copy(values) + if self._fuzzy_mode: + self.subfield_vals_save = self.subfield_vals + if self.subfield_defaults[idx] not in values: + self.subfield_defaults[idx] = None + self.idx_inuse[idx] = self.idx[idx] = 0 + else: + self.idx_inuse[idx] = self.idx[idx] = self.subfield_vals[idx].index(self.subfield_defaults[idx]) + elif extremums is not None: + assert len(extremums) == 2 and extremums[0] <= extremums[1] + assert self.is_compatible(extremums[0], self.subfield_sizes[idx]) + assert self.is_compatible(extremums[1], self.subfield_sizes[idx]) + self.subfield_vals[idx] = None + if self.subfield_vals_save is not None: + self.subfield_vals_save[idx] = None + self.subfield_extrems[idx] = copy.copy(extremums) + if self._fuzzy_mode: + self.subfield_extrems_save = self.subfield_extrems + if self.subfield_defaults[idx] < extremums[0] or self.subfield_defaults[idx] > extremums[1]: + self.subfield_defaults[idx] = None + self.idx_inuse[idx] = self.idx[idx] = 0 + else: + self.idx_inuse[idx] = self.idx[idx] = self.subfield_defaults[idx] - extremums[0] + else: + raise ValueError + + self.current_val_update_pending = True + def get_subfield(self, idx): + if isinstance(idx, str): + idx = self.idx_from_desc(idx) if idx == -1: idx = len(self.subfield_sizes) - 1 if self.subfield_vals[idx] is None: @@ -1884,7 +2293,7 @@ def get_subfield(self, idx): def set_bitfield(self, sf_values=None, sf_val_extremums=None, sf_limits=None, sf_sizes=None, - sf_descs=None, sf_defaults=None): + sf_descs=None, sf_val_descs=None, sf_defaults=None): if sf_limits is not None: self.subfield_limits = copy.copy(sf_limits) @@ -1904,17 +2313,14 @@ def set_bitfield(self, sf_values=None, sf_val_extremums=None, sf_limits=None, sf if sf_val_extremums is None: sf_val_extremums = [None for i in range(len(self.subfield_limits))] elif len(sf_val_extremums) != len(self.subfield_limits): - raise DataModelDefinitionError + raise DataModelDefinitionError('inconsistent number of subfields') if sf_descs is not None: assert(len(self.subfield_limits) == len(sf_descs)) self.subfield_descs = copy.copy(sf_descs) - if sf_defaults is not None: - assert len(sf_defaults) == len(self.subfield_limits) - self.subfield_defaults = copy.copy(sf_defaults) - else: - self.subfield_defaults = [None for i in range(len(self.subfield_limits))] + if sf_val_descs is not None: + self.subfield_value_descs = copy.copy(sf_val_descs) self.size = self.subfield_limits[-1] self.nb_bytes = int(math.ceil(self.size / 8.0)) @@ -1924,8 +2330,6 @@ def set_bitfield(self, sf_values=None, sf_val_extremums=None, sf_limits=None, sf else: self.padding_size = 8 - (self.size % 8) - self._reset_idx() - self.subfield_vals = [] self.subfield_extrems = [] @@ -1944,11 +2348,10 @@ def set_bitfield(self, sf_values=None, sf_val_extremums=None, sf_limits=None, sf for v in values: if self.is_compatible(v, size): l.append(v) - default = self.subfield_defaults[idx] - if default is not None: - assert default in l - l.remove(default) - l.insert(self.idx[idx], default) + else: + s = 'value "{:d}" is out of range for the subfield {:d} [size: {:d} bit(s)]'\ + .format(v, idx, size) + raise ValueError(s) self.subfield_vals.append(l) self.subfield_extrems.append(None) else: @@ -1958,7 +2361,7 @@ def set_bitfield(self, sf_values=None, sf_val_extremums=None, sf_limits=None, sf assert(mini != maxi) self.subfield_extrems.append([mini, maxi]) else: - s = '*** ERROR: builtins.min({:d}) / builtins.max({:d}) values are out of range!'.format(mini, maxi) + s = 'builtins.min({:d}) / builtins.max({:d}) values are out of range'.format(mini, maxi) raise ValueError(s) self.subfield_vals.append(None) else: @@ -1966,13 +2369,30 @@ def set_bitfield(self, sf_values=None, sf_val_extremums=None, sf_limits=None, sf self.subfield_extrems.append([mini, maxi]) self.subfield_vals.append(None) - default = self.subfield_defaults[idx] - if default is not None: - self.idx[idx] = default - mini - self.subfield_fuzzy_vals.append(None) prev_lim = lim + if sf_defaults is not None: + self._check_constraints(sf_defaults) + self.subfield_defaults = copy.copy(sf_defaults) + else: + self.subfield_defaults = [None for i in range(len(self.subfield_limits))] + + self._reset_idx() + + def _check_constraints(self, sf_values): + if len(sf_values) != len(self.subfield_limits): + raise DataModelDefinitionError + + for i, default in enumerate(sf_values): + if default is not None: + if self.subfield_extrems[i] is None: + if default not in self.subfield_vals[i]: + raise DataModelDefinitionError + else: + mini, maxi = self.subfield_extrems[i] + if mini > default or maxi < default: + raise DataModelDefinitionError @property def bit_length(self): @@ -2039,11 +2459,30 @@ def extend(self, bitfield, rightside=True): desc_extension = bitfield.subfield_descs elif self.subfield_descs is not None and bitfield.subfield_descs is None: desc_extension = [None for i in bitfield.subfield_limits] + else: + desc_extension = bitfield.subfield_descs if rightside: self.subfield_descs += desc_extension else: self.subfield_descs = desc_extension + self.subfield_descs + if self.subfield_value_descs is not None or bitfield.subfield_value_descs is not None: + if self.subfield_value_descs is None: + self.subfield_value_descs = {} + desc_ext = {} if bitfield.subfield_value_descs is None else bitfield.subfield_value_descs + + if rightside: + offset = len(self.subfield_sizes) + for k, v in desc_ext.items(): + self.subfield_value_descs[k+offset] = v + else: + offset = len(bitfield.subfield_sizes) + new_dict = {} + for k, v in self.subfield_value_descs.items(): + new_dict[k+offset] = v + new_dict.update(desc_ext) + self.subfield_value_descs = new_dict + if rightside: self.subfield_sizes += bitfield.subfield_sizes self.subfield_vals += bitfield.subfield_vals @@ -2085,9 +2524,19 @@ def extend_left(self, bitfield): def set_size_from_constraints(self, size=None, encoded_size=None): raise DataModelDefinitionError - def pretty_print(self, max_size=None): + if self.subfield_value_descs is None: + def value_desc(subfield_idx, val): + return '' + else: + def value_desc(subfield_idx, val): + if subfield_idx in self.subfield_value_descs: + desc = self.subfield_value_descs[subfield_idx].get(val) + return '' if desc is None else ' [' + desc + ']' + else: + return '' + current_raw_val = self.get_current_raw_val() first_pass = True @@ -2110,12 +2559,14 @@ def pretty_print(self, max_size=None): if values is None: mini, maxi = extrems - string += bin(mini+self.idx_inuse[i])[2:].zfill(sz) + val = mini+self.idx_inuse[i] else: index = 0 if len(values) == 1 else self.idx_inuse[i] - string += bin(values[index])[2:].zfill(sz) + val = values[index] - if self.padding_size != 0: + string += bin(val)[2:].zfill(sz) + value_desc(i, val) + + if self.padding_size != 0 and self.show_padding: if self.padding == 1: # in the case the padding has been modified, following an absorption, # to something not standard because of AbsCsts.Contents == False, @@ -2131,6 +2582,10 @@ def pretty_print(self, max_size=None): else: string += ' |-)' + + if not self.show_padding and self.lsb_padding: + current_raw_val = current_raw_val >> self.padding_size + return string + ' ' + str(current_raw_val) @@ -2160,9 +2615,11 @@ def _enable_fuzz_mode(self, fuzz_magnitude=1.0): l = [] self.subfield_fuzzy_vals[idx] = l - # we substract 1 because after a get_value() call idx is incremented in advance + # we substract 1 because after a get_value() call, self.idx is incremented in advance # max is needed because self.idx[0] is equal to 0 in this case - curr_idx = builtins.max(self.idx[idx]-1, 0) + # curr_idx = builtins.max(self.idx[idx]-1, 0) + + curr_idx = self.idx_inuse[idx] curr_values = self.subfield_vals[idx] if curr_values is not None: @@ -2172,7 +2629,8 @@ def _enable_fuzz_mode(self, fuzz_magnitude=1.0): current = mini + curr_idx # append first a normal value, as it will be used as a - # reference when the other fields are fuzzed + # reference when the other fields are fuzzed. + # self._reset_idx() will init self.idx accordingly to avoid nominal value in fuzz mode l.append(current) if self.subfield_extrems[idx] is not None: @@ -2258,43 +2716,43 @@ def __compute_total_possible_values(self): count_of_possible_values = property(fget=__compute_total_possible_values) def rewind(self): - if self.current_idx > 0 and not self.exhausted: - if self.idx[self.current_idx] > 1: - self.idx[self.current_idx] -= 1 - elif self.idx[self.current_idx] == 1: - self.current_idx -= 1 + def sf_possible_values(sf_index): + vals = self.subfield_vals[sf_index] + if vals is not None: + return len(vals) else: - ValueError - - elif self.exhausted: - assert(self.current_idx == 0) - for i in range(len(self.subfield_limits)): - if self.subfield_vals[i] is None: - last = self.subfield_extrems[i][1] - self.subfield_extrems[i][0] - else: - last = len(self.subfield_vals[i]) - 1 - self.idx[i] = last - - self.current_idx = len(self.subfield_limits) - 1 - - elif self.current_idx == 0: - if self.idx[self.current_idx] > 1: - self.idx[self.current_idx] -= 1 - elif self.idx[self.current_idx] == 1: - if not self._fuzzy_mode: - self.idx[self.current_idx] = 0 + mini, maxi = self.subfield_extrems[sf_index] + return maxi - mini + 1 + + if not self.exhausted: + if self.idx[self.current_subfield] > 1: + self.idx[self.current_subfield] -= 1 + elif self.idx[self.current_subfield] == 1: + if self.current_subfield > 0: + self.current_subfield -= 1 + self.idx[self.current_subfield] = sf_possible_values(self.current_subfield) - 1 + elif self.idx[self.current_subfield] == 0: + if self.current_subfield > 0: + self.current_subfield -= 1 + self.idx[self.current_subfield] = sf_possible_values(self.current_subfield) - 1 else: pass - else: - pass + if self.exhaustion_cpt > 0: + # meaning this is not the first value, thus the previous code had an effect + self.exhaustion_cpt = self.count_of_possible_values-1 - self.drawn_val = None - - if self.exhausted: + else: + assert self.current_subfield == 0 + last_sf = len(self.subfield_limits) - 1 + self.current_subfield = last_sf + self.idx[last_sf] = sf_possible_values(last_sf) - 1 self.exhausted = False - self.exhaustion_cpt = 0 + self.exhaustion_cpt = self.count_of_possible_values-1 + if sf_possible_values(self.current_subfield) == 1: + self.rewind() + self.drawn_val = None def _read_value_from(self, blob, size, endian, constraints): """ @@ -2370,6 +2828,7 @@ def do_absorb(self, blob, constraints, off=0, size=None): self.orig_idx = copy.deepcopy(self.idx) self.orig_subfield_vals = copy.deepcopy(self.subfield_vals) self.orig_drawn_val = self.drawn_val + self.orig_subfield_defaults = copy.copy(self.subfield_defaults) self.orig_padding = self.padding self.padding_one = self.__class__.padding_one @@ -2379,8 +2838,6 @@ def do_absorb(self, blob, constraints, off=0, size=None): self.drawn_val, orig_val = self._read_value_from(blob, self.nb_bytes, self.endian, constraints) - insert_idx = 0 - first_pass = True limits = self.subfield_limits[:-1] limits.insert(0, 0) for lim, sz, values, extrems, i in zip(limits, self.subfield_sizes, self.subfield_vals, @@ -2399,14 +2856,24 @@ def do_absorb(self, blob, constraints, off=0, size=None): else: if constraints[AbsCsts.Contents] and val not in values: raise ValueError("Value for subfield number {:d} does not match the constraints!".format(i+1)) - values.insert(insert_idx, val) + if val in values: + self.idx[i] = values.index(val) + else: + values.insert(self.idx[i], val) - if first_pass: - first_pass = False - insert_idx = 1 + self.subfield_defaults[i] = val return blob, off, self.nb_bytes + def update_raw_value(self, val): + enc_sz = (val.bit_length() + 7) // 8 + if self.nb_bytes < enc_sz: + raise ValueError + if self.lsb_padding: + val = val << self.padding_size + endianess = 'big' if self.endian == VT.BigEndian else 'little' + blob = val.to_bytes(self.nb_bytes, endianess) + self.do_absorb(blob, AbsNoCsts()) def do_revert_absorb(self): ''' @@ -2416,6 +2883,7 @@ def do_revert_absorb(self): self.idx = self.orig_idx self.subfield_vals = self.orig_subfield_vals self.drawn_val = self.orig_drawn_val + self.subfield_defaults = self.orig_subfield_defaults self.padding = self.orig_padding self.padding_one = self.__class__.padding_one @@ -2427,6 +2895,7 @@ def do_cleanup_absorb(self): del self.orig_idx del self.orig_subfield_vals del self.orig_drawn_val + del self.orig_subfield_defaults del self.orig_padding def get_value(self): @@ -2449,54 +2918,78 @@ def get_value(self): val = 0 prev_lim = 0 - update_current_idx = False + update_current_subfield = False + + # Avoid value redundancy when walking through the possible values + # do our best when default value are used (without impacting perfo) + if self.current_subfield > 0 and not self._fuzzy_mode: + default = self.subfield_defaults[self.current_subfield] + if default is None: + vals = self.subfield_vals[self.current_subfield] + if vals is not None: + if len(vals) > 1 and self.idx[self.current_subfield] == 0: + self.idx[self.current_subfield] = 1 + else: + pass + else: + if self.idx[self.current_subfield] == 0: + self.idx[self.current_subfield] = 1 + else: + pass + + else: + if self.subfield_vals[self.current_subfield] is not None: + default_idx = self.subfield_vals[self.current_subfield].index(default) + if self.idx[self.current_subfield] == default_idx \ + and default_idx+1 < len(self.subfield_vals[self.current_subfield]): + self.idx[self.current_subfield] += 1 + else: + pass + else: + min, max = self.subfield_extrems[self.current_subfield] + if self.idx[self.current_subfield] == default - min and default < max: + self.idx[self.current_subfield] += 1 + else: + pass self.idx_inuse = copy.copy(self.idx) for lim, values, extrems, i in zip(self.subfield_limits, self.subfield_vals, self.subfield_extrems, range(len(self.subfield_limits))): if self.determinist: - if i == self.current_idx: + if i == self.current_subfield: + index = self.idx[self.current_subfield] if values is None: mini, maxi = extrems - v = mini + self.idx[self.current_idx] + v = mini + index if v >= maxi: - update_current_idx = True + update_current_subfield = True else: - self.idx[self.current_idx] += 1 + self.idx[self.current_subfield] += 1 + self.idx_inuse[self.current_subfield] = index val += v << prev_lim else: - if len(values) == 1: - index = 0 - else: - index = self.idx[self.current_idx] if index >= len(values) - 1: - update_current_idx = True + update_current_subfield = True else: - self.idx[self.current_idx] += 1 - self.idx_inuse[self.current_idx] = index + self.idx[self.current_subfield] += 1 + self.idx_inuse[self.current_subfield] = index val += values[index] << prev_lim else: - if self._fuzzy_mode: - cursor = 0 - else: - if values is not None and len(values) == 1: - cursor = 0 - else: - if i > self.current_idx and self.subfield_defaults[i] is None: - # Note on the use of builtins.max(): in the - # case of values, idx is always > 1, - # whereas when it is extrems, idx can - # be 0. - cursor = builtins.max(self.idx[i] - 1, 0) - else: - cursor = self.idx[i] - self.idx_inuse[i] = cursor + default = self.subfield_defaults[i] if values is None: mini, maxi = extrems + cursor = self.idx_inuse[i] = default-mini if default is not None else 0 val += (mini + cursor) << prev_lim else: - val += (values[cursor]) << prev_lim + if default is not None: + self.idx_inuse[i] = values.index(default) + sf_val = default + else: + self.idx_inuse[i] = 0 + sf_val = values[0] + + val += sf_val << prev_lim else: if values is None: mini, maxi = extrems @@ -2524,23 +3017,23 @@ def get_value(self): self.exhausted = False - if update_current_idx: + if update_current_subfield: self.exhausted = False - self.current_idx += 1 - if self.current_idx >= len(self.idx): + self.current_subfield += 1 + if self.current_subfield >= len(self.idx): self._reset_idx(reset_idx_inuse=False) self.exhausted = True else: while True: - if self.subfield_vals[self.current_idx] is None: - last = self.subfield_extrems[self.current_idx][1] - self.subfield_extrems[self.current_idx][0] + if self.subfield_vals[self.current_subfield] is None: + last = self.subfield_extrems[self.current_subfield][1] - self.subfield_extrems[self.current_subfield][0] else: - last = len(self.subfield_vals[self.current_idx]) - 1 + last = len(self.subfield_vals[self.current_subfield]) - 1 - if self.idx[self.current_idx] > last: - self.current_idx += 1 - if self.current_idx >= len(self.idx): + if self.idx[self.current_subfield] >= last: + self.current_subfield += 1 + if self.current_subfield >= len(self.idx): self._reset_idx(reset_idx_inuse=False) self.exhausted = True break @@ -2596,17 +3089,19 @@ def _encode_bitfield(self, val): # littleendian-encoded if self.endian == VT.LittleEndian: l = l[::-1] - - if sys.version_info[0] > 2: - return struct.pack('{:d}s'.format(self.nb_bytes), bytes(l)) - else: - return struct.pack('{:d}s'.format(self.nb_bytes), str(bytearray(l))) + + return struct.pack('{:d}s'.format(self.nb_bytes), bytes(l)) def get_current_raw_val(self): if self.drawn_val is None: self.get_value() return self.drawn_val - + + def set_default_value(self, sf_values): + self._check_constraints(sf_values) + self.subfield_defaults = sf_values + self.reset_state() + def is_exhausted(self): return self.exhausted diff --git a/fuddly_shell.py b/fuddly_shell.py index 82b7642..e8bcb7d 100755 --- a/fuddly_shell.py +++ b/fuddly_shell.py @@ -26,7 +26,23 @@ import sys from framework.plumbing import * -fmk = FmkPlumbing() +import argparse + +parser = argparse.ArgumentParser(description='Arguments for Fuddly Shell') + +group = parser.add_argument_group('Miscellaneous Options') +group.add_argument('-f', '--fmkdb', metavar='PATH', help='Path to an alternative fmkDB.db. Create ' + 'it if it does not exist.') +group.add_argument('--external-display', action='store_true', help='Display information on another terminal.') +group.add_argument('--quiet', action='store_true', help='Limit the information displayed at startup.') + +args = parser.parse_args() + +fmkdb = args.fmkdb +external_display = args.external_display +quiet = args.quiet + +fmk = FmkPlumbing(external_term=external_display, fmkdb_path=fmkdb, quiet=quiet) fmk.start() shell = FmkShell("Fuddly Shell", fmk) diff --git a/libs/debug_facility.py b/libs/debug_facility.py index f8b7bf8..fcdf4ed 100644 --- a/libs/debug_facility.py +++ b/libs/debug_facility.py @@ -35,7 +35,7 @@ MW_DEBUG = False # related to knowledge infrastructure -KNOW_DEBUG = True +KNOW_DEBUG = False try: from xtermcolor import colorize diff --git a/libs/external_modules.py b/libs/external_modules.py index 13e3b6c..3f02711 100644 --- a/libs/external_modules.py +++ b/libs/external_modules.py @@ -27,13 +27,6 @@ import xtermcolor from xtermcolor import colorize xtermcolor.isatty = lambda x: True - - if sys.version_info[0] <= 2: - def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): - if isinstance(string, unicode): - string = str(string) - return xtermcolor.colorize(string, rgb=rgb, ansi=ansi, bg=bg, ansi_bg=ansi_bg, fd=fd) - except ImportError: print("WARNING [FMK]: python-xtermcolor module is not installed, colors won't be available!") def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): @@ -85,6 +78,7 @@ class Color(object): ND_SEPARATOR = 0x008000 ND_ENCODED = 0xFFA500 ND_CUSTO = 0x800080 + ND_HLIGHT = 0xEF0000 ANALYSIS_CONFIRM = 0xEF0000 ANALYSIS_FALSEPOSITIVE = 0x00FF00 @@ -107,6 +101,7 @@ class FontStyle: import graphviz except ImportError: graphviz_module = False + graphviz = None print('WARNING [FMK]: python(3)-graphviz module is not installed, Scenario could not be visualized!') sqlite3_module = True @@ -114,6 +109,7 @@ class FontStyle: import sqlite3 except ImportError: sqlite3_module = False + sqlite3 = None print('WARNING [FMK]: SQLite3 not installed, FmkDB will not be available!') cups_module = True @@ -121,6 +117,7 @@ class FontStyle: import cups except ImportError: cups_module = False + cups = None print('WARNING [FMK]: python(3)-cups module is not installed, Printer targets will not be available!') crcmod_module = True @@ -128,6 +125,7 @@ class FontStyle: import crcmod except ImportError: crcmod_module = False + crcmod = None print('WARNING [FMK]: python(3)-crcmod module is not installed, the CRC()' \ ' generator template will not be available!') @@ -136,6 +134,7 @@ class FontStyle: import paramiko as ssh except ImportError: ssh_module = False + ssh = None print('WARNING [FMK]: python(3)-paramiko module is not installed! ' 'Should be installed for ssh-based monitoring.') @@ -144,5 +143,15 @@ class FontStyle: import serial except ImportError: serial_module = False + serial = None print('WARNING [FMK]: python(3)-serial module is not installed! ' 'Should be installed for serial-based Target.') + +csp_module = True +try: + import constraint +except ImportError: + csp_module = False + constraint = None + print('WARNING [FMK]: python-constraint module is not installed! ' + 'Should be installed to support constraint-based nodes.') diff --git a/libs/utils.py b/libs/utils.py index facdb5b..e13e069 100644 --- a/libs/utils.py +++ b/libs/utils.py @@ -30,27 +30,35 @@ import inspect import uuid +from framework.global_resources import config_folder +from framework.config import config +import shlex + +term = config("FmkPlumbing", path=[config_folder]).terminal class Term(object): - def __init__(self, name=None, keepterm=False, xterm_args=None, xterm_prg_name='x-terminal-emulator'): - self.name = name + def __init__(self, title=None, keepterm=False): + self.title = title self.keepterm = keepterm - self.xterm_args = xterm_args - self.xterm_prg_name = xterm_prg_name def start(self): self.pipe_path = os.sep + os.path.join('tmp', 'fuddly_term_'+str(uuid.uuid4())) if not os.path.exists(self.pipe_path): os.mkfifo(self.pipe_path) - self.cmd = [self.xterm_prg_name] - if self.name is not None: - self.cmd.extend(['-title',self.name]) - if self.xterm_args: - self.cmd.extend(self.xterm_args) + self.cmd = [term.name] + if self.title is not None: + self.cmd.extend([term.title_arg, self.title]) if self.keepterm: - self.cmd.append('--hold') - self.cmd.extend(['-e', 'tail -f {:s}'.format(self.pipe_path)]) + self.cmd.append(term.hold_arg) + if term.extra_args: + self.cmd.extend(shlex.split(term.extra_args)) + if term.exec_arg: + self.cmd.append(term.exec_arg) + if term.exec_arg_type == "list": + self.cmd.extend(['tail', '-f', self.pipe_path]) + elif term.exec_arg_type == "string": + self.cmd.append(f"tail -f {self.pipe_path}") self._p = None def _launch_term(self): @@ -66,6 +74,8 @@ def stop(self): pass def print(self, s, newline=False): + if not isinstance(s, str): + s = str(s) s += '\n' if newline else '' if self._p is None or self._p.poll() is not None: self._launch_term() @@ -76,33 +86,130 @@ def print_nl(self, s): self.print(s, newline=True) -def ensure_dir(f): - d = os.path.dirname(f) - if not os.path.exists(d): - os.makedirs(d) +class ExternalDisplay(object): + + def __init__(self): + self._disp = None + + @property + def disp(self): + return self._disp -def ensure_file(f): - if not os.path.isfile(f): - open(f, 'a').close() + @property + def is_terminal(self): + return isinstance(self._disp, Term) -def chunk_lines(string, length): + @property + def is_enabled(self): + return self.disp is not None + + def stop(self): + if self._disp: + self._disp.stop() + self._disp = None + + def start_term(self, title=None, keepterm=False): + self._disp = Term(title=title, keepterm=keepterm) + self._disp.start() + self._disp.print('') + + +class Task(object): + + period = None + fmkops = None + feedback_gate = None + targets = None + dm = None + prj = None + + def __call__(self, args): + pass + + def setup(self): + pass + + def cleanup(self): + pass + + def __init__(self, period=None, init_delay=0, + new_window=False, new_window_title=None): + self.period = period + self.init_delay = init_delay + self.fmkops = None + self.feedback_gate = None + self.targets = None + self.dm = None + self.prj = None + # When a task is used in the context of a FmkTask, this attribute is initialized to a + # threading event by the FmkTask. Then when set, it should be understood by the task that + # the framework want it to stop. + self.stop_event = None + + self._new_window = new_window + self._new_window_title = new_window_title + + def _setup(self): + if self._new_window: + nm = self.__class__.__name__ if self._new_window_title is None else self._new_window_title + self.term = Term(title=nm, keepterm=True) + self.term.start() + + self.setup() + + def _cleanup(self): + self.cleanup() + if self._new_window and self.term is not None: + self.term.stop() + + def __str__(self): + if self.period is None: + desc = 'Oneshot Task' + else: + desc = 'Periodic Task (period={}s)'.format(self.period) + return desc + + def print(self, msg): + if self._new_window: + self.term.print(msg) + else: + print(msg) + + def print_nl(self, msg): + if self._new_window: + self.term.print_nl(msg) + else: + print(msg) + +class Accumulator: + + def __init__(self): + self.content = '' + + def accumulate(self, msg): + self.content += msg + + def clear(self): + self.content = '' + +def chunk_lines(string, length, prefix=''): l = string.split(' ') chk_list = [] full_line = '' for wd in l: full_line += wd + ' ' if len(full_line) > (length - 1): - chk_list.append(full_line) + chk_list.append(prefix+full_line) full_line = '' if full_line: - chk_list.append(full_line) + chk_list.append(prefix+full_line) # remove last space char if chk_list: chk_list[-1] = (chk_list[-1])[:-1] return chk_list def find_file(filename, root_path): - for (dirpath, dirnames, filenames) in os.walk(root_path): + for (dirpath, dirnames, filenames) in os.walk(os.path.expanduser(root_path)): if filename in filenames: return dirpath + os.sep + filename else: @@ -125,12 +232,6 @@ def retrieve_app_handler(filename): app_name = result.group(1).split()[0] return app_name - -if sys.version_info[0] > 2: - def get_caller_object(stack_frame=2): - caller_frame_record = inspect.stack()[stack_frame] - return caller_frame_record.frame.f_locals['self'] -else: - def get_caller_object(stack_frame=2): - caller_frame_record = inspect.stack()[stack_frame] - return caller_frame_record[0].f_locals['self'] +def get_caller_object(stack_frame=2): + caller_frame_record = inspect.stack()[stack_frame] + return caller_frame_record.frame.f_locals['self'] diff --git a/projects/generic/standard_proj.py b/projects/generic/standard_proj.py index 99b4f61..b18a5be 100644 --- a/projects/generic/standard_proj.py +++ b/projects/generic/standard_proj.py @@ -23,6 +23,7 @@ import socket +from framework.comm_backends import Shell_Backend from framework.plumbing import * from framework.targets.local import LocalTarget from framework.targets.network import NetworkTarget @@ -33,7 +34,7 @@ # If you only want one default DM, provide its name directly as follows: # project.default_dm = 'mydf' -logger = Logger('standard', record_data=False, explicit_data_recording=True, export_orig=False, +logger = Logger('standard', record_data=False, explicit_data_recording=True, enable_file_logging=False) printer1_tg = PrinterTarget(tmpfile_ext='.png') @@ -97,7 +98,7 @@ def start(self, fmk_ops, dm, monitor, target, logger, user_input): self.init_gen_len = len(self.gen_ids) self.current_gen_id = self.gen_ids.pop(0) - # fmk_ops.set_fuzz_delay(5) + # fmk_ops.set_sending_delay(5) return True def stop(self, fmk_ops, dm, monitor, target, logger): diff --git a/projects/specific/usb_proj.py b/projects/specific/usb_proj.py index fabc851..63e9932 100644 --- a/projects/specific/usb_proj.py +++ b/projects/specific/usb_proj.py @@ -31,7 +31,7 @@ project = Project() project.default_dm = 'usb' -logger = Logger('bin', record_data=False, explicit_data_recording=True, export_orig=False) +logger = Logger('bin', record_data=False, explicit_data_recording=True) rpyc_module = True try: @@ -97,7 +97,7 @@ def send_data(self, data, from_fmk=False): self.cnx.root.connect(conf_desc_str_list=[data.to_bytes()]) - def is_target_ready_for_new_data(self): + def is_feedback_received(self): time.sleep(3) self.cnx.root.disconnect() time.sleep(1) diff --git a/projects/tuto_proj.py b/projects/tuto_proj.py index 5f53f9a..dfada9e 100644 --- a/projects/tuto_proj.py +++ b/projects/tuto_proj.py @@ -23,6 +23,7 @@ import socket +from framework.comm_backends import Serial_Backend from framework.plumbing import * from framework.targets.debug import TestTarget from framework.targets.network import NetworkTarget @@ -30,15 +31,17 @@ from framework.knowledge.feedback_handler import TestFbkHandler from framework.scenario import * from framework.global_resources import UI -from framework.evolutionary_helpers import DefaultPopulation +from framework.evolutionary_helpers import DefaultPopulation, CrossoverHelper +from framework.data import DataProcess project = Project() project.default_dm = ['mydf', 'myproto'] project.map_targets_to_scenario('ex1', {0: 7, 1: 8, None: 8}) -logger = Logger(record_data=False, explicit_data_recording=False, export_orig=False, - export_raw_data=False, enable_file_logging=False) +logger = Logger(record_data=False, explicit_data_recording=False, + export_raw_data=False, enable_file_logging=False, + highlight_marked_nodes=True) ### KNOWLEDGE ### @@ -171,17 +174,38 @@ def cbk_print(env, step): sc_proj1 = Scenario('proj1', anchor=open_step, user_context=UI(prj='proj1')) sc_proj2 = sc_proj1.clone('proj2') -sc_proj2.set_user_context(UI(prj='proj2')) +sc_proj2.user_context = UI(prj='proj2') -project.register_scenarios(sc_proj1, sc_proj2) + +step1 = Step(DataProcess(process=['tTYPE'], seed='4tg1')) +step2 = Step(DataProcess(process=['tTYPE#2'], seed='4tg2')) + +step1.connect_to(step2, dp_completed_guard=True) +step2.connect_to(FinalStep(), dp_completed_guard=True) + +sc_proj3 = Scenario('proj3', anchor=step1) + +project.register_scenarios(sc_proj1, sc_proj2, sc_proj3) ### EVOLUTIONNARY PROCESS EXAMPLE ### +init_dp1 = DataProcess([('tTYPE', UI(fuzz_mag=0.2))], seed='exist_cond') +init_dp1.append_new_process([('tSTRUCT', UI(deep=True))]) + +init_dp2 = DataProcess([('tTYPE#2', UI(fuzz_mag=0.2))], seed='exist_cond') +init_dp2.append_new_process([('tSTRUCT#2', UI(deep=True))]) + project.register_evolutionary_processes( - ('evol', DefaultPopulation, - {'init_process': [('SEPARATOR', UI(random=True)), 'tTYPE'], - 'size': 10, - 'max_generation_nb': 10}) + ('evol1', DefaultPopulation, + {'init_process': init_dp1, + 'max_size': 80, + 'max_generation_nb': 3, + 'crossover_algo': CrossoverHelper.crossover_algo1}), + ('evol2', DefaultPopulation, + {'init_process': init_dp2, + 'max_size': 80, + 'max_generation_nb': 3, + 'crossover_algo': CrossoverHelper.get_configured_crossover_algo2()}) ) ### OPERATOR DEFINITION ### @@ -203,9 +227,9 @@ def start(self, fmk_ops, dm, monitor, target, logger, user_input): self.detected_error = 0 if self.mode == 1: - fmk_ops.set_fuzz_delay(0) + fmk_ops.set_sending_delay(0) else: - fmk_ops.set_fuzz_delay(0.5) + fmk_ops.set_sending_delay(0.5) return True diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..89e18f1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +cexprtk==0.4.0 +ConfigParser==5.3.0 +crcmod==1.7 +ddt==1.6.0 +graphviz==0.20.1 +matplotlib==3.7.0 +mock==5.0.1 +paramiko==3.0.0 +pyserial==3.5 +python_constraint==1.4.0 +pyxdg==0.28 +rpyc==5.3.0 +sphinx_rtd_theme==1.2.0 +xtermcolor==1.3 diff --git a/test/integration/test_integration.py b/test/integration/test_integration.py index c96cd4e..326ac84 100644 --- a/test/integration/test_integration.py +++ b/test/integration/test_integration.py @@ -28,14 +28,10 @@ import unittest import ddt -import ddt - sys.path.append('.') from framework.value_types import * -import data_models.tutorial.example as example - from framework.fuzzing_primitives import * from framework.plumbing import * from framework.data_model import * @@ -44,14 +40,202 @@ from test import ignore_data_model_specifics, run_long_tests, exit_on_import_error def setUpModule(): - global fmk, dm, results + global fmk, dm, results, node_nterm, node_simple, node_typed + fmk = FmkPlumbing(exit_on_error=exit_on_import_error, debug_mode=True) fmk.start() - fmk.run_project(name='tuto', dm_name='example') - dm = example.data_model + fmk.run_project(name='tuto', dm_name=['mydf']) results = collections.OrderedDict() fmk.prj.reset_knowledge() + ### Node graph: TVE ### + + evt1 = Node('EVT1') + evt1.set_values(value_type=SINT16_be(values=[-4])) + evt1.set_fuzz_weight(10) + + evt2 = Node('EVT2') + evt2.set_values(value_type=UINT16_le(min=50, max=2**16-1)) + # evt2.set_values(value_type=UINT16_le()) + evt2.set_fuzz_weight(9) + + sep1 = Node('sep1', values=["+"]) + sep2 = Node('sep2', values=["*"]) + + sub1 = Node('SUB1') + sub1.set_subnodes_with_csts([ + 1, ['u>', [sep1, 3], [evt1, 2], [sep1, 3]] + ]) + + sp = Node('S', values=[' ']) + + ssub = Node('SSUB') + ssub.set_subnodes_basic([sp, evt2, sp]) + + sub2 = Node('SUB2') + sub2.set_subnodes_with_csts([ + 1, ['u>', [sep2, 3], [ssub, 1], [sep2, 3]] + ]) + + sep = Node('sep', values=[' -=||=- ']) + prefix = Node('Pre', values=['[1] ', '[2] ', '[3] ', '[4] ']) + prefix.make_determinist() + + te3 = Node('EVT3') + te3.set_values(value_type=BitField(subfield_sizes=[4,4], endian=VT.LittleEndian, + subfield_values=[[0x5, 0x6], [0xF, 0xC]])) + te3.set_fuzz_weight(8) + + te4 = Node('EVT4') + te4.set_values(value_type=BitField(subfield_sizes=[4,4], endian=VT.LittleEndian, + subfield_val_extremums=[[4, 8], [3, 15]])) + te4.set_fuzz_weight(7) + + te5 = Node('EVT5') + te5.set_values(value_type=INT_str(values=[9])) + te5.cc.set_specific_fuzzy_values([666]) + te5.set_fuzz_weight(6) + + te6 = Node('EVT6') + vt = BitField(subfield_limits=[2,6,8,10], subfield_values=[[2,1],[2,15,3],[2,3,0],[1]], + padding=0, lsb_padding=True, endian=VT.LittleEndian) + te6.set_values(value_type=vt) + te6.set_fuzz_weight(5) + + te7 = Node('EVT7') + vt = BitField(subfield_sizes=[4,4,4], + subfield_values=[[4,2,1], None, [2,3,0]], + subfield_val_extremums=[None, [3, 15], None], + padding=0, lsb_padding=False, endian=VT.BigEndian) + te7.set_values(value_type=vt) + te7.set_fuzz_weight(4) + + suffix = Node('suffix', subnodes=[sep, te3, sep, te4, sep, te5, sep, te6, sep, te7]) + + typed_node = Node('TVE', subnodes=[prefix, sub1, sep, sub2, suffix]) + + ### Node Graph: Simple ### + + tval1_bottom = Node('TV1_bottom') + vt = UINT16_be(values=[1,2,3,4,5,6]) + + tval1_bottom.set_values(value_type=vt) + tval1_bottom.make_determinist() + + sep_bottom = Node('sep_bottom', values=[' .. ']) + sep_bottom_alt = Node('sep_bottom_alt', values=[' ;; ']) + + tval2_bottom = Node('TV2_bottom') + vt = UINT16_be(values=[0x42,0x43,0x44]) + tval2_bottom.set_values(value_type=vt) + + alt_tag = Node('AltTag', values=[' |AltTag| ', ' +AltTag+ ']) + alt_tag_cpy = alt_tag.get_clone('AltTag_cpy') + + bottom = Node('Bottom_NT') + bottom.set_subnodes_with_csts([ + 1, ['u>', [sep_bottom, 1], [tval1_bottom, 1], [sep_bottom, 1], [tval2_bottom, 1]] + ]) + + val1_bottom2 = Node('V1_bottom2', values=['=BOTTOM_2=', '**BOTTOM_2**', '~~BOTTOM_2~~']) + val1_bottom2.add_conf('ALT') + val1_bottom2.set_values(['=ALT_BOTTOM_2=', '**ALT_BOTTOM_2**', '~~ALT_BOTTOM_2~~', '__ALT_BOTTOM_2__'], conf='ALT') + val1_bottom2.add_conf('ALT_2') + val1_bottom2.set_values(['=2ALT2_BOTTOM_2=', '**2ALT2_BOTTOM_2**', '~~2ALT2_BOTTOM_2~~'], conf='ALT_2') + val1_bottom2.set_fuzz_weight(2) + + val1_bottom2_cpy = val1_bottom2.get_clone('V1_bottom2_cpy') + + bottom2 = Node('Bottom_2_NT') + bottom2.set_subnodes_with_csts([ + 5, ['u>', [sep_bottom, 1], [val1_bottom2, 1]], + 1, ['u>', [sep_bottom_alt, 1], [val1_bottom2_cpy, 2], [sep_bottom_alt, 1]] + ]) + bottom2.add_conf('ALT') + bottom2.set_subnodes_with_csts([ + 5, ['u>', [alt_tag, 1], [val1_bottom2, 1], [alt_tag, 1]], + 1, ['u>', [alt_tag_cpy, 2], [val1_bottom2_cpy, 2], [alt_tag_cpy, 2]] + ], conf='ALT') + + tval2_bottom3 = Node('TV2_bottom3') + vt = UINT32_be(values=[0xF, 0x7]) + tval2_bottom3.set_values(value_type=vt) + bottom3 = Node('Bottom_3_NT') + bottom3.set_subnodes_with_csts([ + 1, ['u>', [sep_bottom, 1], [tval2_bottom3, 1]] + ]) + + val1_middle = Node('V1_middle', values=['=MIDDLE=', '**MIDDLE**', '~~MIDDLE~~']) + sep_middle = Node('sep_middle', values=[' :: ']) + alt_tag2 = Node('AltTag-Mid', values=[' ||AltTag-Mid|| ', ' ++AltTag-Mid++ ']) + + val1_middle_cpy1 = val1_middle.get_clone('V1_middle_cpy1') + val1_middle_cpy2 = val1_middle.get_clone('V1_middle_cpy2') + + middle = Node('Middle_NT') + middle.set_subnodes_with_csts([ + 5, ['u>', [val1_middle, 1], [sep_middle, 1], [bottom, 1]], + 3, ['u>', [val1_middle_cpy1, 2], [sep_middle, 1], [bottom2, 1]], + 1, ['u>', [val1_middle_cpy2, 3], [sep_middle, 1], [bottom3, 1]] + ]) + middle.add_conf('ALT') + middle.set_subnodes_with_csts([ + 5, ['u>', [alt_tag2, 1], [val1_middle, 1], [sep_middle, 1], [bottom, 1], [alt_tag2, 1]] + ], conf='ALT') + + val1_top = Node('V1_top', values=['=TOP=', '**TOP**', '~~TOP~~']) + sep_top = Node('sep_top', values=[' -=|#|=- ', ' -=|@|=- ']) + + prefix1 = Node('prefix1', values=[" ('_') ", " (-_-) ", " (o_o) "]) + prefix2 = Node('prefix2', values=[" |X| ", " |Y| ", " |Z| "]) + + e_simple = Node('Simple') + e_simple.set_subnodes_with_csts([ + 1, ['u>', [prefix1, 1], [prefix2, 1], [sep_top, 1], [val1_top, 1], [sep_top, 1], [middle, 1]] + ]) + + ### Node Graph: NonTerm ### + + e = Node('TV2') + vt = UINT16_be(values=[1,2,3,4,5,6]) + e.set_values(value_type=vt) + sep3 = Node('sep3', values=[' # ']) + nt = Node('Bottom_NT') + nt.set_subnodes_with_csts([ + 1, ['u>', [e, 1], [sep3, 1], [e, 1]] + ]) + + sep = Node('sep', values=[' # ']) + sep2 = Node('sep2', values=[' -|#|- ']) + + e_val1 = Node('V1', values=['A', 'B', 'C']) + e_val1_cpy = e_val1.get_clone('V1_cpy') + e_typedval1 = Node('TV1', value_type=UINT16_be(values=[1,2,3,4,5,6])) + e_val2 = Node('V2', values=['X', 'Y', 'Z']) + e_val3 = Node('V3', values=['<', '>']) + + e_val_random = Node('Rnd', values=['RANDOM']) + e_val_random2 = Node('Rnd2', values=['RANDOM']) + + e_nonterm = Node('NonTerm') + e_nonterm.set_subnodes_with_csts([ + 100, ['u>', [e_val1, 1, 6], [sep, 1], [e_typedval1, 1, 6], + [sep2, 1], + 'u=+(2,3,3)', [e_val1_cpy, 1], [e_val2, 1, 3], [e_val3, 1], + 'u>', [sep2, 1], + 'u=..', [e_val1, 1, 6], [sep, 1], [e_typedval1, 1, 6]], + 50, ['u>', [e_val_random, 0, 1], [sep, 1], [nt, 1]], + 90, ['u>', [e_val_random2, 3]] + ]) + + + node_simple = e_simple + node_simple.set_env(Env()) + node_nterm = e_nonterm + node_nterm.set_env(Env()) + node_typed = typed_node + node_typed.set_env(Env()) + def tearDownModule(): global fmk fmk.stop() @@ -63,40 +247,339 @@ def tearDownModule(): class TestBasics(unittest.TestCase): @classmethod def setUpClass(cls): - cls.dm = example.data_model - cls.dm.load_data_model(fmk._name2dm) + + tx = Node('TX') + tx_h = Node('h', values=['/TX']) + + ku = Node('KU') + kv = Node('KV') + + ku_h = Node('KU_h', values=[':KU:']) + kv_h = Node('KV_h', values=[':KV:']) + + tux_subparts_1 = ['POWN', 'TAILS', 'WORLD1', 'LAND321'] + tux_subparts_2 = ['YYYY', 'ZZZZ', 'XXXX'] + ku.set_values(tux_subparts_1) + kv.set_values(tux_subparts_2) + + + tux_subparts_3 = ['[<]MARCHONS', '[<]TESTONS'] + kv.add_conf('ALT') + kv.set_values(tux_subparts_3, conf='ALT') + + tux_subparts_4 = [u'[\u00c2]PLIP', u'[\u00c2]GLOUP'] + ku.add_conf('ALT') + ku.set_values(value_type=String(values=tux_subparts_4, codec='utf8'), conf='ALT') + + idx = Node('IDX') + idx.set_values(value_type=SINT16_be(min=4,max=40)) + + tx.set_subnodes_basic([tx_h, idx, ku_h, ku, kv_h, kv]) + tx_cpy = tx.get_clone('TX_cpy') + + tc = Node('TC') + tc_h = Node('h', values=['/TC']) + + ku2 = Node('KU', base_node=ku) + kv2 = Node('KV', base_node=kv) + + ku_h2 = Node('KU_h', base_node=ku_h) + kv_h2 = Node('KV_h', base_node=kv_h) + + tc.set_subnodes_basic([tc_h, ku_h2, ku2, kv_h2, kv2]) + + + mark3 = Node('MARK3', values=[' ~(X)~ ']) + + tc.add_conf('ALT') + tc.set_subnodes_basic([mark3, tc_h, ku2, kv_h2], conf='ALT') + tc_cpy1= tc.get_clone('TC_cpy1') + tc_cpy2= tc.get_clone('TC_cpy2') + + mark = Node('MARK', values=[' [#] ']) + + idx2 = Node('IDX2', base_node=idx) + tux = Node('TUX') + tux_h = Node('h', values=['TUX']) + + # set 'mutable' attribute to False + tux_h.clear_attr(NodeInternals.Mutable) + tux_h_cpy = tux_h.get_clone('h_cpy') + + tux.set_subnodes_with_csts([ + 100, ['u>', [tux_h, 1], [idx2, 1], [mark, 1], + 'u=+(1,2)', [tc_cpy2, 2], [tx_cpy, 1, 2], + 'u>', [mark, 1], [tx, 1], [tc_cpy1, 1], + 'u=..', [tux_h, 1], [idx2, 1]], + + 1, ['u>', [mark, 1], + 's=..', [tux_h_cpy, 1, 3], [tc, 3], + 'u>', [mark, 1], [tx, 1], [idx2, 1]], + + 15, ['u>', [mark, 1], + 'u=.', [tux_h_cpy, 1, 3], [tc, 3], + 'u=.', [mark, 1], [tx, 1], [idx2, 1]] + ]) + + + mark2 = Node('MARK2', values=[' ~(..)~ ']) + + tux.add_conf('ALT') + tux.set_subnodes_with_csts( + [1, ['u>', [mark2, 1], + 'u=+(4000,1)', [tux_h, 1], [mark, 1], + 'u>', [mark2, 1], + 'u=.', [tux_h, 1], [tc, 10], + 'u>', [mark, 1], [tx, 1], [idx2, 1]] + ], conf='ALT') + tux.set_attr(MH.Attr.DEBUG, conf='ALT') + + concat = Node('CONCAT') + length = Node('LEN') + node_ex1 = Node('EX1') + + fct = lambda x: b' @ ' + x + b' @ ' + concat.set_func(fct, tux) + + fct = lambda x: b'___' + bytes(chr(x[1]), internal_repr_codec) + b'___' + + concat.add_conf('ALT') + concat.set_func(fct, tux, conf='ALT') + + fct2 = lambda x: len(x) + length.set_func(fct2, tux) + + node_ex1.set_subnodes_basic([concat, tux, length]) + node_ex1.set_env(Env()) + + cls.node_tux = tux.get_clone() + cls.node_ex1 = node_ex1 + def setUp(self): pass - def test_01(self): + def test_node_alt_conf(self): + + print('\n### TEST 8: set_current_conf()') + + node_ex1 = self.node_ex1.get_clone() # fmk.dm.get_atom('EX1') + + node_ex1.show() + + print('\n*** test 8.0:') + + res01 = True + l = sorted(node_ex1.get_nodes_names()) + for k in l: + print(k) + if 'EX1' != k[0][:len('EX1')]: + res01 = False + break + + l2 = sorted(node_ex1.get_nodes_names(conf='ALT')) + for k in l2: + print(k) + if 'EX1' != k[0][:len('EX1')]: + res01 = False + break + + self.assertTrue(res01) + + res02 = False + for k in l2: + if 'MARK2' in k[0]: + for k in l2: + if 'MARK3' in k[0]: + res02 = True + break + break + + self.assertTrue(res02) + + print('\n*** test 8.1:') + + res1 = True + + msg = node_ex1.to_bytes(conf='ALT') + if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg: + res1 = False + print(msg) + node_ex1.unfreeze_all() + msg = node_ex1.to_bytes(conf='ALT') + if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg: + res1 = False + print(msg) + node_ex1.unfreeze_all() + msg = node_ex1.to_bytes() + if b' ~(..)~ ' in msg or b' ~(X)~ ' in msg: + res1 = False + print(msg) + node_ex1.unfreeze_all() + msg = node_ex1.to_bytes(conf='ALT') + if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg: + res1 = False + print(msg) + node_ex1.unfreeze_all() + + self.assertTrue(res1) + + print('\n*** test 8.2:') + + print('\n***** test 8.2.0: subparts:') + + node_ex1 = self.node_ex1.get_clone() + + res2 = True + + print(node_ex1.to_bytes()) + + node_ex1.set_current_conf('ALT', root_regexp=None) + + nonascii_test_str = u'\u00c2'.encode(internal_repr_codec) + + node_ex1.unfreeze_all() + msg = node_ex1.to_bytes() + if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg or b'[<]' not in msg or nonascii_test_str not in msg: + res2 = False + print(msg) + node_ex1.unfreeze_all() + msg = node_ex1.to_bytes() + if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg or b'[<]' not in msg or nonascii_test_str not in msg: + res2 = False + print(msg) + + node_ex1.set_current_conf('MAIN', reverse=True, root_regexp=None) + + node_ex1.unfreeze_all() + msg = node_ex1.to_bytes() + if b' ~(..)~ ' in msg or b' ~(X)~ ' in msg or b'[<]' in msg or nonascii_test_str in msg: + res2 = False + print(msg) + + node_ex1 = self.node_ex1.get_clone() + + node_ex1.set_current_conf('ALT', root_regexp='(TC)|(TC_.*)/KV') + node_ex1.set_current_conf('ALT', root_regexp='TUX$') + + node_ex1.unfreeze_all() + msg = node_ex1.to_bytes() + if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg or b'[<]' not in msg or nonascii_test_str not in msg: + res2 = False + print(msg) + + self.assertTrue(res2) + + print('\n***** test 8.2.1: subparts equality:') + + val1 = node_ex1.get_first_node_by_path('TUX$').to_bytes() + val2 = node_ex1.get_first_node_by_path('CONCAT$').to_bytes() + print(b' @ ' + val1 + b' @ ') + print(val2) + + res21 = b' @ ' + val1 + b' @ ' == val2 + + self.assertTrue(res21) + + print('\n*** test 8.3:') + + node_ex1 = self.node_ex1.get_clone() + + res3 = True + l = sorted(node_ex1.get_nodes_names(conf='ALT')) + for k in l: + print(k) + if 'EX1' != k[0][:len('EX1')]: + res3 = False + break + + self.assertTrue(res3) + + print('\n*** test 8.4:') + + print(node_ex1.to_bytes()) + res4 = True + l = sorted(node_ex1.get_nodes_names()) + for k in l: + print(k) + if 'EX1' != k[0][:len('EX1')]: + res4 = False + break + + self.assertTrue(res4) + + print('\n*** test 8.5:') + + node_ex1 = self.node_ex1.get_clone() + + res5 = True + node_ex1.unfreeze_all() + msg = node_ex1.get_first_node_by_path('TUX$').to_bytes(conf='ALT', recursive=False) + if b' ~(..)~ ' not in msg or b' ~(X)~ ' in msg: + res5 = False + print(msg) + + node_ex1.unfreeze_all() + msg = node_ex1.get_first_node_by_path('TUX$').to_bytes(conf='ALT', recursive=True) + if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg: + res5 = False + print(msg) + + self.assertTrue(res5) + + print('\n*** test 8.6:') + + node_ex1 = self.node_ex1.get_clone() + + crit = NodeInternalsCriteria(mandatory_attrs=[NodeInternals.Mutable], + node_kinds=[NodeInternals_NonTerm]) + + node_ex1.unfreeze_all() + + tux2 = self.node_tux.get_clone() + l = tux2.get_reachable_nodes(internals_criteria=crit, owned_conf='ALT') + + for e in l: + print(e.get_path_from(tux2)) + + if len(l) == 4: + res6 = True + else: + res6 = False + + self.assertTrue(res6) + + + def test_node_paths(self): + + print('\n### TEST 12: get_all_path() test') + + print('\n*** test 12.1:') + + node_ex1 = self.node_ex1.get_clone() + for i in node_ex1.iter_paths(only_paths=True): + print(i) + + print('\n******\n') + + node_ex1.get_value() + for i in node_ex1.iter_paths(only_paths=True): + print(i) + + print('\n******\n') + + node_ex1.unfreeze_all() + node_ex1.get_value() + for i in node_ex1.iter_paths(only_paths=True): + print(i) - # print('\n### TEST 0: generate one EX1 ###') - # - node_ex1 = dm.get_atom('EX1') + + node_ex1 = self.node_ex1.get_clone() print('Flatten 1: ', repr(node_ex1.to_bytes())) print('Flatten 1: ', repr(node_ex1.to_bytes())) l = node_ex1.get_value() hk = list(node_ex1.iter_paths(only_paths=True)) - # print(l) - # - # print('\n\n ####### \n\n') - # - # print(l[0]) - # print(b' @ ' + b''.join(flatten(l[1])) + b' @ ') - # print(l[1]) - # - # print('\n\n ####### \n\n') - # - # - # res1 = b' @ ' + b''.join(flatten(l[1])) + b' @ ' == l[0] - # print('*** Is the concatenation (first list element) correct? %r' % res1) - # - # res2 = len(b''.join(flatten(l[1]))) == int(l[2]) - # print('*** Is length of the concatenation correct? %r' % res2) - # - # results['test0'] = res1 and res2 print('\n### TEST 1: cross check self.node.get_all_paths().keys() and get_nodes_names() ###') @@ -110,7 +593,7 @@ def test_01(self): for k in l: print(k) - res1 = len(hk) == len(l) + self.assertEqual(len(hk), len(l)) res2 = False for i in range(len(hk)): @@ -120,7 +603,7 @@ def test_01(self): else: res2 = True - results['test1'] = res1 and res2 + self.assertTrue(res2) print('\n### TEST 2: generate two different EX1 ###') @@ -132,11 +615,11 @@ def test_01(self): print(node_ex1.get_value()) val2 = node_ex1.to_bytes() - results['test2'] = val1 != val2 + self.assertTrue(val1 != val2) print('\n### TEST 3: generate 4 identical TUX (with last one flatten) ###') - tux = dm.get_atom('TUX') + tux = self.node_tux.get_clone() val1 = tux.get_value() print(val1) @@ -147,8 +630,7 @@ def test_01(self): print(repr(tux.to_bytes())) - res = val1 == val2 and val1 == val3 - results['test3'] = res + self.assertTrue(val1 == val2 and val1 == val3) print('\n### TEST 4: generate 2 different flatten TUX ###') @@ -159,51 +641,37 @@ def test_01(self): val2 = repr(tux.to_bytes()) print(val2) - res = val1 != val2 - results['test4'] = res + self.assertTrue(val1 != val2) - print('\n### TEST 5: test get_node_by_path() ###') - print('\n*** test 5.1: get_node_by_path() with exact path') + def test_node_search_by_path_01(self): - tux2 = dm.get_atom('TUX') + print('\n### Test get_first_node_by_path() ###') - print('* Shall return None:') - val1 = tux2.get_node_by_path(path='EX1') - val2 = tux2.get_node_by_path(path='CONCAT') - print(val1) - print(val2) + tux2 = self.node_tux.get_clone() - print('* Shall not return None:') - val3 = tux2.get_node_by_path('TUX/TX') - val4 = tux2.get_node_by_path('TUX/TC') - print(val3) - print(val4) + print('\n*** 1: call 3 times get_first_node_by_path()') - res1 = val1 == None and val2 == None and val3 != None and val4 != None - - print('\n*** test 5.2: call 3 times get_node_by_path()') - - print('name: %s, result: %s' % ('TUX', tux2.get_node_by_path('TUX').get_path_from(tux2))) - print('name: %s, result: %s' % ('TX', tux2.get_node_by_path('TX').get_path_from(tux2))) - print('name: %s, result: %s' % ('KU', tux2.get_node_by_path('KU', conf='ALT').get_path_from(tux2))) + print('name: %s, result: %s' % ('TUX', tux2.get_first_node_by_path('TUX').get_path_from(tux2))) + print('name: %s, result: %s' % ('TX', tux2.get_first_node_by_path('TX').get_path_from(tux2))) + print('name: %s, result: %s' % ('KU', tux2.get_first_node_by_path('KU', conf='ALT').get_path_from(tux2))) print('name: %s, result: %s' % ( - 'MARK3', tux2.get_node_by_path('MARK3', conf='ALT').get_path_from(tux2, conf='ALT'))) + 'MARK3', tux2.get_first_node_by_path('MARK3', conf='ALT').get_path_from(tux2, conf='ALT'))) - print('\n*** test 5.3: call get_node_by_path() with real regexp') + print('\n*** 2: call get_first_node_by_path() with real regexp') - print('--> ' + tux2.get_node_by_path('TX.*KU').get_path_from(tux2)) + print('--> ' + tux2.get_first_node_by_path('TX.*KU').get_path_from(tux2)) - print('\n*** test 5.4: call get_reachable_nodes()') + print('\n*** 3: call get_reachable_nodes()') - node_ex1 = dm.get_atom('EX1') + node_ex1 = self.node_ex1.get_clone() l = node_ex1.get_reachable_nodes(path_regexp='TUX') for i in l: print(i.get_path_from(node_ex1)) print('\n') - node_ex1 = dm.get_atom('EX1') + node_ex1 = self.node_ex1.get_clone() l = node_ex1.get_reachable_nodes(path_regexp='T[XC]/KU') for i in l: print(i.get_path_from(node_ex1)) @@ -213,12 +681,15 @@ def test_01(self): else: res2 = False - print(res1, res2) + self.assertTrue(res2) - results['test5'] = res1 and res2 + def test_node_search_misc_01(self): print('\n### TEST 6: get_reachable_nodes()') + node_ex1 = self.node_ex1.get_clone() + tux2 = self.node_tux.get_clone() + for e in sorted(tux2.get_nodes_names()): print(e) @@ -233,7 +704,7 @@ def test_01(self): l2 = tux2.get_reachable_nodes(internals_criteria=c2) - res61 = len(l2) > len(l1) + self.assertTrue(len(l2) > len(l1)) print('len(l1): %d, len(l2): %d' % (len(l1), len(l2))) @@ -242,351 +713,126 @@ def test_01(self): res62 = False l = tux2.get_reachable_nodes(internals_criteria=c2, conf='ALT') for k in l: - print(k.get_path_from(tux2, conf='ALT')) - if 'MARK3' in k.get_path_from(tux2, conf='ALT'): - res62 = True - break - - # l = tux2.get_reachable_nodes(node_kinds=[NodeInternals_NonTerm], conf='ALT') - # for k in l: - # print(k.get_path_from(tux2, conf='ALT')) - - print('\n*** test 6.3:') - - c3 = NodeInternalsCriteria(node_kinds=[NodeInternals_Func]) - - l3 = node_ex1.get_reachable_nodes(internals_criteria=c3) - print("*** %d Func Node found" % len(l3)) - print(l3) - - res63 = len(l3) == 2 - - print(res61, res62, res63) - - results['test6'] = res61 and res62 and res63 - - print('\n### TEST 7: get_reachable_nodes() and change_subnodes_csts()') - - print('*** junk test:') - - tux2.get_node_by_path('TUX$').cc.change_subnodes_csts([('u=+', 'u>'), ('u=.', 'u>')]) - print(tux2.to_bytes()) - - print('\n*** test 7.1:') - - print('> l1:') - - tux2 = dm.get_atom('TUX') - # attr = Elt_Attributes(defaults=False) - # attr.conform_to_nonterm_node() - - # node_kind = [NodeInternals_NonTerm] - - crit = NodeInternalsCriteria(node_kinds=[NodeInternals_NonTerm]) - - l1 = tux2.get_reachable_nodes(internals_criteria=crit) - - # tux2.cc.get_subnodes_csts_copy() - # exit() - - res1 = True - for e in l1: - print(e.get_path_from(tux2)) - e.cc.change_subnodes_csts([('*', 'u=.')]) - csts1, _ = e.cc.get_subnodes_csts_copy() - print(csts1) - - e.cc.change_subnodes_csts([('*', 'u=.'), ('u=.', 'u>')]) - csts2, _ = e.cc.get_subnodes_csts_copy() - print(csts2) - - print('\n') - - # val = cmp(csts1, csts2) - val = (csts1 > csts2) - (csts1 < csts2) - if val != 0: - res1 = False - - print('> l2:') - - l2 = tux2.get_reachable_nodes(internals_criteria=crit) - for e in l2: - print(e.get_path_from(tux2)) - - print('\n*** test 7.2:') - - res2 = len(l2) == len(l1) - print('len(l2) == len(l1)? %r' % res2) - - print('\n*** test 7.3:') - - tux = dm.get_atom('TUX') - l1 = tux.get_reachable_nodes(internals_criteria=crit) - c_l1 = [] - for e in l1: - order, attrs = e.cc.get_subnodes_csts_copy() - - e.cc.change_subnodes_csts([('u=.', 'u>'), ('u>', 'u=.')]) - csts1, _ = e.cc.get_subnodes_csts_copy() - print(csts1) - print('\n') - c_l1.append(csts1) - - e.set_subnodes_full_format(order, attrs) - - l2 = tux.get_reachable_nodes(internals_criteria=crit) - c_l2 = [] - for e in l2: - orig = e.cc.get_subnodes_csts_copy() - - e.cc.change_subnodes_csts([('u>', 'u=.'), ('u=.', 'u>')]) - csts2, _ = e.cc.get_subnodes_csts_copy() - print(csts2) - print('\n') - c_l2.append(csts2) - - zip_l = zip(c_l1, c_l2) - - test = 1 - for zl1, zl2 in zip_l: - - for ze1, ze2 in zip(zl1, zl2): - test = (ze1 > ze2) - (ze1 < ze2) - if test != 0: - print(ze1) - print('########') - print(ze2) - print('########') - break - - if test != 0: - res3 = False - else: - res3 = True - - - # val = cmp(c_l1, c_l2) - val = (c_l1 > c_l2) - (c_l1 < c_l2) - if val != 0: - res3 = False - else: - res3 = True - - print(res1, res2, res3) - - results['test7'] = res1 and res2 and res3 - - print('\n### TEST 8: set_current_conf()') - - node_ex1 = dm.get_atom('EX1') - - print('\n*** test 8.0:') - - res01 = True - l = sorted(node_ex1.get_nodes_names()) - for k in l: - print(k) - if 'EX1' != k[0][:len('EX1')]: - res01 = False - break - - l2 = sorted(node_ex1.get_nodes_names(conf='ALT')) - for k in l2: - print(k) - if 'EX1' != k[0][:len('EX1')]: - res01 = False - break - - res02 = False - for k in l2: - if 'MARK2' in k[0]: - for k in l2: - if 'MARK3' in k[0]: - res02 = True - break - break - - res0 = res01 and res02 - - print('\n*** test 8.1:') - - res1 = True - - msg = node_ex1.to_bytes(conf='ALT') - if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg: - res1 = False - print(msg) - node_ex1.unfreeze_all() - msg = node_ex1.to_bytes(conf='ALT') - if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg: - res1 = False - print(msg) - node_ex1.unfreeze_all() - msg = node_ex1.to_bytes() - if b' ~(..)~ ' in msg or b' ~(X)~ ' in msg: - res1 = False - print(msg) - node_ex1.unfreeze_all() - msg = node_ex1.to_bytes(conf='ALT') - if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg: - res1 = False - print(msg) - node_ex1.unfreeze_all() - - print('\n*** test 8.2:') - - print('\n***** test 8.2.0: subparts:') - - node_ex1 = dm.get_atom('EX1') - - res2 = True - - print(node_ex1.to_bytes()) - - node_ex1.set_current_conf('ALT', root_regexp=None) + print(k.get_path_from(tux2, conf='ALT')) + if 'MARK3' in k.get_path_from(tux2, conf='ALT'): + res62 = True + break - nonascii_test_str = u'\u00c2'.encode(internal_repr_codec) + self.assertTrue(res62) - node_ex1.unfreeze_all() - msg = node_ex1.to_bytes() - if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg or b'[<]' not in msg or nonascii_test_str not in msg: - res2 = False - print(msg) - node_ex1.unfreeze_all() - msg = node_ex1.to_bytes() - if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg or b'[<]' not in msg or nonascii_test_str not in msg: - res2 = False - print(msg) + # l = tux2.get_reachable_nodes(node_kinds=[NodeInternals_NonTerm], conf='ALT') + # for k in l: + # print(k.get_path_from(tux2, conf='ALT')) - node_ex1.set_current_conf('MAIN', reverse=True, root_regexp=None) + print('\n*** test 6.3:') - node_ex1.unfreeze_all() - msg = node_ex1.to_bytes() - if b' ~(..)~ ' in msg or b' ~(X)~ ' in msg or b'[<]' in msg or nonascii_test_str in msg: - res2 = False - print(msg) + c3 = NodeInternalsCriteria(node_kinds=[NodeInternals_Func]) - node_ex1 = dm.get_atom('EX1') + l3 = node_ex1.get_reachable_nodes(internals_criteria=c3) + print("*** %d Func Node found" % len(l3)) + print(l3) - node_ex1.set_current_conf('ALT', root_regexp='(TC)|(TC_.*)/KV') - node_ex1.set_current_conf('ALT', root_regexp='TUX$') + self.assertTrue(len(l3) == 2) - node_ex1.unfreeze_all() - msg = node_ex1.to_bytes() - if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg or b'[<]' not in msg or nonascii_test_str not in msg: - res2 = False - print(msg) - print('\n***** test 8.2.1: subparts equality:') + def test_node_search_and_update(self): - val1 = node_ex1.get_node_by_path('TUX$').to_bytes() - val2 = node_ex1.get_node_by_path('CONCAT$').to_bytes() - print(b' @ ' + val1 + b' @ ') - print(val2) + print('\n### TEST 7: get_reachable_nodes() and change_subnodes_csts()') - res21 = b' @ ' + val1 + b' @ ' == val2 + node_ex1 = self.node_ex1.get_clone() + tux2 = self.node_tux.get_clone() - print(res2, res21) + print('*** junk test:') - print('\n*** test 8.3:') + tux2.get_first_node_by_path('TUX$').cc.change_subnodes_csts([('u=+', 'u>'), ('u=.', 'u>')]) + print(tux2.to_bytes()) - node_ex1 = dm.get_atom('EX1') + print('\n*** test 7.1:') - res3 = True - l = sorted(node_ex1.get_nodes_names(conf='ALT')) - for k in l: - print(k) - if 'EX1' != k[0][:len('EX1')]: - res3 = False - break + print('> l1:') - print('\n*** test 8.4:') + tux2 = self.node_tux.get_clone() + # attr = Elt_Attributes(defaults=False) + # attr.conform_to_nonterm_node() - print(node_ex1.to_bytes()) - res4 = True - l = sorted(node_ex1.get_nodes_names()) - for k in l: - print(k) - if 'EX1' != k[0][:len('EX1')]: - res4 = False - break + # node_kind = [NodeInternals_NonTerm] - print('\n*** test 8.5:') + crit = NodeInternalsCriteria(node_kinds=[NodeInternals_NonTerm]) - node_ex1 = dm.get_atom('EX1') + l1 = tux2.get_reachable_nodes(internals_criteria=crit) - res5 = True - node_ex1.unfreeze_all() - msg = node_ex1.get_node_by_path('TUX$').to_bytes(conf='ALT', recursive=False) - if b' ~(..)~ ' not in msg or b' ~(X)~ ' in msg: - res5 = False - print(msg) + # tux2.cc.get_subnodes_csts_copy() + # exit() - node_ex1.unfreeze_all() - msg = node_ex1.get_node_by_path('TUX$').to_bytes(conf='ALT', recursive=True) - if b' ~(..)~ ' not in msg or b' ~(X)~ ' not in msg: - res5 = False - print(msg) + res1 = True + for e in l1: + print(e.get_path_from(tux2)) + e.cc.change_subnodes_csts([('*', 'u=.')]) + csts1, _ = e.cc.get_subnodes_csts_copy() + print(csts1) - print('\n*** test 8.6:') + e.cc.change_subnodes_csts([('*', 'u=.'), ('u=.', 'u>')]) + csts2, _ = e.cc.get_subnodes_csts_copy() + print(csts2) - node_ex1 = dm.get_atom('EX1') + print('\n') - # attr3 = Elt_Attributes(defaults=False) - # attr3.conform_to_nonterm_node() - # attr3.enable_conf('ALT') + # val = cmp(csts1, csts2) + val = (csts1 > csts2) - (csts1 < csts2) + if val != 0: + res1 = False - # node_kind3 = [NodeInternals_NonTerm] + self.assertTrue(res1) - crit = NodeInternalsCriteria(mandatory_attrs=[NodeInternals.Mutable], - node_kinds=[NodeInternals_NonTerm]) + print('> l2:') - node_ex1.unfreeze_all() + l2 = tux2.get_reachable_nodes(internals_criteria=crit) + for e in l2: + print(e.get_path_from(tux2)) - l = tux2.get_reachable_nodes(internals_criteria=crit, owned_conf='ALT') + print('\n*** test 7.2:') - for e in l: - print(e.get_path_from(tux2)) + self.assertEqual(len(l2), len(l1)) - if len(l) == 4: - res6 = True - else: - res6 = False + print('\n*** test 7.3:') - print('Results:') - print(res0, res1, res2, res21, res3, res4, res5, res6) + tux = self.node_tux.get_clone() + l1 = tux.get_reachable_nodes(internals_criteria=crit, respect_order=True) + c_l1 = [] + for e in l1: + order, attrs = e.cc.get_subnodes_csts_copy() - results['test8'] = res0 and res1 and res2 and res21 and res3 and res4 and res5 and res6 + e.cc.change_subnodes_csts([('u=.', 'u>'), ('u>', 'u=.')]) + csts1, _ = e.cc.get_subnodes_csts_copy() + print(csts1) + print('\n') + c_l1.append(csts1) - print('\n### TEST 9: test the constraint type: =+(w1,w2,...)\n' \ - '--> can be False in really rare case') + e.set_subnodes_full_format(order, attrs) - node_ex1 = dm.get_atom('EX1') + l2 = tux.get_reachable_nodes(internals_criteria=crit, respect_order=True) + c_l2 = [] + for e in l2: + orig = e.cc.get_subnodes_csts_copy() - res = True - for i in range(20): - node_ex1.unfreeze_all() - msg = node_ex1.get_node_by_path('TUX$').to_bytes(conf='ALT', recursive=True) - if b' ~(..)~ TUX ~(..)~ ' not in msg: - res = False - break - # print(msg) + e.cc.change_subnodes_csts([('u>', 'u=.'), ('u=.', 'u>')]) + csts2, _ = e.cc.get_subnodes_csts_copy() + print(csts2) + print('\n') + c_l2.append(csts2) - results['test9'] = res + self.assertEqual((c_l1 > c_l2) - (c_l1 < c_l2), 0) - print('\n### TEST 10: test fuzzing primitives') - print('\n*** test 10.1: fuzz_data_tree()') + def test_node_alternate_conf(self): - node_ex1 = dm.get_atom('EX1') - fuzz_data_tree(node_ex1) - node_ex1.get_value() + nonascii_test_str = u'\u00c2'.encode(internal_repr_codec) print('\n### TEST 11: test terminal Node alternate conf') print('\n*** test 11.1: value type Node') - node_ex1 = dm.get_atom('EX1') + node_ex1 = self.node_ex1.get_clone() res1 = True msg = node_ex1.to_bytes(conf='ALT') @@ -594,18 +840,24 @@ def test_01(self): res1 = False print(msg) + self.assertTrue(res1) + node_ex1.unfreeze_all() msg = node_ex1.to_bytes(conf='ALT') if b'[<]' not in msg or nonascii_test_str not in msg: res1 = False print(msg) + self.assertTrue(res1) + node_ex1.unfreeze_all() - msg = node_ex1.get_node_by_path('TUX$').to_bytes(conf='ALT', recursive=False) + msg = node_ex1.get_first_node_by_path('TUX$').to_bytes(conf='ALT', recursive=False) if b'[<]' in msg or nonascii_test_str in msg or b' ~(..)~ TUX ~(..)~ ' not in msg: res1 = False print(msg) + self.assertTrue(res1) + print('\n*****\n') crit = NodeInternalsCriteria(mandatory_attrs=[NodeInternals.Mutable], @@ -618,14 +870,11 @@ def test_01(self): for e in l: print(e.get_path_from(node_ex1)) - if len(l) == 10: - res2 = True - else: - res2 = False + self.assertEqual(len(l), 10) print('\n*** test 11.2: func type Node') - node_ex1 = dm.get_atom('EX1') + node_ex1 = self.node_ex1.get_clone() res3 = True msg = node_ex1.to_bytes(conf='ALT') @@ -640,63 +889,71 @@ def test_01(self): print(msg) node_ex1.unfreeze_all() - msg = node_ex1.get_node_by_path('TUX$').to_bytes(conf='ALT', recursive=False) + msg = node_ex1.get_first_node_by_path('TUX$').to_bytes(conf='ALT', recursive=False) if b'___' in msg: res3 = False print(msg) - print(res1, res2, res3) - results['test11'] = res1 and res2 and res3 + self.assertTrue(res3) - print('\n### TEST 12: get_all_path() test') - print('\n*** test 12.1:') + def test_fuzzing_primitives(self): + print('\n### TEST 10: test fuzzing primitives') - node_ex1 = dm.get_atom('EX1') - for i in node_ex1.iter_paths(only_paths=True): - print(i) + print('\n*** test 10.1: fuzz_data_tree()') - print('\n******\n') + node_ex1 = self.node_ex1.get_clone() + node_ex1.show() - node_ex1.get_value() - for i in node_ex1.iter_paths(only_paths=True): - print(i) + fuzz_data_tree(node_ex1) + node_ex1.show() - print('\n******\n') - node_ex1.unfreeze_all() - node_ex1.get_value() - for i in node_ex1.iter_paths(only_paths=True): - print(i) + def test_node_nt_pick_section(self): + print('\n### TEST 9: test the constraint type: =+(w1,w2,...)\n' \ + '--> can be False in really rare case') + + nonascii_test_str = u'\u00c2'.encode(internal_repr_codec) + node_ex1 = self.node_ex1.get_clone() + + res = True + for i in range(20): + node_ex1.unfreeze_all() + msg = node_ex1.get_first_node_by_path('TUX$').to_bytes(conf='ALT', recursive=True) + if b' ~(..)~ TUX ~(..)~ ' not in msg: + res = False + break + # print(msg) - print('\n### SUMMARY ###') + self.assertTrue(res) - for k, v in results.items(): - print('is %s OK? %r' % (k, v)) - for v in results.values(): - self.assertTrue(v) class TestMisc(unittest.TestCase): @classmethod def setUpClass(cls): - cls.dm = example.data_model - cls.dm.load_data_model(fmk._name2dm) + pass def setUp(self): pass - def _loop_nodes(self, node, cpt=20, criteria_func=None, transform=lambda x: x): + def _loop_nodes(self, node, cpt=20, criteria_func=None, transform=lambda x: x, + result_vector=None): stop_loop = False for i in range(cpt): if stop_loop: break node.unfreeze() - print("[#%d] %r" % (i, transform(node.to_bytes()))) + val = transform(node.to_bytes()) + print("[#%d] %r" % (i, val)) + # node.show() + if result_vector and i < len(result_vector): + print('*** Check value with result_vector[{}]'.format(i)) + self.assertEqual(val, result_vector[i]) if node.env.exhausted_node_exists(): for e in node.env.get_exhausted_nodes(): - criteria_func(e) + # criteria_func(e) if criteria_func(e): print('--> exhausted node: ', e.name) stop_loop = True @@ -710,7 +967,7 @@ def test_Node_unfreeze_dont_change_state(self): ''' unfreeze(dont_change_state) ''' - simple = self.dm.get_atom('Simple') + simple = node_simple.get_clone() simple.make_determinist(recursive=True) for i in range(15): @@ -729,7 +986,7 @@ def test_Node_unfreeze_dont_change_state(self): self.assertTrue(res1) def test_TypedNode_1(self): - evt = dm.get_atom('TVE') + evt = node_typed.get_clone() evt.get_value() print('=======[ PATHS ]========') @@ -754,7 +1011,7 @@ def test_TypedNode_1(self): print('') - evt = dm.get_atom('TVE') + evt = node_typed.get_clone() evt.make_finite(all_conf=True, recursive=True) evt.make_determinist(all_conf=True, recursive=True) evt.show() @@ -787,15 +1044,14 @@ def test_TypedNode_1(self): orig_node_val)) print(' node corrupted value: (hexlified) {0!s:s}, {0!s:s}'.format(binascii.hexlify(node.to_bytes()), node.to_bytes())) + # node.show() else: turn_nb_list.append(i) print('\n--> Fuzzing terminated!\n') break print('\nTurn number when Node has changed: %r, number of test cases: %d' % (turn_nb_list, i)) - good_list = [1, 13, 25, 37, 49, 55, 61, 73, 85, 97, 109, 121, 133, 145, 157, 163, 175, 187, - 199, 208, 217, 233, 248] - + good_list = [1, 12, 22, 32, 42, 48, 54, 64, 74, 84, 95, 105, 115, 125, 135, 141, 151, 161, 171, 180, 189, 203, 218] msg = "If Fuzzy_.values have been modified in size, the good_list should be updated.\n" \ "If BitField are in random mode [currently put in determinist mode], the fuzzy_mode can produce more" \ " or less value depending on drawn value when .get_value() is called (if the drawn value is" \ @@ -803,31 +1059,10 @@ def test_TypedNode_1(self): self.assertTrue(turn_nb_list == good_list, msg=msg) - def test_Node_Attr_01(self): - ''' - Value Node make_random()/make_determinist() - TODO: NEED assertion - ''' - evt = dm.get_atom('TVE') - - for i in range(10): - evt.unfreeze() - print(evt.to_bytes()) - - evt.get_node_by_path('Pre').make_random() - - print('******') - - for i in range(10): - evt.unfreeze() - print(evt.to_bytes()) - - # self.assertEqual(idx, ) def test_NonTerm_Attr_01(self): ''' make_determinist()/finite() on NonTerm Node - TODO: NEED assertion ''' loop_count = 50 @@ -835,40 +1070,40 @@ def test_NonTerm_Attr_01(self): print('\n -=[ determinist & finite (loop count: %d) ]=- \n' % loop_count) - nt = dm.get_atom('NonTerm') + nt = node_nterm.get_clone() nt.make_finite(all_conf=True, recursive=True) nt.make_determinist(all_conf=True, recursive=True) nb = self._loop_nodes(nt, loop_count, criteria_func=crit_func) - self.assertEqual(nb, 18) + self.assertEqual(nb, 32) print('\n -=[ determinist & infinite (loop count: %d) ]=- \n' % loop_count) - nt = dm.get_atom('NonTerm') + nt = node_nterm.get_clone() nt.make_infinite(all_conf=True, recursive=True) nt.make_determinist(all_conf=True, recursive=True) self._loop_nodes(nt, loop_count, criteria_func=crit_func) print('\n -=[ random & infinite (loop count: %d) ]=- \n' % loop_count) - nt = dm.get_atom('NonTerm') + nt = node_nterm.get_clone() # nt.make_infinite(all_conf=True, recursive=True) nt.make_random(all_conf=True, recursive=True) self._loop_nodes(nt, loop_count, criteria_func=crit_func) print('\n -=[ random & finite (loop count: %d) ]=- \n' % loop_count) - nt = dm.get_atom('NonTerm') + nt = node_nterm.get_clone() nt.make_finite(all_conf=True, recursive=True) nt.make_random(all_conf=True, recursive=True) nb = self._loop_nodes(nt, loop_count, criteria_func=crit_func) - self.assertEqual(nb, 18) + self.assertAlmostEqual(nb, 3) def test_BitField_Attr_01(self): ''' make_determinist()/finite() on BitField Node - TODO: NEED assertion + TODO: need more assertion ''' loop_count = 80 @@ -876,9 +1111,10 @@ def test_BitField_Attr_01(self): print('\n -=[ random & infinite (loop count: %d) ]=- \n' % loop_count) t = BitField(subfield_limits=[2, 6, 10, 12], - subfield_values=[[4, 2, 1], [2, 15, 16, 3], None, [1]], + subfield_values=[[2, 1], [2, 15, 3], None, [1]], subfield_val_extremums=[None, None, [3, 11], None], - padding=0, lsb_padding=True, endian=VT.LittleEndian) + padding=0, lsb_padding=True, endian=VT.LittleEndian, + determinist=True, show_padding=True) node = Node('BF', value_type=t) node.set_env(Env()) node.make_random(all_conf=True, recursive=True) @@ -889,7 +1125,8 @@ def test_BitField_Attr_01(self): node_copy = Node('BF_copy', base_node=node, ignore_frozen_state=True) node_copy.set_env(Env()) node_copy.make_determinist(all_conf=True, recursive=True) - self._loop_nodes(node_copy, loop_count, criteria_func=lambda x: True, transform=binascii.b2a_hex) + self._loop_nodes(node_copy, loop_count, criteria_func=lambda x: True, transform=binascii.b2a_hex, + result_vector=[b'a04c', b'904c', b'e04f']) print('\n -=[ determinist & finite (loop count: %d) ]=- \n' % loop_count) @@ -897,7 +1134,8 @@ def test_BitField_Attr_01(self): node_copy2.set_env(Env()) node_copy2.make_determinist(all_conf=True, recursive=True) node_copy2.make_finite(all_conf=True, recursive=True) - self._loop_nodes(node_copy2, loop_count, criteria_func=lambda x: True, transform=binascii.b2a_hex) + it_df = self._loop_nodes(node_copy2, loop_count, criteria_func=lambda x: True, transform=binascii.b2a_hex, + result_vector=[b'a04c', b'904c', b'e04f']) print('\n -=[ random & finite (loop count: %d) ]=- \n' % loop_count) @@ -905,9 +1143,12 @@ def test_BitField_Attr_01(self): node_copy3.set_env(Env()) node_copy3.make_random(all_conf=True, recursive=True) node_copy3.make_finite(all_conf=True, recursive=True) - self._loop_nodes(node_copy3, loop_count, criteria_func=lambda x: True, transform=binascii.b2a_hex) + it_rf = self._loop_nodes(node_copy3, loop_count, criteria_func=lambda x: True, transform=binascii.b2a_hex) - def test_BitField(self): + self.assertEqual(it_df, it_rf) + self.assertEqual(it_df, 12) + + def test_BitField_Node(self): loop_count = 20 e_bf = Node('BF') @@ -1006,16 +1247,18 @@ def test_BitField_basic_features(self): # Note that 4 in subfield 1 and 16 in subfield 2 are ignored # --> 6 different values are output before looping - t = BitField(subfield_limits=[2, 6, 8, 10], subfield_values=[[4, 2, 1], [2, 15, 16, 3], [2, 3, 0], [1]], + t = BitField(subfield_limits=[2, 6, 8, 10], subfield_values=[[2, 1], [2, 15, 3], [2, 3, 0], [1]], padding=0, lsb_padding=True, endian=VT.LittleEndian, determinist=True) for i in range(30): val = binascii.b2a_hex(t.get_value()) print('*** [%d] ' % i, val) + print(t.pretty_print(), ' --> ', t.get_current_raw_val()) print('\n********\n') val = collections.OrderedDict() t.switch_mode() + print(t.subfield_vals) for i in range(30): val[i] = binascii.b2a_hex(t.get_value()) print(t.pretty_print(), ' --> ', t.get_current_raw_val()) @@ -1023,8 +1266,8 @@ def test_BitField_basic_features(self): print(list(val.values())[:15]) self.assertEqual(list(val.values())[:15], - [b'c062', b'0062', b'4062', b'806f', b'8060', b'8063', b'8061', - b'8064', b'806e', b'8072', b'8042', b'8052', b'80e2', b'8022', b'80a2']) + [b'c042', b'0042', b'4042', b'804f', b'8040', b'8043', b'8041', b'8044', + b'804e', b'8072', b'8052', b'80c2', b'8002', b'8082', b'c042']) print('\n********\n') @@ -1063,31 +1306,39 @@ def test_BitField_basic_features(self): subfield_values=[None, None, None, [3]], padding=0, lsb_padding=False, endian=VT.BigEndian, determinist=True) - val = {} + val = collections.OrderedDict() for i in range(30): val[i] = binascii.b2a_hex(t.get_value()) print('*** [%d] ' % i, val[i]) + print(t.pretty_print(), ' --> ', t.get_current_raw_val()) if t.is_exhausted(): break - if val[0] != b'0311' or val[1] != b'0312' or val[2] != b'0316' or val[3] != b'031a' \ - or val[4] != b'031e' or val[5] != b'0322' or val[6] != b'0326' or val[7] != b'032a' \ - or val[8] != b'032e' or val[9] != b'0332' or val[10] != b'0372' or val[11] != b'03b2' or val[ - 12] != b'03f2': - raise ValueError + print(list(val.values())[:15]) + self.assertEqual(list(val.values())[:15], + [b'0311', b'0312', b'0315', b'0319', b'031d', b'0321', b'0325', b'0329', + b'032d', b'0331', b'0351', b'0391', b'03d1']) print('\n********\n') t.reset_state() - print(binascii.b2a_hex(t.get_value())) - print(binascii.b2a_hex(t.get_value())) + val1 = t.get_value() + val2 = t.get_value() + print(binascii.b2a_hex(val1)) + print(binascii.b2a_hex(val2)) print('--> rewind') t.rewind() - print(binascii.b2a_hex(t.get_value())) + val3 = t.get_value() + print(binascii.b2a_hex(val3)) + self.assertEqual(val2, val3) print('--> rewind') t.rewind() - print(binascii.b2a_hex(t.get_value())) - print(binascii.b2a_hex(t.get_value())) + val4 = t.get_value() + val5 = t.get_value() + print(binascii.b2a_hex(val4)) + print(binascii.b2a_hex(val5)) + self.assertEqual(val2, val4) + self.assertEqual(val5, b'\x03\x15') print('\n********\n') @@ -1096,30 +1347,48 @@ def test_BitField_basic_features(self): for i in range(30): val = binascii.b2a_hex(t.get_value()) print('*** [%d] ' % i, val) + print(t.pretty_print(), ' --> ', t.get_current_raw_val()) if t.is_exhausted(): break print('\n********\n') - - print('--> rewind') + print('--> rewind when exhausted') t.rewind() - print(binascii.b2a_hex(t.get_value())) - print(binascii.b2a_hex(t.get_value())) - print(binascii.b2a_hex(t.get_value())) + t.rewind() + t.rewind() + t.rewind() + val1 = t.get_value() + val2 = t.get_value() + val3 = t.get_value() + val4 = t.get_value() + print(binascii.b2a_hex(val1)) + print(binascii.b2a_hex(val2)) + print(binascii.b2a_hex(val3)) + print(binascii.b2a_hex(val4)) + + self.assertEqual([val1, val2, val3, val4], + [b'\x03\x31', b'\x03\x51', b'\x03\x91', b'\x03\xd1']) print('\n******** Fuzzy mode\n') t.reset_state() t.switch_mode() - print(binascii.b2a_hex(t.get_value())) - print(binascii.b2a_hex(t.get_value())) + val1 = t.get_value() + val2 = t.get_value() + print(binascii.b2a_hex(val1)) + print(binascii.b2a_hex(val2)) print('--> rewind') t.rewind() - print(binascii.b2a_hex(t.get_value())) + val3 = t.get_value() + print(binascii.b2a_hex(val3)) + self.assertEqual(val2, val3) print('--> rewind') t.rewind() - print(binascii.b2a_hex(t.get_value())) - print(binascii.b2a_hex(t.get_value())) + val4 = t.get_value() + val5 = t.get_value() + print(binascii.b2a_hex(val4)) + print(binascii.b2a_hex(val5)) + self.assertEqual(val2, val4) print('\n********\n') @@ -1134,11 +1403,22 @@ def test_BitField_basic_features(self): print('\n********\n') - print('--> rewind') + print('--> rewind when exhausted') t.rewind() - print(binascii.b2a_hex(t.get_value())) - print(binascii.b2a_hex(t.get_value())) - print(binascii.b2a_hex(t.get_value())) + t.rewind() + t.rewind() + t.rewind() + val1 = t.get_value() + val2 = t.get_value() + val3 = t.get_value() + val4 = t.get_value() + print(binascii.b2a_hex(val1)) + print(binascii.b2a_hex(val2)) + print(binascii.b2a_hex(val3)) + print(binascii.b2a_hex(val4)) + + self.assertEqual([val1, val2, val3, val4], + [b'\x03\xd1', b'\x03\x51', b'\x00\x11', b'\x02\x11']) def test_BitField_various_features(self): @@ -1228,25 +1508,28 @@ def test_BitField_absorb(self): subfield_val_extremums=[None, [14, 15], None], padding=1, endian=VT.BigEndian, lsb_padding=True) bfield_1 = Node('bfield_1', value_type=vt) - # bfield.set_env(Env()) + bfield_1.set_env(Env()) vt = BitField(subfield_sizes=[4, 4, 4], subfield_values=[[3, 2, 0xe, 1], None, [10, 13, 3]], subfield_val_extremums=[None, [14, 15], None], padding=0, endian=VT.BigEndian, lsb_padding=True) bfield_2 = Node('bfield_2', value_type=vt) + bfield_2.set_env(Env()) vt = BitField(subfield_sizes=[4, 4, 4], subfield_values=[[3, 2, 0xe, 1], None, [10, 13, 3]], subfield_val_extremums=[None, [14, 15], None], padding=1, endian=VT.BigEndian, lsb_padding=False) bfield_3 = Node('bfield_3', value_type=vt) + bfield_3.set_env(Env()) vt = BitField(subfield_sizes=[4, 4, 4], subfield_values=[[3, 2, 0xe, 1], None, [10, 13, 3]], subfield_val_extremums=[None, [14, 15], None], padding=0, endian=VT.BigEndian, lsb_padding=False) bfield_4 = Node('bfield_4', value_type=vt) + bfield_4.set_env(Env()) # '?\xef' (\x3f\xe0) + padding 0b1111 msg = struct.pack('>H', 0x3fe0 + 0b1111) @@ -1285,73 +1568,17 @@ def test_BitField_absorb(self): self.assertEqual(status, AbsorbStatus.FullyAbsorbed) self.assertEqual(size, len(msg)) - def test_MISC(self): - ''' - TODO: assertion + purpose - ''' - loop_count = 20 - - e = Node('VT1') - vt = UINT16_be(values=[1, 2, 3, 4, 5, 6]) - e.set_values(value_type=vt) - e.set_env(Env()) - e.make_determinist(all_conf=True, recursive=True) - e.make_finite(all_conf=True, recursive=True) - self._loop_nodes(e, loop_count, criteria_func=lambda x: True) - - e2 = Node('VT2', base_node=e, ignore_frozen_state=True) - e2.set_env(Env()) - e2.make_determinist(all_conf=True, recursive=True) - e2.make_finite(all_conf=True, recursive=True) - self._loop_nodes(e2, loop_count, criteria_func=lambda x: True) - - print('\n****\n') - - sep = Node('sep', values=[' # ']) - nt = Node('NT') - nt.set_subnodes_with_csts([ - 1, ['u>', [e, 3], [sep, 1], [e2, 2]] - ]) - nt.set_env(Env()) - - self._loop_nodes(nt, loop_count, criteria_func=lambda x: True) - - print('\n****\n') - - v = dm.get_atom('V1_middle') - v.make_finite() - - e = Node('NT') - e.set_subnodes_with_csts([ - 1, ['u>', [v, 2]] - ]) - e.set_env(Env()) - e.make_determinist(recursive=True) - self._loop_nodes(e, loop_count, criteria_func=lambda x: True) - - print('\n****\n') - - self._loop_nodes(e, loop_count, criteria_func=lambda x: True) - - print('\n****\n') - - e = dm.get_atom('Middle_NT') - e.make_finite(all_conf=True, recursive=True) - e.make_determinist(all_conf=True, recursive=True) - self._loop_nodes(e, loop_count, criteria_func=lambda x: x.name == 'Middle_NT') - class TestModelWalker(unittest.TestCase): @classmethod def setUpClass(cls): - cls.dm = example.data_model - cls.dm.load_data_model(fmk._name2dm) + pass def setUp(self): pass def test_NodeConsumerStub_1(self): - nt = self.dm.get_atom('Simple') + nt = node_simple.get_clone() default_consumer = NodeConsumerStub() for rnode, consumed_node, orig_node_val, idx in ModelWalker(nt, default_consumer, make_determinist=True, max_steps=200): @@ -1359,7 +1586,7 @@ def test_NodeConsumerStub_1(self): self.assertEqual(idx, 49) def test_NodeConsumerStub_2(self): - nt = self.dm.get_atom('Simple') + nt = node_simple.get_clone() default_consumer = NodeConsumerStub(max_runs_per_node=-1, min_runs_per_node=2) for rnode, consumed_node, orig_node_val, idx in ModelWalker(nt, default_consumer, make_determinist=True, max_steps=200): @@ -1367,16 +1594,16 @@ def test_NodeConsumerStub_2(self): self.assertEqual(idx, 35) def test_BasicVisitor(self): - nt = self.dm.get_atom('Simple') - default_consumer = BasicVisitor(respect_order=True) + nt = node_simple.get_clone() + default_consumer = BasicVisitor(respect_order=True, consider_side_effects_on_sibbling=False) for rnode, consumed_node, orig_node_val, idx in ModelWalker(nt, default_consumer, make_determinist=True, max_steps=200): print(colorize('[%d] ' % idx + repr(rnode.to_bytes()), rgb=Color.INFO)) - self.assertEqual(idx, 37) + self.assertEqual(idx, 55) print('***') - nt = self.dm.get_atom('Simple') - default_consumer = BasicVisitor(respect_order=False) + nt = node_simple.get_clone() + default_consumer = BasicVisitor(respect_order=False, consider_side_effects_on_sibbling=False) for rnode, consumed_node, orig_node_val, idx in ModelWalker(nt, default_consumer, make_determinist=True, max_steps=200): print(colorize('[%d] ' % idx + repr(rnode.to_bytes()), rgb=Color.INFO)) @@ -1385,8 +1612,8 @@ def test_BasicVisitor(self): def test_NonTermVisitor(self): print('***') idx = 0 - simple = self.dm.get_atom('Simple') - nonterm_consumer = NonTermVisitor(respect_order=True) + simple = node_simple.get_clone() + nonterm_consumer = NonTermVisitor(respect_order=True, consider_side_effects_on_sibbling=False) for rnode, consumed_node, orig_node_val, idx in ModelWalker(simple, nonterm_consumer, make_determinist=True, max_steps=20): print(colorize('[%d] ' % idx + repr(rnode.to_bytes()), rgb=Color.INFO)) @@ -1394,8 +1621,8 @@ def test_NonTermVisitor(self): print('***') idx = 0 - simple = self.dm.get_atom('Simple') - nonterm_consumer = NonTermVisitor(respect_order=False) + simple = node_simple.get_clone() + nonterm_consumer = NonTermVisitor(respect_order=False, consider_side_effects_on_sibbling=False) for rnode, consumed_node, orig_node_val, idx in ModelWalker(simple, nonterm_consumer, make_determinist=True, max_steps=20): print(colorize('[%d] ' % idx + repr(rnode.to_bytes()), rgb=Color.INFO)) @@ -1404,27 +1631,28 @@ def test_NonTermVisitor(self): print('***') results = [ - b' [!] ++++++++++ [!] ::>:: [!] ? [!] ', - b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::>:: [!] ? [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::>:: [!] ? [!] ', - b' [!] >>>>>>>>>> [!] ::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::>:: [!] ', + b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::>:: [!] ? [!] ', + b' [!] ++++++++++ [!] ::>:: [!] ? [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::>:: [!] ', + b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::>:: [!] ', + b' [!] >>>>>>>>>> [!] ::>:: [!] ', ] idx = 0 data = fmk.dm.get_external_atom(dm_name='mydf', data_id='shape') - nonterm_consumer = NonTermVisitor(respect_order=True) + nonterm_consumer = NonTermVisitor(respect_order=True, consider_side_effects_on_sibbling=False) for rnode, consumed_node, orig_node_val, idx in ModelWalker(data, nonterm_consumer, make_determinist=True, max_steps=50): print(colorize('[%d] ' % idx + rnode.to_ascii(), rgb=Color.INFO)) + # print(colorize(repr(rnode.to_bytes()), rgb=Color.INFO)) self.assertEqual(rnode.to_bytes(), results[idx-1]) self.assertEqual(idx, 6) print('***') idx = 0 data = fmk.dm.get_external_atom(dm_name='mydf', data_id='shape') - nonterm_consumer = NonTermVisitor(respect_order=False) + nonterm_consumer = NonTermVisitor(respect_order=False, consider_side_effects_on_sibbling=False) for rnode, consumed_node, orig_node_val, idx in ModelWalker(data, nonterm_consumer, make_determinist=True, max_steps=50): print(colorize('[%d] ' % idx + rnode.to_ascii(), rgb=Color.INFO)) @@ -1480,51 +1708,7 @@ def test_basics(self): bv_data = data.get_clone() nt_data = data.get_clone() - raw_vals = [ - b' [!] ++++++++++ [!] ::?:: [!] ', - b' [!] ++++++++++ [!] ::=:: [!] ', - b' [!] ++++++++++ [!] ::\xff:: [!] ', - b' [!] ++++++++++ [!] ::\x00:: [!] ', - b' [!] ++++++++++ [!] ::\x01:: [!] ', - b' [!] ++++++++++ [!] ::\x80:: [!] ', - b' [!] ++++++++++ [!] ::\x7f:: [!] ', - b' [!] ++++++++++ [!] ::IAA::AAA::AAA::AAA::>:: [!] ', # [8] could change has it is a random corrupt_bit - b' [!] ++++++++++ [!] ::AAAA::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA' + b'XXX'*100 + b'::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::\x00\x00\x00::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::A%n::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::A%s::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA' + b'%n' * 400 + b'::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA' + b'%s' * 400 + b'::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA' + b'\"%n\"' * 400 + b'::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA' + b'\"%s\"' * 400 + b'::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA' + b'\r\n' * 100 + b'::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::../../../../../../etc/password::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::..\\..\\..\\..\\..\\..\\Windows\\system.ini::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::file%n%n%n%nname.txt::AAA::AAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::?:: [!] ', - b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::=:: [!] ', - b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\xff:: [!] ', - b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\x00:: [!] ', - b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\x01:: [!] ', - b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\x80:: [!] ', - b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\x7f:: [!] ', - b' [!] ++++++++++ [!] ::AAQ::AAA::>:: [!] ', # [30] could change has it is a random corrupt_bit - b' [!] ++++++++++ [!] ::AAAA::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA' + b'XXX'*100 + b'::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::\x00\x00\x00::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::A%n::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::A%s::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA' + b'%n' * 400 + b'::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA' + b'%s' * 400 + b'::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA' + b'\"%n\"' * 400 + b'::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA' + b'\"%s\"' * 400 + b'::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::AAA' + b'\r\n' * 100 + b'::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::../../../../../../etc/password::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::..\\..\\..\\..\\..\\..\\Windows\\system.ini::AAA::>:: [!] ', - b' [!] ++++++++++ [!] ::file%n%n%n%nname.txt::AAA::>:: [!] ', + raw_vals = [ b' [!] ++++++++++ [!] ::AAA::AAA::?:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::=:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::\xff:: [!] ', @@ -1532,51 +1716,20 @@ def test_basics(self): b' [!] ++++++++++ [!] ::AAA::AAA::\x01:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::\x80:: [!] ', b' [!] ++++++++++ [!] ::AAA::AAA::\x7f:: [!] ', - - b' [!] >>>>>>>>>> [!] ::?:: [!] ', - b' [!] >>>>>>>>>> [!] ::=:: [!] ', - b' [!] >>>>>>>>>> [!] ::\xff:: [!] ', - b' [!] >>>>>>>>>> [!] ::\x00:: [!] ', - b' [!] >>>>>>>>>> [!] ::\x01:: [!] ', - b' [!] >>>>>>>>>> [!] ::\x80:: [!] ', - b' [!] >>>>>>>>>> [!] ::\x7f:: [!] ', - b' [!] >>>>>>>>>> [!] ::QAA::AAA::AAA::AAA::>:: [!] ', # [59] could change has it is a random corrupt_bit - b' [!] >>>>>>>>>> [!] ::AAAA::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA' + b'XXX'*100 + b'::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::\x00\x00\x00::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::A%n::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::A%s::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA' + b'%n' * 400 + b'::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA' + b'%s' * 400 + b'::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA' + b'\"%n\"' * 400 + b'::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA' + b'\"%s\"' * 400 + b'::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA' + b'\r\n' * 100 + b'::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::../../../../../../etc/password::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::..\\..\\..\\..\\..\\..\\Windows\\system.ini::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::file%n%n%n%nname.txt::AAA::AAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::?:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::=:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\xff:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\x00:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\x01:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\x80:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\x7f:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAC::AAA::>:: [!] ', # [81] could change has it is a random corrupt_bit - b' [!] >>>>>>>>>> [!] ::AAAA::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA' + b'XXX'*100 + b'::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::\x00\x00\x00::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::A%n::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::A%s::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA' + b'%n' * 400 + b'::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA' + b'%s' * 400 + b'::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA' + b'\"%n\"' * 400 + b'::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA' + b'\"%s\"' * 400 + b'::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::AAA' + b'\r\n' * 100 + b'::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::../../../../../../etc/password::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::..\\..\\..\\..\\..\\..\\Windows\\system.ini::AAA::>:: [!] ', - b' [!] >>>>>>>>>> [!] ::file%n%n%n%nname.txt::AAA::>:: [!] ', + b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::?:: [!] ', + b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::=:: [!] ', + b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\xff:: [!] ', + b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\x00:: [!] ', + b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\x01:: [!] ', + b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\x80:: [!] ', + b' [!] ++++++++++ [!] ::AAA::AAA::AAA::AAA::\x7f:: [!] ', + b' [!] ++++++++++ [!] ::?:: [!] ', + b' [!] ++++++++++ [!] ::=:: [!] ', + b' [!] ++++++++++ [!] ::\xff:: [!] ', + b' [!] ++++++++++ [!] ::\x00:: [!] ', + b' [!] ++++++++++ [!] ::\x01:: [!] ', + b' [!] ++++++++++ [!] ::\x80:: [!] ', + b' [!] ++++++++++ [!] ::\x7f:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::?:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::=:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::\xff:: [!] ', @@ -1584,26 +1737,44 @@ def test_basics(self): b' [!] >>>>>>>>>> [!] ::AAA::AAA::\x01:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::\x80:: [!] ', b' [!] >>>>>>>>>> [!] ::AAA::AAA::\x7f:: [!] ', + b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::?:: [!] ', + b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::=:: [!] ', + b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\xff:: [!] ', + b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\x00:: [!] ', + b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\x01:: [!] ', + b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\x80:: [!] ', + b' [!] >>>>>>>>>> [!] ::AAA::AAA::AAA::AAA::\x7f:: [!] ', + b' [!] >>>>>>>>>> [!] ::?:: [!] ', + b' [!] >>>>>>>>>> [!] ::=:: [!] ', + b' [!] >>>>>>>>>> [!] ::\xff:: [!] ', + b' [!] >>>>>>>>>> [!] ::\x00:: [!] ', + b' [!] >>>>>>>>>> [!] ::\x01:: [!] ', + b' [!] >>>>>>>>>> [!] ::\x80:: [!] ', + b' [!] >>>>>>>>>> [!] ::\x7f:: [!] ', ] + # Note that the result of the TC that performs a random bitflip could collide with the one + # playing on letter case, resulting in less test cases (at worst 4 less in total) + # In this case assert won't be validated + tn_consumer = TypedNodeDisruption(respect_order=True, ignore_separator=True) ic = NodeInternalsCriteria(mandatory_attrs=[NodeInternals.Mutable], negative_attrs=[NodeInternals.Separator], node_kinds=[NodeInternals_TypedValue], - negative_node_subkinds=[String]) + negative_node_subkinds=[String, Filename]) tn_consumer.set_node_interest(internals_criteria=ic) for rnode, consumed_node, orig_node_val, idx in ModelWalker(data, tn_consumer, make_determinist=True, max_steps=200): val = rnode.to_bytes() - print(colorize('[%d] ' % idx + repr(val), rgb=Color.INFO)) - if idx not in [8, 30, 59, 81]: - self.assertEqual(val, raw_vals[idx - 1]) + # print(colorize('{!r}'.format(val), rgb=Color.INFO)) + print(colorize('[{:d}] {!r}'.format(idx, val), rgb=Color.INFO)) + self.assertEqual(val, raw_vals[idx - 1]) - self.assertEqual(idx, 102) # should be even + self.assertEqual(idx, 42) print('***') idx = 0 - bv_consumer = BasicVisitor(respect_order=True) + bv_consumer = BasicVisitor(respect_order=True, consider_side_effects_on_sibbling=False) for rnode, consumed_node, orig_node_val, idx in ModelWalker(bv_data, bv_consumer, make_determinist=True, max_steps=100): @@ -1612,7 +1783,7 @@ def test_basics(self): print('***') idx = 0 - nt_consumer = NonTermVisitor(respect_order=True) + nt_consumer = NonTermVisitor(respect_order=True, consider_side_effects_on_sibbling=False) for rnode, consumed_node, orig_node_val, idx in ModelWalker(nt_data, nt_consumer, make_determinist=True, max_steps=100): @@ -1621,7 +1792,7 @@ def test_basics(self): def test_TypedNodeDisruption_1(self): - nt = self.dm.get_atom('Simple') + nt = node_simple.get_clone() tn_consumer = TypedNodeDisruption() ic = NodeInternalsCriteria(negative_node_subkinds=[String]) tn_consumer.set_node_interest(internals_criteria=ic) @@ -1631,7 +1802,7 @@ def test_TypedNodeDisruption_1(self): self.assertEqual(idx, 21) def test_TypedNodeDisruption_2(self): - nt = self.dm.get_atom('Simple') + nt = node_simple.get_clone() tn_consumer = TypedNodeDisruption(max_runs_per_node=3, min_runs_per_node=3) ic = NodeInternalsCriteria(negative_node_subkinds=[String]) tn_consumer.set_node_interest(internals_criteria=ic) @@ -1645,14 +1816,16 @@ def test_TypedNodeDisruption_3(self): Test case similar to test_TermNodeDisruption_1() but with more powerfull TypedNodeDisruption. ''' - nt = self.dm.get_atom('Simple') + nt = node_simple.get_clone() tn_consumer = TypedNodeDisruption(max_runs_per_node=1) # ic = NodeInternalsCriteria(negative_node_subkinds=[String]) # tn_consumer.set_node_interest(internals_criteria=ic) for rnode, consumed_node, orig_node_val, idx in ModelWalker(nt, tn_consumer, make_determinist=True, max_steps=-1): print(colorize('[%d] ' % idx + repr(rnode.to_bytes()), rgb=Color.INFO)) - self.assertEqual(idx, 444) + self.assertAlmostEqual(idx, 346, delta=2) + # almostequal because collision in String test cases can lead to less test cases + # (related to random bitflip test case that could collide with case_sensitive test case) def test_TypedNodeDisruption_BitfieldCollapse(self): ''' @@ -1663,8 +1836,8 @@ def test_TypedNodeDisruption_BitfieldCollapse(self): data.freeze() data.show() - print('\norig value: ' + repr(data['smscmd/TP-DCS'].to_bytes())) - # self.assertEqual(data['smscmd/TP-DCS'].to_bytes(), b'\xF6') + print('\norig value: ' + repr(data['smscmd/TP-DCS'][0].to_bytes())) + # self.assertEqual(data['smscmd/TP-DCS'][0].to_bytes(), b'\xF6') corrupt_table = { 1: b'\x06', @@ -1682,19 +1855,19 @@ def test_TypedNodeDisruption_BitfieldCollapse(self): # tn_consumer.set_node_interest(internals_criteria=ic) for rnode, consumed_node, orig_node_val, idx in ModelWalker(data, tn_consumer, make_determinist=True, max_steps=7): - print(colorize('\n[%d] ' % idx + repr(rnode['smscmd/TP-DCS$'].to_bytes()), rgb=Color.INFO)) + print(colorize('\n[%d] ' % idx + repr(rnode['smscmd/TP-DCS$'][0].to_bytes()), rgb=Color.INFO)) print('node name: ' + consumed_node.name) print('original value: {!s} ({!s})'.format(binascii.b2a_hex(orig_node_val), bin(struct.unpack('B', orig_node_val)[0]))) print('corrupted value: {!s} ({!s})'.format(binascii.b2a_hex(consumed_node.to_bytes()), bin(struct.unpack('B', consumed_node.to_bytes())[0]))) - print('result: {!s} ({!s})'.format(binascii.b2a_hex(rnode['smscmd/TP-DCS$'].to_bytes()), - bin(struct.unpack('B', rnode['smscmd/TP-DCS$'].to_bytes())[0]))) - rnode['smscmd/TP-DCS$'].show() - self.assertEqual(rnode['smscmd/TP-DCS'].to_bytes(), corrupt_table[idx]) + print('result: {!s} ({!s})'.format(binascii.b2a_hex(rnode['smscmd/TP-DCS$'][0].to_bytes()), + bin(struct.unpack('B', rnode['smscmd/TP-DCS$'][0].to_bytes())[0]))) + rnode['smscmd/TP-DCS$'][0].show() + self.assertEqual(rnode['smscmd/TP-DCS'][0].to_bytes(), corrupt_table[idx]) def test_AltConfConsumer_1(self): - simple = self.dm.get_atom('Simple') + simple = node_simple.get_clone() consumer = AltConfConsumer(max_runs_per_node=-1, min_runs_per_node=-1) consumer.set_node_interest(owned_confs=['ALT']) @@ -1704,7 +1877,7 @@ def test_AltConfConsumer_1(self): self.assertEqual(idx, 15) def test_AltConfConsumer_2(self): - simple = self.dm.get_atom('Simple') + simple = node_simple.get_clone() consumer = AltConfConsumer(max_runs_per_node=2, min_runs_per_node=1) consumer.set_node_interest(owned_confs=['ALT']) @@ -1714,7 +1887,7 @@ def test_AltConfConsumer_2(self): self.assertEqual(idx, 8) def test_AltConfConsumer_3(self): - simple = self.dm.get_atom('Simple') + simple = node_simple.get_clone() consumer = AltConfConsumer(max_runs_per_node=-1, min_runs_per_node=-1) consumer.set_node_interest(owned_confs=['ALT', 'ALT_2']) @@ -1724,7 +1897,7 @@ def test_AltConfConsumer_3(self): self.assertEqual(idx, 24) def test_AltConfConsumer_4(self): - simple = self.dm.get_atom('Simple') + simple = node_simple.get_clone() consumer = AltConfConsumer(max_runs_per_node=-1, min_runs_per_node=-1) consumer.set_node_interest(owned_confs=['ALT_2', 'ALT']) @@ -1766,7 +1939,7 @@ def test_USB(self): print(colorize('number of confs: %d' % idx, rgb=Color.INFO)) - self.assertIn(idx, [542]) + self.assertIn(idx, [479]) @ddt.ddt @@ -1825,19 +1998,19 @@ def test_djobs(self): d3 = d.get_clone() d.freeze() - d['.*/value$'].unfreeze() + d['.*/value$'][0].unfreeze() d_raw = d.to_bytes() d.show() d2.freeze() - d2['.*/value$'].unfreeze() - d2['.*/value$'].freeze() + d2['.*/value$'][0].unfreeze() + d2['.*/value$'][0].freeze() d2_raw = d2.to_bytes() d2.show() d3.freeze() - d3['.*/value$'].unfreeze() - d3['.*/len$'].unfreeze() + d3['.*/value$'][0].unfreeze() + d3['.*/len$'][0].unfreeze() d3_raw = d3.to_bytes() d3.show() @@ -2095,11 +2268,11 @@ def verif_val_and_print(*arg, **kwargs): self.assertEqual(len(msg), size) self.assertTrue(self.helper1_called) self.assertTrue(self.helper2_called) - self.assertEqual(top.get_node_by_path("top/middle2/str10").to_bytes(), b'THE_END') + self.assertEqual(top.get_first_node_by_path("top/middle2/str10").to_bytes(), b'THE_END') # Because constraints are untighten on this node, its nominal # size of 4 is set to 5 when absorbing b'COOL!' - self.assertEqual(top.get_node_by_path("top/middle1/cool").to_bytes(), b'COOL!') + self.assertEqual(top.get_first_node_by_path("top/middle1/cool").to_bytes(), b'COOL!') self.assertEqual(status2, AbsorbStatus.FullyAbsorbed) @@ -2107,9 +2280,9 @@ def verif_val_and_print(*arg, **kwargs): del self.helper2_called print('\n*** test __getitem__() ***\n') - print(top["top/middle2"]) + print(top["top/middle2"][0]) print('\n***\n') - print(repr(top["top/middle2"])) + print(repr(top["top/middle2"][0])) def test_show(self): @@ -2575,7 +2748,7 @@ def test_collapse_padding(self): self.assertEqual(result, raw) - def test_search_primitive(self): + def test_node_search_primitive_01(self): data = fmk.dm.get_external_atom(dm_name='mydf', data_id='exist_cond') data.freeze() @@ -2608,7 +2781,98 @@ def test_search_primitive(self): # corrupted_data.unfreeze(recursive=True, reevaluate_constraints=True) # corrupted_data.show() + def test_node_search_primitive_02(self): + + ex_node = fmk.dm.get_atom('ex') + ex_node.show() + + n = ex_node.get_first_node_by_path('ex/data_group/data1') + print('\n*** node', n.name) + + + ic = NodeInternalsCriteria(required_csts=[SyncScope.Existence]) + + def exec_search(**kwargs): + l1 = ex_node.get_reachable_nodes(**kwargs) + print("\n*** Number of node(s) found: {:d} ***".format(len(l1))) + + res = [] + for n in l1: + print(' |_ ' + n.name) + res.append(n.name) + + return res + + exec_search(path_regexp='^ex/data_group/data.*') + + l = ex_node['data_group/data1'][0].get_all_paths_from(ex_node) + print(l) + + + def test_node_search_primitive_03(self): + test_node = fmk.dm.get_atom('TestNode') + test_node.show() + + rexp = 'TestNode/middle/(USB_desc/|val2$)' + + l = test_node[rexp] + for n in l: + print('Node name: {}, value: {}'.format(n.name, n.to_bytes())) + + test_node[rexp] = Node('ignored_name', values=['TEST']) + test_node.show() + + self.assertEqual(len(l), 4) + + for n in l: + print('Node name: {}, value: {}'.format(n.name, n.to_bytes())) + self.assertEqual(n.to_bytes(), b'TEST') + + self.assertEqual(test_node['Unknown path'], None) + + def test_node_search_performance(self): + + ex_node = fmk.dm.get_atom('ex') + ex_node.show() + + t0 = datetime.datetime.now() + for _ in range(30): + l0 = list(ex_node.iter_nodes_by_path(path_regexp='.*', flush_cache=True, resolve_generator=True)) + now = datetime.datetime.now() + print('\n*** Execution time of .iter_nodes_by_path(flush_cache=True): {}'.format((now - t0).total_seconds())) + + for n in l0: + print(n.name) + + t0 = datetime.datetime.now() + for _ in range(30): + l1 = list(ex_node.iter_nodes_by_path(path_regexp='.*', flush_cache=False, resolve_generator=True)) + now = datetime.datetime.now() + print('\n*** Execution time of .iter_nodes_by_path(flush_cache=False): {}'.format((now - t0).total_seconds())) + + for n in l1: + print(n.name) + + t0 = datetime.datetime.now() + for _ in range(30): + nd = ex_node.get_first_node_by_path(path_regexp='.*', flush_cache=False, resolve_generator=True) + now = datetime.datetime.now() + print('\n*** Execution time of .get_first_node_by_path(flush_cache=False): {}'.format((now - t0).total_seconds())) + + t0 = datetime.datetime.now() + for _ in range(30): + l2 = ex_node.get_reachable_nodes(path_regexp='.*', respect_order=True, resolve_generator=True) + now = datetime.datetime.now() + print('\n*** Execution time of .get_reachable_nodes: {}'.format((now - t0).total_seconds())) + + for n in l2: + print(n.name) + + self.assertEqual(l0, l1) + self.assertEqual(l1, l2) + +@ddt.ddt class TestNode_NonTerm(unittest.TestCase): @classmethod def setUpClass(cls): @@ -2617,6 +2881,221 @@ def setUpClass(cls): def setUp(self): pass + @ddt.data((True, True),(True,False),(False,True),(False,False)) + @ddt.unpack + def test_combinatory_1(self, mimick_twalk, full_comb_mode): + test_desc = \ + {'name': 'test', + 'custo_set': MH.Custo.NTerm.CycleClone, + 'contents': [ + {'name': 'prefix', + 'qty': (0,4), + 'default_qty': 1, + 'contents': String(values=['-', '+'])}, + + {'section_type': MH.Pick, + 'weights': (3,2,1), + 'contents': [ + {'name': 'pre1', 'contents': String(values=['x'])}, + {'name': 'pre2', 'contents': String(values=['y'])}, + {'name': 'pre3', 'contents': String(values=['z'])} + ]}, + + {'name': 'digit', + 'qty': (0,10), + 'default_qty': 2, + 'contents': [ + {'weight': 50, + 'contents': [ + {'name': 'n1', 'contents': String(values=['1'])} + ]}, + {'weight': 40, + 'contents': [ + {'name': 'n2', 'contents': String(values=['2'])} + ]}, + {'weight': 30, + 'contents': [ + {'name': 'n3', 'contents': String(values=['3'])} + ]}, + {'weight': 20, + 'contents': [ + {'name': 'n4', 'contents': String(values=['4'])} + ]}, + ]}, + {'section_type': MH.Pick, + 'weights': (2,1,3), + 'contents': [ + {'name': 'suf2', 'contents': String(values=['b'])}, + {'name': 'suf3', 'contents': String(values=['c'])}, + {'name': 'suf1', 'contents': String(values=['a'])} + ]} + ]} + + + mb = NodeBuilder(add_env=True) + nd = mb.create_graph_from_desc(test_desc) + + data_ref = [ + b'-x12a', + b'-+-+x12a', + b'x12a', + b'-x1234123412a', + b'-xa', + b'-y12a', + b'-+-+y12a', + b'y12a', + b'-y1234123412a', + b'-ya', + b'-z12a', + b'-+-+z12a', + b'z12a', + b'-z1234123412a', + b'-za', + b'-x12b', + b'-+-+x12b', + b'x12b', + b'-x1234123412b', + b'-xb', + b'-x12c', + b'-+-+x12c', + b'x12c', + b'-x1234123412c', + b'-xc', + ] + + # mimick_twalk = True + # full_comb_mode = True + + nd.make_finite() + nd.custo.full_combinatory_mode = full_comb_mode + for i in range(1, 200): + print(f'\n###### data #{i}') + if mimick_twalk: # with fix_all + nd.unfreeze(recursive=False) + nd.freeze() + nd.unfreeze(recursive=True, reevaluate_constraints=True, ignore_entanglement=True) + nd.freeze() + else: + nd.walk(recursive=False) + data = nd.to_bytes() + print(data) + if not full_comb_mode: + self.assertEqual(data, data_ref[i-1], f'i = {i-1}') + + if nd.is_exhausted(): + break + + if full_comb_mode: + self.assertEqual(i, 45) + else: + self.assertEqual(i, 25) + + + + @ddt.data( + (True, True, False), (True, False, False), (False, True, False), (False, False, False), + (True, True, True), (True, False, True), (False, True, True), (False, False, True) + ) + @ddt.unpack + def test_combinatory_2(self, mimick_twalk, clone_mode, full_comb_mode): + + test_desc = \ + {'name': 'test', + 'custo_set': MH.Custo.NTerm.CycleClone, + 'custo_clear': MH.Custo.NTerm.MutableClone, + 'contents': [ + {'name': 'scst1', 'qty': 2, 'contents': String(values=['s'])}, + {'name': 'scst2', 'qty': 2, 'contents': String(values=['s'])}, + + {'name': 'pre3', + 'qty': (1,4), + 'default_qty': 3, + 'contents': String(values=['-', '+'])}, + + {'name': 'mcst1', 'qty': 2, 'contents': String(values=['s'])}, + {'name': 'mcst2', 'qty': 2, 'contents': String(values=['s'])}, + + {'name': 'digit1', + 'qty': (0,10), + 'default_qty': 2, + 'contents': String(values=['1']), + }, + + {'name': 'mcst3', 'qty': 2, 'contents': String(values=['s'])}, + + {'name': 'digit2', + 'qty': (3,7), + 'default_qty': 5, + 'contents': String(values=['2']), + }, + {'name': 'digit3', + 'qty': (0,1), + 'default_qty': 0, + 'contents': String(values=['3']), + }, + + {'section_type': MH.Pick, + 'weights': (2,1,3), + 'contents': [ + {'name': 'suf2', 'contents': String(values=['b'])}, + {'name': 'suf3', 'contents': String(values=['c'])}, + {'name': 'suf1', 'contents': String(values=['a'])} + ]}, + + {'name': 'ecst1', 'qty': 2, 'contents': String(values=['s'])}, + {'name': 'ecst2', 'qty': 2, 'contents': String(values=['s'])}, + + ]} + + mb = NodeBuilder(add_env=True) + nd = mb.create_graph_from_desc(test_desc) + + data_ref = [ + b'ssss-+-ssss11ss22222assss', + b'ssss-+-+ssss11ss22222assss', + b'ssss-ssss11ss22222assss', + b'ssss-+-ssss1111111111ss22222assss', + b'ssss-+-ssssss22222assss', + b'ssss-+-ssss11ss2222222assss', + b'ssss-+-ssss11ss222assss', + b'ssss-+-ssss11ss222223assss', + ] + + nd.make_finite() + nd.custo.full_combinatory_mode = full_comb_mode + for i in range(1, 200): + if clone_mode: + nd = nd.get_clone() + print(f'\n###### data #{i}') + if mimick_twalk: # with fix_all + nd.unfreeze(recursive=False) + nd.freeze() + nd.unfreeze(recursive=True, reevaluate_constraints=True, ignore_entanglement=True) + nd.freeze() + else: + nd.walk(recursive=False) + data = nd.to_bytes() + print(data) + if not full_comb_mode: + idx = (i-1) % 8 + if i > 16: + str_ref = data_ref[idx][:-5] + b'cssss' + elif i > 8: + str_ref = data_ref[idx][:-5] + b'bssss' + else: + str_ref = data_ref[idx] + + self.assertEqual(data, str_ref, f'i = {i-1}') + + if nd.is_exhausted(): + break + + if full_comb_mode: + self.assertEqual(i, 162) # 3 x 54 + else: + self.assertEqual(i, 24) # 3 x 8 + + def test_infinity(self): infinity_desc = \ {'name': 'infinity', @@ -2758,7 +3237,7 @@ def test_encoding_attr(self): {'name': 'crc', 'contents': CRC(vt=UINT32_be, after_encoding=False), 'node_args': ['enc_data', 'data2'], - 'absorb_csts': AbsFullCsts(contents=False)}, + 'absorb_csts': AbsFullCsts(content=False, similar_content=False)}, {'name': 'enc_data', 'encoder': GZIP_Enc(6), 'set_attrs': NodeInternals.Abs_Postpone, @@ -2766,7 +3245,7 @@ def test_encoding_attr(self): {'name': 'len', 'contents': LEN(vt=UINT8, after_encoding=False), 'node_args': 'data1', - 'absorb_csts': AbsFullCsts(contents=False)}, + 'absorb_csts': AbsFullCsts(content=False, similar_content=False)}, {'name': 'data1', 'contents': String(values=['Test!', 'Hello World!'], codec='utf-16-le')}, ]}, @@ -2784,8 +3263,8 @@ def test_encoding_attr(self): node.show() print('\nData:') print(node.to_bytes()) - self.assertEqual(struct.unpack('B', node['enc/enc_data/len$'].to_bytes())[0], - len(node['enc/enc_data/data1$'].get_raw_value())) + self.assertEqual(struct.unpack('B', node['enc/enc_data/len$'][0].to_bytes())[0], + len(node['enc/enc_data/data1$'][0].get_raw_value())) raw_data = b'Plop\x8c\xd6/\x06x\x9cc\raHe(f(aPd\x00\x00\x0bv\x01\xc7Blue' status, off, size, name = node_abs.absorb(raw_data, constraints=AbsFullCsts()) @@ -2800,6 +3279,63 @@ def test_encoding_attr(self): self.assertEqual(status, AbsorbStatus.FullyAbsorbed) self.assertEqual(raw_data, raw_data_abs) + def test_node_addition(self): + + # Notes: + # Current test cases are agnostic to differences between min and max value as the NT nodes are + # in determinist mode, meaning that we only see the usage of min qty in the current test cases. + + new_node_min_qty = 2 + new_node_max_qty = 2 + + print('\n*** Test Case 1 ***\n') + + ex_node = fmk.dm.get_atom('ex') + ex_node.show() + new_node = Node('my_node2', values=['New node added!!!!']) + data2_node = ex_node['ex/data_group/data2'][0] + ex_node['ex/data_group'][0].add(new_node, after=data2_node, min=new_node_min_qty, max=new_node_max_qty) + ex_node.show() + + nt_node = ex_node['ex/data_group'][0] + self.assertEqual(nt_node.get_subnode_idx(new_node), + nt_node.get_subnode_idx(ex_node['ex/data_group/data2'][0])+1) + + ex_node.unfreeze() + ex_node.show() + self.assertEqual(nt_node.get_subnode_idx(new_node), + nt_node.get_subnode_idx(ex_node['ex/data_group/data2:3'][0])+1) + + print('\n*** Test Case 2 ***\n') + + ex_node = fmk.dm.get_atom('ex') + ex_node.show() + new_node = Node('my_node2', values=['New node added!!!!']) + ex_node['ex/data_group'][0].add(new_node, idx=0, min=new_node_min_qty, max=new_node_max_qty) + ex_node.show() + + nt_node = ex_node['ex/data_group'][0] + self.assertEqual(nt_node.get_subnode_idx(new_node), 0) + + ex_node.unfreeze() + ex_node.show() + self.assertEqual(nt_node.get_subnode_idx(new_node), 0) + + print('\n*** Test Case 3 ***\n') + + ex_node = fmk.dm.get_atom('ex') + ex_node.show() + new_node = Node('my_node2', values=['New node added!!!!']) + ex_node['ex/data_group'][0].add(new_node, idx=None, min=new_node_min_qty, max=new_node_max_qty) + ex_node.show() + + nt_node = ex_node['ex/data_group'][0] + self.assertEqual(nt_node.get_subnode_idx(new_node), nt_node.get_subnode_qty()-new_node_min_qty) + + ex_node.unfreeze() + ex_node.show() + self.assertEqual(nt_node.get_subnode_idx(new_node), nt_node.get_subnode_qty()-new_node_min_qty) + class TestNode_TypedValue(unittest.TestCase): @classmethod @@ -2809,6 +3345,173 @@ def setUpClass(cls): def setUp(self): pass + def test_bitfield(self): + + bf = BitField(subfield_sizes=[4, 4, 4], + subfield_values=[[4, 2, 1], None, [10, 11, 15]], + subfield_val_extremums=[None, [5, 9], None], + padding=0, lsb_padding=False, endian=VT.BigEndian, + defaults=[2,8,15]) + node = Node('BF', vt=bf) + node.set_env(Env()) + + node_abs = node.get_clone() + + node.show() + b1 = node.to_bytes() + + node.set_default_value([1,5,11]) + node.show() + b2 = node.to_bytes() + + self.assertEqual(b1, b'\x0f\x82') + self.assertEqual(b2, b'\x0b\x51') + + raw_data = b'\x0f\x74' + status, _, _, _ = node_abs.absorb(raw_data, constraints=AbsFullCsts()) + self.assertTrue(status, AbsorbStatus.FullyAbsorbed) + + node_abs.show() + b3 = node_abs.to_bytes() + node_abs.reset_state() + b4 = node_abs.to_bytes() + + self.assertEqual(b3, b'\x0f\x74') + self.assertEqual(b4, b'\x0f\x74') + + def test_integer(self): + node = Node('Int1', vt=UINT8(min=9, max=40, determinist=True, default=21)) + node.set_env(Env()) + + node.show() + i1 = node.get_raw_value() + + node.set_default_value(35) + i2 = node.get_raw_value() + node.walk() + i3 = node.get_raw_value() + + node.reset_state() + i4 = node.get_raw_value() + + print('\n***', i1, i2, i3, i4) + + self.assertEqual(i1, 21) + self.assertEqual(i2, 35) + self.assertNotEqual(i3, 35) + self.assertEqual(i4, 35) + + node = Node('Int2', vt=UINT8(values=[9,10,21,32,40], determinist=True, default=21)) + node.set_env(Env()) + + node_abs = node.get_clone() + + node.show() + i1 = node.get_raw_value() + + self.assertRaises(DataModelDefinitionError, node.set_default_value, 35) + + node.set_default_value(32) + i2 = node.get_raw_value() + + print('\n***', i1, i2) + + self.assertEqual(i1, 21) + self.assertEqual(i2, 32) + + raw_data = b'\x28' # == 40 + status, _, _, _ = node_abs.absorb(raw_data, constraints=AbsFullCsts()) + self.assertTrue(status, AbsorbStatus.FullyAbsorbed) + + node_abs.show() + i3 = node_abs.get_raw_value() + node_abs.reset_state() + i4 = node_abs.get_raw_value() + + self.assertEqual(i3, 40) + self.assertEqual(i4, 40) + + + def test_str_basics(self): + node = Node('test', vt=String(min_sz=2, max_sz=100, alphabet='ABCDEFGH', default='CAFE')) + node.set_env(Env()) + str0 = node.to_str() + + node.set_default_value('BABA') + + str1 = node.to_str() + node.walk() + str2 = node.to_str() + node.reset_state() + str3 = node.to_str() + node.walk() + node.walk() + node.reset_state() + str4 = node.to_str() + + print('*** node.to_str():\n{}\n{}\n{}\n{}\n{}'.format(str0, str1, str2, str3, str4)) + + self.assertEqual(str0, 'CAFE') + self.assertEqual(str1, 'BABA') + self.assertEqual(str3, 'BABA') + self.assertEqual(str4, 'BABA') + self.assertNotEqual(str2, 'BABA') + + node = Node('test', vt=String(values=['ABC', 'GAG'], min_sz=2, max_sz=100, alphabet='ABCDEFGH', default='CAFE')) + node.set_env(Env()) + + node_abs = node.get_clone() + + str0 = node.to_str() + node.walk() + str1 = node.to_str() + + print('*** node.to_str():\n{}\n{}'.format(str0,str1)) #, str1, str2, str3, str4)) + + self.assertEqual(str0, 'CAFE') + self.assertEqual(str1, 'ABC') + + raw_data = b'FACE' + status, _, _, _ = node_abs.absorb(raw_data, constraints=AbsFullCsts()) + self.assertTrue(status, AbsorbStatus.FullyAbsorbed) + + node_abs.show() + str2 = node_abs.to_str() + node_abs.reset_state() + str3 = node_abs.to_str() + + self.assertEqual(str2, 'FACE') + self.assertEqual(str3, 'FACE') + + + def test_filename(self): + + node1 = Node('fname', vt=Filename(values=['myfile.txt'], case_sensitive=False)) + node1.set_env(Env()) + + node2 = Node('fname', vt=Filename(values=['myfile.txt'], case_sensitive=False, uri_parsing=True)) + node2.set_env(Env()) + + node3 = Node('fname', vt=Filename(values=['base/myfile.txt'], case_sensitive=False)) + node3.set_env(Env()) + + node4 = Node('fpath', vt=FolderPath(values=['base/myfolder'], case_sensitive=False)) + node4.set_env(Env()) + + node_list = [(node1, 28), + (node2, 20), + (node3, 19), + (node4, 19)] + + for node, nb_tc in node_list: + tn_consumer = TypedNodeDisruption(fuzz_magnitude=1.0) + for rnode, consumed_node, orig_node_val, idx in ModelWalker(node, tn_consumer, make_determinist=True, max_steps=-1): + data = rnode.to_bytes() + sz = len(data) + print(colorize('[{:d}] ({:04d}) {!r}'.format(idx, sz, data), rgb=Color.INFO)) + + self.assertEqual(idx, nb_tc) + def test_str_alphabet(self): alphabet1 = 'ABC' @@ -2839,8 +3542,7 @@ def test_str_alphabet(self): alphabet = alphabet1 + alphabet2 for l in raw_data: - if sys.version_info[0] > 2: - l = chr(l) + l = chr(l) self.assertTrue(l in alphabet) print('\n*** Test with following data:') @@ -2880,13 +3582,16 @@ def test_str_alphabet(self): def test_encoded_str_1(self): - class EncodedStr(String): + class MyEncoder(Encoder): def encode(self, val): return val + b'***' def decode(self, val): return val[:-3] + @from_encoder(MyEncoder) + class EncodedStr(String): pass + data = ['Test!', u'Hell\u00fc World!'] enc_desc = \ {'name': 'enc', @@ -2894,7 +3599,7 @@ def decode(self, val): {'name': 'len', 'contents': LEN(vt=UINT8, after_encoding=False), 'node_args': 'user_data', - 'absorb_csts': AbsFullCsts(contents=False)}, + 'absorb_csts': AbsFullCsts(content=False, similar_content=False)}, {'name': 'user_data', 'contents': EncodedStr(values=data, codec='utf8')}, {'name': 'compressed_data', @@ -2909,8 +3614,8 @@ def decode(self, val): node_abs.set_env(Env()) node.show() - self.assertEqual(struct.unpack('B', node['enc/len$'].to_bytes())[0], - len(node['enc/user_data$'].get_raw_value())) + self.assertEqual(struct.unpack('B', node['enc/len$'][0].to_bytes())[0], + len(node['enc/user_data$'][0].get_raw_value())) raw_data = b'\x0CHell\xC3\xBC World!***' + \ b'x\x9c\xf3H\xcd\xc9\xf9\xa3\x10\x9e_\x94\x93\xa2\x08\x00 \xb1\x04\xcb' @@ -2943,39 +3648,6 @@ def decode(self, val): gsm_dec = gsm_t.decode(gsm_enc) self.assertEqual(msg, gsm_dec) - # msg = u'où ça'.encode(internal_repr_codec) #' b'o\xf9 \xe7a' - # vtype = UTF16_LE(max_sz=20) - # enc = vtype.encode(msg) - # dec = vtype.decode(enc) - # self.assertEqual(msg, dec) - # - # msg = u'où ça'.encode(internal_repr_codec) - # vtype = UTF16_BE(max_sz=20) - # enc = vtype.encode(msg) - # dec = vtype.decode(enc) - # self.assertEqual(msg, dec) - # - # msg = u'où ça'.encode(internal_repr_codec) - # vtype = UTF8(max_sz=20) - # enc = vtype.encode(msg) - # dec = vtype.decode(enc) - # self.assertEqual(msg, dec) - # - # msg = u'où ça'.encode(internal_repr_codec) - # vtype = Codec(max_sz=20, encoding_arg=None) - # enc = vtype.encode(msg) - # dec = vtype.decode(enc) - # self.assertEqual(msg, dec) - # - # msg = u'où ça'.encode(internal_repr_codec) - # vtype = Codec(max_sz=20, encoding_arg='utf_32') - # enc = vtype.encode(msg) - # dec = vtype.decode(enc) - # self.assertEqual(msg, dec) - # utf32_enc = b"\xff\xfe\x00\x00o\x00\x00\x00\xf9\x00\x00\x00 " \ - # b"\x00\x00\x00\xe7\x00\x00\x00a\x00\x00\x00" - # self.assertEqual(enc, utf32_enc) - msg = b'Hello World!' * 10 vtype = GZIP(max_sz=20) enc = vtype.encode(msg) @@ -3195,13 +3867,10 @@ def test_generic_generators(self): raw = d.to_bytes() print(raw) - if sys.version_info[0] > 2: - retr_off = raw[-1] - else: - retr_off = struct.unpack('B', raw[-1])[0] + retr_off = raw[-1] print('\nRetrieved offset is: %d' % retr_off) - int_idx = d['off_gen/body$'].get_subnode_idx(d['off_gen/body/int']) + int_idx = d['off_gen/body$'][0].get_subnode_idx(d['off_gen/body/int'][0]) off = int_idx * 3 + 10 # +10 for 'prefix' delta self.assertEqual(off, retr_off) @@ -3347,7 +4016,7 @@ def test_zip_specifics(self): self.assertEqual(zip_buff, abs_buff) # abszip.show() - flen_before = len(abszip['ZIP/file_list/file/data'].to_bytes()) + flen_before = len(abszip['ZIP/file_list/file/data'][0].to_bytes()) print('file data len before: ', flen_before) off_before = abszip['ZIP/cdir/cdir_hdr:2/file_hdr_off'] @@ -3355,17 +4024,17 @@ def test_zip_specifics(self): if off_before is not None: # Make modification of the ZIP and verify that some other ZIP # fields are automatically updated - off_before = off_before.to_bytes() + off_before = off_before[0].to_bytes() print('offset before:', off_before) - csz_before = abszip['ZIP/file_list/file/header/common_attrs/compressed_size'].to_bytes() + csz_before = abszip['ZIP/file_list/file/header/common_attrs/compressed_size'][0].to_bytes() print('compressed_size before:', csz_before) - abszip['ZIP/file_list/file/header/common_attrs/compressed_size'].set_current_conf('MAIN') + abszip['ZIP/file_list/file/header/common_attrs/compressed_size'][0].set_current_conf('MAIN') NEWVAL = b'TEST' - print(abszip['ZIP/file_list/file/data'].absorb(NEWVAL, constraints=AbsNoCsts())) + print(abszip['ZIP/file_list/file/data'][0].absorb(NEWVAL, constraints=AbsNoCsts())) - flen_after = len(abszip['ZIP/file_list/file/data'].to_bytes()) + flen_after = len(abszip['ZIP/file_list/file/data'][0].to_bytes()) print('file data len after: ', flen_after) abszip.unfreeze(only_generators=True) @@ -3374,9 +4043,9 @@ def test_zip_specifics(self): # print('\n******\n') # abszip.show() - off_after = abszip['ZIP/cdir/cdir_hdr:2/file_hdr_off'].to_bytes() + off_after = abszip['ZIP/cdir/cdir_hdr:2/file_hdr_off'][0].to_bytes() print('offset after: ', off_after) - csz_after = abszip['ZIP/file_list/file/header/common_attrs/compressed_size'].to_bytes() + csz_after = abszip['ZIP/file_list/file/header/common_attrs/compressed_size'][0].to_bytes() print('compressed_size after:', csz_after) # Should not be equal in the general case @@ -3499,11 +4168,17 @@ def test_xml_helpers(self): data_sizes = [211, 149, 184] for i in range(100): - data = fmk.get_data(['XML5', ('tWALK', UI(path='xml5/command/start-tag/content/attr1/cmd_val'))]) + # fmk.lg.export_raw_data = True + data = fmk.process_data( + ['XML5', ('tWALK', UI(path='xml5/command/start-tag/content/attr1/cmd_val', + consider_sibbling_change=False))]) if data is None: break - assert len(data.to_bytes()) == data_sizes[i] + go_on = fmk.send_data_and_log([data]) + bstr_len = len(data.to_bytes()) + assert bstr_len == data_sizes[i], f'i: {i}, len(data.to_bytes()): {bstr_len}' + if not go_on: raise ValueError else: @@ -3513,10 +4188,11 @@ def test_xml_helpers(self): specific_cases_checked = False for i in range(100): - data = fmk.get_data(['XML5', ('tTYPE', UI(path='xml5/command/LOGIN/start-tag/content/attr1/val'))]) + data = fmk.process_data( + ['XML5', ('tTYPE', UI(path='xml5/command/LOGIN/start-tag/content/attr1/val'))]) if data is None: break - node_to_check = data.content['xml5/command/LOGIN/start-tag/content/attr1/val'] + node_to_check = data.content['xml5/command/LOGIN/start-tag/content/attr1/val'][0] if node_to_check.to_bytes() == b'None': # one case should trigger this condition specific_cases_checked = True @@ -3526,8 +4202,9 @@ def test_xml_helpers(self): else: raise ValueError - assert i == 22, 'number of test cases: {:d}'.format(i) - assert specific_cases_checked + # number of test cases + self.assertIn(i, [21,22]) + self.assertTrue(specific_cases_checked) class TestFMK(unittest.TestCase): @classmethod @@ -3548,16 +4225,16 @@ def test_generic_disruptors_01(self): print(gen_disruptors) for dis in gen_disruptors: - if dis in ['tCROSS']: + if dis in ['CROSS']: continue print("\n\n---[ Tested Disruptor %r ]---" % dis) if dis == 'EXT': act = [dmaker_type, (dis, UI(cmd='/bin/cat', file_mode=True))] - d = fmk.get_data(act) + d = fmk.process_data(act) else: act = [dmaker_type, dis] - d = fmk.get_data(act) + d = fmk.process_data(act) if d is not None: fmk._log_data(d) print("\n---[ Pretty Print ]---\n") @@ -3570,7 +4247,7 @@ def test_generic_disruptors_01(self): def test_separator_disruptor(self): for i in range(100): - d = fmk.get_data(['SEPARATOR', 'tSEP']) + d = fmk.process_data(['SEPARATOR', 'tSEP']) if d is None: break fmk._setup_new_sending() @@ -3589,10 +4266,10 @@ def test_struct_disruptor(self): outcomes = [] - act = [('EXIST_COND', UI(determinist=True)), 'tWALK', 'tSTRUCT'] + act = [('EXIST_COND', UI(determinist=True)), ('tWALK', UI(consider_sibbling_change=False)), 'tSTRUCT'] for i in range(4): for j in range(10): - d = fmk.get_data(act) + d = fmk.process_data(act) if d is None: print('--> Exiting (need new input)') break @@ -3613,7 +4290,7 @@ def test_struct_disruptor(self): idx = 0 act = [('SEPARATOR', UI(determinist=True)), ('tSTRUCT', UI(deep=True))] for j in range(10): - d = fmk.get_data(act) + d = fmk.process_data(act) if d is None: print('--> Exiting (need new input)') break @@ -3633,9 +4310,9 @@ def test_typednode_disruptor(self): expected_outcomes = [] outcomes = [] - act = ['OFF_GEN', ('tTYPE', UI(runs_per_node=1))] + act = ['OFF_GEN', ('tTYPE', UI(min_node_tc=1, max_node_tc=4))] for j in range(100): - d = fmk.get_data(act) + d = fmk.process_data(act) if d is None: print('--> Exiting (need new input)') break @@ -3674,12 +4351,12 @@ def test_operator_2(self): myop = fmk.get_operator(name='MyOp') fmk.launch_operator('MyOp') - fbk = fmk.feedback_gate.get_feedback_from(myop)[0]['content'] + fbk = fmk.last_feedback_gate.get_feedback_from(myop)[0]['content'] print(fbk) self.assertIn(b'You win!', fbk) fmk.launch_operator('MyOp') - fbk = fmk.feedback_gate.get_feedback_from(myop)[0]['content'] + fbk = fmk.last_feedback_gate.get_feedback_from(myop)[0]['content'] print(fbk) self.assertIn(b'You loose!', fbk) @@ -3689,7 +4366,7 @@ def test_scenario_infra_01a(self): base_qty = 0 for i in range(100): - data = fmk.get_data(['SC_NO_REGEN']) + data = fmk.process_data(['SC_NO_REGEN']) data_list = fmk._send_data([data]) # needed to make the scenario progress if not data_list: base_qty = i @@ -3703,12 +4380,12 @@ def test_scenario_infra_01a(self): self.assertEqual(code_vector, ['DataUnusable', 'HandOver', 'DataUnusable', 'HandOver', 'DPHandOver', 'NoMoreData']) - self.assertEqual(base_qty, 55) + self.assertEqual(base_qty, 51) print('\n*** test scenario SC_AUTO_REGEN via _send_data()') for i in range(base_qty * 3): - data = fmk.get_data(['SC_AUTO_REGEN']) + data = fmk.process_data(['SC_AUTO_REGEN']) data_list = fmk._send_data([data]) if not data_list: raise ValueError @@ -3721,7 +4398,7 @@ def test_scenario_infra_01b(self): base_qty = 0 for i in range(100): - data = fmk.get_data(['SC_NO_REGEN']) + data = fmk.process_data(['SC_NO_REGEN']) go_on = fmk.send_data_and_log([data]) if not go_on: base_qty = i @@ -3737,12 +4414,12 @@ def test_scenario_infra_01b(self): self.assertEqual(code_vector, ['DataUnusable', 'HandOver', 'DataUnusable', 'HandOver', 'DPHandOver', 'NoMoreData']) - self.assertEqual(base_qty, 55) + self.assertEqual(base_qty, 51) print('\n*** test scenario SC_AUTO_REGEN via send_data_and_log()') for i in range(base_qty * 3): - data = fmk.get_data(['SC_AUTO_REGEN']) + data = fmk.process_data(['SC_AUTO_REGEN']) go_on = fmk.send_data_and_log([data]) if not go_on: raise ValueError @@ -3763,7 +4440,7 @@ def test_scenario_infra_02(self): now = datetime.datetime.now() for i in range(10): prev_data = data - data = fmk.get_data(['SC_EX1']) + data = fmk.process_data(['SC_EX1']) ok = fmk.send_data_and_log([data]) # needed to make the scenario progress if not ok: raise ValueError @@ -3778,7 +4455,7 @@ def test_scenario_infra_02(self): data = None steps = [] for i in range(4): - data = fmk.get_data(['SC_EX2']) + data = fmk.process_data(['SC_EX2']) if i == 3: self.assertTrue(data is None) if data is not None: @@ -3800,7 +4477,7 @@ def test_scenario_infra_02(self): def test_scenario_infra_03(self): steps = [] for i in range(6): - data = fmk.get_data(['SC_EX3']) + data = fmk.process_data(['SC_EX3']) steps.append(data.origin.current_step) ok = fmk.send_data_and_log([data]) # needed to make the scenario progress if not ok: @@ -3823,7 +4500,7 @@ def walk_scenario(name, iter_num): steps = [] scenario = None for i in range(iter_num): - data = fmk.get_data([name]) + data = fmk.process_data([name]) if i == 1: scenario = data.origin steps.append(data.origin.current_step) @@ -3866,3 +4543,130 @@ def walk_scenario(name, iter_num): self.assertEqual(scenario.env.cbk_true_cpt, 1) self.assertEqual(scenario.env.cbk_false_cpt, 4) self.assertEqual(str(steps[-1]), '4DEFAULT') + + @unittest.skipIf(not run_long_tests, "Long test case") + def test_evolutionary_fuzzing(self): + fmk.reload_all(tg_ids=[7]) + fmk.process_data_and_send(DataProcess(['SC_EVOL1']), verbose=False, max_loop=-1) + fmk.process_data_and_send(DataProcess(['SC_EVOL2']), verbose=False, max_loop=-1) + + +class TestConstBackend(unittest.TestCase): + @classmethod + def setUpClass(cls): + fmk.run_project(name='tuto', tg_ids=0, dm_name='mydf') + fmk.prj.reset_target_mappings() + fmk.disable_fmkdb() + + def setUp(self): + fmk.reload_all(tg_ids=[0]) + fmk.prj.reset_target_mappings() + + def test_twalkcsp_operator(self): + idx = 0 + expected_idx = 8 + expected_outcomes = [b'x = 3y + z (x:123, y:40, z:3)', + b'x = 3y + z (x:120, y:39, z:3)', + b'x = 3y + z (x:122, y:40, z:2)', + b'x = 3y + z (x:121, y:40, z:1)', + b'x = 3y + z [x:123, y:40, z:3]', + b'x = 3y + z [x:120, y:39, z:3]', + b'x = 3y + z [x:122, y:40, z:2]', + b'x = 3y + z [x:121, y:40, z:1]'] + outcomes = [] + + act = [('CSP', UI(determinist=True)), ('tWALKcsp')] + for j in range(20): + d = fmk.process_data(act) + if d is None: + print('--> Exit (need new input)') + break + fmk._setup_new_sending() + fmk._log_data(d) + outcomes.append(d.to_bytes()) + # d.show() + idx += 1 + + self.assertEqual(idx, expected_idx) + self.assertEqual(outcomes, expected_outcomes) + + def test_twalk_operator(self): + idx = 0 + expected_idx = 13 + expected_outcomes = [b'x = 3y + z (x:123, y:40, z:3)', + b'x = 3y + z (X:123, y:40, z:3)', + b'x = 3y + z (x:123, y:40, z:3)', # redundancy + b'x = 3y + z (x:124, y:40, z:3)', + b'x = 3y + z (x:125, y:40, z:3)', + b'x = 3y + z (x:126, y:40, z:3)', + b'x = 3y + z (x:127, y:40, z:3)', + b'x = 3y + z (x:128, y:40, z:3)', + b'x = 3y + z (x:129, y:40, z:3)', + b'x = 3y + z (x:130, y:40, z:3)', + b'x = 3y + z (x:120, y:39, z:3)', + b'x = 3y + z (x:121, y:40, z:1)', + b'x = 3y + z (x:122, y:40, z:2)'] + outcomes = [] + + act = [('CSP', UI(determinist=True)), ('tWALK', UI(path='csp/variables/x'))] + for j in range(20): + d = fmk.process_data(act) + if d is None: + print('--> Exit (need new input)') + break + fmk._setup_new_sending() + fmk._log_data(d) + outcomes.append(d.to_bytes()) + # d.show() + idx += 1 + + self.assertEqual(idx, expected_idx) + self.assertEqual(outcomes, expected_outcomes) + + idx = 0 + expected_idx = 2 + expected_outcomes = [b'x = 3y + z [x:123, y:40, z:3]', + b'x = 3y + z (x:123, y:40, z:3)'] + outcomes = [] + + act = [('CSP', UI(determinist=True)), ('tWALK', UI(path='csp/delim_1'))] + for j in range(20): + d = fmk.process_data(act) + if d is None: + print('--> Exit (need new input)') + break + fmk._setup_new_sending() + fmk._log_data(d) + outcomes.append(d.to_bytes()) + # d.show() + idx += 1 + + self.assertEqual(idx, expected_idx) + self.assertEqual(outcomes, expected_outcomes) + + + def test_tconst_operator(self): + idx = 0 + expected_idx = 362 + expected_outcomes = [b'x = 3y + z (x:123, y:40, z:3-', + b'x = 3y + z [x:123, y:40, z:3)', + b'x = 3y + z [x:123, y:40, z:3-', + b'x = 3y + z (x:130, y:40, z:3)', + b'x = 3y + z (x:130, y:39, z:3)', + b'x = 3y + z (x:130, y:38, z:3)'] + outcomes = [] + + act = [('CSP', UI(determinist=True)), ('tCONST')] + for j in range(500): + d = fmk.process_data(act) + if d is None: + print('--> Exit (need new input)') + break + fmk._setup_new_sending() + fmk._log_data(d) + outcomes.append(d.to_bytes()) + # d.show() + idx += 1 + + self.assertEqual(idx, expected_idx) + self.assertEqual(outcomes[:6], expected_outcomes) diff --git a/test/unit/test_node_builder.py b/test/unit/test_node_builder.py index a83b9d0..ee82f37 100644 --- a/test/unit/test_node_builder.py +++ b/test/unit/test_node_builder.py @@ -2,10 +2,9 @@ import framework.value_types as vt import unittest import ddt -import six from test import mock -ASCII_EXT = ''.join([(chr if sys.version_info[0] == 2 else six.unichr)(i) for i in range(0, 0xFF + 1)]) +ASCII_EXT = ''.join([chr(i) for i in range(0, 0xFF + 1)]) @ddt.ddt diff --git a/test/unit/test_plotty.py b/test/unit/test_plotty.py new file mode 100644 index 0000000..887e0b4 --- /dev/null +++ b/test/unit/test_plotty.py @@ -0,0 +1,125 @@ + +import tools.plotty.plotty as sut +import unittest +import ddt + + +@ddt.ddt +class PlottyTest(unittest.TestCase): + +#region Formula + + @ddt.data( + {'expression': "a", 'variables': set(["a"])}, + {'expression': "a + b", 'variables': set(["a", "b"])}, + {'expression': "exp((a + b) / c)", 'variables': set(["a", "b", "c"]), 'functions': set(["exp"])}, + { + 'expression': "sqrt((1-a*exp(2t) + w^pi) / (sin(2x / pi) + cos(pi/y)))", + 'variables': set(["a", "t", "w", "pi", "x", "y"]), + 'functions': set(["sqrt", "exp", "sin", "cos"]) + } + ) + @ddt.unpack + def test_should_find_all_variables_when_given_well_formed_expression(self, expression, variables=set(), functions=set()): + found_variables, found_functions = sut.collect_names(expression) + + self.assertSetEqual(found_variables, variables) + self.assertSetEqual(found_functions, functions) + + + @ddt.data( + {'formula': "a ~ b"}, + {'formula': "a + b ~ c"}, + {'formula': "exp((a + b) / c) ~ cos(a + sin(b))"}, + {'formula': "sqrt((1-a*exp(2t) + w^pi) ~ (sin(2x / pi) + cos(pi/y) )) "} + ) + @ddt.unpack + def test_should_return_true_when_given_valid_formula(self, formula): + _, _, valid_formula = sut.split_formula(formula) + + self.assertTrue(valid_formula) + + + @ddt.data( + {'formula': "a = b"}, + {'formula': "f(a) = b"}, + {'formula': "exp((a + b) / c) cos(a + sin(b))"}, + {'formula': "a ~ b ~ c"}, + ) + @ddt.unpack + def test_should_return_false_when_given_invalid_formula(self, formula): + _, _, valid_formula = sut.split_formula(formula) + + self.assertFalse(valid_formula) + + + @ddt.data( + {'formula': "a ~ b", 'left_expr': "a", 'right_expr': "b"}, + {'formula': "a + b ~ c", 'left_expr': "a+b", 'right_expr': "c"}, + { + 'formula': "exp((a + b) / c) ~ cos(a + sin(b))", + 'left_expr': "exp((a+b)/c)", + 'right_expr': "cos(a+sin(b))" + }, + { + 'formula': "sqrt ((1 -a *exp(2t) + w^ pi) ~ ( sin(2x / pi) + cos(pi/y) )) ", + 'left_expr': "sqrt((1-a*exp(2t)+w^pi)", + 'right_expr': "(sin(2x/pi)+cos(pi/y)))" + } + ) + @ddt.unpack + def test_should_properly_split_and_trim_when_given_valid_formula(self, formula, left_expr, right_expr): + left, right, _ = sut.split_formula(formula) + + self.assertEqual(left, left_expr) + self.assertEqual(right, right_expr) + +#endregion + +#region Interval + @ddt.data( + {'interval': "1..4", 'expected_set': set(range(1,4))}, + {'interval': "0..1", 'expected_set': set(range(1))}, + {'interval': "547..960", 'expected_set': set(range(547,960))}, + ) + @ddt.unpack + def test_should_retrieve_all_integers_given_well_formed_interval(self, interval, expected_set): + result = sut.parse_interval(interval) + + self.assertSetEqual(result, expected_set) + + + @ddt.data( + {'interval': ""}, + {'interval': "not..an_interval"}, + {'interval': "definitely..not..an_interval"}, + {'interval': "10..1"}, + {'interval': "100..1i0"}, + {'interval': "10.."}, + {'interval': "..10"}, + ) + @ddt.unpack + def test_should_output_empty_set_on_invalid_intervals(self, interval): + result = sut.parse_interval(interval) + + self.assertSetEqual(result, set()) + + + @ddt.data( + {'interval_union': "1..4", 'expected_set': set(range(1,4))}, + {'interval_union': "0..5, 5..10", 'expected_set': set(range(0,10))}, + {'interval_union': "0..120, 100..120", 'expected_set': set(range(120))}, + { + 'interval_union': "0..120, 130..140, 150..200", + 'expected_set': set(range(120)).union(set(range(130,140))).union(set(range(150,200))) + }, + ) + @ddt.unpack + def test_should_properly_merge_intervals_of_intervals_union(self, interval_union, expected_set): + result = sut.parse_interval_union(interval_union) + self.assertSetEqual(result, expected_set) + +#endregion + + def dummy_test(): + pass \ No newline at end of file diff --git a/tools/fmkdb.py b/tools/fmkdb.py index 1059bad..354b3d5 100755 --- a/tools/fmkdb.py +++ b/tools/fmkdb.py @@ -33,11 +33,12 @@ sys.path.insert(0,parentdir) from framework.database import Database +from framework.global_resources import get_user_input from libs.external_modules import * import argparse -parser = argparse.ArgumentParser(description='Argument for FmkDB toolkit script') +parser = argparse.ArgumentParser(description='Arguments for FmkDB toolkit script') group = parser.add_argument_group('Miscellaneous Options') group.add_argument('--fmkdb', metavar='PATH', help='Path to an alternative fmkDB.db') @@ -55,6 +56,11 @@ help='Restrict the data to be displayed to a specific project. ' 'Supported by: --info-by-date, --info-by-ids, ' '--data-with-impact, --data-without-fbk, --data-with-specific-fbk') +group.add_argument('--fbk-status-formula', metavar='STATUS_REF', default='? < 0', + help='Restrict the data to be displayed to specific feedback status. ' + 'This option provides the formula to be used for feedback status ' + 'filtering (the character "?" should be used in place of the status value that will be checked). ' + 'Supported by: --data-with-impact') group = parser.add_argument_group('Fuddly Database Visualization') group.add_argument('-s', '--all-stats', action='store_true', help='Show all statistics') @@ -68,12 +74,13 @@ help='''Display information on data sent between START and END ''' '''(date format 'Year/Month/Day' or 'Year/Month/Day-Hour' or 'Year/Month/Day-Hour:Minute')''') -group.add_argument('--info-by-ids', nargs=2, metavar=('FIRST_DATA_ID','LAST_DATA_ID'), type=int, +group.add_argument('-ids', '--info-by-ids', nargs=2, metavar=('FIRST_DATA_ID','LAST_DATA_ID'), type=int, help='''Display information on all the data included within the specified data ID range''') group.add_argument('-wf', '--with-fbk', action='store_true', help='Display full feedback (expect --data-id)') group.add_argument('-wd', '--with-data', action='store_true', help='Display data content (expect --data-id)') +group.add_argument('-wa', '--with-async-data', action='store_true', help='Display any related async data (expect --data-id)') group.add_argument('--without-fmkinfo', action='store_true', help='Do not display fmkinfo (expect --data-id)') group.add_argument('--without-analysis', action='store_true', @@ -132,17 +139,17 @@ def handle_confirmation(): try: - if sys.version_info[0] == 2: - cont = raw_input("\n*** Press [ENTER] to continue ('C' to CANCEL) ***\n") - else: - cont = input("\n*** Press [ENTER] to continue ('C' to CANCEL) ***\n") + cont = get_user_input(colorize("\n*** Press [ENTER] to continue ('C' to CANCEL) ***\n", + rgb=Color.PROMPT)) except KeyboardInterrupt: cont = 'c' - except: + except Exception as e: + print(f'Unexpected exception received: {e}') cont = 'c' finally: if cont.lower() == 'c': print(colorize("*** Operation Cancelled ***", rgb=Color.ERROR)) + fmkdb.stop() sys.exit(-1) def handle_date(date_str): @@ -156,6 +163,7 @@ def handle_date(date_str): date = datetime.datetime.strptime(date_str, "%Y/%m/%d-%H:%M") except ValueError: print(colorize("*** ERROR: Unrecognized Dates ***", rgb=Color.ERROR)) + fmkdb.stop() sys.exit(-1) return date @@ -186,6 +194,7 @@ def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): prj_name = args.project with_fbk = args.with_fbk with_data = args.with_data + with_async_data = args.with_async_data without_fmkinfo = args.without_fmkinfo without_analysis = args.without_analysis limit_data_sz = args.limit @@ -206,6 +215,7 @@ def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): raw_impact_analysis = args.data_with_impact_raw data_without_fbk = args.data_without_fbk fbk_src = args.fbk_src + fbk_status_formula = args.fbk_status_formula data_with_specific_fbk = args.data_with_specific_fbk add_analysis = args.add_analysis disprove_impact = args.disprove_impact @@ -270,6 +280,7 @@ def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): fmkdb.display_data_info(data_ID, with_data=with_data, with_fbk=with_fbk, with_fmkinfo=not without_fmkinfo, with_analysis=not without_analysis, + with_async_data=with_async_data, fbk_src=fbk_src, limit_data_sz=limit_data_sz, raw=raw_data, page_width=page_width, colorized=colorized, decoding_hints=decoding_hints, dm_list=dm_list) @@ -280,7 +291,9 @@ def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): end = handle_date(data_info_by_date[1]) fmkdb.display_data_info_by_date(start, end, with_data=with_data, with_fbk=with_fbk, - with_fmkinfo=not without_fmkinfo, fbk_src=fbk_src, + with_fmkinfo=not without_fmkinfo, + with_async_data=with_async_data, + fbk_src=fbk_src, prj_name=prj_name, limit_data_sz=limit_data_sz, raw=raw_data, page_width=page_width, colorized=colorized, decoding_hints=decoding_hints, dm_list=dm_list) @@ -291,7 +304,9 @@ def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): last_id=data_info_by_range[1] fmkdb.display_data_info_by_range(first_id, last_id, with_data=with_data, with_fbk=with_fbk, - with_fmkinfo=not without_fmkinfo, fbk_src=fbk_src, + with_fmkinfo=not without_fmkinfo, + with_async_data=with_async_data, + fbk_src=fbk_src, prj_name=prj_name, limit_data_sz=limit_data_sz, raw=raw_data, page_width=page_width, colorized=colorized, decoding_hints=decoding_hints, dm_list=dm_list) @@ -308,11 +323,14 @@ def colorize(string, rgb=None, ansi=None, bg=None, ansi_bg=None, fd=1): if remove_data is not None: for i in range(remove_data[0], remove_data[1]+1): fmkdb.remove_data(i, colorized=colorized) + fmkdb.shrink_db() else: fmkdb.remove_data(remove_one_data, colorized=colorized) + fmkdb.shrink_db() elif impact_analysis or raw_impact_analysis: - fmkdb.get_data_with_impact(prj_name=prj_name, fbk_src=fbk_src, verbose=verbose, + fmkdb.get_data_with_impact(prj_name=prj_name, fbk_src=fbk_src, fbk_status_formula=fbk_status_formula, + verbose=verbose, raw_analysis=raw_impact_analysis, colorized=colorized) diff --git a/tools/plotty/plotty.py b/tools/plotty/plotty.py new file mode 100755 index 0000000..eab6138 --- /dev/null +++ b/tools/plotty/plotty.py @@ -0,0 +1,748 @@ +#!/usr/bin/env python + +import argparse +from datetime import datetime +import inspect +import os +import sys +from typing import Any, Optional +from enum import Enum + +import cexprtk +from matplotlib.axes import Axes +from matplotlib.figure import Figure +from matplotlib.dates import DateFormatter, date2num +import matplotlib.pyplot as plt + + +ARG_INVALID_FMDBK = -1 +ARG_INVALID_ID = -2 +ARG_INVALID_FORMULA = -3 +ARG_INVALID_VAR_NAMES = -5 +ARG_INVALID_POI = -6 + +UNION_DELIMITER = ',' +INTERVAL_OPERATOR = '..' +STEP_OPERATOR = '|' + +class GridMatch(Enum): + AUTO = 1 + POI = 2 + ALL = 3 + +currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +rootdir = os.path.dirname(os.path.dirname(currentdir)) +sys.path.insert(0, rootdir) + +from framework.database import Database +from libs.external_modules import * + +def print_info(msg: str): + print(colorize(f"*** INFO: {msg} *** ", rgb=Color.INFO)) + +def print_warning(msg: str): + print(colorize(f"*** WARNING: {msg} *** ", rgb=Color.WARNING)) + +def print_error(msg: str): + print(colorize(f"*** ERROR: {msg} *** ", rgb=Color.ERROR)) + + +#region Argparse + +parser = argparse.ArgumentParser(description='Arguments for Plotty') + +group = parser.add_argument_group('Main parameters') + +group.add_argument( + '-ids', + '--id-range', + type=str, + help='The ID range to take into account should be: ' + 'either ..[|], ' + 'or ..[|], ..., ..[|]', + required=True +) + +group.add_argument( + '-df', + '--date-format', + type=str, + default='%H:%M:%S.%f', + help='Wanted date format, in a strftime format (1989 C standard). Default is %%H:%%M:%%S.%%f', + required=False +) + +group.add_argument( + '-db', + '--fmkdb', + metavar='PATH', + default=[], + action='extend', + nargs="+", + help='Path to any fmkDB.db files. There can be many if using the --other_id_range option.' + ' Default is fuddly/data/directory/fmkDB.db', + required=False +) + +group = parser.add_argument_group('Display Options') + +group.add_argument( + '-f', + '--formula', + default='SENT_DATE~ID', + type=str, + help='The formula to plot, in the form "y ~ x"', + required=False +) + +group.add_argument( + '-poi', + '--points-of-interest', + type=int, + default=0, + help='How many point of interest the plot should show. Default is none', + required=False +) + +group.add_argument( + '-gm', + '--grid-match', + type=str, + default='all', + help="Should the plot grid specifically match some element. Possible options are 'all', " + "'poi' and 'auto'. Default is 'all'", + choices=['all', 'poi', 'auto'], + required=False +) + +group.add_argument( + '-hp', + '--hide-points', + action='store_true', + help='Should the graph display every point above the line, or just the line. Default is to display the points', + required=False +) + +group = parser.add_argument_group('Labels Configuration') + +group.add_argument( + '-l', + '--labels', + dest='annotations', + action='extend', + nargs='+', + help=''' + Display the specified labels for each Data ID represented in the curve. + ('t' for TYPE, 'g' for TARGET, 's' for SIZE, 'a' for ACK_DATE) + ''', + required=False +) + +group.add_argument( + '-al', + '--async-labels', + dest='async_annotations', + action='extend', + nargs='+', + help=''' + Display the specified labels for each Async Data ID represented in the curve. + ('i' for 'ID', 't' for TYPE, 'g' for TARGET, 's' for SIZE) + ''', + required=False +) + +group = parser.add_argument_group('Multiple Curves Options') + +group.add_argument( + '-o', + '--other-id-range', + type=str, + action='append', + help='Other ranges of IDs to plot against the main one. All other options apply to it', + required=False +) + +group.add_argument( + '-s', + '--vertical-shift', + type=float, + default=1, + help='When --other-id-range is used, specify the spacing between the curves. The shift is ' + 'computed as the multiplication between the original curve height and this value', + required=False +) + +#endregion + + +#region Plot + +def sort_points_by_interest(x_data: list[float], y_data: list[float]) -> list[tuple[float, float]]: + backward_difference = [0] + for i in range(1, len(y_data)): + backward_difference.append(y_data[i] - y_data[i-1]) + + result = zip(x_data, y_data, backward_difference) + result = sorted(result, key=lambda tup: tup[2], reverse=True) + result = list(map(lambda tup: (tup[0], tup[1]), result)) + + return result + + +def add_point(axes: Axes, x: float, y: float, color: str): + axes.plot(x, y, 'o', color=color) + + +def add_annotation(axes: Axes, x: float, y: float, value: str): + text_height = value.count('\n') + 1 + axes.annotate( + f"{value}", + xy=(x, y), xycoords='data', + xytext=(-10, 20 * text_height), textcoords='offset pixels', + horizontalalignment='right', verticalalignment='top' + ) + + +def add_points_of_interest( + axes: Axes, + x_data: list[float], + y_data: list[float], + points_of_interest: int +) -> set[tuple[float, float]]: + + points = sort_points_by_interest(x_data, y_data) + plotted_points = set() + + for i in range(points_of_interest): + if i >= len(points): + break + x, y = points[i] + add_point(axes, x, y, 'red') + plotted_points.add((x,y)) + + return plotted_points + + +def plot_line( + axes: Axes, + x_data: list[float], + y_data: list[float], + annotations: list[str], + args: dict[Any] +) -> set[tuple[float, float]]: + + axes.plot(x_data, y_data, '-') + + if not args['hide_points']: + for (x, y) in zip(x_data, y_data): + add_point(axes, x, y, 'b') + + if args['annotations'] is not None: + for i, (x, y) in enumerate(zip(x_data, y_data)): + add_annotation(axes, x, y, annotations[i]) + + if args['poi'] != 0: + return add_points_of_interest(axes, x_data, y_data, args['poi']) + + return set() + + +def plot_async_data( + axes: Axes, + x_data: list[float], + y_data: list[float], + annotations: list[str], + args: dict[Any] +): + + for (x,y) in zip(x_data, y_data): + axes.plot(x, y, 'g^') + + if args['async_annotations'] is not None: + for i, (x, y) in enumerate(zip(x_data, y_data)): + add_annotation(axes, x, y, annotations[i]) + + +def set_grid( + axes: Axes, + grid_match: GridMatch, + plotted_poi: set[tuple[float, float]], + plotted_points: set[tuple[float, float]] +): + + if grid_match == GridMatch.AUTO: + return + + if grid_match == GridMatch.POI: + new_xticks, new_yticks = zip(*plotted_poi) + axes.xaxis.set_ticks(new_xticks) + axes.yaxis.set_ticks(new_yticks) + return + + if grid_match == GridMatch.ALL: + new_xticks, new_yticks = zip(*plotted_points) + axes.xaxis.set_ticks(new_xticks) + axes.yaxis.set_ticks(new_yticks) + + +def post_process_plot( + figure: Figure, + x_true_type: Optional[type], + y_true_type: Optional[type], + plotted_poi: set[tuple[float, float]], + plotted_points: set[tuple[float, float]], + args: dict[Any] +): + axes: Axes = figure.get_axes()[0] + + set_grid(axes, args['grid_match'], plotted_poi, plotted_points) + + if x_true_type is not None and x_true_type == datetime: + formatter = DateFormatter(args['date_format']) + axes.xaxis.set_major_formatter(formatter) + axes.tick_params(axis='x', which='major', labelrotation=30) + + if y_true_type is not None and y_true_type == datetime: + formatter = DateFormatter(args['date_format']) + axes.yaxis.set_major_formatter(formatter) + axes.tick_params(axis='y', which='major', reset=True) + + +#endregion + + +#region Formula + +def collect_names(expression: str) -> tuple[set[str], set[str]]: + variable_names = set() + function_names = set() + on_build_name = "" + for char in expression: + if char.isalpha() or char == '_': + on_build_name += char + else: + if on_build_name != "": + if char == '(': + function_names.add(on_build_name) + else: + variable_names.add(on_build_name) + on_build_name = "" + + if on_build_name != "": + variable_names.add(on_build_name) + return (variable_names, function_names) + + +def split_formula(formula: str) -> tuple[str, str, bool]: + parts = formula.split('~') + if len(parts) != 2: + return ("", "", False) + + parts = list(map(lambda s: "".join(s.split(' ')), parts)) + + return (parts[0], parts[1], True) + +#endregion + + +#region Interval + +def try_parse_int(s: str) -> Optional[int]: + try: + int_value = int(s) + return int_value + except ValueError: + print_error(f"Value '{s}' is not a valid integer") + return None + + +def parse_int_range(int_range: str) -> Optional[range]: + + step = 1 + bounds_and_step = int_range.split(STEP_OPERATOR) + if len(bounds_and_step) == 2: + parsed_step = try_parse_int(bounds_and_step[1]) + if parsed_step is None: + print_warning(f"Ignoring interval '{int_range}': invalid step '{bounds_and_step[1]}'") + return None + step = parsed_step + + bounds = bounds_and_step[0].split(INTERVAL_OPERATOR) + if len(bounds) == 1: + value = try_parse_int(bounds[0]) + if value is not None: + return range(value, value+1) + + print_warning(f"Ignoring interval '{int_range}' : invalid integer '{bounds[0]}'") + return None + + if len(bounds) == 2: + lower_bound = try_parse_int(bounds[0]) + upper_bound = try_parse_int(bounds[1]) + if lower_bound is None: + print_warning(f"Ignoring interval '{int_range}' : invalid integer '{bounds[0]}'") + return None + + if upper_bound is None: + print_warning(f"Ignoring interval '{int_range}' : invalid integer '{bounds[1]}'") + return None + + if lower_bound >= upper_bound: + print_warning(f"Ignoring interval '{int_range}'") + return None + + return range(lower_bound, upper_bound, step) + + print_warning(f"Invalid interval found: '{int_range}'") + return None + + +def parse_int_range_union(int_range_union: str) -> list[range]: + result = [] + parts = int_range_union.split(UNION_DELIMITER) + for part in parts: + int_range = parse_int_range(part) + if int_range is not None: + result.append(int_range) + return result + +#endregion + + +def convert_non_operable_types(variables_values: list[dict[str, Any]]): + for instanciation in variables_values: + for key, value in instanciation.items(): + if isinstance(value, datetime): + instanciation[key] = date2num(value) + + +def solve_expression(expression: str, variables_values: list[dict[str, Any]]) -> list[float]: + results = [] + for variables_value in variables_values: + result = cexprtk.evaluate_expression(expression, variables_value) + results.append(result) + return results + + +def belongs_condition_sql_string(column_label: str, int_ranges: list[range]): + result = "false" + for int_range in int_ranges: + sub_condition = "" + if int_range.start == int_range.stop: + sub_condition = f" OR {column_label} = {list(int_range)}" + else: + if int_range.step == 1: + sub_condition = f" OR {column_label} >= {int_range.start} AND {column_label} < {int_range.stop}" + + else: + sql_range = ','.join(list(map(str, list(int_range)))) + sub_condition = f" OR {column_label} IN ({sql_range})" + result += sub_condition + return result + + +def request_from_database( + fmkdb_path: str, + int_ranges: list[range], + column_names: list[str], + annotation_column_names: list[str], + async_annotation_column_names: list[str], +) -> tuple[Optional[list[dict[str, Any]]], Optional[list[dict[str,Any]]], list[dict[str, Any]], list[dict[str, Any]]]: + + if len(column_names) == 0: + return (None, None, [], []) + + fmkdb = Database(fmkdb_path) + ok = fmkdb.start() + if not ok: + print_error(f"The database {fmkdb_path} is invalid!") + sys.exit(ARG_INVALID_FMDBK) + + async_data_column_names = fmkdb.column_names_from('ASYNC_DATA') + for c in column_names: + if c not in async_data_column_names: + compatible_async = False + break + else: + compatible_async = True + + id_ranges_check_str = belongs_condition_sql_string("ID", int_ranges) + async_id_ranges_check_str = id_ranges_check_str.replace("ID", "CURRENT_DATA_ID") + + requested_data_columns_str = ', '.join(column_names) + data_statement = f"SELECT {requested_data_columns_str} FROM DATA " \ + f"WHERE {id_ranges_check_str}" + matching_data = fmkdb.execute_sql_statement(data_statement) + + matching_data_annotations = [] + if annotation_column_names is not None: + requested_data_annotations_columns_str = ', '.join(annotation_column_names) + data_annotation_statement = f"SELECT {requested_data_annotations_columns_str} FROM DATA " \ + f"WHERE {id_ranges_check_str}" + matching_data_annotations = fmkdb.execute_sql_statement(data_annotation_statement) + + + matching_async_data_annotations = [] + matching_async_data = None + if compatible_async: + # async data 'CURRENT_DATA_ID' is considered to be their ID for plotting + requested_async_data_columns_str = ', '.join(column_names).replace('ID', 'CURRENT_DATA_ID') + async_data_statement = f"SELECT {requested_async_data_columns_str} FROM ASYNC_DATA " \ + f"WHERE {async_id_ranges_check_str}" + matching_async_data = fmkdb.execute_sql_statement(async_data_statement) + + if async_annotation_column_names is not None: + requested_async_data_annotations_columns_str = ', '.join(async_annotation_column_names) + async_data_annotation_statement = f"SELECT {requested_async_data_annotations_columns_str} FROM ASYNC_DATA " \ + f"WHERE {async_id_ranges_check_str}" + matching_async_data_annotations = fmkdb.execute_sql_statement(async_data_annotation_statement) + + fmkdb.stop() + + if matching_data is None or matching_data == []: + return (None, None, [], []) + + data = [] + for line in matching_data: + if None in line: + continue + line_values = dict() + for index, value in enumerate(line): + line_values[column_names[index]] = value + data.append(line_values) + + if matching_async_data is None: + return (data, matching_data_annotations, [], []) + + async_data = [] + for line in matching_async_data: + if None in line: + continue + line_values = dict() + for index, value in enumerate(line): + # CURRENT_DATA_ID is matched to ID variable name + line_values[column_names[index]] = value + async_data.append(line_values) + + return (data, matching_data_annotations, async_data, matching_async_data_annotations) + + +def parse_arguments() -> dict[Any]: + result = dict() + + args = parser.parse_args() + + fmkdb = args.fmkdb + if not fmkdb: + fmkdb = [Database.get_default_db_path()] + for db in fmkdb: + if db is not None and not os.path.isfile(os.path.expanduser(db)): + print_error(f"'{db}' does not exist") + sys.exit(ARG_INVALID_FMDBK) + result['fmkdb'] = fmkdb + + id_range = args.id_range + if id_range is None: + print_error("Please provide a valid ID interval") + print_info("ID interval can be provided in the form '1..5,9..10,7..8'") + sys.exit(ARG_INVALID_ID) + result['id_range'] = id_range + + formula = args.formula + if formula is None: + print_error("Please provide a valid formula") + print_info("Formula can be provided on the form 'a+b~c*d'") + print_info("for a plot of a+b in function of c*d'") + sys.exit(ARG_INVALID_FORMULA) + result['formula'] = formula + + poi = args.points_of_interest + if poi < 0: + print_error("Please provide a positive or zero number of point of interest") + sys.exit(ARG_INVALID_POI) + result['poi'] = poi + + grid_match_str = args.grid_match + if grid_match_str is None or grid_match_str == 'all': + grid_match = GridMatch.ALL + elif grid_match_str == 'auto': + grid_match = GridMatch.AUTO + elif grid_match_str == 'poi': + grid_match = GridMatch.POI + if poi == 0: + parser.error("--points-of-interest must be set to use --grid-match 'poi' option") + else: + parser.error(f"Unknown Grid Match value '{grid_match_str}'") + + result['grid_match'] = grid_match + + result['hide_points'] = args.hide_points + + if args.annotations is not None: + labels = [] + for l in args.annotations: + labels.append( + {'t': 'TYPE', + 'g': 'TARGET', + 's': 'SIZE', + 'a': 'ACK_DATE'}.get(l, None) + ) + if None in labels: + print_warning('Unknown labels have been discarded') + labels = list(filter(lambda x: x is not None, labels)) + result['annotations'] = labels + else: + result['annotations'] = None + + if args.async_annotations is not None: + async_labels = [] + for l in args.async_annotations: + async_labels.append( + {'i': 'ID', + 't': 'TYPE', + 'g': 'TARGET', + 's': 'SIZE'}.get(l, None) + ) + if None in async_labels: + print_warning('Unknown async labels have been discarded') + async_labels = list(filter(lambda x: x is not None, async_labels)) + result['async_annotations'] = async_labels + else: + result['async_annotations'] = None + + result['other_id_range'] = args.other_id_range + max_fmkdb = len(args.other_id_range) if args.other_id_range else 0 + if len(result['fmkdb']) not in {1, max_fmkdb + 1}: + parser.error('Number of given fmkdbs must be one, or match the total number of given ranges') + + result['vertical_shift'] = args.vertical_shift + + result['date_format'] = args.date_format + + return result + + +def plot_formula( + axes: Axes, + formula: str, + id_range: list[range], + align_to: Optional[tuple[Any, Any]], + index: int, + args: dict[Any] +) -> Optional[tuple[str, str, Optional[type], Optional[type], set[tuple[float,float]], set[tuple[float,float]]]]: + + y_expression, x_expression, valid_formula = split_formula(formula) + + x_variable_names, x_function_names = collect_names(x_expression) + y_variable_names, y_function_names = collect_names(y_expression) + + if not valid_formula: + print_error("Given formula or variables names are invalid") + return None + + db = args['fmkdb'][index % len(args['fmkdb'])] + variable_names = x_variable_names.union(y_variable_names) + variables_values, annotations_values, async_variables_values, async_annotations_values = \ + request_from_database(db, id_range, list(variable_names), args['annotations'], args['async_annotations']) + if variables_values is None: + print_error(f"Cannot gather database information for range '{id_range}', skipping it") + return None + + variables_true_types = {} + if not variables_values: + print_error(f"No valid values to display given the formula") + return None + + for variable, value in variables_values[0].items(): + variables_true_types[variable] = type(value) + convert_non_operable_types(variables_values) + convert_non_operable_types(async_variables_values) + + x_values = solve_expression(x_expression, variables_values) + y_values = solve_expression(y_expression, variables_values) + annotations = [] + if annotations_values is not None: + for annotation_values in annotations_values: + annotation_str = '\n'.join([str(value) for value in annotation_values]) + annotations.append(annotation_str) + + x_async_values = solve_expression(x_expression, async_variables_values) + y_async_values = solve_expression(y_expression, async_variables_values) + async_annotations = [] + for async_annotation_values in async_annotations_values: + annotation_str = '\n'.join([str(value) for value in async_annotation_values]) + async_annotations.append(annotation_str) + + if align_to is not None: + sorted_points = sorted(zip(x_values, y_values), key=lambda p: p[0]) + first = sorted_points[0] + last = sorted_points[-1] + curve_height = abs(last[1] - first[1]) + shift_x = align_to[0] - first[0] + shift_y = align_to[1] - first[1] + index * curve_height * args['vertical_shift'] + x_values = list(map(lambda x: x + shift_x, x_values)) + y_values = list(map(lambda y: y + shift_y, y_values)) + x_async_values = list(map(lambda x: x + shift_x, x_async_values)) + y_async_values = list(map(lambda y: y + shift_y, y_async_values)) + + plotted_poi = plot_line(axes, x_values, y_values, annotations, args) + plot_async_data(axes, x_async_values, y_async_values, async_annotations, args) + + x_conversion_type = None + if len(x_variable_names) == 1: + elmt = next(iter(x_variable_names)) + x_conversion_type = variables_true_types[elmt] + y_conversion_type = None + if len(y_variable_names) == 1: + elmt = next(iter(y_variable_names)) + y_conversion_type = variables_true_types[elmt] + + all_plotted_points = set(zip(x_values, y_values)).union(set(zip(x_async_values, y_async_values))) + + return x_expression, y_expression, x_conversion_type, y_conversion_type, plotted_poi, all_plotted_points + + +def main(): + args = parse_arguments() + + figure = plt.figure() + axes: Axes = figure.add_subplot(111) + + all_plotted_poi: set[tuple[float,float]] = set() + all_plotted_points: set[tuple[float,float]] = set() + + id_range = parse_int_range_union(args['id_range']) + plot_result = plot_formula(axes, args['formula'], id_range, None, 0, args) + if plot_result is None: + sys.exit(ARG_INVALID_VAR_NAMES) + + x_expression, y_expression, x_conversion_type, y_conversion_type, plotted_poi, plotted_points = plot_result + origin = sorted(list(plotted_points), key=lambda p: p[0])[0] + all_plotted_poi = all_plotted_poi.union(plotted_poi) + all_plotted_points = all_plotted_points.union(plotted_points) + + if args['other_id_range'] is not None: + for index, other_id_range in enumerate(args['other_id_range']): + + id_range = parse_int_range_union(other_id_range) + plot_result = plot_formula(axes, args['formula'], id_range, origin, index+1, args) + if plot_result is None: + continue + _, _, _, _, plotted_poi, plotted_points = plot_result + all_plotted_poi = all_plotted_poi.union(plotted_poi) + all_plotted_points = all_plotted_points.union(plotted_points) + + post_process_plot(figure, x_conversion_type, y_conversion_type, all_plotted_poi, all_plotted_points, args) + + axes.set_title(f"{args['formula']}") + axes.set_xlabel(x_expression) + axes.set_ylabel(y_expression) + axes.grid() + plt.show() + + sys.exit(0) + + +if __name__ == "__main__": + main() + \ No newline at end of file