Skip to content

Commit

Permalink
Merge branch 'release-0.24.1'
Browse files Browse the repository at this point in the history
  • Loading branch information
@k0retux
k0retux committed Jun 19, 2016
2 parents 2dcc410 + ac41a5f commit 38468f5
Show file tree
Hide file tree
Showing 29 changed files with 1,205 additions and 464 deletions.
2 changes: 1 addition & 1 deletion data_models/example.py
View file
Expand Up @@ -217,7 +217,7 @@ def build_data_model(self):
e_pre2 = Node('pre2', values=[' [1] ', ' [2] ', ' [3] ', ' [4] '])
e_post = Node('post', values=[' [A]', ' [B]', ' [C]', ' [D]'])

e_jpg = self.get_external_node(dm_name='jpg', data_id='JPG')
e_jpg = self.get_external_node(dm_name='jpg', data_id='jpg')

e_mid = Node('mid', subnodes=[e_pre2, e_jpg, e_post])

Expand Down
334 changes: 145 additions & 189 deletions data_models/file_formats/jpg.py
View file
Expand Up @@ -21,212 +21,168 @@
#
################################################################################

import sys
import os
import copy
import re
import functools
import struct

from framework.plumbing import *
from framework.data_model import *
from framework.data_model_helpers import *
from framework.value_types import *
from framework.fuzzing_primitives import *
from framework.basic_primitives import *


markers = {
'SOF': {0: struct.pack('>H', 0xFFC0),
1: struct.pack('>H', 0xFFC1),
2: struct.pack('>H', 0xFFC2),
3: struct.pack('>H', 0xFFC3)},
'DHT': struct.pack('>H', 0xFFC4),
'DAC': struct.pack('>H', 0xFFCC),
'SOI': struct.pack('>H', 0xFFD8),
'SOE': struct.pack('>H', 0xFFD9),
'SOS': struct.pack('>H', 0xFFDA),
'DQT': struct.pack('>H', 0xFFDB),
'DNL': struct.pack('>H', 0xFFDC),
'JFIF': struct.pack('>H', 0xFFE0),
'EXIF': struct.pack('>H', 0xFFE1),
'COM': struct.pack('>H', 0xFFFE),
'SOF': {0: 0xFFC0,
1: 0xFFC1,
2: 0xFFC2,
3: 0xFFC3},
'DHT': 0xFFC4,
'DAC': 0xFFCC,
'SOI': 0xFFD8,
'SOE': 0xFFD9,
'SOS': 0xFFDA,
'DQT': 0xFFDB,
'DNL': 0xFFDC,
'JFIF': 0xFFE0,
'EXIF': 0xFFE1,
'COM': 0xFFFE,
}


### NEED TO BE REVAMPED
### BY USING ModelHelper()
class JPG_DataModel(DataModel):

file_extension = 'jpg'
name = 'jpg'

def build_data_model(self):

self.jpg_dict = self.import_file_contents(extension='jpg')
nodes = list(self.jpg_dict.values())
self.register_nodes(*nodes)

def absorb(self, buff, idx):

for k, sof in markers['SOF'].items():
if not re.search(sof, buff, re.S):
continue

g = re.search(b'(?P<before_sof>.*?)(?P<sof>' + sof + b')(?P<after_sof>.*)', buff, re.S)
if g:
e_sof_marker = Node('SOF_marker')
# e.add_conf(conf)
e_sof_marker.set_values(value_type=UINT16_be(int_list=[struct.unpack('>H', sof)[0]]))
break
def absorb(self, data, idx):
nm = 'jpg_{:0>2d}'.format(idx)
jpg = self.jpg.get_clone(nm, new_env=True)
jpg.set_current_conf('ABS', recursive=True)
status, off, size, name = jpg.absorb(data, constraints=AbsNoCsts(size=True, struct=True,
contents=True))

print('{:s} Absorb Status: {!r}, {:d}, {:d}, {:s}'.format(nm, status, off, size, name))
print(' \_ length of original jpg: {:d}'.format(len(data)))
print(' \_ remaining: {!r}'.format(data[size:size+1000]))

if status == AbsorbStatus.FullyAbsorbed:
x = jpg['.*/SOF_hdr/X'].get_raw_value()
y = jpg['.*/SOF_hdr/Y'].get_raw_value()
d_priv = {'height':y, 'width':x}
jpg.set_private(d_priv)
print("--> Create {:s} from provided JPG sample [x:{:d}, y:{:d}].".format(nm, x, y))
return jpg
else:
return None

before_sof = g.group('before_sof')
sof = g.group('sof')
after_sof = g.group('after_sof')

e_before_sof = Node('before_SOF')
#e_before_sof.add_conf(conf)
e_before_sof.set_values([before_sof])

Lf, P, Y, X, Nf = struct.unpack_from('>HBHHB', after_sof, 0)

sof_start_len = struct.calcsize('>HBHHB')

e_after_sofhdr = Node('after_SOF')
e_after_sofhdr.set_values([after_sof[Lf:]])

e_Lf = Node('Lf')
e_Lf.set_values(value_type=UINT16_be(int_list=[Lf]))
e_P = Node('P')
e_P.set_values(value_type=UINT8(int_list=[P]))
e_X = Node('X')
e_X.set_values(value_type=UINT16_be(int_list=[X]))
# We add the maximum image dimension supported by the
# 'display' program (maybe a JPG standard constraint)
e_X.cc.set_specific_fuzzy_values([65500])
e_Y = Node('Y')
e_Y.set_values(value_type=UINT16_be(int_list=[Y]))
e_Y.cc.set_specific_fuzzy_values([65500])
e_Nf = Node('Nf')
e_Nf.set_values(value_type=UINT8(int_list=[Nf]))

e_SOF_C_struct = Node('SOF_C_struct')
sof_comp_len = 3
l = []
for i in range(Nf):
C = struct.unpack_from('B', after_sof, sof_start_len + i*sof_comp_len)[0]
HV = struct.unpack_from('B', after_sof, sof_start_len + 1 + i*sof_comp_len)[0]
H = HV >> 4
V = HV & 0x0F
Tq = struct.unpack_from('B', after_sof, sof_start_len + 2 + i*sof_comp_len)[0]

e_C = Node('C%d' % i)
e_C.set_values(value_type=UINT8(int_list=[C]))
e_HV = Node('H&V%d' % i)
e_HV.set_values(value_type=BitField(subfield_sizes=[4,4], subfield_val_lists=[[V], [H]]))
e_HV.make_determinist()
e_Tq = Node('Tq%d' % i)
e_Tq.set_values(value_type=UINT8(int_list=[Tq]))

l.extend([e_C, e_HV, e_Tq])

# c_struct.add_conf(conf) TODO: ajouter conf
e_SOF_C_struct.set_subnodes_basic(l)

e_sof_hdr = Node('SOF_hdr')
# e_sof_hdr.add_conf(conf) TODO: ajouter conf
e_sof_hdr.set_subnodes_basic([e_sof_marker, e_Lf, e_P, e_Y, e_X, e_Nf, e_SOF_C_struct])


##
## Dissect SOS segment
##
after_sof_seg = after_sof[Lf:]
subparts = re.search(b'(?P<between_sof_sos>.*?)(?P<sos>' + markers['SOS'] + b')(?P<after_sos>.*)', after_sof_seg, re.S)
between_sof_sos = subparts.group('between_sof_sos')
sos = subparts.group('sos')
after_sos = subparts.group('after_sos')

before_sos = before_sof + sof + after_sof[:Lf] + between_sof_sos
e_before_sos = Node('before_SOS')
e_before_sos.set_values([before_sos])

e_between_sof_sos = Node('between_SOF_SOS')
e_between_sof_sos.set_values([between_sof_sos])


## BEGIN DISSECTION: SOS segment ##
e_sos_marker = Node('SOS_marker')
e_sos_marker.set_values(value_type=UINT16_be(int_list=[struct.unpack('>H', markers['SOS'])[0]]))

Ls, Ns = struct.unpack_from('>HB', after_sos, 0)
sos_start_len = struct.calcsize('>HB')

e_Ls = Node('Ls')
e_Ls.set_values(value_type=UINT16_be(int_list=[Ls]))
e_Ns = Node('Ns')
e_Ns.set_values(value_type=UINT8(int_list=[Ns]))

e_SOS_C_struct = Node('Comp_params')
sos_comp_len = 2
l = []
for i in range(Ns):
Cs_val = struct.unpack_from('B', after_sos, sos_start_len + i*sos_comp_len)[0]
TdTa = struct.unpack_from('B', after_sos, sos_start_len + 1 + i*sos_comp_len)[0]
Td_val = TdTa >> 4
Ta_val = TdTa & 0x0F

e_Cs = Node('C%d' % i)
e_Cs.set_values(value_type=UINT8(int_list=[Cs_val]))
e_TdTa = Node('Td&Ta%d' % i)
e_TdTa.set_values(value_type=BitField(subfield_sizes=[4,4], subfield_val_lists=[[Ta_val], [Td_val]]))
e_TdTa.make_determinist()
l.extend([e_Cs, e_TdTa])

e_SOS_C_struct.set_subnodes_basic(l)

sos_comp_struct_len = sos_comp_len * Ns

Ss, Se, Ahl = struct.unpack_from('>BBB', after_sos, sos_start_len + sos_comp_struct_len)
Ah = Ahl >> 4
Al = Ahl & 0x0F

e_Ss = Node('Ss')
e_Ss.set_values(value_type=UINT8(int_list=[Ss]))
e_Se = Node('Se')
e_Se.set_values(value_type=UINT8(int_list=[Se]))
e_Ahl = Node('Ah&Al%d' % i)
e_Ahl.set_values(value_type=BitField(subfield_sizes=[4,4], subfield_val_lists=[[Al], [Ah]]))
e_Ahl.make_determinist()

e_sos_hdr = Node('SOS_hdr')
e_sos_hdr.set_subnodes_basic([e_sos_marker, e_Ls, e_Ns, e_SOS_C_struct, e_Ss, e_Se, e_Ahl])
## END DISSECTION: SOS segment ##

e_after_soshdr = Node('afterSOS')
e_after_soshdr.set_values([after_sos[Ls:]])

##
## Top Elts
##

if idx == 0:
jpg_id = 'JPG'
else:
jpg_id = 'JPG_{:0>2d}'.format(idx)

## In this Node, SOF & SOS segments are dissected
e_jpg = Node(jpg_id)
e_jpg.set_subnodes_basic([e_before_sof, e_sof_hdr, e_between_sof_sos, e_sos_hdr, e_after_soshdr])

d_priv = {'height':Y, 'width':X}
e_jpg.set_private(d_priv)
return Node(nm, values=['JPG ABSORBSION FAILED'])

return e_jpg
def build_data_model(self):

jpg_desc = \
{'name': 'jpg',
'contents': [
{'name': 'SOF_hdr',
'contents': [
{'name': 'before_SOF',
'contents': String(size=0),
'absorb_csts': AbsNoCsts(),
'set_attrs': MH.Attr.Abs_Postpone, # Only works if it is side-by-side with the
# the node that will provide the constraint
'mutable': False},

{'name': 'F_marker',
'contents': UINT16_be(int_list=[m for m in markers['SOF'].values()])},
{'name': 'Lf',
'contents': MH.LEN(vt=UINT16_be, base_len=8),
'node_args': 'F_CompGroup',
'alt': [
{'conf': 'ABS',
'contents': UINT16_be()}
]},
{'name': 'P',
'contents': UINT8(int_list=[8,12])},
{'name': 'Y',
'contents': UINT16_be(maxi=65535)},
{'name': 'X',
'contents': UINT16_be(mini=1, maxi=65535)},
{'name': 'Nf',
'contents': MH.QTY(node_name='F_Comp', vt=UINT8),
'node_args': 'F_CompGroup',
'alt': [
{'conf': 'ABS',
'contents': UINT8(mini=1, maxi=255)},
]},
{'name': 'F_CompGroup',
'custo_clear': MH.Custo.NTerm.MutableClone,
'contents': [
{'name': 'F_Comp',
'qty': (1, 255),
'contents': [
{'name': 'Cf',
'contents': UINT8(mini=0, maxi=255)},
{'name': 'H&V',
'contents': BitField(subfield_sizes=[4,4], endian=VT.BigEndian,
subfield_val_extremums=[[1,4], [1,4]],
subfield_descs=['H sampling', 'V sampling'])},
{'name': 'Tq',
'contents': UINT8(mini=0, maxi=3)},
]}
]},
]},

{'name': 'SOS_hdr',
'contents': [
{'name': 'between_SOF_SOS',
'contents': String(),
'random': True,
'absorb_csts': AbsNoCsts(),
'set_attrs': MH.Attr.Abs_Postpone,
'mutable': False},

{'name': 'S_marker',
'contents': UINT16_be(int_list=[markers['SOS']])},
{'name': 'Ls',
'contents': MH.LEN(vt=UINT16_be, base_len=6),
'node_args': 'S_CompGroup',
'alt': [
{'conf': 'ABS',
'contents': UINT16_be()}
]},
{'name': 'Ns',
'contents': MH.QTY(node_name='S_Comp', vt=UINT8),
'node_args': 'S_CompGroup',
'alt': [
{'conf': 'ABS',
'contents': UINT8(mini=1, maxi=255)},
]},
{'name': 'S_CompGroup',
'custo_clear': MH.Custo.NTerm.MutableClone,
'contents': [
{'name': 'S_Comp',
'qty': (1, 4),
'contents': [
{'name': 'Cs',
'contents': UINT8()},
{'name': 'Td&Ta',
'contents': BitField(subfield_sizes=[4, 4], endian=VT.BigEndian,
subfield_val_extremums=[[0, 3], [0, 3]],
subfield_descs=['DC entropy', 'AC entropy'])},
]}
]},
{'name': 'Ss',
'contents': UINT8(mini=0, maxi=63)},
{'name': 'Se',
'contents': UINT8(mini=0, maxi=63)},
{'name': 'Ah&Al',
'contents': BitField(subfield_sizes=[4, 4], endian=VT.BigEndian,
subfield_val_extremums=[[0, 13], [0, 13]],
subfield_descs=['approx high', 'approx low'])},
]},

{'name': 'afterSOS',
'mutable': False,
'contents': String(min_sz=0),
'absorb_csts': AbsNoCsts()}
]}

mh = ModelHelper(delayed_jobs=True)
self.jpg = mh.create_graph_from_desc(jpg_desc)

self.jpg_dict = self.import_file_contents(extension='jpg')
self.register(self.jpg, *self.jpg_dict.values())


data_model = JPG_DataModel()
Expand Down
2 changes: 1 addition & 1 deletion data_models/file_formats/pdf.py
View file
Expand Up @@ -1177,7 +1177,7 @@ def build_data_model(self):

PDFObj.external_pdf_objs = gather_pdf_objects()

e_jpg = self.get_external_node(dm_name='jpg', data_id='JPG')
e_jpg = self.get_external_node(dm_name='jpg', data_id='jpg_00')

PDFObj.jpg_node = e_jpg

Expand Down

0 comments on commit 38468f5

Please sign in to comment.