Merge pull request #51 from trailofbits/48-optional-manticore

Makes Manticore an optional dependency, and various logging fixes

CHANGELOG.md

@@ -2,7 +2,15 @@
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/).
 
-## [Unreleased](https://github.com/trailofbits/manticore/compare/0.2.2...HEAD)
+## [Unreleased](https://github.com/trailofbits/etheno/compare/v0.2.0...HEAD)
+
+### 0.2.1 — 2019-02-07
+
+Bugfix release.
+
+- Manticore is now an optional requirement
+- Improvements and bugfixes to the logger integration with Manticore
+- Added a workaround to the examples for a bug in Truffle
 
 ## 0.2.0 — 2018-11-02
 

Dockerfile

@@ -53,7 +53,7 @@ COPY etheno/*.py /home/etheno/etheno/etheno/
 RUN mkdir -p /home/etheno/examples
 COPY examples /home/etheno/examples/
 
-RUN cd etheno && pip3 install --user .
+RUN cd etheno && pip3 install --user '.[manticore]'
 
 USER root
 

etheno/__main__.py

@@ -8,18 +8,24 @@
 from .client import RpcProxyClient
 from .differentials import DifferentialTester
 from .echidna import echidna_exists, EchidnaPlugin, install_echidna
-from .etheno import app, EthenoView, GETH_DEFAULT_RPC_PORT, ManticoreClient, ETHENO, VERSION_NAME
+from .etheno import app, EthenoView, GETH_DEFAULT_RPC_PORT, ETHENO, VERSION_NAME
 from .genesis import Account, make_accounts, make_genesis
 from .synchronization import AddressSynchronizingClient, RawTransactionClient
 from .utils import clear_directory, decode_value, find_open_port, format_hex_address, ynprompt
 from . import Etheno
 from . import ganache
 from . import geth
 from . import logger
-from . import manticoreutils
 from . import parity
 from . import truffle
 
+try:
+    from .manticoreclient import ManticoreClient
+    from . import manticoreutils
+    MANTICORE_INSTALLED = True
+except ModuleNotFoundError:
+    MANTICORE_INSTALLED = False
+
 def main(argv = None):
     parser = argparse.ArgumentParser(description='An Ethereum JSON RPC multiplexer and Manticore wrapper')
     parser.add_argument('--debug', action='store_true', default=False, help='Enable debugging from within the web server')
@@ -238,12 +244,19 @@ def main(argv = None):
 
     manticore_client = None
     if args.manticore:
+        if not MANTICORE_INSTALLED:
+            ETHENO.logger.error('Manticore is not installed! Running Etheno with Manticore requires Manticore version 0.2.2 or newer. Reinstall Etheno with Manticore support by running `pip3 install --user \'etheno[manticore]\'`, or install Manticore separately with `pip3 install --user \'manticore\'`')
+            sys.exit(1)
+        new_enough = manticoreutils.manticore_is_new_enough()
+        if new_enough is None:
+            ETHENO.logger.warning(f"Unknown Manticore version {manticoreutils.manticore_version()}; it may not be new enough to have Etheno support!")
+        elif not new_enough:
+            ETHENO.logger.error(f"The version of Manticore installed is {manticoreutils.manticore_version()}, but the minimum required version with Etheno support is 0.2.2. We will try to proceed, but things might not work correctly! Please upgrade Manticore.")
         manticore_client = ManticoreClient()
         ETHENO.add_client(manticore_client)
         if args.manticore_max_depth is not None:
             manticore_client.manticore.register_detector(manticoreutils.StopAtDepth(args.manticore_max_depth))
         manticore_client.manticore.verbosity(getattr(logger, args.log_level))
-        manticore_client.reassign_manticore_loggers()
 
     if args.truffle:
         truffle_controller = truffle.Truffle(parent_logger=ETHENO.logger)

etheno/etheno.py

@@ -4,19 +4,14 @@
 VERSION_ID=67
 
 import logging
-import sha3
 from threading import Thread
-import time
 
 from flask import Flask, g, jsonify, request, abort
 from flask.views import MethodView
 
-from manticore.ethereum import ManticoreEVM
-import manticore
-
 from . import logger
 from . import threadwrapper
-from .client import EthenoClient, JSONRPCError, RpcProxyClient, SelfPostingClient, DATA, QUANTITY, transaction_receipt_succeeded, jsonrpc
+from .client import JSONRPCError, SelfPostingClient
 from .utils import format_hex_address
 
 app = Flask(__name__)
@@ -30,17 +25,6 @@ def to_account_address(raw_address):
     addr = "%x" % raw_address
     return "0x%s%s" % ('0'*(40 - len(addr)), addr)
 
-def encode_hex(data):
-    if data is None:
-        return None
-    elif isinstance(data, int) or isinstance(data, long):
-        encoded = hex(data)
-        if encoded[-1] == 'L':
-            encoded = encoded[:-1]
-        return encoded
-    else:
-        return "0x%s" % data.encode('hex')
-
 _CONTROLLER = threadwrapper.MainThreadController()
 
 @app.route('/shutdown')
@@ -53,124 +37,6 @@ def _etheno_shutdown():
     shutdown()
     return ''
 
-class ManticoreClient(EthenoClient):
-    def __init__(self, manticore=None):
-        self._assigned_manticore = manticore
-        self._manticore = None
-        self.contracts = []
-        self.short_name = 'Manticore'
-        self._accounts_to_create = []
-
-    @property
-    def manticore(self):
-        if self._manticore is None:
-            if self._assigned_manticore is None:
-                # we do lazy evaluation of ManticoreClient.manticore so self.log_directory will be assigned already
-                if self.log_directory is None:
-                    workspace = None
-                else:
-                    workspace = self.log_directory
-                self._assigned_manticore = ManticoreEVM(workspace_url=workspace)
-            self._manticore = threadwrapper.MainThreadWrapper(self._assigned_manticore, _CONTROLLER)
-            self._finalize_manticore()
-        return self._manticore
-
-    def _finalize_manticore(self):
-        if not self._manticore:
-            return
-        for balance, address in self._accounts_to_create:
-            self._manticore.create_account(balance=balance, address=address)
-        self._accounts_to_create = []
-        self.logger.cleanup_empty = True
-
-    def create_account(self, balance, address):            
-        self._accounts_to_create.append((balance, address))
-        self._finalize_manticore()
-
-    def reassign_manticore_loggers(self):
-        # Manticore uses a global to track its loggers:
-        for name in manticore.utils.log.all_loggers:
-            manticore_logger = logging.getLogger(name)
-            for handler in list(manticore_logger.handlers):
-                manticore_logger.removeHandler(handler)
-            logger.EthenoLogger(name, parent=self.logger, cleanup_empty=True)
-
-    @jsonrpc(from_addr = QUANTITY, to = QUANTITY, gas = QUANTITY, gasPrice = QUANTITY, value = QUANTITY, data = DATA, nonce = QUANTITY, RETURN = DATA)
-    def eth_sendTransaction(self, from_addr, to = None, gas = 90000, gasPrice = None, value = 0, data = None, nonce = None, rpc_client_result = None):
-        if to is None or to == 0:
-            # we are creating a new contract
-            if rpc_client_result is not None:
-                tx_hash = rpc_client_result['result']
-                while True:
-                    receipt = self.etheno.master_client.post({
-                        'id' : "%s_receipt" % rpc_client_result['id'],
-                        'method' : 'eth_getTransactionReceipt',
-                        'params' : [tx_hash]
-                    })
-                    if 'result' in receipt and receipt['result']:
-                        address = int(receipt['result']['contractAddress'], 16)
-                        break
-                    # The transaction is still pending
-                    time.sleep(1.0)
-            else:
-                address = None
-            contract_address = self.manticore.create_contract(owner = from_addr, balance = value, init=data)
-            self.contracts.append(contract_address)
-            self.logger.info(f"Manticore contract created: {encode_hex(contract_address.address)}")
-            #self.logger.info("Block number: %s" % self.manticore.world.block_number())
-        else:
-            self.manticore.transaction(address = to, data = data, caller=from_addr, value = value)
-        # Just mimic the result from the master client
-        # We need to return something valid to appease the differential tester
-        return rpc_client_result
-
-    @jsonrpc(TX_HASH = QUANTITY)
-    def eth_getTransactionReceipt(self, tx_hash, rpc_client_result = None):
-        # Mimic the result from the master client
-        # to appease the differential tester
-        return rpc_client_result
-
-    def multi_tx_analysis(self, contract_address = None, tx_limit=None, tx_use_coverage=True, args=None):
-        if contract_address is None:
-            for contract_address in self.contracts:
-                self.multi_tx_analysis(contract_address = contract_address, tx_limit = tx_limit, tx_use_coverage = tx_use_coverage, args = args)
-            return
-
-        tx_account = self.etheno.accounts
-
-        prev_coverage = 0
-        current_coverage = 0
-        tx_no = 0
-        while (current_coverage < 100 or not tx_use_coverage) and not self.manticore.is_shutdown():
-            try:
-                self.logger.info("Starting symbolic transaction: %d" % tx_no)
-
-                # run_symbolic_tx
-                symbolic_data = self.manticore.make_symbolic_buffer(320)
-                symbolic_value = self.manticore.make_symbolic_value()
-                self.manticore.transaction(caller=tx_account[min(tx_no, len(tx_account) - 1)],
-                                 address=contract_address,
-                                 data=symbolic_data,
-                                 value=symbolic_value)
-                self.logger.info("%d alive states, %d terminated states" % (self.manticore.count_running_states(), self.manticore.count_terminated_states()))
-            except NoAliveStates:
-                break
-
-            # Check if the maximun number of tx was reached
-            if tx_limit is not None and tx_no + 1 >= tx_limit:
-                break
-
-            # Check if coverage has improved or not
-            if tx_use_coverage:
-                prev_coverage = current_coverage
-                current_coverage = self.manticore.global_coverage(contract_address)
-                found_new_coverage = prev_coverage < current_coverage
-
-                if not found_new_coverage:
-                    break
-
-            tx_no += 1
-
 class EthenoPlugin(object):
     _etheno = None
     logger = None
@@ -299,6 +165,8 @@ def estimate_gas(self, transaction):
         return None
 
     def post(self, data):
+        self.logger.debug(f"Handling JSON RPC request {data}")
+
         for plugin in self.plugins:
             plugin.before_post(data)
 
@@ -332,6 +200,7 @@ def post(self, data):
                     ret = e
 
         self.rpc_client_result = ret
+        self.logger.debug(f"Result from the master client ({self.master_client}): {ret}")
 
         results = []
 
@@ -357,6 +226,7 @@ def post(self, data):
             except JSONRPCError as e:
                 self.logger.error(e)
                 results.append(e)
+            self.logger.debug(f"Result from client {client}: {results[-1]}")
 
         if ret is None:
             return None
@@ -487,6 +357,8 @@ def post(self):
 
         ret = ETHENO.post(data)
 
+        ETHENO.logger.debug(f"Returning {ret}")
+
         if ret is None:
             return None
 

etheno/ganache.py

@@ -13,9 +13,8 @@ def __init__(self, args=None, port=8546):
         super().__init__("http://127.0.0.1:%d/" % port)
         self.port = port
         if args is None:
-            self.args = []
-        else:
-            self.args = ['/usr/bin/env', 'ganache-cli', '-d', '-p', str(port)] + args
+            args = []
+        self.args = ['/usr/bin/env', 'ganache-cli', '-d', '-p', str(port)] + args
         self.ganache = None
         self._client = None
     def start(self):

etheno/logger.py

@@ -83,21 +83,47 @@ def format(self, *args, **kwargs):
         else:
             return self._parent_formatter.format(*args, **kwargs)
 
+ETHENO_LOGGERS = {}
+
+_LOGGING_GETLOGGER = logging.getLogger
+def getLogger(name):
+    if name in ETHENO_LOGGERS:
+        # TODO: Only enable this if Etheno was run as a standalone application
+        ret = ETHENO_LOGGERS[name]
+    else:
+        ret = _LOGGING_GETLOGGER(name)
+    # ####BEGIN####
+    # Horrible hack to workaround Manticore's global logging system.
+    # This can be removed after https://github.com/trailofbits/manticore/issues/1369
+    # is resolved.
+    if name.startswith('manticore'):
+        ret.propagate = False
+    # ####END####
+    return ret
+logging.getLogger = getLogger
+
 class EthenoLogger(object):
     DEFAULT_FORMAT='$RESET$LEVELCOLOR$BOLD%(levelname)-8s $BLUE[$RESET$WHITE%(asctime)14s$BLUE$BOLD]$NAME$RESET %(message)s'
 
-    def __init__(self, name, log_level=None, parent=None, cleanup_empty=False):
+    def __init__(self, name, log_level=None, parent=None, cleanup_empty=False, displayname=None):
+        if name in ETHENO_LOGGERS:
+            raise Exception(f'An EthenoLogger instance for name {name} already exists: {ETHENO_LOGGERS[name]}')
+        ETHENO_LOGGERS[name] = self
         self._directory = None
         self.parent = parent
         self.cleanup_empty = cleanup_empty
         self.children = []
         self._descendant_handlers = []
+        if displayname is None:
+            self.displayname = name
+        else:
+            self.displayname = displayname
         if log_level is None:
             if parent is None:
                 raise ValueError('A logger must be provided a parent if `log_level` is None')
             log_level = parent.log_level
         self._log_level = log_level
-        self._logger = logging.getLogger(name)
+        self._logger = _LOGGING_GETLOGGER(name)
         self._handlers = [logging.StreamHandler()]
         if log_level is not None:
             self.log_level = log_level
@@ -136,7 +162,7 @@ def directory(self):
         return self._directory
 
     def _add_child(self, child):
-        if child in self.children:
+        if child in self.children or any(c for c in self.children if c.name == child.name):
             raise ValueError("Cannot double-add child logger %s to logger %s" % (child.name, self.name))
         self.children.append(child)
         if self.directory is not None:
@@ -155,7 +181,7 @@ def _name_format(self):
             ret = self.parent._name_format()
         else:
             ret = ''
-        return ret + "[$RESET$WHITE%s$BLUE$BOLD]" % self._logger.name
+        return ret + "[$RESET$WHITE%s$BLUE$BOLD]" % self.displayname
 
     def addHandler(self, handler, include_descendants=True, set_log_level=True):
         if set_log_level:
@@ -249,6 +275,10 @@ def log_level(self, level):
     def __getattr__(self, name):
         return getattr(self._logger, name)
 
+    def __repr__(self):
+        return f'{type(self).__name__}(name={self.name!r}, log_level={self.log_level!r}, parent={self.parent!r}, cleanup_empty={self.cleanup_empty!r}, displayname={self.displayname!r})'
+
+
 class StreamLogger(threading.Thread):
     def __init__(self, logger, *streams, newline_char=b'\n'):
         super().__init__(daemon=True)