Merge pull request #418 from CIRCL/domain_object

Domain object

.gitignore

@@ -35,9 +35,11 @@ var/www/server.crt
 var/www/server.key
 
 # Local config
-bin/packages/config.cfg
-bin/packages/config.cfg.backup
 configs/keys
+bin/packages/core.cfg
+bin/packages/config.cfg.backup
+configs/core.cfg
+configs/core.cfg.backup
 configs/update.cfg
 update/current_version
 files

HOWTO.md

@@ -25,7 +25,7 @@ Feed data to AIL:
 
 3. Launch pystemon ``` ./pystemon ```
 
-4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly
+4. Edit your configuration file ```configs/core.cfg``` and modify the pystemonpath path accordingly
 
 5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ```
 
@@ -123,7 +123,7 @@ There are two types of installation. You can install a *local* or a *remote* Spl
       (for a linux docker, the localhost IP is *172.17.0.1*; Should be adapted for other platform)
     - Restart the tor proxy: ``sudo service tor restart``
 
-3. *(AIL host)* Edit the ``/bin/packages/config.cfg`` file:
+3. *(AIL host)* Edit the ``/configs/core.cfg`` file:
     - In the crawler section, set ``activate_crawler`` to ``True``
     - Change the IP address of Splash servers if needed (remote only)
     - Set ``splash_onion_port`` according to your Splash servers port numbers that will be used.
@@ -134,7 +134,7 @@ There are two types of installation. You can install a *local* or a *remote* Spl
 
 - *(Splash host)* Launch all Splash servers with:
 ```sudo ./bin/torcrawler/launch_splash_crawler.sh -f <config absolute_path> -p <port_start> -n <number_of_splash>```
-With ``<port_start>`` and ``<number_of_splash>`` matching those specified at ``splash_onion_port`` in the configuration file of point 3 (``/bin/packages/config.cfg``)
+With ``<port_start>`` and ``<number_of_splash>`` matching those specified at ``splash_onion_port`` in the configuration file of point 3 (``/configs/core.cfg``)
 
 All Splash dockers are launched inside the ``Docker_Splash`` screen. You can use ``sudo screen -r Docker_Splash`` to connect to the screen session and check all Splash servers status.
 
@@ -148,7 +148,7 @@ All Splash dockers are launched inside the ``Docker_Splash`` screen. You can use
 - ```crawler_hidden_services_install.sh -y```
 - Add the following line in ``SOCKSPolicy accept 172.17.0.0/16`` in ``/etc/tor/torrc``
 - ```sudo service tor restart```
-- set activate_crawler to True in ``/bin/packages/config.cfg``
+- set activate_crawler to True in ``/configs/core.cfg``
 #### Start
 - ```sudo ./bin/torcrawler/launch_splash_crawler.sh -f $AIL_HOME/configs/docker/splash_onion/etc/splash/proxy-profiles/ -p 8050 -n 1```
 
@@ -166,4 +166,3 @@ Then starting the crawler service (if you follow the procedure above)
 ##### Python 3 Upgrade
 
 To upgrade from an existing AIL installation, you have to launch [python3_upgrade.sh](./python3_upgrade.sh), this script will delete and create a new virtual environment. The script **will upgrade the packages but won't keep your previous data** (neverthless the data is copied into a directory called `old`). If you install from scratch, you don't require to launch the [python3_upgrade.sh](./python3_upgrade.sh).
-

OVERVIEW.md

@@ -261,6 +261,9 @@ Redis and ARDB overview
 | set_pgpdump_name:*name* | *item_path* |
 | | |
 | set_pgpdump_mail:*mail* | *item_path* |
+| | |
+| | |
+| set_domain_pgpdump_**pgp_type**:**key** | **domain** |
 
 ##### Hset date:
 | Key | Field | Value |
@@ -288,11 +291,20 @@ Redis and ARDB overview
 | item_pgpdump_name:*item_path* | *name* |
 | | |
 | item_pgpdump_mail:*item_path* | *mail* |
+| | |
+| | |
+| domain_pgpdump_**pgp_type**:**domain** | **key** |
 
 #### Cryptocurrency
 
 Supported cryptocurrency:
 - bitcoin
+- bitcoin-cash
+- dash
+- etherum
+- litecoin
+- monero
+- zcash
 
 ##### Hset:
 | Key | Field | Value |
@@ -303,7 +315,8 @@ Supported cryptocurrency:
 ##### set:
 | Key | Value |
 | ------ | ------ |
-| set_cryptocurrency_**cryptocurrency name**:**cryptocurrency address** | **item_path** |
+| set_cryptocurrency_**cryptocurrency name**:**cryptocurrency address** | **item_path** | PASTE
+| domain_cryptocurrency_**cryptocurrency name**:**cryptocurrency address** | **domain** | DOMAIN
 
 ##### Hset date:
 | Key | Field | Value |
@@ -318,8 +331,14 @@ Supported cryptocurrency:
 ##### set:
 | Key | Value |
 | ------ | ------ |
-| item_cryptocurrency_**cryptocurrency name**:**item_path** | **cryptocurrency address** |
+| item_cryptocurrency_**cryptocurrency name**:**item_path** | **cryptocurrency address** | PASTE
+| domain_cryptocurrency_**cryptocurrency name**:**item_path** | **cryptocurrency address** | DOMAIN
 
+#### HASH
+| Key | Value |
+| ------ | ------ |
+| hash_domain:**domain** | **hash** |
+| domain_hash:**hash** | **domain** |
 
 ## DB9 - Crawler:
 
@@ -362,6 +381,20 @@ Supported cryptocurrency:
 }
 ```
 
+##### CRAWLER QUEUES:
+| SET - Key | Value |
+| ------ | ------ |
+| onion_crawler_queue | **url**;**item_id** | RE-CRAWL
+| regular_crawler_queue | - |
+|  |  |
+| onion_crawler_priority_queue   | **url**;**item_id** | USER
+| regular_crawler_priority_queue | - |
+|  |  |
+| onion_crawler_discovery_queue   | **url**;**item_id** | DISCOVER
+| regular_crawler_discovery_queue | - |
+
+##### TO CHANGE:
+
 ARDB overview
 
 	----------------------------------------- SENTIMENT ------------------------------------

bin/Decoder.py

@@ -18,6 +18,7 @@
 
 from Helper import Process
 from packages import Paste
+from packages import Item
 
 import re
 import signal
@@ -120,6 +121,12 @@ def save_hash(decoder_name, message, date, decoded):
     serv_metadata.zincrby('nb_seen_hash:'+hash, message, 1)# hash - paste map
     serv_metadata.zincrby(decoder_name+'_hash:'+hash, message, 1) # number of b64 on this paste
 
+    # Domain Object
+    if Item.is_crawled(message):
+        domain = Item.get_item_domain(message)
+        serv_metadata.sadd('hash_domain:{}'.format(domain), hash) # domain - hash map
+        serv_metadata.sadd('domain_hash:{}'.format(hash), domain) # hash - domain map
+
 
 def save_hash_on_disk(decode, type, hash, json_data):
 

bin/Helper.py

@@ -20,10 +20,10 @@
 import json
 
 
-class PubSub(object):
+class PubSub(object): ## TODO: remove config, use ConfigLoader by default
 
     def __init__(self):
-        configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+        configfile = os.path.join(os.environ['AIL_HOME'], 'configs/core.cfg')
         if not os.path.exists(configfile):
             raise Exception('Unable to find the configuration file. \
                             Did you set environment variables? \
@@ -58,7 +58,6 @@ def setup_subscribe(self, conn_name):
             for address in addresses.split(','):
                 new_sub = context.socket(zmq.SUB)
                 new_sub.connect(address)
-                # bytes64 encode bytes to ascii only bytes
                 new_sub.setsockopt_string(zmq.SUBSCRIBE, channel)
                 self.subscribers.append(new_sub)
 
@@ -112,7 +111,7 @@ def subscribe(self):
 class Process(object):
 
     def __init__(self, conf_section, module=True):
-        configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+        configfile = os.path.join(os.environ['AIL_HOME'], 'configs/core.cfg')
         if not os.path.exists(configfile):
             raise Exception('Unable to find the configuration file. \
                             Did you set environment variables? \

bin/LAUNCH.sh

@@ -222,7 +222,7 @@ function launching_scripts {
 
 function launching_crawler {
     if [[ ! $iscrawler ]]; then
-        CONFIG=$AIL_BIN/packages/config.cfg
+        CONFIG=$AIL_HOME/configs/core.cfg
         lport=$(awk '/^\[Crawler\]/{f=1} f==1&&/^splash_port/{print $3;exit}' "${CONFIG}")
 
         IFS='-' read -ra PORTS <<< "$lport"

bin/MISP_The_Hive_feeder.py

@@ -8,20 +8,20 @@
 This module send tagged pastes to MISP or THE HIVE Project
 
 """
-
-import redis
-import sys
 import os
+import sys
+import uuid
+import redis
 import time
 import json
-import configparser
 
 from pubsublogger import publisher
 from Helper import Process
 from packages import Paste
 import ailleakObject
 
-import uuid
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
 
 from pymisp import PyMISP
 
@@ -133,26 +133,10 @@ def feeder(message, count=0):
 
     config_section = 'MISP_The_hive_feeder'
 
-    configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
-    if not os.path.exists(configfile):
-        raise Exception('Unable to find the configuration file. \
-                        Did you set environment variables? \
-                        Or activate the virtualenv.')
-
-    cfg = configparser.ConfigParser()
-    cfg.read(configfile)
+    config_loader = ConfigLoader.ConfigLoader()
 
-    r_serv_db = redis.StrictRedis(
-        host=cfg.get("ARDB_DB", "host"),
-        port=cfg.getint("ARDB_DB", "port"),
-        db=cfg.getint("ARDB_DB", "db"),
-        decode_responses=True)
-
-    r_serv_metadata = redis.StrictRedis(
-        host=cfg.get("ARDB_Metadata", "host"),
-        port=cfg.getint("ARDB_Metadata", "port"),
-        db=cfg.getint("ARDB_Metadata", "db"),
-        decode_responses=True)
+    r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+    r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
 
     # set sensor uuid
     uuid_ail = r_serv_db.get('ail:uuid')
@@ -212,7 +196,9 @@ def feeder(message, count=0):
 
     refresh_time = 3
     ## FIXME: remove it
-    PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes"))
+    PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes"))
+    config_loader = None
+
     time_1 = time.time()
 
     while True:

bin/Mixer.py

@@ -29,16 +29,20 @@
 The mapping can be done via the variable FEED_QUEUE_MAPPING
 
 """
+import os
+import sys
+
 import base64
 import hashlib
-import os
 import time
 from pubsublogger import publisher
 import redis
-import configparser
 
 from Helper import Process
 
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
 
 # CONFIG #
 refresh_time = 30
@@ -52,37 +56,22 @@
 
     p = Process(config_section)
 
-    configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
-    if not os.path.exists(configfile):
-        raise Exception('Unable to find the configuration file. \
-                        Did you set environment variables? \
-                        Or activate the virtualenv.')
-
-    cfg = configparser.ConfigParser()
-    cfg.read(configfile)
+    config_loader = ConfigLoader.ConfigLoader()
 
     # REDIS #
-    server = redis.StrictRedis(
-        host=cfg.get("Redis_Mixer_Cache", "host"),
-        port=cfg.getint("Redis_Mixer_Cache", "port"),
-        db=cfg.getint("Redis_Mixer_Cache", "db"),
-        decode_responses=True)
-
-    server_cache = redis.StrictRedis(
-        host=cfg.get("Redis_Log_submit", "host"),
-        port=cfg.getint("Redis_Log_submit", "port"),
-        db=cfg.getint("Redis_Log_submit", "db"),
-        decode_responses=True)
+    server = config_loader.get_redis_conn("Redis_Mixer_Cache")
+    server_cache = config_loader.get_redis_conn("Redis_Log_submit")
 
     # LOGGING #
     publisher.info("Feed Script started to receive & publish.")
 
     # OTHER CONFIG #
-    operation_mode = cfg.getint("Module_Mixer", "operation_mode")
-    ttl_key = cfg.getint("Module_Mixer", "ttl_duplicate")
-    default_unnamed_feed_name = cfg.get("Module_Mixer", "default_unnamed_feed_name")
+    operation_mode = config_loader.get_config_int("Module_Mixer", "operation_mode")
+    ttl_key = config_loader.get_config_int("Module_Mixer", "ttl_duplicate")
+    default_unnamed_feed_name = config_loader.get_config_str("Module_Mixer", "default_unnamed_feed_name")
 
-    PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes")) + '/'
+    PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
+    config_loader = None
 
     # STATS #
     processed_paste = 0