-
Notifications
You must be signed in to change notification settings - Fork 282
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chg: [update + show decoded items] add background update
- Loading branch information
Showing
11 changed files
with
288 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,77 @@ | ||
#!/usr/bin/env python3 | ||
# -*-coding:UTF-8 -* | ||
|
||
import os | ||
import re | ||
import sys | ||
import time | ||
import redis | ||
import datetime | ||
|
||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) | ||
import Item | ||
import Tag | ||
from Cryptocurrency import cryptocurrency | ||
from Pgp import pgp | ||
|
||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) | ||
import ConfigLoader | ||
import Decoded | ||
import Domain | ||
|
||
def update_update_stats(): | ||
nb_updated = int(r_serv_db.get('update:nb_elem_converted')) | ||
progress = int((nb_updated * 100) / nb_elem_to_update) | ||
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress)) | ||
r_serv_db.set('ail:current_background_script_stat', progress) | ||
|
||
def update_domain_by_item(domain_obj, item_id): | ||
domain_name = domain_obj.get_domain_name() | ||
# update domain tags | ||
for tag in Tag.get_item_tags(item_id): | ||
if tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"': | ||
Tag.add_domain_tag(tag, domain_name, Item.get_item_date(item_id)) | ||
|
||
# update domain correlation | ||
item_correlation = Item.get_item_all_correlation(item_id) | ||
|
||
for correlation_name in item_correlation: | ||
for correlation_type in item_correlation[correlation_name]: | ||
if correlation_name in ('pgp', 'cryptocurrency'): | ||
for correl_value in item_correlation[correlation_name][correlation_type]: | ||
if correlation_name=='pgp': | ||
pgp.save_domain_correlation(domain_name, correlation_type, correl_value) | ||
if correlation_name=='cryptocurrency': | ||
cryptocurrency.save_domain_correlation(domain_name, correlation_type, correl_value) | ||
if correlation_name=='decoded': | ||
for decoded_item in item_correlation['decoded']: | ||
Decoded.save_domain_decoded(domain_name, decoded_item) | ||
|
||
if __name__ == '__main__': | ||
|
||
start_deb = time.time() | ||
|
||
config_loader = ConfigLoader.ConfigLoader() | ||
r_serv_db = config_loader.get_redis_conn("ARDB_DB") | ||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") | ||
config_loader = None | ||
|
||
nb_elem_to_update = int( r_serv_db.get('update:nb_elem_to_convert') ) | ||
|
||
while True: | ||
domain = r_serv_onion.spop('domain_update_v2.4') | ||
if domain is not None: | ||
print(domain) | ||
domain = Domain.Domain(domain) | ||
for domain_history in domain.get_domain_history(): | ||
|
||
domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag | ||
if "items" in domain_item: | ||
for item_dict in domain_item['items']: | ||
update_domain_by_item(domain, item_dict['id']) | ||
|
||
r_serv_db.incr('update:nb_elem_converted') | ||
update_update_stats() | ||
|
||
else: | ||
sys.exit(0) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.