Merge branch 'testing'

AUTHORS

@@ -5,14 +5,15 @@ IRMA is a project co-funded by the following actors:
     * GOVCERT.LU (governmental CERT of Luxembourg)
     * Airbus Group
     * QuarksLab
-    * Orange DSI Groupe
+    * Orange Group IS&T
 
 The PRIMARY AUTHORS are (and/or have been):
 
     * Alexandre Quint - Lead Developer, QuarksLab
     * Fernand Lone-Sang - QuarksLab
-    * Bruno Dorsemaine - Orange DSI Groupe
+    * Bruno Dorsemaine - Orange Group IS&T
     * David Carle - QuarksLab
+    * Guillaume Dedrie - QuarksLab
 
 And here is an inevitably incomplete list of MUCH-APPRECIATED CONTRIBUTORS --
 people who have submitted patches, reported bugs, helped answer newbie questions, 

Makefile

@@ -3,7 +3,7 @@ DESTDIR=/
 DISTDIR=$(CURDIR)/deb_dist
 BUILDIR=$(CURDIR)/debian/irma-brain
 PROJECT=irma-brain
-VERSION=1.0.4
+VERSION=1.1.0
 
 all:
 	@echo "make source - Create source package"
@@ -40,6 +40,5 @@ clean:
 	rm -rf $(CURDIR)/debian/irma-brain-logrotate
 	rm -rf $(CURDIR)/debian/irma-brain-rabbitmq
 	rm -rf $(CURDIR)/debian/irma-brain-ftpd
-	rm -rf $(CURDIR)/debian/irma-brain-redis
 	rm -rf build/ MANIFEST $(DISTDIR) $(BUILDIR) 
 	find . -name '*.pyc' -delete

brain/controllers/frontendtasks.py

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2013-2014 QuarksLab.
+# This file is part of IRMA project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the top-level directory
+# of this distribution and at:
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# No part of the project, including this file, may be copied,
+# modified, propagated, or distributed except according to the
+# terms contained in the LICENSE file.
+
+import celery
+import config.parser as config
+from brain.helpers.celerytasks import async_call
+
+frontend_app = celery.Celery('frontendtasks')
+config.conf_frontend_celery(frontend_app)
+config.configure_syslog(frontend_app)
+
+
+def scan_launched(frontend_scanid):
+    async_call(frontend_app,
+               "frontend.tasks",
+               "scan_launched",
+               args=[frontend_scanid])
+
+
+def scan_result(frontend_scanid, filename, probe, result):
+    async_call(frontend_app,
+               "frontend.tasks",
+               "scan_result",
+               args=[frontend_scanid, filename, probe, result])

brain/controllers/ftpctrl.py

@@ -0,0 +1,27 @@
+#
+# Copyright (c) 2013-2014 QuarksLab.
+# This file is part of IRMA project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the top-level directory
+# of this distribution and at:
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# No part of the project, including this file, may be copied,
+# modified, propagated, or distributed except according to the
+# terms contained in the LICENSE file.
+
+from lib.irma.ftp.handler import FtpTls
+import config.parser as config
+
+
+def flush_dir(ftpuser, scanid):
+    print("Flushing dir {0}".format(scanid))
+    conf_ftp = config.brain_config['ftp_brain']
+    with FtpTls(conf_ftp.host,
+                conf_ftp.port,
+                conf_ftp.username,
+                conf_ftp.password) as ftps:
+        ftps.deletepath("{0}/{1}".format(ftpuser, scanid), deleteParent=True)

brain/controllers/jobctrl.py

@@ -0,0 +1,66 @@
+#
+# Copyright (c) 2013-2014 QuarksLab.
+# This file is part of IRMA project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the top-level directory
+# of this distribution and at:
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# No part of the project, including this file, may be copied,
+# modified, propagated, or distributed except according to the
+# terms contained in the LICENSE file.
+
+from brain.controllers import scanctrl
+from brain.models.sqlobjects import Job
+from brain.helpers.sql import session_query, session_transaction
+from lib.common.compat import timestamp
+
+
+def new(scan_id, filename, probe):
+    with session_transaction() as session:
+        j = Job(filename, probe, scan_id)
+        j.save(session)
+        session.commit()
+        return j.id
+
+
+def _finish(job_id, status):
+    with session_transaction() as session:
+        job = Job.load(job_id, session)
+        job.status = status
+        job.ts_end = timestamp()
+        job.update(['status', 'ts_end'], session)
+        scan_id = job.scan.id
+    scanctrl.check_finished(scan_id)
+
+
+def success(job_id):
+    _finish(job_id, Job.success)
+
+
+def error(job_id):
+    _finish(job_id, Job.error)
+
+
+def set_taskid(job_id, task_id):
+    with session_transaction() as session:
+        job = Job.load(job_id, session)
+        job.task_id = task_id
+
+
+def info(job_id):
+    with session_query() as session:
+        job = Job.load(job_id, session)
+        frontend_scan_id = job.scan.scan_id
+        filename = job.filename
+        probe = job.probename
+        return (frontend_scan_id, filename, probe)
+
+
+def duration(job_id):
+    with session_query() as session:
+        job = Job.load(job_id, session)
+        return (job.ts_end - job.ts_start)

brain/controllers/probetasks.py

@@ -0,0 +1,81 @@
+#
+# Copyright (c) 2013-2014 QuarksLab.
+# This file is part of IRMA project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the top-level directory
+# of this distribution and at:
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# No part of the project, including this file, may be copied,
+# modified, propagated, or distributed except according to the
+# terms contained in the LICENSE file.
+
+import celery
+import time
+import config.parser as config
+from brain.helpers.celerytasks import route, async_call
+
+
+results_app = celery.Celery('resultstasks')
+config.conf_results_celery(results_app)
+
+probe_app = celery.Celery('probetasks')
+config.conf_results_celery(probe_app)
+
+# Time to cache the probe list
+# to avoid asking to rabbitmq
+PROBELIST_CACHE_TIME = 60
+cache_probelist = {'list': None, 'time': None}
+
+
+def get_probelist():
+    # get active queues list from probe celery app
+    now = time.time()
+    result_queue = config.brain_config['broker_probe'].queue
+    if cache_probelist['time'] is not None:
+        cache_time = now - cache_probelist['time']
+    if cache_probelist['time'] is None or cache_time > PROBELIST_CACHE_TIME:
+        slist = list()
+        i = probe_app.control.inspect()
+        queues = i.active_queues()
+        if queues:
+            for infolist in queues.values():
+                for info in infolist:
+                    if info['name'] not in slist:
+                        # exclude only predefined result queue
+                        if info['name'] != result_queue:
+                            slist.append(info['name'])
+        if len(slist) != 0:
+            # activate cache only on non empty list
+            cache_probelist['time'] = now
+        cache_probelist['list'] = slist
+    return cache_probelist['list']
+
+
+# ============
+#  Task calls
+# ============
+
+def job_launch(ftpuser, frontend_scanid, filename, probe, job_id):
+    """ send a task to the brain to flush the scan files"""
+    hook_success = route(
+        results_app.signature("brain.tasks.job_success",
+                              [job_id]))
+    hook_error = route(
+        results_app.signature("brain.tasks.job_error",
+                              [job_id]))
+    task = async_call(probe_app,
+                      "probe.tasks",
+                      "probe_scan",
+                      args=(ftpuser, frontend_scanid, filename),
+                      queue=probe,
+                      link=hook_success,
+                      link_error=hook_error)
+    return task.id
+
+
+def job_cancel(job_list):
+    probe_app.control.revoke(job_list, terminate=True)

brain/controllers/scanctrl.py

@@ -0,0 +1,108 @@
+#
+# Copyright (c) 2013-2014 QuarksLab.
+# This file is part of IRMA project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the top-level directory
+# of this distribution and at:
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# No part of the project, including this file, may be copied,
+# modified, propagated, or distributed except according to the
+# terms contained in the LICENSE file.
+
+from brain.models.sqlobjects import Scan
+from brain.helpers.sql import session_query, session_transaction
+from lib.irma.common.utils import IrmaScanStatus
+from lib.irma.common.exceptions import IrmaValueError
+
+
+def new(frontend_scan_id, user_id, nb_files):
+    with session_transaction() as session:
+        scan = Scan(frontend_scan_id, user_id, nb_files)
+        scan.save(session)
+        session.commit()
+        return scan.id
+
+
+def get_scan_id(frontend_scan_id, user_id):
+    with session_query() as session:
+        scan = Scan.get_scan(frontend_scan_id, user_id, session)
+        return scan.id
+
+
+def get_nbjobs(scan_id):
+    with session_query() as session:
+        scan = Scan.load(scan_id, session)
+        return scan.nb_jobs
+
+
+def _set_status(scan_id, code):
+    with session_transaction() as session:
+        scan = Scan.load(scan_id, session)
+        scan.status = code
+        session.commit()
+
+
+def cancelling(scan_id):
+    _set_status(scan_id, IrmaScanStatus.cancelling)
+
+
+def cancelled(scan_id):
+    _set_status(scan_id, IrmaScanStatus.cancelled)
+
+
+def launched(scan_id):
+    _set_status(scan_id, IrmaScanStatus.launched)
+
+
+def progress(scan_id):
+    with session_query() as session:
+        scan = Scan.load(scan_id, session)
+        if IrmaScanStatus.is_error(scan.status):
+            status_str = IrmaScanStatus.label[scan.status]
+            raise IrmaValueError(status_str)
+        status = IrmaScanStatus.label[scan.status]
+        progress_details = None
+        if scan.status == IrmaScanStatus.launched:
+            (total, finished, success) = scan.progress()
+            progress_details = {}
+            progress_details['total'] = total
+            progress_details['finished'] = finished
+            progress_details['successful'] = success
+        return (status, progress_details)
+
+
+def get_pending_jobs(scan_id):
+    with session_query() as session:
+        scan = Scan.load(scan_id, session)
+        return scan.get_pending_jobs_taskid()
+
+
+def check_finished(scan_id):
+    with session_transaction() as session:
+        scan = Scan.load(scan_id, session)
+        if scan.status == IrmaScanStatus.processed:
+            return True
+        if scan.finished():
+            scan.status = IrmaScanStatus.processed
+            return True
+        return False
+
+
+def flush(scan_id):
+    with session_transaction() as session:
+        scan = Scan.load(scan_id, session)
+        if scan.status == IrmaScanStatus.flushed:
+            return
+        for job in scan.jobs:
+            session.delete(job)
+        scan.status = IrmaScanStatus.flushed
+
+
+def error(scan_id, code):
+    with session_transaction() as session:
+        scan = Scan.load(scan_id, session)
+        scan.status = code

brain/controllers/userctrl.py

@@ -0,0 +1,39 @@
+#
+# Copyright (c) 2013-2014 QuarksLab.
+# This file is part of IRMA project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the top-level directory
+# of this distribution and at:
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# No part of the project, including this file, may be copied,
+# modified, propagated, or distributed except according to the
+# terms contained in the LICENSE file.
+
+from brain.models.sqlobjects import User
+from brain.helpers.sql import session_query, session_transaction
+
+
+def get_userid(rmqvhost):
+    with session_query() as session:
+        user = User.get_by_rmqvhost(rmqvhost, session)
+        return user.id
+
+
+def get_quota(user_id):
+    with session_query() as session:
+        user = User.load(user_id, session)
+        quota = user.quota
+        remaining = None
+        if quota is not None:
+            remaining = user.remaining_quota(session)
+        return (remaining, quota)
+
+
+def get_ftpuser(user_id):
+    with session_query() as session:
+        user = User.load(user_id, session)
+        return user.ftpuser