remove separate scanner instances

migrate dependency and script management to poetry
add pre-commit hooks for linting and formatting
remove celery, and associated code
decouple scanning logic from database logic as much as possible

.flake8

@@ -2,6 +2,9 @@
 exclude =
     .flake8
     .git
-ignore = E722,  # ignore bare excepts until I can get around to fixing them
-         W504   # line break after binary operator
-max-line-length = 119
+extend-ignore =
+    # black compatibility
+    E203, E704,
+    # ignore bare excepts until I can get around to fixing them
+    E722,
+max-line-length = 120

.git-blame-ignore-revs

@@ -0,0 +1,7 @@
+# .git-blame-ignore-revs
+# run pre-commit against all files
+376413493eca8298b8f2c17c8f02fc0d04285136
+# format all files with black
+1583d515c0a767e2b69b3cdcd785008d2dd0cd0b
+# format all files with isort
+73b2a811edc4f366516c3a63da73fc5e769761e7

.github/workflows/test.yml

@@ -2,35 +2,41 @@ on:
   push:
     branches:
       - master
+      - next
   pull_request:
     branches:
       - master
+      - next
 
 name: Run tests
 
 jobs:
   test:
-    name: nose tests
+    name: lint and test
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.6', '3.7' ]
+        python-version: ["3.11"]
     env:
-      HTTPOBS_BROKER_URL: fakebrokerurl
       HTTPOBS_DATABASE_HOST: fakehost
       HTTPOBS_DATABASE_PASS: foo
       HTTPOBS_DATABASE_USER: bar
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - name: Install poetry
+        run: pipx install poetry
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          python -m pip install . -r requirements.txt
-      - name: Run flake8 tests
-        run: flake8 --config .flake8 httpobs
-      - name: Run nose tests
-        run: nosetests httpobs/tests -e insert_test_result -e scored_test -e select_test_results -e test_retrieve --with-coverage --cover-package=httpobs
+          cache: "poetry"
+      - run: poetry install
+
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit|${{ hashFiles('.pre-commit-config.yaml') }}
+      - name: Run pre-commit against all files
+        run: poetry run pre-commit run --all-files
 
+      - name: Run nose tests
+        run: poetry run nosetests httpobs/tests --with-coverage --cover-package=httpobs

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+default_language_version:
+  python: python3.11
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 23.12.1
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8

CODE_OF_CONDUCT.md

@@ -1,8 +1,8 @@
 # Community Participation Guidelines
 
-This repository is governed by Mozilla's code of conduct and etiquette guidelines. 
+This repository is governed by Mozilla's code of conduct and etiquette guidelines.
 For more details, please read the
-[Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 
+[Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/).
 
 ## How to Report
 For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page.

httpobs/Dockerfile → Dockerfile

@@ -1,6 +1,6 @@
 # http-observatory
 
-FROM python:3.5
+FROM python:3.11
 MAINTAINER https://github.com/mozilla/http-observatory
 
 RUN groupadd --gid 1001 app && \
@@ -9,13 +9,13 @@ RUN install -o app -g app -d /var/run/httpobs /var/log/httpobs
 
 WORKDIR /app
 
-COPY . httpobs
+COPY pyproject.toml poetry.lock .
+RUN pip install poetry && \
+    poetry config virtualenvs.create false && \
+    poetry install
 
-RUN pip install --upgrade --no-cache-dir \
-    -r httpobs/requirements.txt \
-    -r httpobs/database/requirements.txt \
-    -r httpobs/scanner/requirements.txt \
-    -r httpobs/website/requirements.txt
+COPY httpobs httpobs
+RUN poetry install --no-dev
 
 ENV PYTHONPATH $PYTHONPATH:/app
 

README.md

@@ -1,4 +1,4 @@
-# Mozilla HTTP Observatory - [![Build Status](https://travis-ci.org/april/http-observatory.svg?branch=master)](https://travis-ci.org/april/http-observatory) [![Requirements Status](https://requires.io/github/mozilla/http-observatory/requirements.svg?branch=master)](https://requires.io/github/mozilla/http-observatory/requirements/?branch=master)
+# Mozilla HTTP Observatory
 
 The Mozilla HTTP Observatory is a set of tools to analyze your website and inform you if you are utilizing the many available methods to secure it.
 
@@ -16,31 +16,54 @@ Sites can be scanned using:
 * [observatory-cli](https://github.com/mozilla/observatory-cli) - the official node.js command line interface
 * [java-http-observatory-api](https://github.com/stoennies/java-http-observatory-api) - a third party java library and command line interface
 
-## Contributing
+## Development
 
 ### Prerequisites
-* Python 3.7
+* Python 3.11
 * Git
-* pip3
+* pip
 
 #### Notes
 
-These instructions assume that you have a working Python3.7 development environment with `pip3` installed and capable of building requirements, which may require installing an additional python OS package (`-dev`, `-devel`).
+These instructions assume that you have a working Python3.11 development environment with `pip` installed and capable of building requirements, which may require installing an additional python OS package (`-dev`, `-devel`).
 
-If this is not appropriate for your environment, you may install the appropriate requirements using your OS package manager (or other means) and skip the `pip3 -r requirements` command.
+```bash
+# Clone the code
+$ git clone https://github.com/mozilla/http-observatory.git
+$ cd http-observatory
+# Install poetry
+$ pip install poetry
+# Install the project dependencies and scripts
+$ poetry install
+# Activate the virtual environment
+$ poetry shell
+# Install the pre-commit hooks
+$ pre-commit install
+# copy and edit the config file
+$ cp httpobs/conf/httpobs.conf ~/.httpobs.conf
+$ nano ~/.httpobs.conf
+# start the dev server
+$ httpobs-server
+```
+
+### Running tests
+
+```bash
+$ nosetests httpobs/tests --with-coverage --cover-package=httpobs
+```
 
 ## Running a scan from the local codebase, without DB, for continuous integration
 ```bash
 # Install the HTTP Observatory
 $ git clone https://github.com/mozilla/http-observatory.git
 $ cd http-observatory
-$ pip3 install --upgrade .
-$ pip3 install --upgrade -r requirements.txt
+$ pip install poetry
+$ poetry install
 ```
 
-### Using the local scanner function calls
+### Using the scanner function calls
 ```python
->>> from httpobs.scanner.local import scan
+>>> from httpobs.scanner import scan
 >>> scan('observatory.mozilla.org')  # a scan with default options
 >>> scan('observatory.mozilla.org',  # all the custom options
          http_port=8080,             # http server runs on port 8080
@@ -53,75 +76,11 @@ $ pip3 install --upgrade -r requirements.txt
 
 ### The same, but with the local CLI
 ```bash
+$ poetry shell
 $ httpobs-local-scan --http-port 8080 --https-port 8443 --path '/foo/bar' \
     --cookies '{"foo": "bar"}' --headers '{"X-Foo": "bar"}' --no-verify mozilla.org
 ```
 
-## Running a local scanner with Docker
-* Install [Docker Toolbox](https://docs.docker.com/toolbox/overview/) and [VirtualBox](https://www.virtualbox.org/wiki/Downloads)
-
-```bash
-# Install the HTTP Observatory client and requests library
-$ git clone https://github.com/mozilla/http-observatory.git
-$ cd http-observatory
-$ pip3 install .
-$ pip3 install --upgrade requests
-
-# Create docker machine
-$ docker-machine create --driver virtualbox --virtualbox-disk-size "40000" http-observatory
-
-# Save the URL to the API in your .profile, .bash_profile, or whatever
-$ echo export HTTPOBS_API_URL=http://$(docker-machine ip http-observatory):57001/api/v1 >> ~/.profile
-$ . ~/.profile
-
-# Start up the docker instance and install all the pieces
-$ eval $(docker-machine env http-observatory)
-$ docker-compose up -d
-```
-
-## Creating a local installation (tested on Ubuntu 15)
-```
-# Install git, postgresql, and redis
-# sudo -s
-# apt-get install -y git libpq-dev postgresql redis-server
-
-# Clone the repo
-# cd /opt
-# git clone https://github.com/mozilla/http-observatory.git
-# cd http-observatory
-
-# Install the observatory and scanner
-# pip install .
-# pip3 install -r requirements.txt
-
-# Install the database
-# su - postgres
-$ createdb http_observatory
-$ psql http_observatory < httpobs/database/schema.sql
-$ psql http_observatory
-http_observatory=# \password httpobsapi
-http_observatory=# \password httpobsscanner
-# vi /etc/postgresql/9.4/main/postgresql.conf (set max_connections = 512, shared_buffers = 256MB)
-# service postgresql restart
-
-# Create the httpobs user, and log/pid directories
-# useradd -m httpobs
-# install -m 750 -o httpobs -g httpobs -d /var/run/httpobs /var/log/httpobs
-
-# Update the environmental variables
-# su - httpobs
-$ echo export HTTPOBS_API_URL="http://localhost:57001/api/v1" >> ~/.profile
-
-# Start the scanner
-$ cd /opt/http-observatory
-$ HTTPOBS_DATABASE_USER="httpobsscanner" HTTPOBS_DATABASE_PASS="....." \
-    /opt/http-observatory/httpobs/scripts/httpobs-scan-worker
-
-# Start the API (in another terminal)
-# HTTPOBS_DATABASE_USER="httpobsapi" HTTPOBS_DATABASE_PASS="....." \
-    uwsgi --http :57001 --wsgi-file httpobs/website/main.py --processes 8 --callable app --master
-```
-
 ## Authors
 
 * April King

docker-compose.yml

@@ -1,44 +1,23 @@
 version: '2'
 services:
   website:
-    build: ./httpobs
-    command: uwsgi --http :57001 --wsgi-file /app/httpobs/website/main.py --processes 8 --callable app --master
+    build: .
+    command: uwsgi --http :57001 --wsgi-file /app/httpobs/website/main.py --processes 1 --callable app --master
     depends_on:
       - postgres
     environment:
-      - HTTPOBS_BROKER_URL=redis://redis:6379/0
       - HTTPOBS_DATABASE_HOST=postgres
       - HTTPOBS_DATABASE_PASS=httpobsapipassword
       - HTTPOBS_DATABASE_USER=httpobsapi
     links:
       - postgres
     ports:
       - "57001:57001"
-    restart: always
-
-  # celery task for scanner
-  scanner:
-    build: ./httpobs
-    command: /app/httpobs/scripts/httpobs-scan-worker
-    depends_on:
-      - postgres
-      - redis
-    environment:
-      - HTTPOBS_BROKER_URL=redis://redis:6379/0
-      - HTTPOBS_DATABASE_HOST=postgres
-      - HTTPOBS_DATABASE_PASS=httpobsscannerpassword
-      - HTTPOBS_DATABASE_USER=httpobsscanner
-      - HTTPOBS_MAX_CONCURRENCY=16
-    links:
-      - postgres
-      - redis
+    restart: unless-stopped
 
   postgres:
     build: ./httpobs/database
     environment:
       - POSTGRES_USER=httpobs
       - POSTGRES_PASSWORD=totallyfakepassword
       - POSTGRES_DB=http_observatory
-
-  redis:
-    image: redis

httpobs/__init__.py

@@ -1,2 +1,12 @@
 SOURCE_URL = 'https://github.com/mozilla/http-observatory'
 VERSION = '0.9.3'
+
+# The various statuses
+STATE_ABORTED = 'ABORTED'
+STATE_FAILED = 'FAILED'
+STATE_FINISHED = 'FINISHED'
+STATE_PENDING = 'PENDING'
+STATE_STARTING = 'STARTING'
+STATE_RUNNING = 'RUNNING'
+
+STATES = (STATE_ABORTED, STATE_FAILED, STATE_FINISHED, STATE_PENDING, STATE_RUNNING, STATE_STARTING)