Bump django from 3.2.4 to 3.2.24 in /web

.env

@@ -28,3 +28,10 @@ POSTGRES_HOST=db
 #
 MAX_CONCURRENCY=80
 MIN_CONCURRENCY=10
+
+#
+# Rengine web interface super user (for non-interactive install)
+#
+DJANGO_SUPERUSER_USERNAME=rengine
+DJANGO_SUPERUSER_EMAIL=rengine@example.com
+DJANGO_SUPERUSER_PASSWORD=Sm7IJG.IfHAFw9snSKv

Makefile

@@ -1,3 +1,4 @@
+include .env
 .DEFAULT_GOAL:=help
 
 # Credits: https://github.com/sherifabdlnaby/elastdocker/
@@ -25,7 +26,14 @@ build:			## Build all services.
 	${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} build ${SERVICES}
 
 username:		## Generate Username (Use only after make up).
+ifeq ($(isNonInteractive), true)
+	${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py createsuperuser --username ${DJANGO_SUPERUSER_USERNAME} --email ${DJANGO_SUPERUSER_EMAIL} --noinput
+else
 	${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py createsuperuser
+endif
+
+migrate:		## Apply migrations
+	${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py migrate
 
 pull:			## Pull Docker images.
 	docker login docker.pkg.github.com

docker-compose.yml

@@ -9,6 +9,8 @@ services:
       - POSTGRES_USER=${POSTGRES_USER}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_PORT=${POSTGRES_PORT}
+    ports:
+      - "127.0.0.1:5432:5432"
     volumes:
       - postgres_data:/var/lib/postgresql/data/
     networks:
@@ -94,6 +96,7 @@ services:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_PORT=${POSTGRES_PORT}
       - POSTGRES_HOST=${POSTGRES_HOST}
+      - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD}
       # THIS IS A MUST FOR CHECKING UPDATE, EVERYTIME A COMMIT IS MERGED INTO
       # MASTER, UPDATE THIS!!! MAJOR.MINOR.PATCH https://semver.org/
       - RENGINE_CURRENT_VERSION='2.0.3'
@@ -107,7 +110,7 @@ services:
       - tool_config:/root/.config
       - static_volume:/usr/src/app/staticfiles/
     ports:
-      - "8000:8000"
+      - "127.0.0.1:8000:8000"
     depends_on:
       - db
       - celery

install.sh

@@ -1,21 +1,45 @@
 #!/bin/bash
 
+usageFunction()
+{
+  echo " "
+  tput setaf 2;
+  echo "Usage: $0 (-n) (-h)"
+  echo -e "\t-n Non-interactive installation (Optional)"
+  echo -e "\t-h Show usage"
+  exit 1
+}
+
 tput setaf 2;
 cat web/art/reNgine.txt
 
 tput setaf 1; echo "Before running this script, please make sure Docker is running and you have made changes to .env file."
 tput setaf 2; echo "Changing the postgres username & password from .env is highly recommended."
 
 tput setaf 4;
-read -p "Are you sure, you made changes to .env file (y/n)? " answer
-case ${answer:0:1} in
-    y|Y|yes|YES|Yes )
-      echo "Continiuing Installation!"
-    ;;
-    * )
-      nano .env
-    ;;
-esac
+
+isNonInteractive=false
+while getopts nh opt; do
+   case $opt in
+      n) isNonInteractive=true ;;
+      h) usageFunction ;;
+      ?) usageFunction ;;
+   esac
+done
+
+if [ $isNonInteractive = false ]; then
+    read -p "Are you sure, you made changes to .env file (y/n)? " answer
+    case ${answer:0:1} in
+        y|Y|yes|YES|Yes )
+          echo "Continiuing Installation!"
+        ;;
+        * )
+          nano .env
+        ;;
+    esac
+else
+  echo "Non-interactive installation parameter set. Installation begins."
+fi
 
 echo " "
 tput setaf 3;
@@ -26,7 +50,7 @@ echo "#########################################################################"
 
 echo " "
 tput setaf 4;
-echo "Installing reNgine and it's dependencies"
+echo "Installing reNgine and its dependencies"
 
 echo " "
 if [ "$EUID" -ne 0 ]
@@ -60,6 +84,7 @@ else
   tput setaf 2; echo "Docker installed!!!"
 fi
 
+
 echo " "
 tput setaf 4;
 echo "#########################################################################"
@@ -68,12 +93,13 @@ echo "#########################################################################"
 if [ -x "$(command -v docker-compose)" ]; then
   tput setaf 2; echo "docker-compose already installed, skipping."
 else
-  curl -L "https://github.com/docker/compose/releases/download/v2.23.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+  curl -L "https://github.com/docker/compose/releases/download/v2.5.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
   chmod +x /usr/local/bin/docker-compose
   ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
   tput setaf 2; echo "docker-compose installed!!!"
 fi
 
+
 echo " "
 tput setaf 4;
 echo "#########################################################################"
@@ -100,6 +126,8 @@ else
   exit 1
 fi
 
+
+
 echo " "
 tput setaf 4;
 echo "#########################################################################"
@@ -115,9 +143,10 @@ if [ "${failed}" -eq 0 ]; then
   echo "#########################################################################"
   echo "Creating an account"
   echo "#########################################################################"
-  make username
+  make username isNonInteractive=$isNonInteractive
 
   tput setaf 2 && printf "\n%s\n" "Thank you for installing reNgine, happy recon!!"
+  echo "In case you have unapplied migrations (see above in red), run 'make migrate'"
 else
   tput setaf 1 && printf "\n%s\n" "reNgine installation failed!!"
 fi

make.bat

@@ -15,6 +15,8 @@ if "%1" == "up" docker-compose %COMPOSE_ALL_FILES% up -d --build %SERVICES%
 if "%1" == "build" docker-compose %COMPOSE_ALL_FILES% build %SERVICES%
 :: Generate Username (Use only after make up).
 if "%1" == "username" docker-compose %COMPOSE_ALL_FILES% exec web python3 manage.py createsuperuser
+:: Apply migrations
+if "%1" == "migrate" docker-compose %COMPOSE_ALL_FILES% exec web python3 manage.py migrate
 :: Pull Docker images.
 if "%1" == "pull" docker login docker.pkg.github.com & docker-compose %COMPOSE_ALL_FILES% pull
 :: Down all services.

update.sh

@@ -2,6 +2,7 @@
 
 echo "Do you want to apply your local changes after updating? (y/n)"
 read answer
+answer=$(echo $answer | tr '[:upper:]' '[:lower:]')
 
 if [[ $answer == "y" ]]; then
   make down && git stash save && git pull && git stash apply && make build && make up

web/api/views.py

@@ -547,6 +547,7 @@ def post(self, request):
 		h1_team_handle = data.get('h1_team_handle')
 		description = data.get('description')
 		domain_name = data.get('domain_name')
+		organization_name = data.get('organization')
 		slug = data.get('slug')
 
 		# Validate domain name
@@ -563,6 +564,20 @@ def post(self, request):
 		if not domain.insert_date:
 			domain.insert_date = timezone.now()
 		domain.save()
+
+		# Create org object in DB
+		if organization_name:
+			organization_obj = None
+			organization_query = Organization.objects.filter(name=organization_name)
+			if organization_query.exists():
+				organization_obj = organization_query[0]
+			else:
+				organization_obj = Organization.objects.create(
+					name=organization_name,
+					project=project,
+					insert_date=timezone.now())
+			organization_obj.domains.add(domain)
+
 		return Response({
 			'status': True,
 			'message': 'Domain successfully added as target !',
@@ -712,6 +727,7 @@ def post(self, request):
 				task_ids = scan.celery_ids
 				scan.scan_status = ABORTED_TASK
 				scan.stop_scan_date = timezone.now()
+				scan.aborted_by = request.user
 				scan.save()
 				create_scan_activity(
 					scan.id,
@@ -958,7 +974,7 @@ def get(self, request):
 			return Response({'status': False, 'message': 'Not Found'})
 		elif not response:
 			return Response({'status': False, 'message': 'Not Found'})
-		
+
 		# only send latest release
 		response = response[0]
 
@@ -1806,6 +1822,9 @@ def get_queryset(self):
 
 		subdomains = Subdomain.objects.filter(target_domain__project__slug=project)
 
+		if 'is_important' in req.query_params:
+			subdomains = subdomains.filter(is_important=True)
+
 		if target_id:
 			self.queryset = (
 				subdomains

web/celery-entrypoint.sh

@@ -136,12 +136,6 @@ then
   pip install -r /usr/src/github/CMSeeK/requirements.txt
 fi
 
-if [ ! -d "/usr/src/github/Infoga" ]
-then
-  echo "Cloning Infoga"
-  git clone https://github.com/m4ll0k/Infoga /usr/src/github/Infoga
-fi
-
 # clone ctfr
 if [ ! -d "/usr/src/github/ctfr" ]
 then
@@ -162,29 +156,33 @@ exec "$@"
 # httpx seems to have issue, use alias instead!!!
 echo 'alias httpx="/go/bin/httpx"' >> ~/.bashrc
 
+loglevel='info'
+if [ "$DEBUG" == "1" ]; then
+    loglevel='debug'
+fi
 
 # watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --autoscale=10,0 -l INFO -Q scan_queue &
 echo "Starting Workers..."
 echo "Starting Main Scan Worker with Concurrency: $MAX_CONCURRENCY,$MIN_CONCURRENCY"
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --loglevel=info --autoscale=$MAX_CONCURRENCY,$MIN_CONCURRENCY -Q main_scan_queue &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q initiate_scan_queue -n initiate_scan_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q subscan_queue -n subscan_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=20 --loglevel=info -Q report_queue -n report_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q send_notif_queue -n send_notif_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q send_scan_notif_queue -n send_scan_notif_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q send_task_notif_queue -n send_task_notif_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=5 --loglevel=info -Q send_file_to_discord_queue -n send_file_to_discord_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=5 --loglevel=info -Q send_hackerone_report_queue -n send_hackerone_report_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q parse_nmap_results_queue -n parse_nmap_results_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=20 --loglevel=info -Q geo_localize_queue -n geo_localize_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q query_whois_queue -n query_whois_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q remove_duplicate_endpoints_queue -n remove_duplicate_endpoints_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=50 --loglevel=info -Q run_command_queue -n run_command_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q query_reverse_whois_queue -n query_reverse_whois_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q query_ip_history_queue -n query_ip_history_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=info -Q gpt_queue -n gpt_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q dorking_queue -n dorking_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q osint_discovery_queue -n osint_discovery_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q h8mail_queue -n h8mail_worker &
-watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=info -Q theHarvester_queue -n theHarvester_worker
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --loglevel=$loglevel --autoscale=$MAX_CONCURRENCY,$MIN_CONCURRENCY -Q main_scan_queue &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=$loglevel -Q initiate_scan_queue -n initiate_scan_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=$loglevel -Q subscan_queue -n subscan_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=20 --loglevel=$loglevel -Q report_queue -n report_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q send_notif_queue -n send_notif_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q send_scan_notif_queue -n send_scan_notif_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q send_task_notif_queue -n send_task_notif_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=5 --loglevel=$loglevel -Q send_file_to_discord_queue -n send_file_to_discord_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=5 --loglevel=$loglevel -Q send_hackerone_report_queue -n send_hackerone_report_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q parse_nmap_results_queue -n parse_nmap_results_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=20 --loglevel=$loglevel -Q geo_localize_queue -n geo_localize_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q query_whois_queue -n query_whois_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=$loglevel -Q remove_duplicate_endpoints_queue -n remove_duplicate_endpoints_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=50 --loglevel=$loglevel -Q run_command_queue -n run_command_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q query_reverse_whois_queue -n query_reverse_whois_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q query_ip_history_queue -n query_ip_history_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=30 --loglevel=$loglevel -Q gpt_queue -n gpt_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q dorking_queue -n dorking_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q osint_discovery_queue -n osint_discovery_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q h8mail_queue -n h8mail_worker &
+watchmedo auto-restart --recursive --pattern="*.py" --directory="/usr/src/app/reNgine/" -- celery -A reNgine.tasks worker --pool=gevent --concurrency=10 --loglevel=$loglevel -Q theHarvester_queue -n theHarvester_worker
 exec "$@"

web/dashboard/templates/dashboard/admin.html

@@ -216,7 +216,7 @@ <h4>Users</h4>
           .catch(function() {
             swal.insertQueueStep({
               type: 'error',
-              title: 'Oops! Unable to updte user!'
+              title: 'Oops! Unable to update user!'
             })
           })
           resolve()
@@ -246,6 +246,9 @@ <h4>Users</h4>
     <label for="create_username" class="form-label float-start">Username</label>
     <input id="create_username" class="form-control" placeholder="Username" autocomplete="off">
   </div>
+  <i class="fa fa-info-circle text-muted float-end" data-bs-container="#tooltip-container" data-bs-toggle="tooltip" data-bs-placement="bottom" title="- Sys Admin: Sys Admin is a super user that has permission to modify system and scan related configurations, scan engines, create new users, add new tools etc. Super user can initiate scans and subscans effortlessly.
+- Penetration Tester: Penetration Tester will be allowed to modify and initiate scans and subscans, add or update targets, etc. A penetration tester will not be allowed to modify system configurations.
+- Auditor: Auditor can only view and download the report. An auditor can not change any system or scan related configurations nor can initiate any scans or subscans."></i>
   <div class="form mb-3">
     <label for="create_user_role" class="form-label float-start">Role</label>
     <select class="form-select" id="create_user_role">
@@ -318,7 +321,7 @@ <h4>Users</h4>
           .catch(function() {
             swal.insertQueueStep({
               type: 'error',
-              title: 'Oops! Unable to updte user!'
+              title: 'Oops! Unable to update user!'
             })
           })
           resolve()

web/reNgine/context_processors.py

@@ -1,4 +1,5 @@
 from dashboard.models import *
+import requests
 
 def projects(request):
     projects = Project.objects.all()
@@ -11,3 +12,9 @@ def projects(request):
         'projects': projects,
         'current_project': project
     }
+
+def misc(request):
+    externalIp = requests.get('https://checkip.amazonaws.com').text.strip()
+    return {
+        'external_ip': externalIp
+    }