Syncing recent changes.

CHANGELOG.md

@@ -14,8 +14,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Removed support for foreman rules using `uname` of an endpoint (this can be
   simulated by using 3 rules for system name, release and version).
 * GRR server Debian package is removed when github actions are updated. The
-  docker image and docker compose stack (see section "Added") are the
+  docker image and Docker Compose stack (see section "Added") are the
   recommended wait of running GRR in the future.
+* Removed the `provides` field from the `Artifact` message. This change has been
+  done in anticipation of the removal of the same field from the official GitHub
+  repository (ForensicArtifacts/artifacts#275).
 
 
 ### Added

appveyor/e2e_tests/run_docker_compose_e2e_test.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 #
-# Runs the e2e tests in the docker-compose stack.
+# Runs the e2e tests in the Docker Compose stack.
 #
-# This script is executed in the grr docker container or in an
+# This script is executed in the grr Docker container or in an
 # environment with the grr src and develpment environment
 # (grr-python-api, grr-test) available. And assumes the
-# docker-compose stack to be running with exposed ports for
+# Docker Compose stack to be running with exposed ports for
 # the admin API and GRR database.
 #
 # Running this test (from the main folder):
-# - Start the docker compose stack with:
-#     $ docker-compose up
+# - Start the Docker Compose stack with:
+#     $ docker compose up
 #
-# - Build and run the GRR docker container and set the entrypoint
+# - Build and run the GRR Docker container and set the entrypoint
 #   to this script:
 #     $ docker build -f ./Dockerfile . -t local-grr-container
 #     $ docker run \
@@ -25,7 +25,7 @@
 
 set -ex
 
-# The IP address of the client inside the docker-compose stack.
+# The IP address of the client inside the Docker Compose stack.
 readonly CLIENT_IP=${1}
 
 readonly GRR_API="http://host.docker.internal:8000"

colab/grr_colab/fs.py

@@ -13,7 +13,6 @@
 from grr_colab import vfs
 from grr_response_proto import flows_pb2
 from grr_response_proto import jobs_pb2
-from grr_response_server.flows.general import file_finder
 
 
 class FileSystem(object):
@@ -208,9 +207,7 @@ def _collect_file(self, path: Text) -> None:
     args.action.action_type = flows_pb2.FileFinderAction.Action.DOWNLOAD
 
     try:
-      cff = self._client.CreateFlow(
-          name=file_finder.ClientFileFinder.__name__, args=args
-      )
+      cff = self._client.CreateFlow(name='FileFinder', args=args)
     except api_errors.AccessForbiddenError as e:
       raise errors.ApprovalMissingError(self.id, e)
 

colab/grr_colab/vfs.py

@@ -128,7 +128,7 @@ def detach(self) -> None:  # pytype: disable=signature-mismatch  # overriding-re
   def readable(self) -> bool:
     return True
 
-  def read(self, size: int = -1) -> bytes:
+  def read(self, size: int = -1) -> bytes:  # pytype: disable=signature-mismatch
     self._ensure_not_closed()
     size = size or -1
 

docker-compose.yaml → compose.yaml

@@ -1,3 +1,4 @@
+version: "3.8"
 services:
   db:
     image: mysql:8.2
@@ -26,37 +27,50 @@ services:
       retries: 10
 
   grr-admin-ui:
-    image: ghcr.io/google/grr:docker-compose-testing
+    image: ghcr.io/google/grr:latest
     container_name: grr-admin-ui
     hostname: admin-ui
-    restart: always
     depends_on:
       db:
         condition: service_healthy
+      fleetspeak-admin:
+        condition: service_started
     volumes:
-      - ./docker_config_files/server:/configs/
+      - ./docker_config_files:/configs/
+      # Mount a directory for the repacked client installers, so they
+      # can be used in the grr-client container which mounts the same volume.
+      - client_installers:/client_installers
     ports:
       - "8000:8000"
     expose:
       - "8000"
     networks:
       - server-network
-    command:
-      - -component
-      - admin_ui
-      - -config
-      - /configs/grr.server.yaml
-      - --verbose
+    entrypoint: [
+      "/bin/bash",
+      "-c",
+      "/configs/server/repack_clients.sh && grr_server -component admin_ui -config /configs/server/grr.server.yaml --verbose"
+    ]
+    healthcheck:
+      # As soon as any files have been written to the /client_installer we
+      # assume the service is healthy.
+      test: |
+        if [[ -z "$(ls /client_installers)" ]]; then
+          echo "Healthckeck: GRR client installer not available"
+          exit 1
+        fi
+      timeout: 10s
+      retries: 10
 
   grr-fleetspeak-frontend:
-    image: ghcr.io/google/grr:docker-compose-testing
+    image: ghcr.io/google/grr:latest
     container_name: grr-fleetspeak-frontend
     hostname: grr-fleetspeak-frontend
     depends_on:
       db:
         condition: service_healthy
     volumes:
-      - ./docker_config_files/server/:/configs/
+      - ./docker_config_files:/configs
     expose:
       - "11111"
     restart: always
@@ -66,11 +80,11 @@ services:
       - -component
       - frontend
       - -config
-      - /configs/grr.server.yaml
+      - /configs/server/grr.server.yaml
       - --verbose
 
   fleetspeak-admin:
-    image: ghcr.io/google/fleetspeak:cl-601031487
+    image: ghcr.io/google/fleetspeak:latest
     container_name: fleetspeak-admin
     hostname: fleetspeak-admin
     depends_on:
@@ -81,20 +95,20 @@ services:
     expose:
       - "4444"
     volumes:
-      - ./docker_config_files/server/:/configs/
+      - ./docker_config_files:/configs
     entrypoint: [
       "server",
       "-components_config",
-      "/configs/textservices/admin.components.config",
+      "/configs/server/textservices/admin.components.config",
       "-services_config",
-      "/configs/grr_frontend.service",
+      "/configs/server/grr_frontend.service",
       "-alsologtostderr",
       "-v",
       "1000"
     ]
 
   fleetspeak-frontend:
-    image: ghcr.io/google/fleetspeak:cl-601031487
+    image: ghcr.io/google/fleetspeak:latest
     container_name: fleetspeak-frontend
     hostname: fleetspeak-frontend
     depends_on:
@@ -106,23 +120,23 @@ services:
       - "4443"
       - "10000"
     volumes:
-      - ./docker_config_files/server/:/configs/
+      - ./docker_config_files:/configs
     entrypoint: [
       "server",
       "-components_config",
-      "/configs/textservices/frontend.components.config",
+      "/configs/server/textservices/frontend.components.config",
       "-services_config",
-      "/configs/grr_frontend.service",
+      "/configs/server/grr_frontend.service",
       "-alsologtostderr",
       "-v",
       "1000"
     ]
 
   grr-worker:
-    image: ghcr.io/google/grr:docker-compose-testing
+    image: ghcr.io/google/grr:latest
     container_name: grr-worker
     volumes:
-      - ./docker_config_files/server/:/configs/
+      - ./docker_config_files:/configs
     hostname: grr-worker
     depends_on:
       db:
@@ -134,27 +148,34 @@ services:
       - -component
       - worker
       - -config
-      - /configs/grr.server.yaml
+      - /configs/server/grr.server.yaml
       - --verbose
 
   grr-client:
-    image: ghcr.io/google/grr:docker-compose-testing
+    image: ubuntu:22.04
     container_name: grr-client
-    restart: always
     depends_on:
-      - db
-      - fleetspeak-frontend
+      db:
+        condition: service_healthy
+      fleetspeak-frontend:
+        condition: service_started
+      grr-admin-ui:
+        # Service is healthy as soon as client installers are repacked.
+        condition: service_healthy
     volumes:
-      - ./docker_config_files/client/:/configs/
-      # Mount the client_installers folder, to preserve
-      # the repacked templates across restarts.
+      - ./docker_config_files:/configs
+      # Mount the client_installers folder which contains the
+      # repacked templates written by the grr-admin-ui container
       - client_installers:/client_installers
+      # Mount the client_state volume to preserve the clients state
+      # including the client_id across restarts.
+      - client_state:/client_state
     networks:
       - server-network
     entrypoint: [
       "/bin/bash",
       "-c",
-      "/configs/repack_install_client.sh && fleetspeak-client -config /configs/client.config"
+      "/configs/client/install_client.sh && fleetspeak-client -config /configs/client/client.config"
     ]
     healthcheck:
       test: |
@@ -168,5 +189,6 @@ services:
 volumes:
   db_data:
   client_installers:
+  client_state:
 networks:
   server-network:

devenv/config/grr-server.yaml

@@ -42,22 +42,3 @@ FleetspeakFrontend Context:
 Server.fleetspeak_enabled: true
 Server.fleetspeak_server: localhost:4444
 Server.initialized: true
-
-Frontend.certificate: |
-  -----BEGIN CERTIFICATE-----
-  MIICuTCCAaGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAgMREwDwYDVQQDDAhncnJf
-  dGVzdDELMAkGA1UEBhMCVVMwHhcNMjMwMTIyMTEyMTA2WhcNMzMwMTIwMTEyMTA2
-  WjAOMQwwCgYDVQQDDANncnIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
-  AQDRgX3/3lvGJ2wHO502LFBmNOdN3OHqeo8LNpam0wzDYKevZUpebcCl4aiqYU8g
-  t/Cd+F5TCOnjLHRore7c86yzI0cfk2ytP0bTCQsCR6AUXzlSt87J6x510wGgW5oB
-  pEfdTsBHl+bAm3dzJNA0TzNr2i4VfpV9/L1wEw+Se6lC/J74W+Tjm4cHFtIQcwGt
-  547wBU3CN71XFMrV8LhaIT7FV4jOqiGZLCTSSR0143d9TOeEErxwXyqPMPhIF0Xm
-  ihdd9h6VHq/1L6B0qiKTsGnxdtb0KmBIgs/b9i33PEyAWnTTIcw1eK1ryAulCXjF
-  e7BDiOtVbz43AjSm1iwaWGM9AgMBAAGjEDAOMAwGA1UdEwEB/wQCMAAwDQYJKoZI
-  hvcNAQELBQADggEBAIt3sLvzluPDkWvNoDKnil9HQ8zBlP1sxMlwtCvTDZbIiTuM
-  IK+VL1KuNzGEhpeEbziSpN7ZDUT053xpPYnoZZgQlgLBiNmXJaoHOnj+WAewsK0j
-  vJm7mxLgqdjkXBVyc7jIE/yoZJihygjwDiA3YgvMj/lWZfqU6f57XJERnVDlEUAW
-  QP2YYDStQZvQuwdn/Lie3PfNTIgwkFRoFcrd4tQGrWhH7/pEfSetgeGJLbW56xSl
-  GCXRpNm584CHsx3JzkUNgpM6wl+Jc7arcy8uF6bqbQXOFVL2drgzOFWbI+RXS/74
-  TGQFaywDGAmHCMx/vcLacmwycH8tEWVLFP1DbLo=
-  -----END CERTIFICATE-----

docker_config_files/client/client.config

@@ -1,11 +1,9 @@
 server: "fleetspeak-frontend:4443"
-#  .-.
-# (o.o) WARNING: Publicly stored key. For testing only.
-#  |=|           NEVER reuse in production.
-trusted_certs: "-----BEGIN CERTIFICATE-----\nMIIBhjCCASygAwIBAgIQbZTIkKIjOwVDH5kZDEwz+zAKBggqhkjOPQQDAjAjMSEw\nHwYDVQQDExhGbGVldHNwZWFrIEZsZWV0c3BlYWsgQ0EwHhcNMjQwMTEyMTQ1MTU0\nWhcNMzQwMTA5MTQ1MTU0WjAjMSEwHwYDVQQDExhGbGVldHNwZWFrIEZsZWV0c3Bl\nYWsgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARcKcmCDpGj32sDzRUxBO9E\n9eNg92wGHYYbqHJ5DxqQWVyU8lmE7pPyrZAhVvAAIWQN5pL/MwGRDncOhAciseFW\no0IwQDAOBgNVHQ8BAf8EBAMCAoQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU\nWl3keEC1M5wmeN/+sUTqrtOVgpIwCgYIKoZIzj0EAwIDSAAwRQIgGMUGaqhSEt4Q\n4SkeTjeU2lr4UpO5wCTRJ80SVENoZUICIQDL31xpZF25HQroy9ApHYuxn8C7oUES\n2RvOjey+9sHQzg==\n-----END CERTIFICATE-----\n"
+
+trusted_certs: "%TRUSTED_FLEETSPEAK_CERT%"
 client_label: ""
 filesystem_handler: {
-  configuration_directory:"/configs/"
-  state_file:"/tmp/fleetspeak-client.state"
+  configuration_directory:"/configs/client"
+  state_file:"/client_state/fleetspeak-client.state"
 }
 streaming:true

docker_config_files/client/grr.client.yaml

@@ -1,32 +1,13 @@
 Client.fleetspeak_enabled: true
-ClientBuilder.fleetspeak_bundled: true
-ClientBuilder.template_dir: /client_templates
 Client.server_urls:
 - fleetspeak-frontend
 Client.foreman_check_frequency: 10  # seconds
 
+Config.directory: /configs/client
+
 Logging.verbose: true
 Logging.engines: file,stderr
 Logging.path: /tmp/grr-client
 Logging.filename: /tmp/grr-client/grr-client.log
 
-#  .-.
-# (o.o) WARNING: Publicly stored key. For testing only.
-#  |=|           NEVER reuse in production.
-Client.executable_signing_public_key: |
-  -----BEGIN PUBLIC KEY-----
-  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx6YQNUwITzi7l+biDnwv
-  n63Rg3vbfPZexL/0O1XzQw1Z7mFp3uHtnSrkgDmqYIDXwxDXvn8Ck+k8dYt8SZCc
-  Jq4Jd/YkJXaUiM2E/2Y+Gv33ioVaN7QRyVBGRldK7X6a9Z8tEBE8jF3mlzlO2Z16
-  ZCgMLD1I6ZJpHfQFcDGJP7idHY1TVHJ7j9YG8PObi2k9r5E9UBg6DcFD3Rqg5CP/
-  OUtE56B7VW3y8q49c8pw+ZfiQaXd11xMLuMOX9Brlsp/RqFC6wvM1RJc9oR08Bq8
-  je7ZmTVuwGEUR8snL2eqPqhM1UAvelbEF4IVG9E7A043Fhh7qVPxVGqKSkgfwXS0
-  0QIDAQAB
-  -----END PUBLIC KEY-----
-
-Target:Linux:
-  ClientBuilder.fleetspeak_client_config: /configs/client.config
-Target:Windows:
-  ClientBuilder.fleetspeak_client_config: /configs/client.config
-Target:Darwin:
-  ClientBuilder.fleetspeak_client_config: /configs/client.config
+Client.executable_signing_public_key: "%(/configs/public-key.pem|file)"

docker_config_files/client/install_client.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This script is run when the client is started in the Docker Compose stack.
+# It installs the provided debian package if no installers or fleetspeak-client
+# binary are found.
+# The client installers are repacked by the admin ui.
+INSTALLERS_DIR="/client_installers"
+
+if ! command -v fleetspeak-client &> /dev/null
+    then
+        echo "**Installing Client from debian package."
+        dpkg -i  ${INSTALLERS_DIR}/*.deb
+    else
+        echo "** Found fleetspeak-client binary, skipping install."
+fi
+
+echo "** Completed client setup."