Syncing recent changes.

CHANGELOG.md

@@ -16,6 +16,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * GRR server Debian package is removed when github actions are updated. The
   docker image and docker compose stack (see section "Added") are the
   recommended wait of running GRR in the future.
+* Removed the `provides` field from the `Artifact` message. This change has been
+  done in anticipation of the removal of the same field from the official GitHub
+  repository (ForensicArtifacts/artifacts#275).
 
 
 ### Added

colab/grr_colab/fs.py

@@ -13,7 +13,6 @@
 from grr_colab import vfs
 from grr_response_proto import flows_pb2
 from grr_response_proto import jobs_pb2
-from grr_response_server.flows.general import file_finder
 
 
 class FileSystem(object):
@@ -208,9 +207,7 @@ def _collect_file(self, path: Text) -> None:
     args.action.action_type = flows_pb2.FileFinderAction.Action.DOWNLOAD
 
     try:
-      cff = self._client.CreateFlow(
-          name=file_finder.ClientFileFinder.__name__, args=args
-      )
+      cff = self._client.CreateFlow(name='FileFinder', args=args)
     except api_errors.AccessForbiddenError as e:
       raise errors.ApprovalMissingError(self.id, e)
 

colab/grr_colab/vfs.py

@@ -128,7 +128,7 @@ def detach(self) -> None:  # pytype: disable=signature-mismatch  # overriding-re
   def readable(self) -> bool:
     return True
 
-  def read(self, size: int = -1) -> bytes:
+  def read(self, size: int = -1) -> bytes:  # pytype: disable=signature-mismatch
     self._ensure_not_closed()
     size = size or -1
 

devenv/config/grr-server.yaml

@@ -42,22 +42,3 @@ FleetspeakFrontend Context:
 Server.fleetspeak_enabled: true
 Server.fleetspeak_server: localhost:4444
 Server.initialized: true
-
-Frontend.certificate: |
-  -----BEGIN CERTIFICATE-----
-  MIICuTCCAaGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAgMREwDwYDVQQDDAhncnJf
-  dGVzdDELMAkGA1UEBhMCVVMwHhcNMjMwMTIyMTEyMTA2WhcNMzMwMTIwMTEyMTA2
-  WjAOMQwwCgYDVQQDDANncnIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
-  AQDRgX3/3lvGJ2wHO502LFBmNOdN3OHqeo8LNpam0wzDYKevZUpebcCl4aiqYU8g
-  t/Cd+F5TCOnjLHRore7c86yzI0cfk2ytP0bTCQsCR6AUXzlSt87J6x510wGgW5oB
-  pEfdTsBHl+bAm3dzJNA0TzNr2i4VfpV9/L1wEw+Se6lC/J74W+Tjm4cHFtIQcwGt
-  547wBU3CN71XFMrV8LhaIT7FV4jOqiGZLCTSSR0143d9TOeEErxwXyqPMPhIF0Xm
-  ihdd9h6VHq/1L6B0qiKTsGnxdtb0KmBIgs/b9i33PEyAWnTTIcw1eK1ryAulCXjF
-  e7BDiOtVbz43AjSm1iwaWGM9AgMBAAGjEDAOMAwGA1UdEwEB/wQCMAAwDQYJKoZI
-  hvcNAQELBQADggEBAIt3sLvzluPDkWvNoDKnil9HQ8zBlP1sxMlwtCvTDZbIiTuM
-  IK+VL1KuNzGEhpeEbziSpN7ZDUT053xpPYnoZZgQlgLBiNmXJaoHOnj+WAewsK0j
-  vJm7mxLgqdjkXBVyc7jIE/yoZJihygjwDiA3YgvMj/lWZfqU6f57XJERnVDlEUAW
-  QP2YYDStQZvQuwdn/Lie3PfNTIgwkFRoFcrd4tQGrWhH7/pEfSetgeGJLbW56xSl
-  GCXRpNm584CHsx3JzkUNgpM6wl+Jc7arcy8uF6bqbQXOFVL2drgzOFWbI+RXS/74
-  TGQFaywDGAmHCMx/vcLacmwycH8tEWVLFP1DbLo=
-  -----END CERTIFICATE-----

docker-compose.yaml

@@ -1,3 +1,4 @@
+version: "3.8"
 services:
   db:
     image: mysql:8.2
@@ -26,37 +27,48 @@ services:
       retries: 10
 
   grr-admin-ui:
-    image: ghcr.io/google/grr:docker-compose-testing
+    image: ghcr.io/google/grr:latest
     container_name: grr-admin-ui
     hostname: admin-ui
-    restart: always
     depends_on:
       db:
         condition: service_healthy
+      fleetspeak-admin:
+        condition: service_started
     volumes:
-      - ./docker_config_files/server:/configs/
+      - ./docker_config_files:/configs/
+      # Mount a directory for the repacked client installers, so they
+      # can be used in the grr-client container which mounts the same volume.
+      - client_installers:/client_installers
     ports:
       - "8000:8000"
     expose:
       - "8000"
     networks:
       - server-network
-    command:
-      - -component
-      - admin_ui
-      - -config
-      - /configs/grr.server.yaml
-      - --verbose
+    entrypoint: [
+      "/bin/bash",
+      "-c",
+      "/configs/server/repack_clients.sh && grr_server -component admin_ui -config /configs/server/grr.server.yaml --verbose"
+    ]
+    healthcheck:
+      test: |
+        if [[ "$(ls -l /client_installers | wc -l)" == "0" ]]; then
+          echo "Healthckeck: GRR client installer not available"
+          exit 1
+        fi
+      timeout: 10s
+      retries: 10
 
   grr-fleetspeak-frontend:
-    image: ghcr.io/google/grr:docker-compose-testing
+    image: ghcr.io/google/grr:latest
     container_name: grr-fleetspeak-frontend
     hostname: grr-fleetspeak-frontend
     depends_on:
       db:
         condition: service_healthy
     volumes:
-      - ./docker_config_files/server/:/configs/
+      - ./docker_config_files:/configs
     expose:
       - "11111"
     restart: always
@@ -66,11 +78,11 @@ services:
       - -component
       - frontend
       - -config
-      - /configs/grr.server.yaml
+      - /configs/server/grr.server.yaml
       - --verbose
 
   fleetspeak-admin:
-    image: ghcr.io/google/fleetspeak:cl-601031487
+    image: ghcr.io/google/fleetspeak:latest
     container_name: fleetspeak-admin
     hostname: fleetspeak-admin
     depends_on:
@@ -81,20 +93,20 @@ services:
     expose:
       - "4444"
     volumes:
-      - ./docker_config_files/server/:/configs/
+      - ./docker_config_files:/configs
     entrypoint: [
       "server",
       "-components_config",
-      "/configs/textservices/admin.components.config",
+      "/configs/server/textservices/admin.components.config",
       "-services_config",
-      "/configs/grr_frontend.service",
+      "/configs/server/grr_frontend.service",
       "-alsologtostderr",
       "-v",
       "1000"
     ]
 
   fleetspeak-frontend:
-    image: ghcr.io/google/fleetspeak:cl-601031487
+    image: ghcr.io/google/fleetspeak:latest
     container_name: fleetspeak-frontend
     hostname: fleetspeak-frontend
     depends_on:
@@ -106,23 +118,23 @@ services:
       - "4443"
       - "10000"
     volumes:
-      - ./docker_config_files/server/:/configs/
+      - ./docker_config_files:/configs
     entrypoint: [
       "server",
       "-components_config",
-      "/configs/textservices/frontend.components.config",
+      "/configs/server/textservices/frontend.components.config",
       "-services_config",
-      "/configs/grr_frontend.service",
+      "/configs/server/grr_frontend.service",
       "-alsologtostderr",
       "-v",
       "1000"
     ]
 
   grr-worker:
-    image: ghcr.io/google/grr:docker-compose-testing
+    image: ghcr.io/google/grr:latest
     container_name: grr-worker
     volumes:
-      - ./docker_config_files/server/:/configs/
+      - ./docker_config_files:/configs
     hostname: grr-worker
     depends_on:
       db:
@@ -134,27 +146,34 @@ services:
       - -component
       - worker
       - -config
-      - /configs/grr.server.yaml
+      - /configs/server/grr.server.yaml
       - --verbose
 
   grr-client:
-    image: ghcr.io/google/grr:docker-compose-testing
+    image: ubuntu:22.04
     container_name: grr-client
-    restart: always
     depends_on:
-      - db
-      - fleetspeak-frontend
+      db:
+        condition: service_healthy
+      fleetspeak-frontend:
+        condition: service_started
+      grr-admin-ui:
+        # Service is healthy as soon as client installers are repacked.
+        condition: service_healthy
     volumes:
-      - ./docker_config_files/client/:/configs/
-      # Mount the client_installers folder, to preserve
-      # the repacked templates across restarts.
+      - ./docker_config_files:/configs
+      # Mount the client_installers folder which contains the
+      # repacked templates written by the grr-admin-ui container
       - client_installers:/client_installers
+      # Mount the client_state volume to preserve the clients state
+      # including the client_id across restarts.
+      - client_state:/client_state
     networks:
       - server-network
     entrypoint: [
       "/bin/bash",
       "-c",
-      "/configs/repack_install_client.sh && fleetspeak-client -config /configs/client.config"
+      "/configs/client/install_client.sh && fleetspeak-client -config /configs/client/client.config"
     ]
     healthcheck:
       test: |
@@ -168,5 +187,6 @@ services:
 volumes:
   db_data:
   client_installers:
+  client_state:
 networks:
   server-network:

docker_config_files/client/client.config

@@ -1,11 +1,9 @@
 server: "fleetspeak-frontend:4443"
-#  .-.
-# (o.o) WARNING: Publicly stored key. For testing only.
-#  |=|           NEVER reuse in production.
-trusted_certs: "-----BEGIN CERTIFICATE-----\nMIIBhjCCASygAwIBAgIQbZTIkKIjOwVDH5kZDEwz+zAKBggqhkjOPQQDAjAjMSEw\nHwYDVQQDExhGbGVldHNwZWFrIEZsZWV0c3BlYWsgQ0EwHhcNMjQwMTEyMTQ1MTU0\nWhcNMzQwMTA5MTQ1MTU0WjAjMSEwHwYDVQQDExhGbGVldHNwZWFrIEZsZWV0c3Bl\nYWsgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARcKcmCDpGj32sDzRUxBO9E\n9eNg92wGHYYbqHJ5DxqQWVyU8lmE7pPyrZAhVvAAIWQN5pL/MwGRDncOhAciseFW\no0IwQDAOBgNVHQ8BAf8EBAMCAoQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU\nWl3keEC1M5wmeN/+sUTqrtOVgpIwCgYIKoZIzj0EAwIDSAAwRQIgGMUGaqhSEt4Q\n4SkeTjeU2lr4UpO5wCTRJ80SVENoZUICIQDL31xpZF25HQroy9ApHYuxn8C7oUES\n2RvOjey+9sHQzg==\n-----END CERTIFICATE-----\n"
+
+trusted_certs: "%TRUSTED_FLEETSPEAK_CERT%"
 client_label: ""
 filesystem_handler: {
-  configuration_directory:"/configs/"
-  state_file:"/tmp/fleetspeak-client.state"
+  configuration_directory:"/configs/client"
+  state_file:"/client_state/fleetspeak-client.state"
 }
 streaming:true

docker_config_files/client/grr.client.yaml

@@ -1,32 +1,21 @@
 Client.fleetspeak_enabled: true
 ClientBuilder.fleetspeak_bundled: true
-ClientBuilder.template_dir: /client_templates
 Client.server_urls:
 - fleetspeak-frontend
 Client.foreman_check_frequency: 10  # seconds
 
+Config.directory: /configs/client
+
 Logging.verbose: true
 Logging.engines: file,stderr
 Logging.path: /tmp/grr-client
 Logging.filename: /tmp/grr-client/grr-client.log
 
-#  .-.
-# (o.o) WARNING: Publicly stored key. For testing only.
-#  |=|           NEVER reuse in production.
-Client.executable_signing_public_key: |
-  -----BEGIN PUBLIC KEY-----
-  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx6YQNUwITzi7l+biDnwv
-  n63Rg3vbfPZexL/0O1XzQw1Z7mFp3uHtnSrkgDmqYIDXwxDXvn8Ck+k8dYt8SZCc
-  Jq4Jd/YkJXaUiM2E/2Y+Gv33ioVaN7QRyVBGRldK7X6a9Z8tEBE8jF3mlzlO2Z16
-  ZCgMLD1I6ZJpHfQFcDGJP7idHY1TVHJ7j9YG8PObi2k9r5E9UBg6DcFD3Rqg5CP/
-  OUtE56B7VW3y8q49c8pw+ZfiQaXd11xMLuMOX9Brlsp/RqFC6wvM1RJc9oR08Bq8
-  je7ZmTVuwGEUR8snL2eqPqhM1UAvelbEF4IVG9E7A043Fhh7qVPxVGqKSkgfwXS0
-  0QIDAQAB
-  -----END PUBLIC KEY-----
+Client.executable_signing_public_key: "%(/configs/client/public-key.pem|file)"
 
 Target:Linux:
-  ClientBuilder.fleetspeak_client_config: /configs/client.config
+  ClientBuilder.fleetspeak_client_config: /configs/client/client.config
 Target:Windows:
-  ClientBuilder.fleetspeak_client_config: /configs/client.config
+  ClientBuilder.fleetspeak_client_config: /configs/client/client.config
 Target:Darwin:
-  ClientBuilder.fleetspeak_client_config: /configs/client.config
+  ClientBuilder.fleetspeak_client_config: /configs/client/client.config

docker_config_files/client/install_client.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This script is run when the client is started in the docker-compose stack.
+# It installs the provided debian package if no installers or fleetspeak-client
+# binary are found.
+# The client installers are repacked by the admin ui.
+INSTALLERS_DIR="/client_installers"
+
+if ! command -v fleetspeak-client &> /dev/null
+    then
+        echo "**Installing Client from debian package."
+        dpkg -i  ${INSTALLERS_DIR}/grr.client/*.deb
+    else
+        echo "** Found fleetspeak-client binary, skipping install."
+fi
+
+echo "** Completed client setup."

docker_config_files/client/textservices/grr_client.service

@@ -2,10 +2,8 @@ name: "GRR"
 factory: "Daemon"
 config: {
   [type.googleapis.com/fleetspeak.daemonservice.Config]: {
-    argv: "python"
-    argv: "-m"
-    argv: "grr_response_client.client"
-    argv: "--secondary_configs"
-    argv: "/configs/grr.client.yaml"
+    argv: "grrd"
+    argv: "--config"
+    argv: "/configs/client/grr.client.yaml"
   }
 }