Merge remote-tracking branch 'origin/main' into topic/newGeoIpInputType

tenzir · Apr 28, 2024 · 979ae05 · 979ae05
2 parents be87c21 + 697cc98
commit 979ae05
Show file tree

Hide file tree

Showing 35 changed files with 229 additions and 953 deletions.
diff --git a/.github/workflows/docker-config-base.json b/.github/workflows/docker-config-base.json
@@ -49,16 +49,6 @@
         "622024652768.dkr.ecr.eu-west-1.amazonaws.com",
         "docker.io"
       ]
-    },
-    {
-      "name": "tenzir-ee",
-      "target": "tenzir-ee",
-      "registries": []
-    },
-    {
-      "name": "tenzir-node-ee",
-      "target": "tenzir-node-ee",
-      "registries": []
     }
   ]
 }
diff --git a/.github/workflows/docker-manual.yaml b/.github/workflows/docker-manual.yaml
@@ -14,10 +14,6 @@ on:
         type: boolean
         default: false
         required: false
-      push-ee-to-ecr:
-        type: boolean
-        default: false
-        required: false
       tag:
         type: string
         required: false
@@ -89,15 +85,6 @@ jobs:
                 }
               else
                 empty
-              end,
-              if ${{ inputs.push-ee-to-ecr }} then
-                {
-                  "name": "tenzir-ee", "target": "tenzir-ee", "registries": [
-                   "622024652768.dkr.ecr.eu-west-1.amazonaws.com"
-                  ]
-                }
-              else
-                empty
               end
             ] | ."version-build-metadata" = $vs | ."tags" = $ARGS.positional' \
             --arg vs "${{ steps.version.outputs.version-build-metadata }}" \

diff --git a/.github/workflows/nix-config-base.json b/.github/workflows/nix-config-base.json
@@ -13,16 +13,6 @@
         "docker.io"
       ]
     },
-    {
-      "name": "tenzir-ee",
-      "gha-runner-labels": ["ubuntu-latest"],
-      "static": true,
-      "upload-package-to-github": false,
-      "package-stores": [
-        "gcs:tenzir-dist-private/packages/main"
-      ],
-      "image-registries": []
-    },
     {
       "name": "tenzir",
       "gha-runner-labels": ["macOS", "ARM64"],

diff --git a/.github/workflows/nix-manual.yaml b/.github/workflows/nix-manual.yaml
@@ -10,10 +10,6 @@ on:
         type: boolean
         default: false
         required: false
-      build-ee:
-        type: boolean
-        default: false
-        required: false
       container-tag:
         type: string
         required: false
@@ -85,22 +81,6 @@ jobs:
                 }
               else
                 empty
-              end,
-              if ${{ inputs.build-ee }} then
-                {
-                  "name": "tenzir-ee",
-                  "gha-runner-labels": ["ubuntu-latest"],
-                  "static": true,
-                  "upload-package-to-github": false,
-                  "package-stores": [
-                    "gcs:tenzir-dist-private/packages/manual"
-                  ],
-                  "image-registries": [
-                    "622024652768.dkr.ecr.eu-west-1.amazonaws.com"
-                  ]
-                }
-              else
-                empty
               end
             ] | ."tags" = $ARGS.positional' \
             --args -- "${tags[@]}")"

diff --git a/Dockerfile b/Dockerfile
@@ -236,6 +236,7 @@ RUN cmake -S contrib/tenzir-plugins/vast -B build-vast -G Ninja \
 
 FROM tenzir-de AS tenzir-ce
 
+COPY --from=compaction-plugin --chown=tenzir:tenzir /plugin/compaction /
 COPY --from=context-plugin --chown=tenzir:tenzir /plugin/context /
 COPY --from=matcher-plugin --chown=tenzir:tenzir /plugin/matcher /
 COPY --from=pipeline-manager-plugin --chown=tenzir:tenzir /plugin/pipeline-manager /
@@ -268,18 +269,6 @@ RUN /demo-node/setup.bash
 
 ENTRYPOINT ["tenzir-node"]
 
-# -- tenzir-ee -------------------------------------------------------------------
-
-FROM tenzir-ce AS tenzir-ee
-
-COPY --from=compaction-plugin --chown=tenzir:tenzir /plugin/compaction /
-
-# -- tenzir-node-ee ------------------------------------------------------------
-
-FROM tenzir-ee AS tenzir-node-ee
-
-ENTRYPOINT ["tenzir-node"]
-
 # -- tenzir-node -----------------------------------------------------------------
 
 FROM tenzir-node-ce AS tenzir-node

diff --git a/changelog/next/bug-fixes/4139.md b/changelog/next/bug-fixes/4139.md
@@ -0,0 +1 @@
+The python operator now checks for syntax errors on operator start up.
diff --git a/changelog/v4.12.1/bug-fixes/4149.md b/changelog/v4.12.1/bug-fixes/4149.md
@@ -0,0 +1,5 @@
+We fixed a misconfiguration that caused the `publish` and `subscribe` operators
+not to be available in the statically linked Linux builds.
+
+We fixed a crash on startup when selectively enabling or disabling plugins when
+at least two plugins with dependent plugins were disabled.
diff --git a/flake.nix b/flake.nix
@@ -2,10 +2,9 @@
   description = "Tenzir as a standalone app or NixOS module";
 
   nixConfig = {
-    extra-substituters = ["https://tenzir.cachix.org" "https://vast.cachix.org"];
+    extra-substituters = ["https://tenzir.cachix.org"];
     extra-trusted-public-keys = [
       "tenzir.cachix.org-1:+MLwldLx9GLGLsi9mDr5RrVYyI64iVobWpntJaPM50E="
-      "vast.cachix.org-1:0L8rErLUuFAdspyGYYQK3Sgs9PYRMzkLEqS2GxfaQhA="
     ];
   };
 
@@ -73,22 +72,12 @@
             tenzir-de-static = pkgs.pkgsStatic.tenzir-de;
             tenzir = pkgs.tenzir;
             tenzir-static = pkgs.pkgsStatic.tenzir;
-            tenzir-ee = pkgs.tenzir-ee;
-            tenzir-ee-static = pkgs.pkgsStatic.tenzir-ee;
             integration-test-shell = pkgs.mkShell {
               packages = pkgs.tenzir-integration-test-runner;
             };
           }
           // {
             default = self.packages.${system}.tenzir-static;
-            # Legacy aliases for backwards compatibility.
-            vast = self.packages.${system}.tenzir-de;
-            vast-static = self.packages.${system}.tenzir-de-static;
-            vast-ce = self.packages.${system}.tenzir;
-            vast-ce-static = self.packages.${system}.tenzir-static;
-            vast-ee = self.packages.${system}.tenzir-ee;
-            vast-ee-static = self.packages.${system}.tenzir-ee-static;
-            vast-integration-test-shell = self.packages.${system}.integration-test-shell;
           };
         apps.tenzir-de = flake-utils.lib.mkApp {drv = self.packages.${system}.tenzir-de;};
         apps.tenzir-de-static = flake-utils.lib.mkApp {drv = self.packages.${system}.tenzir-de-static;};
@@ -97,11 +86,6 @@
           drv =
             self.packages.${system}.tenzir-static;
         };
-        apps.tenzir-ee = flake-utils.lib.mkApp {drv = self.packages.${system}.tenzir-ee;};
-        apps.tenzir-ee-static = flake-utils.lib.mkApp {
-          drv =
-            self.packages.${system}.tenzir-ee-static;
-        };
         apps.stream-tenzir-de-image = stream-image {
           entrypoint = "tenzir";
           name = "tenzir/tenzir-de";
@@ -146,28 +130,6 @@
           pkg = self.packages.${system}.tenzir-static;
           tag = "latest-slim";
         };
-        apps.stream-tenzir-ee-image = stream-image {
-          entrypoint = "tenzir";
-          name = "tenzir/tenzir-ee";
-          pkg = self.packages.${system}.tenzir-ee;
-        };
-        apps.stream-tenzir-node-ee-image = stream-image {
-          entrypoint = "tenzir-node";
-          name = "tenzir/tenzir-ee";
-          pkg = self.packages.${system}.tenzir-ee;
-        };
-        apps.stream-tenzir-ee-slim-image = stream-image {
-          entrypoint = "tenzir";
-          name = "tenzir/tenzir-ee-slim";
-          pkg = self.packages.${system}.tenzir-ee-static;
-          tag = "latest-slim";
-        };
-        apps.stream-tenzir-node-ee-slim-image = stream-image {
-          entrypoint = "tenzir-node";
-          name = "tenzir/tenzir-ee-slim";
-          pkg = self.packages.${system}.tenzir-ee-static;
-          tag = "latest-slim";
-        };
         apps.default = self.apps.${system}.tenzir-static;
         # Run with `nix run .#sbom`, output is created in sbom/.
         apps.generate-sbom = let
@@ -210,25 +172,13 @@
           '';
         };
         # Legacy aliases for backwards compatibility.
-        apps.vast = self.apps.${system}.tenzir-de;
-        apps.vast-static = self.apps.${system}.tenzir-de-static;
-        apps.vast-ce = self.apps.${system}.tenzir;
-        apps.vast-ce-static = self.apps.${system}.tenzir-static;
-        apps.vast-ee = self.apps.${system}.tenzir-ee;
-        apps.vast-ee-static = self.apps.${system}.tenzir-ee-static;
-        apps.stream-vast-image = self.apps.${system}.stream-tenzir-de-image;
-        apps.stream-vast-slim-image = self.apps.${system}.stream-tenzir-de-slim-image;
-        apps.stream-vast-ce-image = self.apps.${system}.stream-tenzir-image;
-        apps.stream-vast-ce-slim-image = self.apps.${system}.stream-tenzir-slim-image;
-        apps.stream-vast-ee-image = self.apps.${system}.stream-tenzir-ee-image;
-        apps.stream-vast-ee-slim-image = self.apps.${system}.stream-tenzir-ee-slim-image;
         devShell = import ./shell.nix {inherit pkgs;};
         formatter = pkgs.alejandra;
         hydraJobs =
           {packages = self.packages.${system};}
           // (
             let
-              vast-vm-tests =
+              tenzir-vm-tests =
                 nixpkgs.legacyPackages."${system}".callPackage ./nix/nixos-test.nix
                 {
                   # FIXME: the pkgs channel has an issue made the testing creashed
@@ -238,8 +188,8 @@
             in
               pkgs.lib.optionalAttrs pkgs.stdenv.isLinux {
                 inherit
-                  (vast-vm-tests)
-                  vast-vm-systemd
+                  (tenzir-vm-tests)
+                  tenzir-vm-systemd
                   ;
               }
           );

diff --git a/libtenzir/builtins/operators/openapi.cpp b/libtenzir/builtins/operators/openapi.cpp
@@ -40,13 +40,14 @@ auto openapi_record() -> record {
       "manner.\n\n"
       "All API requests must be authenticated with a valid token, which must "
       "be supplied in the `X-Tenzir-Token` request header. The token can be "
-      "generated on the command-line using `tenzir-ctl web generate-token`.";
+      "generated on the command-line using `tenzir-ctl web generate-token`.\n\n"
+      "All endpoints are versioned, and must be prefixed with `/v0`.";
   auto openapi = record{
     {"openapi", "3.0.0"},
     {"info",
      record{
        {"title", "Tenzir REST API"},
-       {"version", "\"0.1\""},
+       {"version", "\"v0\""},
        {"description", std::move(description)},
      }},
     {

diff --git a/libtenzir/builtins/operators/python.cpp b/libtenzir/builtins/operators/python.cpp
@@ -70,8 +70,9 @@ struct config {
 
 auto drain_pipe(bp::ipstream& pipe) -> std::string {
   auto result = std::string{};
-  if (pipe.peek() == bp::ipstream::traits_type::eof())
+  if (pipe.peek() == bp::ipstream::traits_type::eof()) {
     return result;
+  }
   auto line = std::string{};
   while (std::getline(pipe, line)) {
     if (not result.empty()) {
@@ -155,8 +156,9 @@ class python_operator final : public crtp_operator<python_operator> {
         // verified. We truncate it to this length unconditionally for
         // consistency.
         constexpr auto semaphore_name_max_length = 30u;
-        if (sem_name.size() > semaphore_name_max_length)
+        if (sem_name.size() > semaphore_name_max_length) {
           sem_name.erase(semaphore_name_max_length);
+        }
         // The initial venv creation tends to take a very long time, and often
         // causes the pipline creation to take longer then what our FE tolerate
         // in terms of wait time. As a workaround we yield early, so that the
@@ -265,6 +267,13 @@ class python_operator final : public crtp_operator<python_operator> {
       ::close(errpipe.pipe().native_sink());
       co_yield {}; // signal successful startup
       for (auto&& slice : input) {
+        if (!child.running()) {
+          auto python_error = drain_pipe(errpipe);
+          diagnostic::error("{}", python_error)
+            .note("python process exited with error")
+            .emit(ctrl.diagnostics());
+          co_return;
+        }
         if (slice.rows() == 0) {
           co_yield {};
           continue;
@@ -303,13 +312,17 @@ class python_operator final : public crtp_operator<python_operator> {
         auto reader = arrow::ipc::RecordBatchStreamReader::Open(&file);
         if (!reader.status().ok()) {
           auto python_error = drain_pipe(errpipe);
-          diagnostic::error("{}", python_error).emit(ctrl.diagnostics());
+          diagnostic::error("{}", python_error)
+            .note("python process exited with error")
+            .emit(ctrl.diagnostics());
           co_return;
         }
         auto result_batch = (*reader)->ReadNext();
         if (!result_batch.status().ok()) {
           auto python_error = drain_pipe(errpipe);
-          diagnostic::error("{}", python_error).emit(ctrl.diagnostics());
+          diagnostic::error("{}", python_error)
+            .note("python process exited with error")
+            .emit(ctrl.diagnostics());
           co_return;
         }
         // The writer on the other side writes an invalid record batch as
@@ -383,18 +396,20 @@ class plugin final : public virtual operator_plugin<python_operator> {
     -> caf::error override {
     auto create_virtualenv
       = try_get_or<bool>(plugin_config, "create-venvs", true);
-    if (!create_virtualenv)
+    if (!create_virtualenv) {
       return create_virtualenv.error();
-    if (!(*create_virtualenv))
+    }
+    if (!(*create_virtualenv)) {
       config.venv_base_dir = std::nullopt;
-    else if (const auto* cache_dir
-             = get_if<std::string>(&global_config, "tenzir.cache-directory"))
+    } else if (const auto* cache_dir = get_if<std::string>(
+                 &global_config, "tenzir.cache-directory")) {
       config.venv_base_dir
         = (std::filesystem::path{*cache_dir} / "python" / "venvs").string();
-    else
+    } else {
       config.venv_base_dir = (std::filesystem::temp_directory_path() / "tenzir"
                               / "python" / "venvs")
                                .string();
+    }
     auto implicit_requirements_default = std::string{
       detail::install_datadir() / "python"
       / fmt::format("tenzir-{}.{}.{}-py3-none-any.whl[operator]",