Merge origin/main into topic/tui-take-two

libtenzir/builtins/endpoints/ping.cpp

@@ -0,0 +1,103 @@
+//    _   _____   __________
+//   | | / / _ | / __/_  __/     Visibility
+//   | |/ / __ |_\ \  / /          Across
+//   |___/_/ |_/___/ /_/       Space and Time
+//
+// SPDX-FileCopyrightText: (c) 2023 The Tenzir Contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <tenzir/builtin_rest_endpoints.hpp>
+#include <tenzir/plugin.hpp>
+#include <tenzir/version.hpp>
+
+#include <caf/typed_event_based_actor.hpp>
+
+namespace tenzir::plugins::rest_api::ping {
+
+static auto const* SPEC_V0 = R"_(
+/ping:
+  post:
+    summary: Returns a success response
+    description: Returns a success response to indicate that the node is able to respond to requests. The response body includes the current node version.
+    responses:
+      200:
+        description: OK.
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                version:
+                  type: string
+                  description: The version of the responding node.
+                  example: "v2.3.0-rc3-32-g8529a6c43f"
+            example:
+              version: v2.3.0-rc3-32-g8529a6c43f
+      401:
+        description: Not authenticated.
+    )_";
+
+using ping_handler_actor
+  = typed_actor_fwd<>::extend_with<rest_handler_actor>::unwrap;
+
+struct ping_handler_state {
+  static constexpr auto name = "ping-handler";
+  ping_handler_state() = default;
+};
+
+auto ping_handler(ping_handler_actor::stateful_pointer<ping_handler_state> self)
+  -> ping_handler_actor::behavior_type {
+  return {
+    [self](atom::http_request, uint64_t,
+           const tenzir::record&) -> caf::result<rest_response> {
+      TENZIR_DEBUG("{} handles /ping request", *self);
+      return rest_response{record{{{"version", tenzir::version::version}}}};
+    },
+  };
+}
+
+class plugin final : public virtual rest_endpoint_plugin {
+  auto initialize([[maybe_unused]] const record& plugin_config,
+                  [[maybe_unused]] const record& global_config)
+    -> caf::error override {
+    return {};
+  }
+
+  [[nodiscard]] auto name() const -> std::string override {
+    return "ping";
+  };
+
+  [[nodiscard]] auto openapi_specification(api_version version) const
+    -> data override {
+    if (version != api_version::v0)
+      return tenzir::record{};
+    auto result = from_yaml(SPEC_V0);
+    TENZIR_ASSERT(result);
+    return *result;
+  }
+
+  /// List of API endpoints provided by this plugin.
+  [[nodiscard]] auto rest_endpoints() const
+    -> const std::vector<rest_endpoint>& override {
+    static const auto endpoints = std::vector<rest_endpoint>{
+      {
+        .endpoint_id = static_cast<uint64_t>(0),
+        .method = http_method::post,
+        .path = "/ping",
+        .params = std::nullopt,
+        .version = api_version::v0,
+        .content_type = http_content_type::json,
+      },
+    };
+    return endpoints;
+  }
+
+  auto handler(caf::actor_system& system, node_actor) const
+    -> rest_handler_actor override {
+    return system.spawn(ping_handler);
+  }
+};
+
+} // namespace tenzir::plugins::rest_api::ping
+
+TENZIR_REGISTER_PLUGIN(tenzir::plugins::rest_api::ping::plugin)

libtenzir/builtins/operators/serve.cpp

@@ -74,7 +74,7 @@ namespace tenzir::plugins::serve {
 namespace {
 
 constexpr auto SERVE_ENDPOINT_ID = 0;
-constexpr auto FINAL_CONTINUATION_TOKEN = std::string{"__DONE__"};
+constexpr auto FINAL_CONTINUATION_TOKEN = std::string_view{"__DONE__"};
 
 constexpr auto SPEC_V0 = R"_(
 /serve:

libtenzir/src/detail/series_builders.cpp

@@ -13,11 +13,6 @@
 namespace tenzir::detail {
 
 namespace {
-template <class T>
-constexpr auto is_caf_expected = false;
-
-template <class T>
-constexpr auto is_caf_expected<caf::expected<T>> = true;
 
 template <class T>
 concept allowed_common_type

libtenzir/src/http_api.cpp

@@ -18,7 +18,7 @@
 namespace {
 
 // Validates that the argument is a valid JSON object.
-auto validate_json(const std::string& json) -> bool {
+[[maybe_unused]] auto validate_json(const std::string& json) -> bool {
   auto parser = simdjson::dom::parser{};
   auto padded = simdjson::padded_string{json};
   auto obj = simdjson::dom::object{};

nix/tenzir/plugins/source.json

@@ -2,7 +2,7 @@
   "name": "tenzir-plugins",
   "url": "git@github.com:tenzir/tenzir-plugins",
   "ref": "main",
-  "rev": "15b3c31c9c6121af7d89a6dcea8f083e68cf6c88",
+  "rev": "9596f97779401d7548c271ecb22c88ec13f3aa40",
   "submodules": true,
   "shallow": true
 }

plugins/web/src/specification_command.cpp

@@ -136,12 +136,12 @@ on the command-line using the `tenzir rest generate-token` command.)_"},
       record{{"url", "https://tenzir.example.com/api/v0"}},
     }}},
     {"security", list {{
-      record {{"VastToken", list{}}},
+      record {{"TenzirToken", list{}}},
     }}},
     {"components", record{
       {"schemas", std::move(*schemas)},
       {"securitySchemes",
-        record{{"VastToken", record {
+        record{{"TenzirToken", record {
             {"type", "apiKey"},
             {"in", "header"},
             {"name", "X-Tenzir-Token"}

python/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pytenzir"
-version = "4.0.0-rc8"
+version = "4.0.0-rc9"
 description = "A security telemetry engine for detection and response"
 authors = ["Tenzir <engineering@tenzir.com>"]
 maintainers = ["Tenzir <engineering@tenzir.com>"]

tenzir.yaml.example

@@ -503,11 +503,20 @@ caf:
   # User-defined operators.
   operators:
     # The Zeek operator is an example that takes raw bytes in the form of a
-    # PCAP and then parses Zeek's output via the 'zeek-json` format to generate
+    # PCAP and then parses Zeek's output via the `zeek-json` format to generate
     # a stream of events.
-    zeek: >
+    zeek:
       shell "zeek -r - LogAscii::output_to_stdout=T
              JSONStreaming::disable_default_logs=T
              JSONStreaming::enable_log_rotation=F
              json-streaming-logs"
       | read zeek-json
+    # The Suricata operator is analogous to the above Zeek example, with the
+    # difference that we are using Suricata. The commmand line configures
+    # Suricata such that it reads PCAP on stdin and produces EVE JSON logs on
+    # stdout, which we then parse with the `suricata` format.
+    suricata:
+     shell "suricata -r /dev/stdin
+            --set outputs.1.eve-log.filename=/dev/stdout
+            --set logging.outputs.0.console.enabled=no"
+     | read suricata

version.json

@@ -4,14 +4,14 @@
     "annotated git tag without the leading 'v'.",
     "This value gets updated automatically by `scripts/prepare-release`."
   ],
-  "tenzir-version-fallback": "4.0.0-rc8",
+  "tenzir-version-fallback": "4.0.0-rc9",
   "tenzir-version-rev-count_COMMENT": [
     "This value stores the ancestor count of the tagged commit, calculated",
     "with `git rev-list --count HEAD`, then incremented by 1. This operates",
     "under the assumption that the release-preparing PR contains exactly one",
     "commit and is rebased before merging."
   ],
-  "tenzir-version-rev-count": 17446,
+  "tenzir-version-rev-count": 17626,
   "tenzir-partition-version_COMMENT": [
     "The partition version. This number must be bumped alongside the release",
     "version for releases that contain major format changes to the on-disk",

web/blog/shell-yeah-supercharging-zeek-and-suricata-with-tenzir/index.md

@@ -2,6 +2,7 @@
 title: Shell Yeah! Supercharging Zeek and Suricata with Tenzir
 authors: mavam
 date: 2023-07-20
+last_updated: 2023-07-22
 tags: [zeek, suricata, logs, shell]
 comments: true
 ---
@@ -84,13 +85,13 @@ shell script:
 ```yaml title="tenzir.yaml"
 tenzir:
   operators:
-    zeek: >
+    zeek:
      shell "zeek -r - LogAscii::output_to_stdout=T
             JSONStreaming::disable_default_logs=T
             JSONStreaming::enable_log_rotation=F
             json-streaming-logs"
      | read zeek-json
-    suricata: >
+    suricata:
      shell "suricata -r /dev/stdin
             --set outputs.1.eve-log.filename=/dev/stdout
             --set logging.outputs.0.console.enabled=no"

web/docs/operators/transformations/shell.md

@@ -60,7 +60,7 @@ Using [user-defined operators](../user-defined.md), we can expose this
 ```yaml {0} title="tenzir.yaml"
 tenzir:
   operators:
-    jsonize: >
+    jsonize:
       write json | shell "jq -C" | save stdout
 ```
 
@@ -86,7 +86,7 @@ that:
 ```yaml {0} title="tenzir.yaml"
 tenzir:
   operators:
-    zeek: >
+    zeek:
       shell "zeek -r - LogAscii::output_to_stdout=T
              JSONStreaming::disable_default_logs=T
              JSONStreaming::enable_log_rotation=F

web/docs/operators/user-defined.md

@@ -14,7 +14,7 @@ tenzir:
   operators:
     # Aggregate suricata.flow events with matching source and destination IP
     # addresses.
-    summarize-flows: >
+    summarize-flows:
       where #schema == "suricata.flow"
       | summarize
           pkts_toserver=sum(flow.pkts_toserver),

web/openapi/openapi.yaml

@@ -3,11 +3,11 @@ openapi: 3.0.0
 info:
   title: Tenzir Rest API
   version: "\"0.1\""
-  description: "\nThis API can be used to interact with a Tenzir Node in a RESTful manner.\n\nAll API requests must be authenticated with a valid token, which must be\nsupplied in the `X-Tenzir-Token` request header. The token can be generated\non the command-line using the `vast rest generate-token` command."
+  description: "\nThis API can be used to interact with a Tenzir Node in a RESTful manner.\n\nAll API requests must be authenticated with a valid token, which must be\nsupplied in the `X-Tenzir-Token` request header. The token can be generated\non the command-line using the `tenzir rest generate-token` command."
 servers:
-  - url: https://vast.example.com/api/v0
+  - url: https://tenzir.example.com/api/v0
 security:
-  - VastToken:
+  - TenzirToken:
       []
 components:
   schemas:
@@ -113,11 +113,31 @@ components:
         diagnostics:
           $ref: "#/components/schemas/Diagnostics"
   securitySchemes:
-    VastToken:
+    TenzirToken:
       type: apiKey
       in: header
       name: X-Tenzir-Token
 paths:
+  /ping:
+    get:
+      summary: Returns a success response
+      description: Returns a success response to indicate that the node is able to respond to requests. The response body includes the current node version.
+      responses:
+        200:
+          description: OK.
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  version:
+                    type: string
+                    description: The version of the responding node.
+                    example: v2.3.0-rc3-32-g8529a6c43f
+              example:
+                version: v2.3.0-rc3-32-g8529a6c43f
+        401:
+          description: Not authenticated.
   /pipeline/create:
     post:
       summary: Create a new pipeline
@@ -434,6 +454,10 @@ paths:
                     type: string
                     description: A token to access the next pipeline data batch, null if the pipeline is completed.
                     example: 340ce2j
+                  dropped:
+                    type: integer
+                    description: The serve endpoint will drop responses that exceed the configured maximum message size. The number of events dropped this way is returned here. Should be 0 most of the time.
+                    example: 0
                   schemas:
                     type: array
                     items: