Prepare repository for Tenzir v4.0.0-rc12 (#3427)

This commit is was created with /scripts/prepare-release.

Here is a high-level summary of the changes:

* Updated `/version.json` to v4.0.0-rc12 and the expected rev-count of
the merge commit to 17821.
* Generated a new entry in the docs version selector list.
* Removed the docs for the previous release candidate.
* Moved all changelog entries from `/changelog/next` to
`/changelog/v4.0.0-rc12`.
* Moved all changelog entries from `/changelog/v4.0.0-rc11` to
`/changelog/v4.0.0-rc12`.
* Updated the python bindings version in `/python/pyproject.toml` to
v4.0.0-rc12.

python/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pytenzir"
-version = "4.0.0-rc11"
+version = "4.0.0-rc12"
 description = "A security telemetry engine for detection and response"
 authors = ["Tenzir <engineering@tenzir.com>"]
 maintainers = ["Tenzir <engineering@tenzir.com>"]

version.json

@@ -4,14 +4,14 @@
     "annotated git tag without the leading 'v'.",
     "This value gets updated automatically by `scripts/prepare-release`."
   ],
-  "tenzir-version-fallback": "4.0.0-rc11",
+  "tenzir-version-fallback": "4.0.0-rc12",
   "tenzir-version-rev-count_COMMENT": [
     "This value stores the ancestor count of the tagged commit, calculated",
     "with `git rev-list --count HEAD`, then incremented by 1. This operates",
     "under the assumption that the release-preparing PR contains exactly one",
     "commit and is rebased before merging."
   ],
-  "tenzir-version-rev-count": 17726,
+  "tenzir-version-rev-count": 17823,
   "tenzir-partition-version_COMMENT": [
     "The partition version. This number must be bumped alongside the release",
     "version for releases that contain major format changes to the on-disk",

web/versioned_docs/version-Tenzir v4.0-rc11/developer-guides/build-from-source.md → web/versioned_docs/version-Tenzir v4.0-rc12/developer-guides/build-from-source.md

@@ -34,7 +34,7 @@ Every [release](https://github.com/tenzir/tenzir/releases) of Tenzir includes an
 |✓|[OpenSSL](https://www.openssl.org)||Utilities for secure networking and cryptography.|
 |✓|[FlatBuffers](https://google.github.io/flatbuffers/)|>= 1.12.0|Memory-efficient cross-platform serialization library.|
 |✓|[Boost](https://www.boost.org)|>= 1.81.0|Required as a general utility library.|
-|✓|[Apache Arrow](https://arrow.apache.org)|>= 8.0.0|Required for in-memory data representation. Must be built with Compute, Zstd and Parquet enabled.|
+|✓|[Apache Arrow](https://arrow.apache.org)|>= 12.0.0|Required for in-memory data representation. Must be built with Compute, Zstd and Parquet enabled.|
 |✓|[re2](https://github.com/google/re2)||Required for regular expressione evaluation.|
 |✓|[yaml-cpp](https://github.com/jbeder/yaml-cpp)|>= 0.6.2|Required for reading YAML configuration files.|
 |✓|[simdjson](https://github.com/simdjson/simdjson)|>= 3.1.0|Required for high-performance JSON parsing. (Bundled as submodule.)|

web/versioned_docs/version-Tenzir v4.0-rc11/operators/sinks/serve.md → web/versioned_docs/version-Tenzir v4.0-rc12/operators/sinks/serve.md

@@ -42,7 +42,7 @@ tenzir 'from file path/to/conn.log read zeek-tsv | serve zeek-conn-logs'
 curl \
   -X POST \
   -H "Content-Type: application/json" \
-  -d '{"serve_id": "zeek-conn-logs", "continuation_token": null, "timeout": "1s", max_events": 100}' \
+  -d '{"serve_id": "zeek-conn-logs", "continuation_token": null, "timeout": "1s", "max_events": 100}' \
   http://localhost:5160/api/v0/serve
 ```
 

web/versioned_docs/version-Tenzir v4.0-rc11/operators/sinks/to.md → web/versioned_docs/version-Tenzir v4.0-rc12/operators/sinks/to.md

@@ -48,7 +48,7 @@ to stdout write csv
 Write events to the file `path/to/eve.json` formatted as JSON.
 
 ```
-to file path/to/eve.json | write json
+to file path/to/eve.json write json
 ```
 
 [connectors]: ../../connectors.md

web/versioned_docs/version-Tenzir v4.0-rc12/operators/sources/show.md

@@ -0,0 +1,54 @@
+# show
+
+Returns meta information about Tenzir and nodes.
+
+:::caution Experimental
+This operator is experimental and subject to change without notice, even in
+minor or patch releases.
+:::
+
+## Synopsis
+
+```
+show <aspect> [options]
+```
+
+## Description
+
+The `show` operator offers introspection capabilities to look at various
+*aspects* of Tenzir.
+
+### `<aspect>`
+
+Describes the part of Tenzir to look at.
+
+Available aspects:
+
+- `connectors`: shows all available [connectors](../../connectors.md).
+- `fields`: shows all fields of existing tables at a remote node.
+- `formats`: shows all available [formats](../../formats.md).
+- `operators`: shows all available [operators](../../operators.md).
+- `partitions`: shows all table partitions of a remote node.
+- `types`: shows all known types at a remote node.
+
+## Examples
+
+Show all available connectors and formats:
+
+```
+show connectors
+show formats
+```
+
+Show all transformations:
+
+```
+show operators | where transformation == true
+```
+
+Show all tables and partitions at a node:
+
+```
+show tables
+show partitions
+```

web/versioned_docs/version-Tenzir v4.0-rc11/operators/transformations/summarize.md → web/versioned_docs/version-Tenzir v4.0-rc12/operators/transformations/summarize.md

@@ -96,5 +96,5 @@ Create 1-hour groups and produce a summary of network traffic between host
 pairs:
 
 ```
-summarize sum(bytes_in), sum(bytes_out) by src_ip, dest_ip resolution 1 hour
+summarize sum(bytes_in), sum(bytes_out) by ts, src_ip, dest_ip resolution 1 hour
 ```

web/versioned_docs/version-Tenzir v4.0-rc11/user-guides/import-into-a-node.md → web/versioned_docs/version-Tenzir v4.0-rc12/user-guides/import-into-a-node.md

@@ -13,7 +13,7 @@ Consider this example that takes a Zeek conn.log from our M57 dataset:
 ```
 from file Zeek/conn.log read zeek-tsv
 | select id.orig_h, id.resp_h, orig_bytes, resp_bytes
-| where orig_bytes > 1 MiB
+| where orig_bytes > 1 Mi
 | import
 ```
 

web/versioned_docs/version-Tenzir v4.0-rc12/user-guides/show-available-schemas.md

@@ -0,0 +1,53 @@
+---
+sidebar_position: 5
+---
+
+# Show available schemas
+
+When you write a pipeline, you often reference field names. If you do not know
+the shape of your data, you can look up available
+[schemas](../data-model/schemas.md), i.e., the record types describing a table.
+
+Many SQL databases have a `SHOW TABLES` command to show all available table
+names, and `SHOW COLUMNS` to display the individual fiels of a given table.
+
+Similarly, our [`show`](../operators/sources/show.md) operator offers the
+ability for introspection. Use `show fields` to display all schema fields, with
+with one field per event:
+
+```
+show fields | where schema == "zeek.dns" | write json -c
+```
+
+<details>
+<summary>Output</summary>
+
+```json
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "ts", "path": ["ts"], "index": [0], "type": {"kind": "timestamp", "category": "atomic", "lists": 0, "name": "timestamp", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "uid", "path": ["uid"], "index": [1], "type": {"kind": "string", "category": "atomic", "lists": 0, "name": "", "attributes": [{"key": "index", "value": "hash"}]}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "orig_h", "path": ["id", "orig_h"], "index": [2, 0], "type": {"kind": "ip", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "orig_p", "path": ["id", "orig_p"], "index": [2, 1], "type": {"kind": "port", "category": "atomic", "lists": 0, "name": "port", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "resp_h", "path": ["id", "resp_h"], "index": [2, 2], "type": {"kind": "ip", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "resp_p", "path": ["id", "resp_p"], "index": [2, 3], "type": {"kind": "port", "category": "atomic", "lists": 0, "name": "port", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "proto", "path": ["proto"], "index": [3], "type": {"kind": "string", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "trans_id", "path": ["trans_id"], "index": [4], "type": {"kind": "uint", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "rtt", "path": ["rtt"], "index": [5], "type": {"kind": "duration", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "query", "path": ["query"], "index": [6], "type": {"kind": "string", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "qclass", "path": ["qclass"], "index": [7], "type": {"kind": "uint", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "qclass_name", "path": ["qclass_name"], "index": [8], "type": {"kind": "string", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "qtype", "path": ["qtype"], "index": [9], "type": {"kind": "uint", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "qtype_name", "path": ["qtype_name"], "index": [10], "type": {"kind": "string", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "rcode", "path": ["rcode"], "index": [11], "type": {"kind": "uint", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "rcode_name", "path": ["rcode_name"], "index": [12], "type": {"kind": "string", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "AA", "path": ["AA"], "index": [13], "type": {"kind": "bool", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "TC", "path": ["TC"], "index": [14], "type": {"kind": "bool", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "RD", "path": ["RD"], "index": [15], "type": {"kind": "bool", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "RA", "path": ["RA"], "index": [16], "type": {"kind": "bool", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "Z", "path": ["Z"], "index": [17], "type": {"kind": "uint", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "answers", "path": ["answers"], "index": [18], "type": {"kind": "string", "category": "atomic", "lists": 1, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "TTLs", "path": ["TTLs"], "index": [19], "type": {"kind": "duration", "category": "atomic", "lists": 1, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "rejected", "path": ["rejected"], "index": [20], "type": {"kind": "bool", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+{"schema": "zeek.dns", "schema_id": "1581ec5887691e0b", "field": "_write_ts", "path": ["_write_ts"], "index": [21], "type": {"kind": "time", "category": "atomic", "lists": 0, "name": "", "attributes": []}}
+```
+
+</details>

web/versions.json

@@ -1,5 +1,5 @@
 [
-  "Tenzir v4.0-rc11",
+  "Tenzir v4.0-rc12",
   "VAST v3.1",
   "VAST v3.0"
 ]