From c7ec8f634ed6052674762b32fa640087d32f165f Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 19:32:58 +1030 Subject: [PATCH] packetbeat/docs: add documentation for ingest pipelines (#37798) (#37815) Also fix typo in winlogbeat load-ingest-pipelines doc. (cherry picked from commit 842c77c95641708c06b2acbbab2be3c088baf7a8) Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- packetbeat/docs/howto/howto.asciidoc | 2 + .../docs/howto/load-ingest-pipelines.asciidoc | 28 +++++++++++++ packetbeat/docs/modules.asciidoc | 41 +++++++++++++++++++ packetbeat/docs/packetbeat-options.asciidoc | 9 ++++ .../docs/howto/load-ingest-pipelines.asciidoc | 2 +- 5 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 packetbeat/docs/howto/load-ingest-pipelines.asciidoc create mode 100644 packetbeat/docs/modules.asciidoc diff --git a/packetbeat/docs/howto/howto.asciidoc b/packetbeat/docs/howto/howto.asciidoc index cdadf3cb7b3..b7284ab3024 100644 --- a/packetbeat/docs/howto/howto.asciidoc +++ b/packetbeat/docs/howto/howto.asciidoc @@ -23,6 +23,8 @@ include::{libbeat-dir}/howto/load-dashboards.asciidoc[] include::{libbeat-dir}/shared-geoip.asciidoc[] +include::load-ingest-pipelines.asciidoc[] + :standalone: include::{libbeat-dir}/shared-env-vars.asciidoc[] :standalone!: diff --git a/packetbeat/docs/howto/load-ingest-pipelines.asciidoc b/packetbeat/docs/howto/load-ingest-pipelines.asciidoc new file mode 100644 index 00000000000..acca824829c --- /dev/null +++ b/packetbeat/docs/howto/load-ingest-pipelines.asciidoc @@ -0,0 +1,28 @@ +[[load-ingest-pipelines]] +== Load ingest pipelines + +{beatname_uc} modules are implemented using {es} ingest node +pipelines. The events receive their transformations within +{es}. The ingest node pipelines must be loaded +into {es}. This can happen one of several ways. + +[id="{beatname_lc}-load-pipeline-auto"] +[float] +=== On connection to {es} + +{beatname_uc} will send ingest pipelines automatically to {es} if the +{es} output is enabled. + +Make sure the user specified in +{beatname_lc}.yml+ is +<>. + +If {beatname_uc} is sending events to {ls} or another output you need +to load the ingest pipelines with the `setup` command or manually. + +[id="{beatname_lc}-load-pipeline-manual"] +[float] +=== Manually install pipelines + +Pipelines can be loaded them into {es} with the `_ingest/pipeline` REST API +call. The user making the REST API call will need to have the `ingest_admin` +role assigned to them. diff --git a/packetbeat/docs/modules.asciidoc b/packetbeat/docs/modules.asciidoc new file mode 100644 index 00000000000..8e72454f9cf --- /dev/null +++ b/packetbeat/docs/modules.asciidoc @@ -0,0 +1,41 @@ +[id="{beatname_lc}-modules"] +[role="xpack"] += Modules + +[partintro] +-- +This section contains detailed information about the available network packet +log processing modules contained in {beatname_uc}. + +{beatname_uc} modules are implemented using Elasticsearch Ingest Node pipelines. +The events receive their transformations within Elasticsearch. All events are +sent through {beatname_uc}'s "routing" pipeline that routes events to specific +module pipelines based on their network protocol. + +{beatname_uc}'s default config file contains the option to send all events to +the routing pipeline. If you remove this option then the module processing +will not be applied. + +[source,yaml,subs="attributes"] +---- +output.elasticsearch.pipeline: packetbeat-%{[agent.version]}-routing +---- + +The general goal of each module is to transform events by renaming fields to +comply with the {ecs-ref}/index.html[Elastic Common Schema] (ECS). The modules +may also apply additional categorization, tagging, and parsing as necessary. +about how to configure the language in `packetbeat`, refer to <>. + +[id="{beatname_lc}-modules-setup"] +[float] +=== Setup of Ingest Node pipelines + +{beatname_uc}'s Ingest Node pipelines must be installed to Elasticsearch if you +want to apply the module processing to events. The simplest way to get started +is to use the Elasticsearch output and {beatname_uc} will automatically install +the pipelines when it first connects to Elasticsearch. + +Installation Methods + +1. <<{beatname_lc}-load-pipeline-auto>> +2. <<{beatname_lc}-load-pipeline-manual>> diff --git a/packetbeat/docs/packetbeat-options.asciidoc b/packetbeat/docs/packetbeat-options.asciidoc index c5cb4d95d6b..c48b4a1b01d 100644 --- a/packetbeat/docs/packetbeat-options.asciidoc +++ b/packetbeat/docs/packetbeat-options.asciidoc @@ -1650,3 +1650,12 @@ Example configuration: ------------------------------------------------------------------------------------- packetbeat.shutdown_timeout: 5s ------------------------------------------------------------------------------------- + +[float] +==== `overwrite_pipelines` + +By default Ingest pipelines are not updated if a pipeline with the same ID +already exists. If this option is enabled {beatname_uc} overwrites pipelines +every time a new Elasticsearch connection is established. + +The default value is `false`. diff --git a/winlogbeat/docs/howto/load-ingest-pipelines.asciidoc b/winlogbeat/docs/howto/load-ingest-pipelines.asciidoc index fa795f0b6b2..0d7f842249e 100644 --- a/winlogbeat/docs/howto/load-ingest-pipelines.asciidoc +++ b/winlogbeat/docs/howto/load-ingest-pipelines.asciidoc @@ -24,7 +24,7 @@ to load the ingest pipelines with the `setup` command or manually. === setup command On a machine that has {beatname_uc} installed and has {es} configured -as the outup, run the `setup` command with the `--pipelines` option +as the output, run the `setup` command with the `--pipelines` option specified. For example, the following command loads the ingest pipelines: