Add support for azure service bus

Makefile

@@ -13,6 +13,7 @@
 IMAGE_NAME := fluent/fluentd-kubernetes
 X86_IMAGES := \
 	v1.13/debian-azureblob:v1.13.3-debian-azureblob-amd64-1.2,v1.13-debian-azureblob-amd64-1 \
+	v1.13/debian-azureservicebus:v1.13.3-debian-azureservicebus-amd64-1.2,v1.13-debian-azureservicebus-amd64-1 \
 	v1.13/debian-elasticsearch7:v1.13.3-debian-elasticsearch7-amd64-1.2,v1.13-debian-elasticsearch7-amd64-1,v1-debian-elasticsearch-amd64 \
 	v1.13/debian-elasticsearch6:v1.13.3-debian-elasticsearch6-amd64-1.2,v1.13-debian-elasticsearch6-amd64-1 \
 	v1.13/debian-loggly:v1.13.3-debian-loggly-amd64-1.1,v1.13-debian-loggly-amd64-1 \
@@ -34,6 +35,7 @@ X86_IMAGES := \
 
 ARM64_IMAGES := \
 	v1.13/arm64/debian-azureblob:v1.13.3-debian-azureblob-arm64-1.2,v1.13-debian-azureblob-arm64-1 \
+	v1.13/arm64/debian-azureservicebus:v1.13.3-debian-azureservicebus-arm64-1.2,v1.13-debian-azureservicebus-arm64-1 \
 	v1.13/arm64/debian-elasticsearch7:v1.13.3-debian-elasticsearch7-arm64-1.2,v1.13-debian-elasticsearch7-arm64-1,v1-debian-elasticsearch-arm64 \
 	v1.13/arm64/debian-elasticsearch6:v1.13.3-debian-elasticsearch6-arm64-1.2,v1.13-debian-elasticsearch6-arm64-1 \
 	v1.13/arm64/debian-loggly:v1.13.3-debian-loggly-arm64-1.1,v1.13-debian-loggly-arm64-1 \

README.md

@@ -24,6 +24,9 @@ If you want to use above non published images, build it by yourself. Dockefile i
 - `Azureblob`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13.3-debian-azureblob-1.2`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13-debian-azureblob-1`
+- `Azureservicebus`
+  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13.3-debian-azureservicebus-1.2`
+  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13-debian-azureservicebus-1`
 - `Elasticsearch7`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13.3-debian-elasticsearch7-1.2`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13-debian-elasticsearch7-1`
@@ -57,6 +60,9 @@ If you want to use above non published images, build it by yourself. Dockefile i
 - `Azureblob` [Dockerfile](docker-image/v1.13/debian-azureblob/Dockerfile)
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13.3-debian-azureblob-amd64-1.2`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13-debian-azureblob-amd64-1`
+- `Azureservicebus` [Dockerfile](docker-image/v1.13/debian-azureservicebus/Dockerfile)
+  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13.3-debian-azureservicebus-amd64-1.2`
+  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13-debian-azureservicebus-amd64-1`
 - `Elasticsearch7` [Dockerfile](docker-image/v1.13/debian-elasticsearch7/Dockerfile)
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13.3-debian-elasticsearch7-amd64-1.2`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13-debian-elasticsearch7-amd64-1`
@@ -105,6 +111,9 @@ If you want to use above non published images, build it by yourself. Dockefile i
 - `Azureblob` [Dockerfile](docker-image/v1.13/arm64/debian-azureblob/Dockerfile)
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13.3-debian-azureblob-arm64-1.2`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13-debian-azureblob-arm64-1`
+- `Azureservicebus` [Dockerfile](docker-image/v1.13/arm64/debian-azureservicebus/Dockerfile)
+  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13.3-debian-azureservicebus-arm64-1.2`
+  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13-debian-azureservicebus-arm64-1`
 - `Elasticsearch7` [Dockerfile](docker-image/v1.13/arm64/debian-elasticsearch7/Dockerfile)
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13.3-debian-elasticsearch7-arm64-1.2`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.13-debian-elasticsearch7-arm64-1`

docker-image/v1.13/arm64/debian-azureservicebus/.dockerignore

@@ -0,0 +1 @@
+**/*.gitkeep

docker-image/v1.13/arm64/debian-azureservicebus/Dockerfile

@@ -0,0 +1,59 @@
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb
+
+# For multiarch build on Docker hub automated build.
+FROM golang:alpine AS builder
+WORKDIR /go
+ENV QEMU_DOWNLOAD_SHA256 a1ef52971537e11915565233f48aa179839f676008d7911c05b3ae94c08c4f5c
+RUN apk add curl --no-cache
+RUN curl -sL -o qemu-3.0.0+resin-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-3.0.0+resin-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-3.0.0+resin-aarch64.tar.gz -C . && mv qemu-3.0.0+resin-aarch64/qemu-aarch64-static .
+
+FROM fluent/fluentd:v1.13.3-debian-arm64-1.0
+COPY --from=builder /go/qemu-aarch64-static /usr/bin/
+
+LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>"
+USER root
+WORKDIR /home/fluent
+ENV PATH /fluentd/vendor/bundle/ruby/2.6.0/bin:$PATH
+ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.6.0
+ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.6.0
+# skip runtime bundler installation
+ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
+
+COPY Gemfile* /fluentd/
+RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev git" \
+  runtimeDeps="" \
+      && apt-get update \
+     && apt-get upgrade -y \
+     && apt-get install \
+     -y --no-install-recommends \
+     $buildDeps $runtimeDeps net-tools \
+    && gem install bundler --version 2.2.24 \
+    && bundle config silence_root_warning true \
+    && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \
+    && SUDO_FORCE_REMOVE=yes \
+    apt-get purge -y --auto-remove \
+                  -o APT::AutoRemove::RecommendsImportant=false \
+                  $buildDeps \
+ && rm -rf /var/lib/apt/lists/* \
+    && gem sources --clear-all \
+    && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
+# Copy configuration files
+COPY ./conf/fluent.conf /fluentd/etc/
+COPY ./conf/systemd.conf /fluentd/etc/
+COPY ./conf/kubernetes.conf /fluentd/etc/
+COPY ./conf/prometheus.conf /fluentd/etc/
+COPY ./conf/tail_container_parse.conf /fluentd/etc/
+COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/
+RUN touch /fluentd/etc/disable.conf
+
+# Copy plugins
+COPY plugins /fluentd/plugins/
+COPY entrypoint.sh /fluentd/entrypoint.sh
+
+# Environment variables
+ENV FLUENTD_OPT=""
+ENV FLUENTD_CONF="fluent.conf"
+
+# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib
+ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"]

docker-image/v1.13/arm64/debian-azureservicebus/Gemfile

@@ -0,0 +1,20 @@
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb
+
+source "https://rubygems.org"
+
+gem "fluentd", "1.13.3"
+gem "oj", "3.11.0"
+gem "fluent-plugin-multi-format-parser", "~> 1.0.0"
+gem "fluent-plugin-concat", "~> 2.4.0"
+gem "fluent-plugin-grok-parser", "~> 2.6.2"
+gem "fluent-plugin-prometheus", "~> 1.8.5"
+gem 'fluent-plugin-json-in-json-2', ">= 1.0.2"
+gem "fluent-plugin-record-modifier", "~> 2.1.0"
+gem "fluent-plugin-detect-exceptions", "~> 0.0.13"
+gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0"
+gem "fluent-plugin-parser-cri", "~> 0.1.0"
+gem "fluent-plugin-servicebus-queue", :git => "git://github.com/Azure/fluent-plugin-servicebus-queue.git"
+gem "fluent-plugin-kubernetes_metadata_filter", "~> 2.7.1"
+gem "ffi"
+gem "fluent-plugin-systemd", "~> 1.0.5"

docker-image/v1.13/arm64/debian-azureservicebus/Gemfile.lock

@@ -0,0 +1,138 @@
+GIT
+  remote: git://github.com/Azure/fluent-plugin-servicebus-queue.git
+  revision: dd4bdb5a33d4dacdda2bbe845b430715164e32fd
+  specs:
+    fluent-plugin-servicebus-queue (0.1.0)
+      fluentd (>= 0.14.0, < 2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    concurrent-ruby (1.1.9)
+    cool.io (1.7.1)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    ffi (1.15.3)
+    ffi-compiler (1.0.1)
+      ffi (>= 1.0.0)
+      rake
+    fluent-config-regexp-type (1.0.0)
+      fluentd (> 1.0.0, < 2)
+    fluent-plugin-concat (2.4.0)
+      fluentd (>= 0.14.0, < 2)
+    fluent-plugin-detect-exceptions (0.0.13)
+      fluentd (>= 0.10)
+    fluent-plugin-grok-parser (2.6.2)
+      fluentd (>= 0.14.6, < 2)
+    fluent-plugin-json-in-json-2 (1.0.2)
+      fluentd (>= 0.14.0, < 2)
+      yajl-ruby (~> 1.0)
+    fluent-plugin-kubernetes_metadata_filter (2.7.2)
+      fluentd (>= 0.14.0, < 1.14)
+      kubeclient (< 5)
+      lru_redux
+    fluent-plugin-multi-format-parser (1.0.0)
+      fluentd (>= 0.14.0, < 2)
+    fluent-plugin-parser-cri (0.1.1)
+      fluentd (>= 1)
+    fluent-plugin-prometheus (1.8.5)
+      fluentd (>= 1.9.1, < 2)
+      prometheus-client (< 0.10)
+    fluent-plugin-record-modifier (2.1.0)
+      fluentd (>= 1.0, < 2)
+    fluent-plugin-rewrite-tag-filter (2.4.0)
+      fluent-config-regexp-type
+      fluentd (>= 0.14.2, < 2)
+    fluent-plugin-systemd (1.0.5)
+      fluentd (>= 0.14.11, < 2)
+      systemd-journal (~> 1.4.2)
+    fluentd (1.13.3)
+      bundler
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.8.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
+      yajl-ruby (~> 1.0)
+    http (4.4.1)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.2)
+      http-parser (~> 1.2.0)
+    http-accept (1.7.0)
+    http-cookie (1.0.4)
+      domain_name (~> 0.5)
+    http-form_data (2.3.0)
+    http-parser (1.2.3)
+      ffi-compiler (>= 1.0, < 2.0)
+    http_parser.rb (0.7.0)
+    jsonpath (1.1.0)
+      multi_json
+    kubeclient (4.9.2)
+      http (>= 3.0, < 5.0)
+      jsonpath (~> 1.0)
+      recursive-open-struct (~> 1.1, >= 1.1.1)
+      rest-client (~> 2.0)
+    lru_redux (1.1.0)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2021.0704)
+    msgpack (1.4.2)
+    multi_json (1.15.0)
+    netrc (0.11.0)
+    oj (3.11.0)
+    prometheus-client (0.9.0)
+      quantile (~> 0.2.1)
+    public_suffix (4.0.6)
+    quantile (0.2.1)
+    rake (13.0.6)
+    recursive-open-struct (1.1.3)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    serverengine (2.2.4)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.5)
+    systemd-journal (1.4.2)
+      ffi (~> 1.9)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2021.1)
+      tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    webrick (1.7.0)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  ffi
+  fluent-plugin-concat (~> 2.4.0)
+  fluent-plugin-detect-exceptions (~> 0.0.13)
+  fluent-plugin-grok-parser (~> 2.6.2)
+  fluent-plugin-json-in-json-2 (>= 1.0.2)
+  fluent-plugin-kubernetes_metadata_filter (~> 2.7.1)
+  fluent-plugin-multi-format-parser (~> 1.0.0)
+  fluent-plugin-parser-cri (~> 0.1.0)
+  fluent-plugin-prometheus (~> 1.8.5)
+  fluent-plugin-record-modifier (~> 2.1.0)
+  fluent-plugin-rewrite-tag-filter (~> 2.4.0)
+  fluent-plugin-servicebus-queue!
+  fluent-plugin-systemd (~> 1.0.5)
+  fluentd (= 1.13.3)
+  oj (= 3.11.0)
+
+BUNDLED WITH
+   2.2.22

docker-image/v1.13/arm64/debian-azureservicebus/conf/fluent.conf

@@ -0,0 +1,24 @@
+
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb
+
+@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf"
+@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf"
+@include kubernetes.conf
+@include conf.d/*.conf
+
+<match **>
+    @type azure_servicebus_queue
+    @id out_azure_servicebus_queue
+    namespace "#{ENV['AZURESERVICEBUS_NAMESPACE']}"
+    queueName "#{ENV['AZURESERVICEBUS_QUEUENAME']}"
+    accessKeyName "#{ENV['AZURESERIVCEBUS_ACCESSKEYNAME']}"
+    accessKeyValueFile "#{ENV['AZURESERIVCEBUS_ACCESSKEYVALUEFILE']}"
+    timeToLive "#{ENV['AZURESERIVCEBUS_TIMETOLIVE']}"
+    field "#{ENV['AZURESERVICEBUS_FIELD'] || 'message'}"
+    <buffer>
+        @type memory
+        flush_interval 1s
+    </buffer>
+</match>
+

docker-image/v1.13/arm64/debian-azureservicebus/conf/kubernetes.conf

@@ -0,0 +1,39 @@
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb
+
+<label @FLUENT_LOG>
+  <match fluent.**>
+    @type null
+    @id ignore_fluent_logs
+  </match>
+</label>
+
+@include kubernetes/cluster-autoscaler.conf
+@include kubernetes/containers.conf
+@include kubernetes/docker.conf
+@include kubernetes/etcd.conf
+@include kubernetes/glbc.conf
+@include kubernetes/kube-apiserver-audit.conf
+@include kubernetes/kube-apiserver.conf
+@include kubernetes/kube-controller-manager.conf
+@include kubernetes/kube-proxy.conf
+@include kubernetes/kube-scheduler.conf
+@include kubernetes/kubelet.conf
+@include kubernetes/rescheduler.conf
+@include kubernetes/salt.conf
+@include kubernetes/startupscript.conf
+
+
+<filter kubernetes.**>
+  @type kubernetes_metadata
+  @id filter_kube_metadata
+  kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
+  verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
+  ca_file "#{ENV['KUBERNETES_CA_FILE']}"
+  skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
+  skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
+  skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
+  skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
+  watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}"
+</filter>
+

docker-image/v1.13/arm64/debian-azureservicebus/conf/kubernetes/cluster-autoscaler.conf

@@ -0,0 +1,11 @@
+<source>
+  @type tail
+  @id in_tail_cluster_autoscaler
+  multiline_flush_interval 5s
+  path /var/log/cluster-autoscaler.log
+  pos_file /var/log/fluentd-cluster-autoscaler.log.pos
+  tag cluster-autoscaler
+  <parse>
+    @type kubernetes
+  </parse>
+</source>

docker-image/v1.13/arm64/debian-azureservicebus/conf/kubernetes/containers.conf

@@ -0,0 +1,13 @@
+<source>
+  @type tail
+  @id in_tail_container_logs
+  path /var/log/containers/*.log
+  pos_file /var/log/fluentd-containers.log.pos
+  tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}"
+  exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}"
+  read_from_head true
+  <parse>
+    @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}"
+    time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}"
+  </parse>
+</source>

docker-image/v1.13/arm64/debian-azureservicebus/conf/kubernetes/docker.conf

@@ -0,0 +1,11 @@
+<source>
+  @type tail
+  @id in_tail_docker
+  path /var/log/docker.log
+  pos_file /var/log/fluentd-docker.log.pos
+  tag docker
+  <parse>
+    @type regexp
+    expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
+  </parse>
+</source>

docker-image/v1.13/arm64/debian-azureservicebus/conf/kubernetes/etcd.conf

@@ -0,0 +1,10 @@
+<source>
+  @type tail
+  @id in_tail_etcd
+  path /var/log/etcd.log
+  pos_file /var/log/fluentd-etcd.log.pos
+  tag etcd
+  <parse>
+    @type none
+  </parse>
+</source>

docker-image/v1.13/arm64/debian-azureservicebus/conf/kubernetes/glbc.conf

@@ -0,0 +1,11 @@
+<source>
+  @type tail
+  @id in_tail_glbc
+  multiline_flush_interval 5s
+  path /var/log/glbc.log
+  pos_file /var/log/fluentd-glbc.log.pos
+  tag glbc
+  <parse>
+    @type kubernetes
+  </parse>
+</source>