Library to support Telemetry.Metrics as OpenTelemetry metrics

.github/workflows/elixir.yml

@@ -527,6 +527,45 @@ jobs:
       - name: Test
         run: mix test
 
+  opentelemetry-telemetry-metrics:
+    needs: [test-matrix]
+    if: (contains(github.event.pull_request.labels.*.name, 'elixir') && contains(github.event.pull_request.labels.*.name, 'opentelemetry_telemetry_metrics'))
+    env:
+      app: "opentelemetry_telemetry_metrics"
+    defaults:
+      run:
+        working-directory: utilities/${{ env.app }}
+    runs-on: ubuntu-22.04
+    name: Opentelemetry TelemetryMetrics test on Elixir ${{ matrix.elixir_version }} (OTP ${{ matrix.otp_version }})
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.test-matrix.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: erlef/setup-beam@v1
+        with:
+          version-type: strict
+          otp-version: ${{ matrix.otp_version }}
+          elixir-version: ${{ matrix.elixir_version }}
+          rebar3-version: ${{ matrix.rebar3_version }}
+      - name: Cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/deps
+            ~/_build
+          key: ${{ runner.os }}-build-${{ matrix.otp_version }}-${{ matrix.elixir_version }}-v3-${{ hashFiles('**/mix.lock') }}
+      - name: Fetch deps
+        if: steps.deps-cache.outputs.cache-hit != 'true'
+        run: mix deps.get
+      - name: Compile project
+        run: mix compile --warnings-as-errors
+      - name: Check formatting
+        run: mix format --check-formatted
+        if: matrix.check_formatted
+      - name: Test
+        run: mix test --no-start
+
   opentelemetry-tesla:
     needs: [test-matrix]
     if: (contains(github.event.pull_request.labels.*.name, 'elixir') && contains(github.event.pull_request.labels.*.name, 'opentelemetry_tesla'))

CODEOWNERS

@@ -21,5 +21,6 @@
 /instrumentation/opentelemetry_phoenix          @bryannaegele @tsloughter
 /instrumentation/opentelemetry_redix            @andrewhr
 /utilities/opentelemetry_telemetry              @bryannaegele @tsloughter
+/utilities/opentelemetry_telemetry_metrics      @tsloughter
 /utilities/opentelemetry_instrumentation_http   @tsloughter
 /instrumentation/opentelemetry_tesla            @ricardoccpaiva

utilities/opentelemetry_telemetry_metrics/.formatter.exs

@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+]

utilities/opentelemetry_telemetry_metrics/.gitignore

@@ -0,0 +1,26 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+opentelemetry_telemetry_metrics-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/

utilities/opentelemetry_telemetry_metrics/README.md

@@ -0,0 +1,68 @@
+# OtelTelemetryMetrics
+
+A utility library for creating OpenTelemetry metrics from metric definitions
+done with
+[Telemetry.Metrics](https://github.com/beam-telemetry/telemetry_metrics) and
+their corresponding telemetry events.
+
+## Installation
+
+```elixir
+def deps do
+  [
+    {:opentelemetry_telemetry_metrics, "~> 0.1.0"}
+  ]
+end
+```
+
+## Usage
+
+`OtelTelemetryMetrics.start_link/1` creates OpenTelemetry Instruments for
+`Telemetry.Metric` metrics and records to them when their corresponding events
+are triggered.
+
+``` elixir
+metrics = [
+  Metrics.last_value("vm.memory.binary", unit: :byte),
+  Metrics.counter("vm.memory.total"),
+  Metrics.counter("db.query.duration", tags: [:table, :operation]),
+  Metrics.summary("http.request.response_time",
+    tag_values: fn
+      %{foo: :bar} -> %{bar: :baz}
+    end,
+    tags: [:bar],
+    drop: fn metadata ->
+      metadata[:boom] == :pow
+    end
+  ),
+  Metrics.sum("telemetry.event_size.metadata",
+    unit: {:byte, :megabyte},
+    measurement: &__MODULE__.metadata_measurement/2
+  ),
+  Metrics.distribution("phoenix.endpoint.stop.duration",
+    measurement: &__MODULE__.measurement/1
+  )
+]
+
+{:ok, _} = OtelTelemetryMetrics.start_link([metrics: metrics])
+```
+
+Then either in your Application code or a dependency execute `telemetry` events
+containing the measurements. For example, an event that will result in the
+metrics `vm.memory.total` and `vm.memory.binary` being recorded to:
+
+```elixir
+:telemetry.execute([:vm, :memory], %{binary: 100, total: 200}, %{})
+```
+
+OpenTelemetry does not support a `summary` type metric, the `summary`
+`http.request.response_time` is recorded as a single bucket histogram.
+
+In `Telemetry.Metrics` the `counter` type refers to counting the number of times
+an event is triggered, this is represented as a `sum` in OpenTelemetry and when
+recording the value is sent as a `1` every time.
+
+Metrics of type `last_value` are ignored because `last_value` is not yet an
+aggregation supported on synchronous instruments in Erlang/Elixir OpenTelemetry.
+When it is added to the SDK this library will be updated to no longer ignore
+metrics of this type.

utilities/opentelemetry_telemetry_metrics/lib/otel_telemetry_metrics.ex

@@ -0,0 +1,184 @@
+defmodule OtelTelemetryMetrics do
+  @moduledoc """
+  `OtelTelemetryMetrics.start_link/1` creates OpenTelemetry Instruments for
+  `Telemetry.Metric` metrics and records to them when their corresponding
+  events are triggered.
+
+      metrics = [
+        last_value("vm.memory.binary", unit: :byte),
+        counter("vm.memory.total"),
+        counter("db.query.duration", tags: [:table, :operation]),
+        summary("http.request.response_time",
+          tag_values: fn
+            %{foo: :bar} -> %{bar: :baz}
+          end,
+          tags: [:bar],
+          drop: fn metadata ->
+            metadata[:boom] == :pow
+          end
+        ),
+        sum("telemetry.event_size.metadata",
+          measurement: &__MODULE__.metadata_measurement/2
+        ),
+        distribution("phoenix.endpoint.stop.duration",
+          measurement: &__MODULE__.measurement/1
+        )
+      ]
+
+      {:ok, _} = OtelTelemetryMetrics.start_link([metrics: metrics])
+
+  Then either in your Application code or a dependency execute `telemetry`
+  events conataining the measurements. For example, an event that will result
+  in the metrics `vm.memory.total` and `vm.memory.binary` being recorded to:
+
+      :telemetry.execute([:vm, :memory], %{binary: 100, total: 200}, %{})
+
+  OpenTelemetry does not support a `summary` type metric, the `summary`
+  `http.request.response_time` is recorded as a single bucket histogram.
+
+  In `Telemetry.Metrics` the `counter` type refers to counting the number of
+  times an event is triggered, this is represented as a `sum` in OpenTelemetry
+  and when recording the value is sent as a `1` every time.
+
+  Metrics of type `last_value` are ignored because `last_value` is not yet an
+  aggregation supported on synchronous instruments in Erlang/Elixir
+  OpenTelemetry. When it is added to the SDK this library will be updated to
+  no longer ignore metrics of this type.
+  """
+
+  require Logger
+  use GenServer
+
+  @doc """
+  """
+  def start_link(options) do
+    GenServer.start_link(__MODULE__, options)
+  end
+
+  @impl true
+  def init(options) do
+    Process.flag(:trap_exit, true)
+
+    meter = options[:meter] || :opentelemetry_experimental.get_meter()
+    metrics = options[:metrics] || []
+
+    handler_ids = create_instruments_and_attach(meter, metrics)
+
+    {:ok, %{handler_ids: handler_ids}}
+  end
+
+  @impl true
+  def terminate(_, %{handler_ids: handler_ids}) do
+    detach(handler_ids)
+    :ok
+  end
+
+  defp create_instruments_and_attach(meter, metrics) do
+    metrics_by_event = Enum.group_by(metrics, & &1.event_name)
+
+    for {event_name, metrics} <- metrics_by_event do
+      instruments = create_instruments(meter, metrics)
+
+      attach(meter, event_name, instruments)
+    end
+  end
+
+  defp create_instruments(meter, metrics) do
+    for metric <- metrics,
+        instrument = create_instrument(metric, meter, %{unit: unit(metric.unit)}),
+        into: %{} do
+      {metric, instrument}
+    end
+  end
+
+  defp create_instrument(%Telemetry.Metrics.Counter{}=metric, meter, opts) do
+    :otel_counter.create(meter, format_name(metric), opts)
+  end
+
+  # a summary is represented as an explicit histogram with a single bucket
+  defp create_instrument(%Telemetry.Metrics.Summary{}=metric, meter, opts) do
+    :otel_histogram.create(meter, format_name(metric), Map.put(opts, :advisory_params, %{explicit_bucket_boundaries: []}))
+  end
+
+  defp create_instrument(%Telemetry.Metrics.Distribution{}=metric, meter, opts) do
+    :otel_histogram.create(meter, format_name(metric), opts)
+  end
+
+  defp create_instrument(%Telemetry.Metrics.Sum{}=metric, meter, opts) do
+    :otel_counter.create(meter, format_name(metric), opts)
+  end
+
+  # waiting on
+  defp create_instrument(%Telemetry.Metrics.LastValue{}=metric, _meter, _) do
+    Logger.info("Ignoring metric #{inspect(metric.name)} because LastValue aggregation is not supported in this version of OpenTelemetry Elixir")
+    nil
+  end
+
+  defp unit(:unit), do: 1
+  defp unit(unit), do: unit
+
+  defp format_name(metric) do
+    metric.name
+    |> Enum.join(".")
+    |> String.to_atom
+  end
+
+  defp attach(meter, event_name, instruments) do
+    handler_id = handler_id(event_name)
+
+    :ok =
+      :telemetry.attach(handler_id, event_name, &__MODULE__.handle_event/4,
+        %{meter: meter,
+          instruments: instruments})
+
+    handler_id
+  end
+
+  defp detach(handler_ids) do
+    Enum.each(handler_ids, fn id -> :telemetry.detach(id) end)
+  end
+
+  defp handler_id(event_name) do
+    {__MODULE__, event_name, self()}
+  end
+
+  def handle_event(_event_name, measurements, metadata, %{meter: meter,
+                                                          instruments: instruments}) do
+    for {metric, instrument} <- instruments do
+      if value = keep?(metric, metadata) && extract_measurement(metric, measurements, metadata) do
+        ctx = OpenTelemetry.Ctx.get_current()
+        tags = extract_tags(metric, metadata)
+        :otel_meter.record(ctx, meter, instrument, value, tags)
+      end
+    end
+  end
+
+
+  defp keep?(%{keep: nil}, _metadata), do: true
+  defp keep?(%{keep: keep}, metadata), do: keep.(metadata)
+
+  defp extract_measurement(%Telemetry.Metrics.Counter{}, _measurements, _metadata) do
+    1
+  end
+
+  defp extract_measurement(metric, measurements, metadata) do
+    case metric.measurement do
+      nil ->
+        nil
+
+      fun when is_function(fun, 1) ->
+        fun.(measurements)
+
+      fun when is_function(fun, 2) ->
+        fun.(measurements, metadata)
+
+      key ->
+        measurements[key] || 1
+    end
+  end
+
+  defp extract_tags(metric, metadata) do
+    tag_values = metric.tag_values.(metadata)
+    Map.take(tag_values, metric.tags)
+  end
+end

utilities/opentelemetry_telemetry_metrics/mix.exs

@@ -0,0 +1,37 @@
+defmodule OtelTelemetryMetrics.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :opentelemetry_telemetry_metrics,
+      version: "0.1.0",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps(),
+      source_url: "https://github.com/open-telemetry/opentelemetry-erlang-contrib",
+      homepage_url: "http://github.com/open-telemetry/opentelemetry-erlang-contrib",
+      docs: [main: "OtelTelemetryMetrics",
+             extras: ["README.md"]]
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications.
+  def application do
+    [
+      extra_applications: [:logger]
+    ]
+  end
+
+  # Run "mix help deps" to learn about dependencies.
+  defp deps do
+    [
+      {:telemetry, "~> 1.0"},
+      {:telemetry_metrics, "~> 0.6"},
+      {:opentelemetry_api, path: "/home/tristan/Devel/opentelemetry-erlang/apps/opentelemetry_api", override: true},
+      {:opentelemetry_api_experimental, path: "/home/tristan/Devel/opentelemetry-erlang/apps/opentelemetry_api_experimental", override: true},
+      {:opentelemetry, path: "/home/tristan/Devel/opentelemetry-erlang/apps/opentelemetry", override: true},
+      {:opentelemetry_experimental, path: "/home/tristan/Devel/opentelemetry-erlang/apps/opentelemetry_experimental"},
+      {:ex_doc, "~> 0.31", only: :dev, runtime: false}
+    ]
+  end
+end

utilities/opentelemetry_telemetry_metrics/mix.lock

@@ -0,0 +1,12 @@
+%{
+  "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"},
+  "ex_doc": {:hex, :ex_doc, "0.31.2", "8b06d0a5ac69e1a54df35519c951f1f44a7b7ca9a5bb7a260cd8a174d6322ece", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "317346c14febaba9ca40fd97b5b5919f7751fb85d399cc8e7e8872049f37e0af"},
+  "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"},
+  "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"},
+  "makeup_erlang": {:hex, :makeup_erlang, "0.1.5", "e0ff5a7c708dda34311f7522a8758e23bfcd7d8d8068dc312b5eb41c6fd76eba", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "94d2e986428585a21516d7d7149781480013c56e30c6a233534bedf38867a59a"},
+  "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"},
+  "opentelemetry_api": {:hex, :opentelemetry_api, "1.2.2", "693f47b0d8c76da2095fe858204cfd6350c27fe85d00e4b763deecc9588cf27a", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "dc77b9a00f137a858e60a852f14007bb66eda1ffbeb6c05d5fe6c9e678b05e9d"},
+  "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"},
+  "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},
+  "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.2", "2caabe9344ec17eafe5403304771c3539f3b6e2f7fb6a6f602558c825d0d0bfb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9b43db0dc33863930b9ef9d27137e78974756f5f198cae18409970ed6fa5b561"},
+}