Including list of common units for formatting purposes in Backend. (#…

…19138)

* Units introduction.

* Removed MongoDB related code

Removed REST endpoints and further remainings of Mongo code. Added JSON file as a source of suported units.

Copying units json for FE in mvn compile phase

Folder for common resources. Data moved from there to BE resources, by Maven

With units hardcoded in file, some functionality does not have sense anymore and was removed

Units are referred by identifiers

List of fields with default units prepared

* Bringing back static factory method to MappedFieldTypeDTO, as requested in review

* Moved supported units obtaining away from constructor, to get method of provider

* Fixed error

* Refactoring

* Do not use fallback, stop GL if there are problems with units file

* resolved resource loading issue

* moving resource file into class package

* Adding additional resource directory instead of copying units file over.

---------

Co-authored-by: Jan Heise <jan.heise@graylog.com>
Co-authored-by: Dennis Oelkers <dennis@graylog.com>

graylog-shared-resources/units/supported_units.json

@@ -0,0 +1,128 @@
+{
+  "units": {
+    "size": [
+      {
+        "type": "base",
+        "abbrev": "b",
+        "name": "byte",
+        "unit_type": "size"
+      },
+      {
+        "type": "derived",
+        "abbrev": "kb",
+        "name": "kilobyte",
+        "unit_type": "size",
+        "conversion": {
+          "value": 1000,
+          "action": "MULTIPLY"
+        }
+      },
+      {
+        "type": "derived",
+        "abbrev": "Mb",
+        "name": "megabyte",
+        "unit_type": "size",
+        "conversion": {
+          "value": 1000000,
+          "action": "MULTIPLY"
+        }
+      },
+      {
+        "type": "derived",
+        "abbrev": "Gb",
+        "name": "gigabyte",
+        "unit_type": "size",
+        "conversion": {
+          "value": 1000000000,
+          "action": "MULTIPLY"
+        }
+      }
+    ],
+    "time": [
+      {
+        "type": "derived",
+        "abbrev": "ns",
+        "name": "nanosecond",
+        "unit_type": "time",
+        "conversion": {
+          "value": 1000000000,
+          "action": "DIVIDE"
+        }
+      },
+      {
+        "type": "derived",
+        "abbrev": "μs",
+        "name": "microsecond",
+        "unit_type": "time",
+        "conversion": {
+          "value": 1000000,
+          "action": "DIVIDE"
+        }
+      },
+      {
+        "type": "derived",
+        "abbrev": "ms",
+        "name": "millisecond",
+        "unit_type": "time",
+        "conversion": {
+          "value": 1000,
+          "action": "DIVIDE"
+        }
+      },
+      {
+        "type": "base",
+        "abbrev": "s",
+        "name": "second",
+        "unit_type": "time"
+      },
+      {
+        "type": "derived",
+        "abbrev": "min",
+        "name": "minute",
+        "unit_type": "time",
+        "conversion": {
+          "value": 60,
+          "action": "MULTIPLY"
+        }
+      },
+      {
+        "type": "derived",
+        "abbrev": "h",
+        "name": "hour",
+        "unit_type": "time",
+        "conversion": {
+          "value": 3600,
+          "action": "MULTIPLY"
+        }
+      },
+      {
+        "type": "derived",
+        "abbrev": "d",
+        "name": "day",
+        "unit_type": "time",
+        "conversion": {
+          "value": 86400,
+          "action": "MULTIPLY"
+        }
+      },
+      {
+        "type": "derived",
+        "abbrev": "m",
+        "name": "month",
+        "unit_type": "time",
+        "conversion": {
+          "value": 2592000,
+          "action": "MULTIPLY"
+        }
+      }
+    ],
+    "percent": [
+      {
+        "type": "base",
+        "abbrev": "%",
+        "name": "percent",
+        "unit_type": "percent"
+      }
+    ]
+  }
+}

graylog2-server/pom.xml

@@ -875,6 +875,13 @@
                     <include>**/*.properties</include>
                 </includes>
             </resource>
+            <resource>
+                <directory>../graylog-shared-resources</directory>
+                <filtering>true</filtering>
+                <includes>
+                    <include>units/**/*</include>
+                </includes>
+            </resource>
         </resources>
         <testResources>
             <testResource>

graylog2-server/src/main/java/org/graylog/plugins/formatting/units/UnitsModule.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.formatting.units;
+
+import org.graylog.plugins.formatting.units.model.SupportedUnits;
+import org.graylog.plugins.formatting.units.provider.SupportedUnitsProvider;
+import org.graylog2.plugin.inject.Graylog2Module;
+
+public class UnitsModule extends Graylog2Module {
+
+    @Override
+    protected void configure() {
+        super.configure();
+        bind(SupportedUnits.class).toProvider(SupportedUnitsProvider.class).asEagerSingleton();
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/formatting/units/fields/FieldUnitObtainer.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.formatting.units.fields;
+
+import jakarta.inject.Inject;
+import org.graylog.plugins.formatting.units.model.Unit;
+
+import java.util.List;
+import java.util.Optional;
+
+public class FieldUnitObtainer {
+
+    private final List<FieldUnitObtainingMethod> prioritizedMethodList;
+
+    @Inject
+    public FieldUnitObtainer(final HardcodedFieldUnitObtainingMethod hardcodedFieldUnitObtainingMethod) {
+        prioritizedMethodList = List.of(hardcodedFieldUnitObtainingMethod);
+    }
+
+    public Optional<Unit> obtainUnit(final String fieldName) {
+        return prioritizedMethodList.stream()
+                .flatMap(fieldUnitObtainingMethod -> fieldUnitObtainingMethod.obtainUnit(fieldName).stream())
+                .findFirst();
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/formatting/units/fields/FieldUnitObtainingMethod.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.formatting.units.fields;
+
+import org.graylog.plugins.formatting.units.model.Unit;
+
+import java.util.Optional;
+
+public interface FieldUnitObtainingMethod {
+
+    Optional<Unit> obtainUnit(final String fieldName);
+}

graylog2-server/src/main/java/org/graylog/plugins/formatting/units/fields/HardcodedFieldUnitObtainingMethod.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.formatting.units.fields;
+
+import jakarta.inject.Inject;
+import org.graylog.plugins.formatting.units.model.SupportedUnits;
+import org.graylog.plugins.formatting.units.model.Unit;
+import org.graylog.plugins.formatting.units.model.UnitId;
+
+import java.util.Optional;
+
+import static org.graylog.schema.HttpFields.HTTP_BYTES;
+import static org.graylog.schema.HttpFields.HTTP_RESPONSE_BYTES;
+import static org.graylog.schema.NetworkFields.NETWORK_BYTES;
+import static org.graylog.schema.NetworkFields.NETWORK_DATA_BYTES;
+import static org.graylog.schema.NetworkFields.NETWORK_HEADER_BYTES;
+import static org.graylog.schema.SourceFields.SOURCE_BYTES_SENT;
+import static org.graylog2.plugin.Message.FIELD_GL2_ACCOUNTED_MESSAGE_SIZE;
+import static org.graylog2.plugin.Message.FIELD_GL2_PROCESSING_DURATION_MS;
+
+public class HardcodedFieldUnitObtainingMethod implements FieldUnitObtainingMethod {
+
+    private final SupportedUnits supportedUnits;
+
+
+    @Inject
+    public HardcodedFieldUnitObtainingMethod(final SupportedUnits supportedUnits) {
+        this.supportedUnits = supportedUnits;
+    }
+
+    @Override
+    public Optional<Unit> obtainUnit(final String fieldName) {
+        return switch (fieldName) {
+            case FIELD_GL2_PROCESSING_DURATION_MS -> supportedUnits.getUnit(new UnitId("time", "ms"));
+            case FIELD_GL2_ACCOUNTED_MESSAGE_SIZE,
+                    HTTP_BYTES,
+                    HTTP_RESPONSE_BYTES,
+                    NETWORK_BYTES,
+                    NETWORK_DATA_BYTES,
+                    NETWORK_HEADER_BYTES,
+                    SOURCE_BYTES_SENT -> supportedUnits.getUnit(new UnitId("size", "B"));
+            default -> Optional.empty();
+        };
+
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/formatting/units/model/BaseUnit.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.formatting.units.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonTypeName(BaseUnit.TYPE)
+public record BaseUnit(@JsonProperty(value = ABBREVIATION, required = true) String abbrev,
+                       @JsonProperty(value = NAME, required = true) String name,
+                       @JsonProperty(value = UNIT_TYPE, required = true) String unitType
+) implements Unit {
+
+    public static final String TYPE = "base";
+
+}

graylog2-server/src/main/java/org/graylog/plugins/formatting/units/model/Conversion.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.formatting.units.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public record Conversion(
+        @JsonProperty(value = "value", required = true) long value,
+        @JsonProperty(value = "action", required = true) ConversionAction action) {
+
+
+    public enum ConversionAction {
+        MULTIPLY, DIVIDE
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/formatting/units/model/DerivedUnit.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.formatting.units.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonTypeName(DerivedUnit.TYPE)
+public record DerivedUnit(@JsonProperty(value = ABBREVIATION, required = true) String abbrev,
+                          @JsonProperty(value = NAME, required = true) String name,
+                          @JsonProperty(value = UNIT_TYPE, required = true) String unitType,
+                          @JsonProperty(value = "conversion", required = true) Conversion conversion) implements Unit {
+
+    public static final String TYPE = "derived";
+
+}