Merge remote-tracking branch 'er-pa/ursrv-storage' into infra-test-ursrv

* er-pa/ursrv-storage:
  Adjust compatibility env address
  Add copyright notes
  multi-instance support for incoming reports
  code style
  unused param
  wip
  wip
  wip
  cmd/ursrv: Improve data storage and aggregation (fixes #9212)

cmd/ursrv/serve/analytics.go → cmd/ursrv/aggregate/analytics.go

@@ -1,28 +1,23 @@
-// Copyright (C) 2018 The Syncthing Authors.
+// Copyright (C) 2024 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package serve
+package aggregate
 
 import (
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
-)
 
-type analytic struct {
-	Key        string
-	Count      int
-	Percentage float64
-	Items      []analytic `json:",omitempty"`
-}
+	"github.com/syncthing/syncthing/cmd/ursrv/report"
+)
 
-type analyticList []analytic
+type AnalyticList []report.Analytic
 
-func (l analyticList) Less(a, b int) bool {
+func (l AnalyticList) Less(a, b int) bool {
 	if l[a].Key == "Others" {
 		return false
 	}
@@ -32,40 +27,40 @@ func (l analyticList) Less(a, b int) bool {
 	return l[b].Count < l[a].Count // inverse
 }
 
-func (l analyticList) Swap(a, b int) {
+func (l AnalyticList) Swap(a, b int) {
 	l[a], l[b] = l[b], l[a]
 }
 
-func (l analyticList) Len() int {
+func (l AnalyticList) Len() int {
 	return len(l)
 }
 
-// Returns a list of frequency analytics for a given list of strings.
-func analyticsFor(ss []string, cutoff int) []analytic {
+// Returns a list of frequency Analytics for a given list of strings.
+func analyticsFor(ss []string, cutoff int) []report.Analytic {
 	m := make(map[string]int)
 	t := 0
 	for _, s := range ss {
 		m[s]++
 		t++
 	}
 
-	l := make([]analytic, 0, len(m))
+	l := make([]report.Analytic, 0, len(m))
 	for k, c := range m {
-		l = append(l, analytic{
+		l = append(l, report.Analytic{
 			Key:        k,
 			Count:      c,
 			Percentage: 100 * float64(c) / float64(t),
 		})
 	}
 
-	sort.Sort(analyticList(l))
+	sort.Sort(AnalyticList(l))
 
 	if cutoff > 0 && len(l) > cutoff {
 		c := 0
 		for _, i := range l[cutoff:] {
 			c += i.Count
 		}
-		l = append(l[:cutoff], analytic{
+		l = append(l[:cutoff], report.Analytic{
 			Key:        "Others",
 			Count:      c,
 			Percentage: 100 * float64(c) / float64(t),
@@ -76,12 +71,12 @@ func analyticsFor(ss []string, cutoff int) []analytic {
 }
 
 // Find the points at which certain penetration levels are met
-func penetrationLevels(as []analytic, points []float64) []analytic {
+func penetrationLevels(as []report.Analytic, points []float64) []report.Analytic {
 	sort.Slice(as, func(a, b int) bool {
 		return versionLess(as[b].Key, as[a].Key)
 	})
 
-	var res []analytic
+	var res []report.Analytic
 
 	idx := 0
 	sum := 0.0
@@ -145,8 +140,8 @@ func statsForFloats(data []float64) [4]float64 {
 	return res
 }
 
-func group(by func(string) string, as []analytic, perGroup int, otherPct float64) []analytic {
-	var res []analytic
+func group(by func(string) string, as []report.Analytic, perGroup int, otherPct float64) []report.Analytic {
+	var res []report.Analytic
 
 next:
 	for _, a := range as {
@@ -161,19 +156,19 @@ next:
 				continue next
 			}
 		}
-		res = append(res, analytic{
+		res = append(res, report.Analytic{
 			Key:        group,
 			Count:      a.Count,
 			Percentage: a.Percentage,
-			Items:      []analytic{a},
+			Items:      []report.Analytic{a},
 		})
 	}
 
-	sort.Sort(analyticList(res))
+	sort.Sort(AnalyticList(res))
 
 	if otherPct > 0 {
 		// Groups with less than otherPCt go into "Other"
-		other := analytic{
+		other := report.Analytic{
 			Key: "Other",
 		}
 		for i := 0; i < len(res); i++ {

cmd/ursrv/serve/compiler_test.go → cmd/ursrv/aggregate/compiler_test.go

@@ -4,9 +4,11 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package serve
+package aggregate
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestCompilerRe(t *testing.T) {
 	tests := [][3]string{

cmd/ursrv/aggregate/formatting.go

@@ -0,0 +1,45 @@
+// Copyright (C) 2024 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package aggregate
+
+import (
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+var (
+	plusRe  = regexp.MustCompile(`(\+.*|\.dev\..*)$`)
+	plusStr = "(+dev)"
+)
+
+func prettyCase(input string) string {
+	output := ""
+	for i, runeValue := range input {
+		if i == 0 {
+			runeValue = unicode.ToUpper(runeValue)
+		} else if unicode.IsUpper(runeValue) {
+			output += " "
+		}
+		output += string(runeValue)
+	}
+	return output
+}
+
+// transformVersion returns a version number formatted correctly, with all
+// development versions aggregated into one.
+func transformVersion(v string) string {
+	if v == "unknown-dev" {
+		return v
+	}
+	if !strings.HasPrefix(v, "v") {
+		v = "v" + v
+	}
+	v = plusRe.ReplaceAllString(v, " "+plusStr)
+
+	return v
+}

cmd/ursrv/aggregate/util.go

@@ -0,0 +1,52 @@
+// Copyright (C) 2024 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package aggregate
+
+import "github.com/syncthing/syncthing/cmd/ursrv/report"
+
+// add sets a key in a nested map, initializing things if needed as we go.
+func add(storage map[string]map[string]int, parent, child string, value int) {
+	n, ok := storage[parent]
+	if !ok {
+		n = make(map[string]int)
+		storage[parent] = n
+	}
+	n[child] += value
+}
+
+// inc makes sure that even for unused features, we initialize them in the
+// feature map. Furthermore, this acts as a helper that accepts booleans
+// to increment by one, or integers to increment by that integer.
+func inc(storage map[string]int, key string, i interface{}) {
+	cv := storage[key]
+	switch v := i.(type) {
+	case bool:
+		if v {
+			cv++
+		}
+	case int:
+		cv += v
+	}
+	storage[key] = cv
+}
+
+type sortableFeatureList []report.Feature
+
+func (l sortableFeatureList) Less(a, b int) bool {
+	if l[a].Pct != l[b].Pct {
+		return l[a].Pct < l[b].Pct
+	}
+	return l[a].Key > l[b].Key
+}
+
+func (l sortableFeatureList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+
+func (l sortableFeatureList) Len() int {
+	return len(l)
+}