Merge branch 'geoip' into infrastructure

* geoip:
  wip
  build: Use Go 1.22.3 at minimum
  build: Use Go 1.22.3 at minimum
  gui: Add Hindi (hi) translation template (#9530)
  wip
  wip
  wip
  wip
  muddier error handling to satsify deepsource... :/
  Limit maximum database size
  cmd/relaypoolsrv, cmd/ursrv: Automatically manage GeoIP updates

.github/workflows/build-infra-dockers.yaml

@@ -7,7 +7,7 @@ on:
       - infra-*
 
 env:
-  GO_VERSION: "~1.22.0"
+  GO_VERSION: "~1.22.3"
   CGO_ENABLED: "0"
   BUILD_USER: docker
   BUILD_HOST: github.syncthing.net

.github/workflows/build-syncthing.yaml

@@ -12,7 +12,7 @@ env:
   # The go version to use for builds. We set check-latest to true when
   # installing, so we get the latest patch version that matches the
   # expression.
-  GO_VERSION: "~1.22.0"
+  GO_VERSION: "~1.22.3"
 
   # Optimize compatibility on the slow archictures.
   GO386: softfloat
@@ -48,7 +48,7 @@ jobs:
         runner: ["windows-latest", "ubuntu-latest", "macos-latest"]
         # The oldest version in this list should match what we have in our go.mod.
         # Variables don't seem to be supported here, or we could have done something nice.
-        go: ["~1.21.7", "~1.22.0"]
+        go: ["~1.21.7", "~1.22.3"]
     runs-on: ${{ matrix.runner }}
     steps:
       - name: Set git to use LF

Dockerfile.strelaypoolsrv

@@ -11,14 +11,6 @@ LABEL org.opencontainers.image.authors="The Syncthing Project" \
 
 EXPOSE 8080
 
-RUN apk add --no-cache ca-certificates su-exec curl
-ENV PUID=1000 PGID=1000 MAXMIND_KEY=
-
-RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv
-USER 1000
-
 COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv
-COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh
 
-WORKDIR /var/strelaypoolsrv
-ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"]
+ENTRYPOINT ["/bin/strelaypoolsrv", "-listen", ":8080"]

cmd/strelaypoolsrv/main.go

@@ -21,12 +21,12 @@ import (
 	"time"
 
 	lru "github.com/hashicorp/golang-lru/v2"
-	"github.com/oschwald/geoip2-golang"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
 	"github.com/syncthing/syncthing/lib/assets"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
+	"github.com/syncthing/syncthing/lib/geoip"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/relay/client"
 	"github.com/syncthing/syncthing/lib/sync"
@@ -102,11 +102,12 @@ var (
 	debug             bool
 	permRelaysFile    string
 	ipHeader          string
-	geoipPath         string
 	proto             string
 	statsRefresh      = time.Minute
 	requestQueueLen   = 64
 	requestProcessors = 8
+	geoipLicenseKey   = os.Getenv("GEOIP_LICENSE_KEY")
+	geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID"))
 
 	requests chan request
 
@@ -132,34 +133,35 @@ func main() {
 	flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
 	flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays")
 	flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
-	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
 	flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
 	flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
 	flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
 	flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
+	flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database")
 
 	flag.Parse()
 
 	requests = make(chan request, requestQueueLen)
+	geoip := geoip.NewGeoLite2CityProvider(geoipAccountID, geoipLicenseKey, os.TempDir())
 
 	var listener net.Listener
 	var err error
 
 	if permRelaysFile != "" {
-		permanentRelays = loadRelays(permRelaysFile)
+		permanentRelays = loadRelays(permRelaysFile, geoip)
 	}
 
 	testCert = createTestCertificate()
 
 	for i := 0; i < requestProcessors; i++ {
-		go requestProcessor()
+		go requestProcessor(geoip)
 	}
 
 	// Load relays from cache in the background.
 	// Load them in a serial fashion to make sure any genuine requests
 	// are not dropped.
 	go func() {
-		for _, relay := range loadRelays(knownRelaysFile) {
+		for _, relay := range loadRelays(knownRelaysFile, geoip) {
 			resultChan := make(chan result)
 			requests <- request{relay, resultChan, nil}
 			result := <-resultChan
@@ -445,19 +447,19 @@ func handleRegister(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func requestProcessor() {
+func requestProcessor(geoip *geoip.Provider) {
 	for request := range requests {
 		if request.queueTimer != nil {
 			request.queueTimer.ObserveDuration()
 		}
 
 		timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
-		handleRelayTest(request)
+		handleRelayTest(request, geoip)
 		timer.ObserveDuration()
 	}
 }
 
-func handleRelayTest(request request) {
+func handleRelayTest(request request, geoip *geoip.Provider) {
 	if debug {
 		log.Println("Request for", request.relay)
 	}
@@ -470,7 +472,7 @@ func handleRelayTest(request request) {
 	}
 
 	stats := fetchStats(request.relay)
-	location := getLocation(request.relay.uri.Host)
+	location := getLocation(request.relay.uri.Host, geoip)
 
 	mut.Lock()
 	if stats != nil {
@@ -543,7 +545,7 @@ func evict(relay *relay) func() {
 	}
 }
 
-func loadRelays(file string) []*relay {
+func loadRelays(file string, geoip *geoip.Provider) []*relay {
 	content, err := os.ReadFile(file)
 	if err != nil {
 		log.Println("Failed to load relays: " + err.Error())
@@ -567,7 +569,7 @@ func loadRelays(file string) []*relay {
 
 		relays = append(relays, &relay{
 			URL:      line,
-			Location: getLocation(uri.Host),
+			Location: getLocation(uri.Host, geoip),
 			uri:      uri,
 		})
 		if debug {
@@ -600,21 +602,16 @@ func createTestCertificate() tls.Certificate {
 	return cert
 }
 
-func getLocation(host string) location {
+func getLocation(host string, geoip *geoip.Provider) location {
 	timer := prometheus.NewTimer(locationLookupSeconds)
 	defer timer.ObserveDuration()
-	db, err := geoip2.Open(geoipPath)
-	if err != nil {
-		return location{}
-	}
-	defer db.Close()
 
 	addr, err := net.ResolveTCPAddr("tcp", host)
 	if err != nil {
 		return location{}
 	}
 
-	city, err := db.City(addr.IP)
+	city, err := geoip.City(addr.IP)
 	if err != nil {
 		return location{}
 	}

cmd/ursrv/serve/serve.go

@@ -17,6 +17,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"os"
 	"regexp"
 	"sort"
 	"strconv"
@@ -26,20 +27,21 @@ import (
 	"unicode"
 
 	_ "github.com/lib/pq" // PostgreSQL driver
-	"github.com/oschwald/geoip2-golang"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 
+	"github.com/syncthing/syncthing/lib/geoip"
 	"github.com/syncthing/syncthing/lib/upgrade"
 	"github.com/syncthing/syncthing/lib/ur/contract"
 )
 
 type CLI struct {
-	Debug     bool   `env:"UR_DEBUG"`
-	DBConn    string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"`
-	Listen    string `env:"UR_LISTEN" default:"0.0.0.0:8080"`
-	GeoIPPath string `env:"UR_GEOIP" default:"GeoLite2-City.mmdb"`
+	Debug           bool   `env:"UR_DEBUG"`
+	DBConn          string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"`
+	Listen          string `env:"UR_LISTEN" default:"0.0.0.0:8080"`
+	GeoIPLicenseKey string `env:"UR_GEOIP_LICENSE_KEY"`
+	GeoIPAccountID  int    `env:"UR_GEOIP_ACCOUNT_ID"`
 }
 
 //go:embed static
@@ -190,9 +192,9 @@ func (cli *CLI) Run() error {
 	}
 
 	srv := &server{
-		db:        db,
-		debug:     cli.Debug,
-		geoIPPath: cli.GeoIPPath,
+		db:    db,
+		debug: cli.Debug,
+		geoip: geoip.NewGeoLite2CityProvider(cli.GeoIPAccountID, cli.GeoIPLicenseKey, os.TempDir()),
 	}
 	http.HandleFunc("/", srv.rootHandler)
 	http.HandleFunc("/newdata", srv.newDataHandler)
@@ -213,9 +215,9 @@ func (cli *CLI) Run() error {
 }
 
 type server struct {
-	debug     bool
-	db        *sql.DB
-	geoIPPath string
+	debug bool
+	db    *sql.DB
+	geoip *geoip.Provider
 
 	cacheMut        sync.Mutex
 	cachedIndex     []byte
@@ -238,7 +240,7 @@ func (s *server) cacheRefresher() {
 }
 
 func (s *server) refreshCacheLocked() error {
-	rep := getReport(s.db, s.geoIPPath)
+	rep := getReport(s.db, s.geoip)
 	buf := new(bytes.Buffer)
 	err := tpl.Execute(buf, rep)
 	if err != nil {
@@ -492,15 +494,7 @@ type weightedLocation struct {
 	Weight int `json:"weight"`
 }
 
-func getReport(db *sql.DB, geoIPPath string) map[string]interface{} {
-	geoip, err := geoip2.Open(geoIPPath)
-	if err != nil {
-		log.Println("opening geoip db", err)
-		geoip = nil
-	} else {
-		defer geoip.Close()
-	}
-
+func getReport(db *sql.DB, geoip *geoip.Provider) map[string]interface{} {
 	nodes := 0
 	countriesTotal := 0
 	var versions []string

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/lib/pq v1.10.9
 	github.com/maruel/panicparse/v2 v2.3.1
 	github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1
+	github.com/maxmind/geoipupdate/v6 v6.1.0
 	github.com/minio/sha256-simd v1.0.1
 	github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75
 	github.com/oschwald/geoip2-golang v1.9.0
@@ -52,6 +53,7 @@ require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
@@ -60,13 +62,13 @@ require (
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
-	github.com/kr/text v0.2.0 // indirect
 	github.com/nxadm/tail v1.4.11 // indirect
 	github.com/onsi/ginkgo/v2 v2.17.1 // indirect
 	github.com/onsi/gomega v1.31.1 // indirect

go.sum

@@ -22,6 +22,8 @@ github.com/calmh/xdr v1.1.0 h1:U/Dd4CXNLoo8EiQ4ulJUXkgO1/EyQLgDKLgpY1SOoJE=
 github.com/calmh/xdr v1.1.0/go.mod h1:E8sz2ByAdXC8MbANf1LCRYzedSnnc+/sXXJs/PVqoeg=
 github.com/ccding/go-stun v0.1.4 h1:lC0co3Q3vjAuu2Jz098WivVPBPbemYFqbwE1syoka4M=
 github.com/ccding/go-stun v0.1.4/go.mod h1:cCZjJ1J3WFSJV6Wj8Y9Di8JMTsEXh6uv2eNmLzKaUeM=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s=
 github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -34,7 +36,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
 github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -62,6 +63,8 @@ github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -131,6 +134,8 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 h1:NicmruxkeqHjDv03SfSxqmaLuisddudfP3h5wdXFbhM=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw2mDK7CWABMG5r9uyaKC7I=
+github.com/maxmind/geoipupdate/v6 v6.1.0 h1:sdtTHzzQNJlXF5+fd/EoPTucRHyMonYt/Cok8xzzfqA=
+github.com/maxmind/geoipupdate/v6 v6.1.0/go.mod h1:cZYCDzfMzTY4v6dKRdV7KTB6SStxtn3yFkiJ1btTGGc=
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
 github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=

gui/default/assets/lang/lang-hi.json

@@ -0,0 +1,2 @@
+{
+}