fix(chore): Bump storages deps and memory leaks investigation (#366)

* fix(chore): Bump storages deps

* fix(nutsdb): close DB when needed

* fix(go-zero): run with new deps

* feat(tyk): cleanup unnecessary overrided code

* feat(tyk): use io instead of ioutil

* feat(pre-release): tag v1.6.42

README.md

@@ -970,7 +970,7 @@ experimental:
   plugins:
     souin:
       moduleName: github.com/darkweak/souin
-      version: v1.6.41
+      version: v1.6.42
 ```
 After that you can declare either the whole configuration at once in the middleware block or by service. See the examples below.
 ```yaml

go.mod

@@ -1,6 +1,6 @@
 module github.com/darkweak/souin
 
-go 1.16
+go 1.19
 
 require (
 	github.com/buraksezer/olric v0.5.4
@@ -10,16 +10,61 @@ require (
 	github.com/go-chi/stampede v0.5.1
 	github.com/google/uuid v1.3.0
 	github.com/imdario/mergo v0.3.13
-	github.com/pquerna/cachecontrol v0.1.1-0.20230415224848-baaf0ee61529
-	github.com/prometheus/client_golang v1.14.0
-	github.com/prometheus/client_model v0.3.0
-	github.com/redis/go-redis/v9 v9.0.2
-	github.com/xujiajun/nutsdb v0.11.1
-	go.etcd.io/etcd/client/v3 v3.5.7
-	go.uber.org/zap v1.21.0
+	github.com/nutsdb/nutsdb v0.14.0
+	github.com/pquerna/cachecontrol v0.2.0
+	github.com/prometheus/client_golang v1.16.0
+	github.com/prometheus/client_model v0.4.0
+	github.com/redis/go-redis/v9 v9.1.0
+	go.etcd.io/etcd/client/v3 v3.5.9
+	go.uber.org/zap v1.26.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
+require (
+	github.com/antlabs/stl v0.0.1 // indirect
+	github.com/antlabs/timer v0.0.11 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/buraksezer/consistent v0.10.0 // indirect
+	github.com/bwmarrin/snowflake v0.3.0 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-redis/redis/v8 v8.11.5 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v1.0.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/snappy v0.0.3 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
+	github.com/hashicorp/golang-lru v0.5.5-0.20200511160909-eb529947af53 // indirect
+	github.com/hashicorp/logutils v1.0.0 // indirect
+	github.com/hashicorp/memberlist v0.5.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mschoch/smat v0.2.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/redcon v1.6.2 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	github.com/xujiajun/mmap-go v1.0.1 // indirect
+	github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
+	go.opencensus.io v0.22.5 // indirect
+	go.uber.org/multierr v1.10.0 // indirect
+	golang.org/x/mod v0.9.0 // indirect
+	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)
+
 require (
 	github.com/RoaringBitmap/roaring v1.2.3 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
@@ -36,8 +81,8 @@ require (
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
 	github.com/klauspost/compress v1.16.0 // indirect
 	github.com/miekg/dns v1.1.51 // indirect
-	github.com/prometheus/common v0.40.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/tidwall/btree v1.6.0 // indirect
 	golang.org/x/tools v0.6.0 // indirect
 	google.golang.org/genproto v0.0.0-20230227214838-9b19f0bdc514 // indirect

pkg/storage/nutsProvider.go

@@ -11,7 +11,7 @@ import (
 	t "github.com/darkweak/souin/configurationtypes"
 	"github.com/darkweak/souin/pkg/rfc"
 	"github.com/imdario/mergo"
-	"github.com/xujiajun/nutsdb"
+	"github.com/nutsdb/nutsdb"
 	"go.uber.org/zap"
 )
 
@@ -150,7 +150,7 @@ func (provider *Nuts) Prefix(key string, req *http.Request, validator *rfc.Reval
 	_ = provider.DB.View(func(tx *nutsdb.Tx) error {
 		prefix := []byte(key)
 
-		if entries, _, err := tx.PrefixSearchScan(bucket, prefix, "^({|$)", 0, 50); err != nil {
+		if entries, err := tx.PrefixSearchScan(bucket, prefix, "^({|$)", 0, 50); err != nil {
 			return err
 		} else {
 			for _, entry := range entries {
@@ -212,7 +212,7 @@ func (provider *Nuts) Delete(key string) {
 // DeleteMany method will delete the responses in Nuts provider if exists corresponding to the regex key param
 func (provider *Nuts) DeleteMany(key string) {
 	_ = provider.DB.Update(func(tx *nutsdb.Tx) error {
-		if entries, _, err := tx.PrefixSearchScan(bucket, []byte(""), key, 0, nutsLimit); err != nil {
+		if entries, err := tx.PrefixSearchScan(bucket, []byte(""), key, 0, nutsLimit); err != nil {
 			return err
 		} else {
 			for _, entry := range entries {

pkg/storage/nutsProvider_test.go

@@ -30,6 +30,7 @@ func getNutsClientAndMatchedURL(key string) (Storer, configurationtypes.URL) {
 func TestNutsConnectionFactory(t *testing.T) {
 	c := tests.MockConfiguration(tests.NutsConfiguration)
 	r, err := NutsConnectionFactory(c)
+	defer r.(*Nuts).DB.Close()
 
 	if nil != err {
 		errors.GenerateError(t, "Shouldn't have panic")
@@ -42,6 +43,7 @@ func TestNutsConnectionFactory(t *testing.T) {
 
 func TestIShouldBeAbleToReadAndWriteDataInNuts(t *testing.T) {
 	client, matchedURL := getNutsClientAndMatchedURL("Test")
+	defer client.(*Nuts).DB.Close()
 
 	_ = client.Set("Test", []byte(BASE_VALUE), matchedURL, time.Duration(20)*time.Second)
 	time.Sleep(1 * time.Second)
@@ -58,6 +60,7 @@ func TestIShouldBeAbleToReadAndWriteDataInNuts(t *testing.T) {
 func TestNuts_GetRequestInCache(t *testing.T) {
 	c := tests.MockConfiguration(tests.BaseConfiguration)
 	client, _ := NutsConnectionFactory(c)
+	defer client.(*Nuts).DB.Close()
 	res := client.Get(NONEXISTENTKEY)
 	if 0 < len(res) {
 		errors.GenerateError(t, fmt.Sprintf("Key %s should not exist", NONEXISTENTKEY))
@@ -66,6 +69,7 @@ func TestNuts_GetRequestInCache(t *testing.T) {
 
 func TestNuts_GetSetRequestInCache_OneByte(t *testing.T) {
 	client, matchedURL := getNutsClientAndMatchedURL(BYTEKEY)
+	defer client.(*Nuts).DB.Close()
 	_ = client.Set(BYTEKEY, []byte("A"), matchedURL, time.Duration(20)*time.Second)
 	time.Sleep(1 * time.Second)
 
@@ -82,12 +86,14 @@ func TestNuts_GetSetRequestInCache_OneByte(t *testing.T) {
 func TestNuts_SetRequestInCache_TTL(t *testing.T) {
 	key := "MyEmptyKey"
 	client, matchedURL := getNutsClientAndMatchedURL(key)
+	defer client.(*Nuts).DB.Close()
 	nv := []byte("Hello world")
 	setValueThenVerify(client, key, nv, matchedURL, time.Duration(20)*time.Second, t)
 }
 
 func TestNuts_DeleteRequestInCache(t *testing.T) {
 	client, _ := NutsConnectionFactory(tests.MockConfiguration(tests.BaseConfiguration))
+	defer client.(*Nuts).DB.Close()
 	client.Delete(BYTEKEY)
 	time.Sleep(1 * time.Second)
 	if 0 < len(client.Get(BYTEKEY)) {
@@ -97,6 +103,7 @@ func TestNuts_DeleteRequestInCache(t *testing.T) {
 
 func TestNuts_Init(t *testing.T) {
 	client, _ := NutsConnectionFactory(tests.MockConfiguration(tests.BaseConfiguration))
+	defer client.(*Nuts).DB.Close()
 	err := client.Init()
 
 	if nil != err {

plugins/beego/go.mod

@@ -4,11 +4,13 @@ go 1.20
 
 require (
 	github.com/beego/beego/v2 v2.1.1
-	github.com/darkweak/souin v1.6.41
+	github.com/darkweak/souin v1.6.42
 )
 
 require (
 	github.com/RoaringBitmap/roaring v1.4.0 // indirect
+	github.com/antlabs/stl v0.0.1 // indirect
+	github.com/antlabs/timer v0.0.11 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.8.0 // indirect
@@ -24,7 +26,9 @@ require (
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/go-redis/redis/v8 v8.11.5 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/glog v1.1.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -47,6 +51,7 @@ require (
 	github.com/miekg/dns v1.1.55 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
+	github.com/nutsdb/nutsdb v0.14.0 // indirect
 	github.com/onsi/gomega v1.24.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pquerna/cachecontrol v0.2.0 // indirect
@@ -63,14 +68,13 @@ require (
 	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xujiajun/mmap-go v1.0.1 // indirect
-	github.com/xujiajun/nutsdb v0.11.1 // indirect
 	github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
 	go.etcd.io/etcd/client/v3 v3.5.9 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.25.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/crypto v0.12.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.14.0 // indirect
@@ -87,4 +91,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-replace github.com/darkweak/souin v1.6.41 => ../..
+replace github.com/darkweak/souin v1.6.42 => ../..