feat(rfc): Cache invalidation and Cache groups (#363)

* fix(middleware): revalidate when needed #362

* feat(chore): support http-cache-groups RFC as early stage (https://datatracker.ietf.org/doc/draft-nottingham-http-cache-groups)

* feat(chore): support http-invalidation RFC as early stage (https://datatracker.ietf.org/doc/draft-nottingham-http-invalidation)

* fix(plugins): traefik and caddy to use Storers

* fix(ci): debug caddy reload

* fix(plugin): tyk

* feat(release): bump next release version

.github/workflows/non-regression.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Go
         uses: actions/setup-go@v3
         with:
@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Go
         uses: actions/setup-go@v3
         with:
@@ -41,7 +41,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Go
         uses: actions/setup-go@v3
         with:
@@ -59,7 +59,7 @@ jobs:
     steps:
       -
         name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Build the stack
         run: docker network create your_network || true

.github/workflows/plugin_template.yml

@@ -31,7 +31,7 @@ jobs:
           go-version: ${{ inputs.GO_VERSION }}
       -
         name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - 
         name: golangci-lint
         uses: golangci/golangci-lint-action@v3

.github/workflows/plugins-master.yml

@@ -24,7 +24,7 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
       -
         name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Install xcaddy
         run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest

.github/workflows/plugins.yml

@@ -39,7 +39,7 @@ jobs:
           go-version: '1.21'
       -
         name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Install xcaddy
         run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest

.github/workflows/release.yml

@@ -17,7 +17,7 @@ jobs:
         run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       -
         name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -48,7 +48,7 @@ jobs:
         run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       -
         name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -85,7 +85,7 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       -
@@ -113,7 +113,7 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
       -
         name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Generate Tyk amd64 artifacts
         run: cd plugins/tyk && make vendor && docker compose -f docker-compose.yml.artifacts up

.github/workflows/workflow_plugins_generator.sh

@@ -45,7 +45,7 @@ jobs:
           go-version: '$go_version'
       -
         name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Install xcaddy
         run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest

README.md

@@ -45,6 +45,7 @@
 Souin is a new HTTP cache system suitable for every reverse-proxy. It can be either placed on top of your current reverse-proxy whether it's Apache, Nginx or as plugin in your favorite reverse-proxy like Træfik, Caddy or Tyk.  
 Since it's written in go, it can be deployed on any server and thanks to the docker integration, it will be easy to install on top of a Swarm, or a kubernetes instance.  
 It's RFC compatible, supporting Vary, request coalescing, stale cache-control and other specifications related to the [RFC-7234](https://tools.ietf.org/html/rfc7234).  
+It supports the newly written RFCs (currently in draft) [http-cache-groups](https://datatracker.ietf.org/doc/draft-nottingham-http-cache-groups/) and [http-invalidation](https://datatracker.ietf.org/doc/draft-nottingham-http-invalidation/).  
 It also supports the [Cache-Status HTTP response header](https://www.rfc-editor.org/rfc/rfc9211) and the YKey group such as Varnish.  
 It supports the ESI tags, thanks to the [go-esi package](https://github.com/darkweak/go-esi).
 
@@ -969,7 +970,7 @@ experimental:
   plugins:
     souin:
       moduleName: github.com/darkweak/souin
-      version: v1.6.40
+      version: v1.6.41
 ```
 After that you can declare either the whole configuration at once in the middleware block or by service. See the examples below.
 ```yaml

docker-compose.yml.dev

@@ -31,7 +31,7 @@ services:
   etcd:
     image: quay.io/coreos/etcd
     ports:
-      - 2379:2379
+      - 2379
       - 2380
       - 4001
     environment:
@@ -52,6 +52,18 @@ services:
     command: redis-server
     <<: *networks
 
+  caddy:
+    build:
+      context: .
+      dockerfile: ./docker/caddy.Dockerfile
+    depends_on:
+      - etcd
+      - redis
+    ports:
+      - 4443:4443
+      - 2019:2019
+    <<: *networks
+
   traefik:
     image: traefik:latest
     command: --providers.docker

docker/caddy.Dockerfile

@@ -0,0 +1,13 @@
+FROM caddy:2.7-builder-alpine AS app_caddy_builder
+
+COPY . /usr/local/go/src/souin
+WORKDIR /usr/local/go/src/souin/plugins/caddy
+
+RUN xcaddy build v2.7.4 --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../..
+RUN mv ./caddy /usr/bin/caddy
+
+FROM caddy:2-alpine AS app_caddy
+WORKDIR /srv/app
+
+COPY --from=app_caddy_builder --link /usr/bin/caddy /usr/bin/caddy
+COPY ./plugins/caddy/Caddyfile /etc/caddy/Caddyfile

pkg/api/souin.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 
 	"github.com/darkweak/souin/configurationtypes"
 	"github.com/darkweak/souin/pkg/storage"
@@ -17,6 +18,23 @@ type SouinAPI struct {
 	enabled          bool
 	storers          []storage.Storer
 	surrogateStorage providers.SurrogateInterface
+	allowedMethods   []string
+}
+
+type invalidationType string
+
+const (
+	uriInvalidationType       invalidationType = "uri"
+	uriPrefixInvalidationType invalidationType = "uri-prefix"
+	originInvalidationType    invalidationType = "origin"
+	groupInvalidationType     invalidationType = "group"
+)
+
+type invalidation struct {
+	Type      invalidationType `json:"type"`
+	Selectors []string         `json:"selectors"`
+	Groups    []string         `json:"groups"`
+	Purge     bool             `json:"purge"`
 }
 
 func initializeSouin(
@@ -28,11 +46,18 @@ func initializeSouin(
 	if basePath == "" {
 		basePath = "/souin"
 	}
+
+	allowedMethods := configuration.GetDefaultCache().GetAllowedHTTPVerbs()
+	if len(allowedMethods) == 0 {
+		allowedMethods = []string{http.MethodGet, http.MethodHead}
+	}
+
 	return &SouinAPI{
 		basePath,
 		configuration.GetAPI().Souin.Enable,
 		storers,
 		surrogateStorage,
+		allowedMethods,
 	}
 }
 
@@ -103,6 +128,78 @@ func (s *SouinAPI) HandleRequest(w http.ResponseWriter, r *http.Request) {
 			res, _ = json.Marshal(s.GetAll())
 		}
 		w.Header().Set("Content-Type", "application/json")
+	case http.MethodPost:
+		var invalidator invalidation
+		err := json.NewDecoder(r.Body).Decode(&invalidator)
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		keysToInvalidate := []string{}
+		switch invalidator.Type {
+		case groupInvalidationType:
+			keysToInvalidate, _ = s.surrogateStorage.Purge(http.Header{"Surrogate-Key": invalidator.Groups})
+		case uriPrefixInvalidationType, uriInvalidationType:
+			bodyKeys := []string{}
+			listedKeys := s.GetAll()
+			for _, k := range invalidator.Selectors {
+				if !strings.Contains(k, "//") {
+					rq, err := http.NewRequest(http.MethodGet, "//"+k, nil)
+					if err != nil {
+						continue
+					}
+
+					bodyKeys = append(bodyKeys, rq.Host+"-"+rq.URL.Path)
+				}
+			}
+
+			for _, allKey := range listedKeys {
+				for _, bk := range bodyKeys {
+					if invalidator.Type == uriInvalidationType {
+						if strings.Contains(allKey, bk) && strings.Contains(allKey, bk+"-") && strings.HasSuffix(allKey, bk) {
+							keysToInvalidate = append(keysToInvalidate, allKey)
+							break
+						}
+					} else {
+						if strings.Contains(allKey, bk) &&
+							(strings.Contains(allKey, bk+"-") || strings.Contains(allKey, bk+"?") || strings.Contains(allKey, bk+"/") || strings.HasSuffix(allKey, bk)) {
+							keysToInvalidate = append(keysToInvalidate, allKey)
+							break
+						}
+					}
+				}
+			}
+		case originInvalidationType:
+			bodyKeys := []string{}
+			listedKeys := s.GetAll()
+			for _, k := range invalidator.Selectors {
+				if !strings.Contains(k, "//") {
+					rq, err := http.NewRequest(http.MethodGet, "//"+k, nil)
+					if err != nil {
+						continue
+					}
+
+					bodyKeys = append(bodyKeys, rq.Host)
+				}
+			}
+
+			for _, allKey := range listedKeys {
+				for _, bk := range bodyKeys {
+					if strings.Contains(allKey, bk) {
+						keysToInvalidate = append(keysToInvalidate, allKey)
+						break
+					}
+				}
+			}
+		}
+
+		for _, k := range keysToInvalidate {
+			for _, current := range s.storers {
+				current.Delete(k)
+			}
+		}
+		w.WriteHeader(http.StatusOK)
 	case "PURGE":
 		if compile {
 			keysRg := regexp.MustCompile(s.GetBasePath() + "/(.+)")

pkg/middleware/middleware.go

@@ -101,7 +101,6 @@ func NewHTTPCacheHandler(c configurationtypes.AbstractConfigurationInterface) *S
 
 type SouinBaseHandler struct {
 	Configuration            configurationtypes.AbstractConfigurationInterface
-	Storer                   storage.Storer
 	Storers                  []storage.Storer
 	InternalEndpointHandlers *api.MapHandler
 	ExcludeRegex             *regexp.Regexp
@@ -267,16 +266,6 @@ func (s *SouinBaseHandler) Store(
 			if len(fails) > 0 {
 				status += strings.Join(fails, "")
 			}
-
-			// if s.Storer.Set(cachedKey, response, currentMatchedURL, ma) == nil {
-			// 	s.Configuration.GetLogger().Sugar().Debugf("Store the cache key %s into the surrogate keys from the following headers %v", cachedKey, res)
-			// 	go func(rs http.Response, key string) {
-			// 		_ = s.SurrogateKeyStorer.Store(&rs, key)
-			// 	}(res, cachedKey)
-			// 	status += "; stored"
-			// } else {
-			// 	status += "; detail=STORAGE-INSERTION-ERROR"
-			// }
 		}
 	} else {
 		status += "; detail=NO-STORE-DIRECTIVE"
@@ -300,6 +289,7 @@ func (s *SouinBaseHandler) Upstream(
 		return err
 	}
 
+	s.SurrogateKeyStorer.Invalidate(rq.Method, customWriter.Header())
 	if !isCacheableCode(customWriter.statusCode) {
 		customWriter.Headers.Set("Cache-Status", fmt.Sprintf("%s; fwd=uri-miss; key=%s; detail=UNCACHEABLE-STATUS-CODE", rq.Context().Value(context.CacheName), rfc.GetCacheKeyFromCtx(rq.Context())))
 
@@ -332,6 +322,7 @@ func (s *SouinBaseHandler) Revalidate(validator *rfc.Revalidator, next handlerFu
 	s.Configuration.GetLogger().Sugar().Debug("Revalidate the request with the upstream server")
 	prometheus.Increment(prometheus.RequestRevalidationCounter)
 	err := next(customWriter, rq)
+	s.SurrogateKeyStorer.Invalidate(rq.Method, customWriter.Header())
 
 	if err == nil {
 		if validator.IfUnmodifiedSincePresent && customWriter.statusCode != http.StatusNotModified {
@@ -394,7 +385,10 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
 	if !rq.Context().Value(context.SupportedMethod).(bool) {
 		rw.Header().Set("Cache-Status", cacheName+"; fwd=bypass; detail=UNSUPPORTED-METHOD")
 
-		return next(rw, rq)
+		err := next(rw, rq)
+		s.SurrogateKeyStorer.Invalidate(rq.Method, rw.Header())
+
+		return err
 	}
 
 	requestCc, coErr := cacheobject.ParseRequestCacheControl(rq.Header.Get("Cache-Control"))
@@ -403,14 +397,20 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
 	if !modeContext.Bypass_request && (coErr != nil || requestCc == nil) {
 		rw.Header().Set("Cache-Status", cacheName+"; fwd=bypass; detail=CACHE-CONTROL-EXTRACTION-ERROR")
 
-		return next(rw, rq)
+		err := next(rw, rq)
+		s.SurrogateKeyStorer.Invalidate(rq.Method, rw.Header())
+
+		return err
 	}
 
 	rq = s.context.SetContext(rq)
 	if rq.Context().Value(context.IsMutationRequest).(bool) {
 		rw.Header().Set("Cache-Status", cacheName+"; fwd=bypass; detail=IS-MUTATION-REQUEST")
 
-		return nil
+		err := next(rw, rq)
+		s.SurrogateKeyStorer.Invalidate(rq.Method, rw.Header())
+
+		return err
 	}
 	cachedKey := rq.Context().Value(context.Key).(string)
 
@@ -454,6 +454,19 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
 
 				return nil
 			}
+
+			if validator.NeedRevalidation {
+				err := s.Revalidate(validator, next, customWriter, rq, requestCc, cachedKey)
+				_, _ = customWriter.Send()
+
+				return err
+			}
+			if resCc, _ := cacheobject.ParseResponseCacheControl(response.Header.Get("Cache-Control")); resCc.NoCachePresent {
+				err := s.Revalidate(validator, next, customWriter, rq, requestCc, cachedKey)
+				_, _ = customWriter.Send()
+
+				return err
+			}
 			rfc.SetCacheStatusHeader(response)
 			if !modeContext.Strict || rfc.ValidateMaxAgeCachedResponse(requestCc, response) != nil {
 				customWriter.Headers = response.Header