Skip to content

Commit

Permalink
feat(debug): redis provider (#489)
Browse files Browse the repository at this point in the history 
* feat(debug): redis provider

* feat(debug): redis provider

* feat(redis): use scan instead of keys

* feat(chore): use Delete single to purge the mapping itself without touching to stored resources

* feat(storage): migrate keys to xxhash to reduce cost

* feat(redis): introduce ability to use hashtag in the key as prefix

* feat(storage): add ability to hash the key, compress using lz4 values

* feat(chore): compress response

* fix(ci): newman and unit tests

* feat(plugins): sync middleware and storage code to upstream

* fix(review): from hussam-almarzoq

* feat(tyk): ci lint fixes

* feat(tyk): use dc v3 instead of deprecated v3.8

* fix(tyk): remove lz4 dep
  • Loading branch information
@darkweak
darkweak committed Apr 23, 2024
1 parent 27cb6b7 commit d399288
Show file tree
Hide file tree
Showing 125 changed files with 5,104 additions and 393 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/non-regression.yml
View file
Expand Up @@ -49,9 +49,9 @@ jobs:
- name: Build and run the docker stack
run: |
docker network create your_network || true
docker-compose -f docker-compose.yml.test up -d --build --force-recreate --remove-orphans
docker compose -f docker-compose.yml.test up -d --build --force-recreate --remove-orphans
- name: Run pkg storage tests
run: docker-compose -f docker-compose.yml.test exec -T souin go test -v -race ./pkg/storage
run: docker compose -f docker-compose.yml.test exec -T souin go test -v -race ./pkg/storage
validate-prod-container-building:
needs: unit-test-golang-with-services
name: Validate that the container build for prod
Expand All @@ -65,7 +65,7 @@ jobs:
run: docker network create your_network || true
-
name: Build the stack
run: docker-compose -f docker-compose.yml.prod up -d --build --force-recreate --remove-orphans
run: docker compose -f docker-compose.yml.prod up -d --build --force-recreate --remove-orphans
-
name: Souin container healthceck
run: docker-compose -f docker-compose.yml.prod exec -T souin ls
run: docker compose -f docker-compose.yml.prod exec -T souin ls
15 changes: 15 additions & 0 deletions README.md
View file
Expand Up @@ -87,6 +87,9 @@ cache_keys:
disable_host: true # Prevent the host from being used in the cache key
disable_method: true # Prevent the method from being used in the cache key
disable_query: true # Prevent the query string from being used in the cache key
disable_scheme: true # request scheme the query string from being used in the cache key
hash: true # Hash the cache key instead of a plaintext one
hide: true # Prevent the cache key to be in the response Cache-Status header
headers: # Add headers to the key
- Authorization # Add the header value in the key
- Content-Type # Add the header value in the key
Expand All @@ -107,6 +110,9 @@ default_cache:
disable_host: true # Prevent the host from being used in the cache key
disable_method: true # Prevent the method from being used in the cache key
disable_query: true # Prevent the query string from being used in the cache key
disable_scheme: true # Prevent the request scheme string from being used in the cache key
hash: true # Hash the cache key instead of a plaintext one
hide: true # Prevent the cache key to be in the response Cache-Status header
headers: # Add headers to the key
- Authorization # Add the header value in the key
- Content-Type # Add the header value in the key
Expand Down Expand Up @@ -172,6 +178,8 @@ surrogate_keys:
| `cache_keys.{your regexp}.disable_host` | Disable the host part in the key matching the regexp | `true`<br/><br/>`(default: false)` |
| `cache_keys.{your regexp}.disable_method` | Disable the method part in the key matching the regexp | `true`<br/><br/>`(default: false)` |
| `cache_keys.{your regexp}.disable_query` | Disable the query string part in the key matching the regexp | `true`<br/><br/>`(default: false)` |
| `cache_keys.{your regexp}.disable_scheme` | Disable the request scheme string part in the key matching the regexp | `true`<br/><br/>`(default: false)` |
| `cache_keys.{your regexp}.hash` | Hash the key matching the regexp | `true`<br/><br/>`(default: false)` |
| `cache_keys.{your regexp}.headers` | Add headers to the key matching the regexp | `- Authorization`<br/><br/>`- Content-Type`<br/><br/>`- X-Additional-Header` |
| `cache_keys.{your regexp}.hide` | Prevent the key from being exposed in the `Cache-Status` HTTP response header | `true`<br/><br/>`(default: false)` |
| `cdn` | The CDN management, if you use any cdn to proxy your requests Souin will handle that | |
Expand All @@ -197,6 +205,8 @@ surrogate_keys:
| `default_cache.key.disable_host` | Disable the host part in the key | `true`<br/><br/>`(default: false)` |
| `default_cache.key.disable_method` | Disable the method part in the key | `true`<br/><br/>`(default: false)` |
| `default_cache.key.disable_query` | Disable the query string part in the key | `true`<br/><br/>`(default: false)` |
| `default_cache.key.disable_scheme` | Disable the request scheme string part in the key | `true`<br/><br/>`(default: false)` |
| `default_cache.key.hash` | Hash the key name in the storage | `true`<br/><br/>`(default: false)` |
| `default_cache.key.headers` | Add headers to the key matching the regexp | `- Authorization`<br/><br/>`- Content-Type`<br/><br/>`- X-Additional-Header` |
| `default_cache.key.hide` | Prevent the key from being exposed in the `Cache-Status` HTTP response header | `true`<br/><br/>`(default: false)` |
| `default_cache.mode` | RFC respect tweaking | One of `bypass` `bypass_request` `bypass_response` `strict` (default `strict`) |
Expand Down Expand Up @@ -468,8 +478,10 @@ There is the fully configuration below
disable_host
disable_method
disable_query
disable_scheme
headers X-Token Authorization
hide
hash
}
}
cdn {
Expand All @@ -488,6 +500,9 @@ There is the fully configuration below
disable_host
disable_method
disable_query
disable_scheme
hash
hide
headers Content-Type Authorization
}
log_level debug
Expand Down
8 changes: 8 additions & 0 deletions configurationtypes/types.go
View file
Expand Up @@ -49,6 +49,12 @@ func (c *CacheKeys) parseJSON(rootDecoder *json.Decoder) {
case "disable_query":
val, _ := rootDecoder.Token()
key.DisableQuery, _ = strconv.ParseBool(fmt.Sprint(val))
case "disable_scheme":
val, _ := rootDecoder.Token()
key.DisableScheme, _ = strconv.ParseBool(fmt.Sprint(val))
case "hash":
val, _ := rootDecoder.Token()
key.Hash, _ = strconv.ParseBool(fmt.Sprint(val))
case "hide":
val, _ := rootDecoder.Token()
key.Hide, _ = strconv.ParseBool(fmt.Sprint(val))
Expand Down Expand Up @@ -208,6 +214,8 @@ type Key struct {
DisableHost bool `json:"disable_host,omitempty" yaml:"disable_host,omitempty"`
DisableMethod bool `json:"disable_method,omitempty" yaml:"disable_method,omitempty"`
DisableQuery bool `json:"disable_query,omitempty" yaml:"disable_query,omitempty"`
DisableScheme bool `json:"disable_scheme,omitempty" yaml:"disable_scheme,omitempty"`
Hash bool `json:"hash,omitempty" yaml:"hash,omitempty"`
Hide bool `json:"hide,omitempty" yaml:"hide,omitempty"`
Headers []string `json:"headers,omitempty" yaml:"headers,omitempty"`
}
Expand Down
42 changes: 35 additions & 7 deletions context/key.go
View file
Expand Up @@ -12,14 +12,17 @@ const (
Key ctxKey = "souin_ctx.CACHE_KEY"
DisplayableKey ctxKey = "souin_ctx.DISPLAYABLE_KEY"
IgnoredHeaders ctxKey = "souin_ctx.IGNORE_HEADERS"
Hashed ctxKey = "souin_ctx.HASHED"
)

type keyContext struct {
disable_body bool
disable_host bool
disable_method bool
disable_query bool
disable_scheme bool
displayable bool
hash bool
headers []string
overrides []map[*regexp.Regexp]keyContext
}
Expand All @@ -34,6 +37,8 @@ func (g *keyContext) SetupContext(c configurationtypes.AbstractConfigurationInte
g.disable_host = k.DisableHost
g.disable_method = k.DisableMethod
g.disable_query = k.DisableQuery
g.disable_scheme = k.DisableScheme
g.hash = k.Hash
g.displayable = !k.Hide
g.headers = k.Headers

Expand All @@ -46,6 +51,8 @@ func (g *keyContext) SetupContext(c configurationtypes.AbstractConfigurationInte
disable_host: v.DisableHost,
disable_method: v.DisableMethod,
disable_query: v.DisableQuery,
disable_scheme: v.DisableScheme,
hash: v.Hash,
displayable: !v.Hide,
headers: v.Headers,
}})
Expand All @@ -57,11 +64,9 @@ func (g *keyContext) SetContext(req *http.Request) *http.Request {
key := req.URL.Path
var headers []string

scheme := "http-"
if req.TLS != nil {
scheme = "https-"
}
hash := g.hash
query := ""
scheme := ""
body := ""
host := ""
method := ""
Expand All @@ -80,6 +85,13 @@ func (g *keyContext) SetContext(req *http.Request) *http.Request {
host = req.Host + "-"
}

if !g.disable_scheme {
scheme = "http-"
if req.TLS != nil {
scheme = "https-"
}
}

if !g.disable_method {
method = req.Method + "-"
}
Expand All @@ -97,6 +109,7 @@ func (g *keyContext) SetContext(req *http.Request) *http.Request {
host = ""
method = ""
query = ""
scheme = ""
if !v.disable_query && len(req.URL.RawQuery) > 0 {
query = "?" + req.URL.RawQuery
}
Expand All @@ -109,13 +122,22 @@ func (g *keyContext) SetContext(req *http.Request) *http.Request {
if !v.disable_host {
host = req.Host + "-"
}
if !v.disable_scheme {
scheme = "http-"
if req.TLS != nil {
scheme = "https-"
}
}
if len(v.headers) > 0 {
headerValues = ""
for _, hn := range v.headers {
headers = v.headers
headerValues += "-" + req.Header.Get(hn)
}
}
if v.hash {
hash = true
}
hasOverride = true
break
}
Expand All @@ -126,13 +148,19 @@ func (g *keyContext) SetContext(req *http.Request) *http.Request {
}
}

key = method + scheme + host + key + query + body + headerValues

return req.WithContext(
context.WithValue(
context.WithValue(
context.WithValue(
req.Context(),
Key,
method+scheme+host+key+query+body+headerValues,
context.WithValue(
req.Context(),
Key,
key,
),
Hashed,
hash,
),
IgnoredHeaders,
headers,
Expand Down

0 comments on commit d399288

Please sign in to comment.