envoy: add support to bind to privileged ports

images/cilium/Dockerfile

@@ -3,7 +3,7 @@
 
 ARG CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:bcf26b2fb0be1bccdb0bacf0c29fcb5530eb5997@sha256:45e22e685599cfdfbf9d2f0fce9635b3fc599f97c23dc541351d637e9563127c
 ARG CILIUM_RUNTIME_IMAGE=quay.io/cilium/cilium-runtime:0615d2db1a5616db6ebe0d467da69bfc2b552427@sha256:60d433ea2d8cbe9712205323fe00cfd76a210ef0115216dbdd9f22603136e9a4
-ARG CILIUM_ENVOY_IMAGE=quay.io/cilium/cilium-envoy:v1.29.4-fe3f52ea52e1a28e4c2cd295b0884fd697bb9e69@sha256:f48e6e65252f2ff53e8ae91e6521c2ae9ed747e8998e1b9dfc83a2664de3dc35
+ARG CILIUM_ENVOY_IMAGE=quay.io/cilium/cilium-envoy:v1.29.4-174c6af504235ce8f4ce998f98cac88c50202d29@sha256:3c5494946d3d72873d0a40408dba95ed35ee9ed39fbbe46ea031cfbf21c8bc7e
 
 # cilium-envoy from github.com/cilium/proxy
 #

install/kubernetes/Makefile.values

@@ -45,8 +45,8 @@ export CILIUM_NODEINIT_VERSION:=62093c5c233ea914bfa26a10ba41f8780d9b737f
 export CILIUM_NODEINIT_DIGEST:=sha256:e1d442546e868db1a3289166c14011e0dbd32115b338b963e56f830972bc22a2
 
 export CILIUM_ENVOY_REPO:=quay.io/cilium/cilium-envoy
-export CILIUM_ENVOY_VERSION:=v1.29.4-fe3f52ea52e1a28e4c2cd295b0884fd697bb9e69
-export CILIUM_ENVOY_DIGEST:=sha256:f48e6e65252f2ff53e8ae91e6521c2ae9ed747e8998e1b9dfc83a2664de3dc35
+export CILIUM_ENVOY_VERSION:=v1.29.4-174c6af504235ce8f4ce998f98cac88c50202d29
+export CILIUM_ENVOY_DIGEST:=sha256:3c5494946d3d72873d0a40408dba95ed35ee9ed39fbbe46ea031cfbf21c8bc7e
 
 export HUBBLE_UI_BACKEND_REPO:=quay.io/cilium/hubble-ui-backend
 export HUBBLE_UI_BACKEND_VERSION:=v0.13.0

install/kubernetes/cilium/templates/cilium-configmap.yaml

@@ -1253,6 +1253,8 @@ data:
   envoy-log: {{ .Values.envoy.log.path | quote }}
 {{- end }}
 
+  envoy-keep-cap-netbindservice: {{ .Values.envoy.securityContext.capabilities.keepCapNetBindService | quote }}
+
 {{- if hasKey .Values.clustermesh "maxConnectedClusters" }}
   max-connected-clusters: {{ .Values.clustermesh.maxConnectedClusters | quote }}
 {{- end }}

install/kubernetes/cilium/templates/cilium-envoy/daemonset.yaml

@@ -67,6 +67,10 @@ spec:
         command:
         - /usr/bin/cilium-envoy-starter
         args:
+        {{- if .Values.envoy.securityContext.capabilities.keepCapNetBindService }}
+        - '--keep-cap-net-bind-service'
+        {{- end }}
+        - '--'
         - '-c /var/run/cilium/envoy/bootstrap-config.json'
         - '--base-id {{ int .Values.envoy.baseID }}'
         {{- if and (.Values.debug.enabled) (hasKey .Values.debug "verbose") (.Values.debug.verbose) (has "envoy" ( splitList " " .Values.debug.verbose )) }}

install/kubernetes/cilium/values.schema.json

@@ -1984,6 +1984,9 @@
                     ]
                   },
                   "type": "array"
+                },
+                "keepCapNetBindService": {
+                  "type": "boolean"
                 }
               },
               "type": "object"

install/kubernetes/cilium/values.yaml.tmpl

@@ -2118,7 +2118,13 @@ envoy:
       # type available on the system.
       type: 'spc_t'
     capabilities:
-      # -- Capabilities for the `cilium-envoy` container
+      # -- Capabilities for the `cilium-envoy` container.
+      # Even though granted to the container, the cilium-envoy-starter wrapper drops
+      # all capabilities after forking the actual Envoy process.
+      # `NET_BIND_SERVICE` is the only capability that can be passed to the Envoy process by
+      # setting `envoy.capabilities.keepNetBindService=true` (in addition to granting the
+      # capability to the container).
+      # Note: In case of embedded envoy, the capability must  be granted to the cilium-agent container.
       envoy:
         # Used since cilium proxy uses setting IPPROTO_IP/IP_TRANSPARENT
         - NET_ADMIN
@@ -2131,6 +2137,8 @@ envoy:
         # If available, SYS_ADMIN can be removed.
         #- PERFMON
         #- BPF
+      # -- Keep capability `NET_BIND_SERVICE` for Envoy process.
+      keepCapNetBindService: false
   # -- Affinity for cilium-envoy.
   affinity:
     podAntiAffinity:

pkg/envoy/cell.go

@@ -47,6 +47,7 @@ type envoyProxyConfig struct {
 	ProxyAdminPort                    int
 	EnvoyLog                          string
 	EnvoyBaseID                       uint64
+	EnvoyKeepCapNetbindservice        bool
 	ProxyConnectTimeout               uint
 	ProxyGID                          uint
 	ProxyMaxRequestsPerConnection     int
@@ -67,6 +68,7 @@ func (r envoyProxyConfig) Flags(flags *pflag.FlagSet) {
 	flags.Int("proxy-admin-port", 0, "Port to serve Envoy admin interface on.")
 	flags.String("envoy-log", "", "Path to a separate Envoy log file, if any")
 	flags.Uint64("envoy-base-id", 0, "Envoy base ID")
+	flags.Bool("envoy-keep-cap-netbindservice", false, "Keep capability NET_BIND_SERVICE for Envoy process")
 	flags.Uint("proxy-connect-timeout", 2, "Time after which a TCP connect attempt is considered failed unless completed (in seconds)")
 	flags.Uint("proxy-gid", 1337, "Group ID for proxy control plane sockets.")
 	flags.Int("proxy-max-requests-per-connection", 0, "Set Envoy HTTP option max_requests_per_connection. Default 0 (disable)")
@@ -162,6 +164,7 @@ func newEnvoyXDSServer(params xdsServerParams) (XDSServer, error) {
 			runDir:                   option.Config.RunDir,
 			envoyLogPath:             params.EnvoyProxyConfig.EnvoyLog,
 			envoyBaseID:              params.EnvoyProxyConfig.EnvoyBaseID,
+			keepCapNetBindService:    params.EnvoyProxyConfig.EnvoyKeepCapNetbindservice,
 			metricsListenerPort:      params.EnvoyProxyConfig.ProxyPrometheusPort,
 			adminListenerPort:        params.EnvoyProxyConfig.ProxyAdminPort,
 			connectTimeout:           int64(params.EnvoyProxyConfig.ProxyConnectTimeout),

pkg/envoy/embedded_envoy.go

@@ -87,6 +87,7 @@ type embeddedEnvoyConfig struct {
 	runDir                   string
 	logPath                  string
 	baseID                   uint64
+	keepCapNetBindService    bool
 	connectTimeout           int64
 	maxRequestsPerConnection uint32
 	maxConnectionDuration    time.Duration
@@ -150,9 +151,15 @@ func startEmbeddedEnvoy(config embeddedEnvoyConfig) (*EmbeddedEnvoy, error) {
 		}
 		defer logWriter.Close()
 
+		envoyArgs := []string{"-l", mapLogLevel(logging.GetLevel(logging.DefaultLogger)), "-c", bootstrapFilePath, "--base-id", strconv.FormatUint(config.baseID, 10), "--log-format", logFormat}
+		envoyStarterArgs := []string{}
+		if config.keepCapNetBindService {
+			envoyStarterArgs = append(envoyStarterArgs, "--keep-cap-net-bind-service", "--")
+		}
+		envoyStarterArgs = append(envoyStarterArgs, envoyArgs...)
+
 		for {
-			logLevel := logging.GetLevel(logging.DefaultLogger)
-			cmd := exec.Command(ciliumEnvoyStarter, "-l", mapLogLevel(logLevel), "-c", bootstrapFilePath, "--base-id", strconv.FormatUint(config.baseID, 10), "--log-format", logFormat)
+			cmd := exec.Command(ciliumEnvoyStarter, envoyStarterArgs...)
 			cmd.Stderr = logWriter
 			cmd.Stdout = logWriter
 

pkg/envoy/xds_server_ondemand.go

@@ -19,6 +19,7 @@ type onDemandXdsStarter struct {
 	runDir                   string
 	envoyLogPath             string
 	envoyBaseID              uint64
+	keepCapNetBindService    bool
 	metricsListenerPort      int
 	adminListenerPort        int
 	connectTimeout           int64
@@ -64,6 +65,7 @@ func (o *onDemandXdsStarter) startEmbeddedEnvoy(wg *completion.WaitGroup) error
 			runDir:                   o.runDir,
 			logPath:                  o.envoyLogPath,
 			baseID:                   o.envoyBaseID,
+			keepCapNetBindService:    o.keepCapNetBindService,
 			connectTimeout:           o.connectTimeout,
 			maxRequestsPerConnection: o.maxRequestsPerConnection,
 			maxConnectionDuration:    o.maxConnectionDuration,