envoy: add support to bind to privileged ports

Currently, the Envoy process of the Cilium Proxy doesn't have the
capabilities to bind to privileged ports. Even if the respective capabilities
would be configured in the Pods SecurityContext, the `cilium-envoy-starter` would
drop all capabilities before starting the Envoy process.

This commit adds support for binding to privileged ports by introducing a new Helm
value `envoy.securityContext.capabilities.keepCapNetBindService` (`bool` - defaults to `false`).

If set to `true`, the `cilium-envoy-starter` keeps the capability `NET_BIND_SERVICE` when starting
the Envoy process. This way, it's possible to bind to privileged ports.

Note: It's still required to add the capability `NET_BIND_SERVICE` to the respective container

* daemonset mode: `envoy.securityContext.capabilities.envoy`
* embedded mode: `securityContext.capabilities.ciliumAgent`

Signed-off-by: Marco Hofstetter <marco.hofstetter@isovalent.com>

install/kubernetes/cilium/templates/cilium-configmap.yaml

@@ -1253,6 +1253,8 @@ data:
   envoy-log: {{ .Values.envoy.log.path | quote }}
 {{- end }}
 
+  envoy-keep-cap-netbindservice: {{ .Values.envoy.securityContext.capabilities.keepCapNetBindService | quote }}
+
 {{- if hasKey .Values.clustermesh "maxConnectedClusters" }}
   max-connected-clusters: {{ .Values.clustermesh.maxConnectedClusters | quote }}
 {{- end }}

install/kubernetes/cilium/templates/cilium-envoy/daemonset.yaml

@@ -67,6 +67,8 @@ spec:
         command:
         - /usr/bin/cilium-envoy-starter
         args:
+        - '--keep-cap-net-bind-service'
+        - '--'
         - '-c /var/run/cilium/envoy/bootstrap-config.json'
         - '--base-id {{ int .Values.envoy.baseID }}'
         {{- if and (.Values.debug.enabled) (hasKey .Values.debug "verbose") (.Values.debug.verbose) (has "envoy" ( splitList " " .Values.debug.verbose )) }}

install/kubernetes/cilium/values.schema.json

@@ -1984,6 +1984,9 @@
                     ]
                   },
                   "type": "array"
+                },
+                "keepCapNetBindService": {
+                  "type": "boolean"
                 }
               },
               "type": "object"

install/kubernetes/cilium/values.yaml.tmpl

@@ -2118,7 +2118,13 @@ envoy:
       # type available on the system.
       type: 'spc_t'
     capabilities:
-      # -- Capabilities for the `cilium-envoy` container
+      # -- Capabilities for the `cilium-envoy` container.
+      # Even though granted to the container, the cilium-envoy-starter wrapper drops
+      # all capabilities after forking the actual Envoy process.
+      # `NET_BIND_SERVICE` is the only capability that can be passed to the Envoy process by
+      # setting `envoy.capabilities.keepNetBindService=true` (in addition to granting the
+      # capability to the container).
+      # Note: In case of embedded envoy, the capability must  be granted to the cilium-agent container.
       envoy:
         # Used since cilium proxy uses setting IPPROTO_IP/IP_TRANSPARENT
         - NET_ADMIN
@@ -2131,6 +2137,8 @@ envoy:
         # If available, SYS_ADMIN can be removed.
         #- PERFMON
         #- BPF
+      # -- Keep capability `NET_BIND_SERVICE` for Envoy process.
+      keepCapNetBindService: false
   # -- Affinity for cilium-envoy.
   affinity:
     podAntiAffinity:

pkg/envoy/cell.go

@@ -47,6 +47,7 @@ type envoyProxyConfig struct {
 	ProxyAdminPort                    int
 	EnvoyLog                          string
 	EnvoyBaseID                       uint64
+	EnvoyKeepCapNetbindservice        bool
 	ProxyConnectTimeout               uint
 	ProxyGID                          uint
 	ProxyMaxRequestsPerConnection     int
@@ -67,6 +68,7 @@ func (r envoyProxyConfig) Flags(flags *pflag.FlagSet) {
 	flags.Int("proxy-admin-port", 0, "Port to serve Envoy admin interface on.")
 	flags.String("envoy-log", "", "Path to a separate Envoy log file, if any")
 	flags.Uint64("envoy-base-id", 0, "Envoy base ID")
+	flags.Bool("envoy-keep-cap-netbindservice", false, "Keep capability NET_BIND_SERVICE for Envoy process")
 	flags.Uint("proxy-connect-timeout", 2, "Time after which a TCP connect attempt is considered failed unless completed (in seconds)")
 	flags.Uint("proxy-gid", 1337, "Group ID for proxy control plane sockets.")
 	flags.Int("proxy-max-requests-per-connection", 0, "Set Envoy HTTP option max_requests_per_connection. Default 0 (disable)")
@@ -162,6 +164,7 @@ func newEnvoyXDSServer(params xdsServerParams) (XDSServer, error) {
 			runDir:                   option.Config.RunDir,
 			envoyLogPath:             params.EnvoyProxyConfig.EnvoyLog,
 			envoyBaseID:              params.EnvoyProxyConfig.EnvoyBaseID,
+			keepCapNetBindService:    params.EnvoyProxyConfig.EnvoyKeepCapNetbindservice,
 			metricsListenerPort:      params.EnvoyProxyConfig.ProxyPrometheusPort,
 			adminListenerPort:        params.EnvoyProxyConfig.ProxyAdminPort,
 			connectTimeout:           int64(params.EnvoyProxyConfig.ProxyConnectTimeout),

pkg/envoy/embedded_envoy.go

@@ -87,6 +87,7 @@ type embeddedEnvoyConfig struct {
 	runDir                   string
 	logPath                  string
 	baseID                   uint64
+	keepCapNetBindService    bool
 	connectTimeout           int64
 	maxRequestsPerConnection uint32
 	maxConnectionDuration    time.Duration
@@ -150,9 +151,15 @@ func startEmbeddedEnvoy(config embeddedEnvoyConfig) (*EmbeddedEnvoy, error) {
 		}
 		defer logWriter.Close()
 
+		envoyArgs := []string{"-l", mapLogLevel(logging.GetLevel(logging.DefaultLogger)), "-c", bootstrapFilePath, "--base-id", strconv.FormatUint(config.baseID, 10), "--log-format", logFormat}
+		envoyStarterArgs := []string{}
+		if config.keepCapNetBindService {
+			envoyStarterArgs = append(envoyStarterArgs, "--keep-cap-net-bind-service", "--")
+		}
+		envoyStarterArgs = append(envoyStarterArgs, envoyArgs...)
+
 		for {
-			logLevel := logging.GetLevel(logging.DefaultLogger)
-			cmd := exec.Command(ciliumEnvoyStarter, "-l", mapLogLevel(logLevel), "-c", bootstrapFilePath, "--base-id", strconv.FormatUint(config.baseID, 10), "--log-format", logFormat)
+			cmd := exec.Command(ciliumEnvoyStarter, envoyStarterArgs...)
 			cmd.Stderr = logWriter
 			cmd.Stdout = logWriter
 

pkg/envoy/xds_server_ondemand.go

@@ -19,6 +19,7 @@ type onDemandXdsStarter struct {
 	runDir                   string
 	envoyLogPath             string
 	envoyBaseID              uint64
+	keepCapNetBindService    bool
 	metricsListenerPort      int
 	adminListenerPort        int
 	connectTimeout           int64
@@ -64,6 +65,7 @@ func (o *onDemandXdsStarter) startEmbeddedEnvoy(wg *completion.WaitGroup) error
 			runDir:                   o.runDir,
 			logPath:                  o.envoyLogPath,
 			baseID:                   o.envoyBaseID,
+			keepCapNetBindService:    o.keepCapNetBindService,
 			connectTimeout:           o.connectTimeout,
 			maxRequestsPerConnection: o.maxRequestsPerConnection,
 			maxConnectionDuration:    o.maxConnectionDuration,