loader: infrastructure for attaching SKB programs using the tcx API

bpf/lib/overloadable_skb.h

@@ -19,6 +19,9 @@ bpf_clear_meta(struct __sk_buff *ctx)
 	WRITE_ONCE(ctx->cb[2], zero);
 	WRITE_ONCE(ctx->cb[3], zero);
 	WRITE_ONCE(ctx->cb[4], zero);
+
+	/* This needs to be cleared mainly for tcx. */
+	WRITE_ONCE(ctx->tc_classid, zero);
 }
 
 /**

cilium-dbg/cmd/post_uninstall_cleanup.go

@@ -255,6 +255,7 @@ func (c ciliumCleanup) cleanupFuncs() []cleanupFunc {
 		cleanupTCFilters,
 		cleanupXDPs,
 		removeSocketLBPrograms,
+		removeCiliumBPFFS,
 	}
 	if !c.bpfOnly {
 		funcs = append(funcs, cleanupRoutesAndLinks)
@@ -595,3 +596,14 @@ func isCiliumXDP(progId uint32) (bool, error) {
 	return false, nil
 
 }
+
+func removeCiliumBPFFS() error {
+	path := bpf.CiliumPath()
+
+	if err := bpf.Remove(path); err != nil {
+		return err
+	}
+
+	fmt.Printf("removed all cilium bpffs objects under %s\n", path)
+	return nil
+}

daemon/cmd/daemon_main.go

@@ -705,6 +705,9 @@ func InitGlobalFlags(cmd *cobra.Command, vp *viper.Viper) {
 	flags.Bool(option.EnableXDPPrefilter, false, "Enable XDP prefiltering")
 	option.BindEnv(vp, option.EnableXDPPrefilter)
 
+	flags.Bool(option.EnableTCX, false, "Attach endpoint programs using tcx if supported by the kernel")
+	option.BindEnv(vp, option.EnableTCX)
+
 	flags.Bool(option.PreAllocateMapsName, defaults.PreAllocateMaps, "Enable BPF map pre-allocation")
 	option.BindEnv(vp, option.PreAllocateMapsName)
 

install/kubernetes/cilium/templates/cilium-configmap.yaml

@@ -579,6 +579,12 @@ data:
   enable-ipv6-big-tcp: {{ .Values.enableIPv6BIGTCP | quote }}
   enable-ipv6-masquerade: {{ .Values.enableIPv6Masquerade | quote }}
 
+{{- if hasKey .Values "enableTCX" }}
+  enable-tcx: {{ .Values.enableTCX | quote }}
+{{- else }}
+  enable-tcx: "true"
+{{- end }}
+
 {{- if (not (kindIs "invalid" .Values.bpf.masquerade)) }}
   enable-bpf-masquerade: {{ .Values.bpf.masquerade | quote }}
 {{- else if eq $defaultBpfMasquerade "true" }}

install/kubernetes/cilium/values.schema.json

@@ -1499,6 +1499,9 @@
     "enableRuntimeDeviceDetection": {
       "type": "boolean"
     },
+    "enableTCX": {
+      "type": "boolean"
+    },
     "enableXTSocketFallback": {
       "type": "boolean"
     },

install/kubernetes/cilium/values.yaml.tmpl

@@ -1802,6 +1802,8 @@ enableMasqueradeRouteSource: false
 enableIPv4BIGTCP: false
 # -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods
 enableIPv6BIGTCP: false
+# -- Attach endpoint programs using tcx instead of legacy tc hooks on supported kernels.
+enableTCX: true
 egressGateway:
   # -- Enables egress gateway to redirect and SNAT the traffic that leaves the
   # cluster.

pkg/datapath/loader/base.go

@@ -223,6 +223,7 @@ func (l *loader) reinitializeIPSec(ctx context.Context) error {
 				elf:      networkObj,
 				programs: progs,
 				linkDir:  bpffsDeviceLinksDir(bpf.CiliumPath(), device),
+				tcx:      option.Config.EnableTCX,
 			},
 		)
 		if err != nil {

pkg/datapath/loader/loader.go

@@ -331,6 +331,13 @@ func removeObsoleteNetdevPrograms(devices []string) error {
 			continue
 		}
 
+		// Remove the per-device bpffs directory containing pinned links and
+		// per-endpoint maps.
+		bpffsPath := bpffsDeviceDir(bpf.CiliumPath(), l)
+		if err := bpf.Remove(bpffsPath); err != nil {
+			log.WithError(err).WithField(logfields.Device, l.Attrs().Name)
+		}
+
 		ingressFilters, err := netlink.FilterList(l, directionToParent(dirIngress))
 		if err != nil {
 			return fmt.Errorf("listing ingress filters: %w", err)
@@ -357,14 +364,14 @@ func removeObsoleteNetdevPrograms(devices []string) error {
 	}
 
 	for _, dev := range ingressDevs {
-		err = removeTCFilters(dev.Attrs().Name, directionToParent(dirIngress))
+		err = removeTCFilters(dev, directionToParent(dirIngress))
 		if err != nil {
 			log.WithError(err).Errorf("couldn't remove ingress tc filters from %s", dev.Attrs().Name)
 		}
 	}
 
 	for _, dev := range egressDevs {
-		err = removeTCFilters(dev.Attrs().Name, directionToParent(dirEgress))
+		err = removeTCFilters(dev, directionToParent(dirEgress))
 		if err != nil {
 			log.WithError(err).Errorf("couldn't remove egress tc filters from %s", dev.Attrs().Name)
 		}
@@ -405,6 +412,7 @@ func (l *loader) reloadHostDatapath(ctx context.Context, ep datapath.Endpoint, o
 			elf:      objPath,
 			programs: progs,
 			linkDir:  bpffsDeviceLinksDir(bpf.CiliumPath(), host),
+			tcx:      option.Config.EnableTCX,
 		},
 	)
 	if err != nil {
@@ -445,6 +453,7 @@ func (l *loader) reloadHostDatapath(ctx context.Context, ep datapath.Endpoint, o
 			elf:      secondDevObjPath,
 			programs: progs,
 			linkDir:  bpffsDeviceLinksDir(bpf.CiliumPath(), net),
+			tcx:      option.Config.EnableTCX,
 		},
 	)
 	if err != nil {
@@ -467,6 +476,8 @@ func (l *loader) reloadHostDatapath(ctx context.Context, ep datapath.Endpoint, o
 			continue
 		}
 
+		linkDir := bpffsDeviceLinksDir(bpf.CiliumPath(), iface)
+
 		netdevObjPath := path.Join(ep.StateDir(), hostEndpointNetdevPrefix+device+".o")
 		if err := l.patchHostNetdevDatapath(ep, objPath, netdevObjPath, device); err != nil {
 			return err
@@ -486,8 +497,7 @@ func (l *loader) reloadHostDatapath(ctx context.Context, ep datapath.Endpoint, o
 		} else {
 			// Remove any previously attached device from egress path if BPF
 			// NodePort and host firewall are disabled.
-			err := removeTCFilters(device, netlink.HANDLE_MIN_EGRESS)
-			if err != nil {
+			if err := detachSKBProgram(iface, symbolToHostNetdevEp, linkDir, netlink.HANDLE_MIN_EGRESS); err != nil {
 				log.WithField("device", device).Error(err)
 			}
 		}
@@ -497,7 +507,8 @@ func (l *loader) reloadHostDatapath(ctx context.Context, ep datapath.Endpoint, o
 				device:   device,
 				elf:      netdevObjPath,
 				programs: progs,
-				linkDir:  bpffsDeviceLinksDir(bpf.CiliumPath(), iface),
+				linkDir:  linkDir,
+				tcx:      option.Config.EnableTCX,
 			},
 		)
 		if err != nil {
@@ -530,6 +541,7 @@ func (l *loader) reloadHostDatapath(ctx context.Context, ep datapath.Endpoint, o
 func (l *loader) reloadDatapath(ctx context.Context, ep datapath.Endpoint, dirs *directoryInfo) error {
 	// Replace the current program
 	objPath := path.Join(dirs.Output, endpointObj)
+	device := ep.InterfaceName()
 
 	if ep.IsHost() {
 		// TODO: react to changes (using the currently ignored watch channel)
@@ -546,28 +558,33 @@ func (l *loader) reloadDatapath(ctx context.Context, ep datapath.Endpoint, dirs
 		}
 	} else {
 		progs := []progDefinition{{progName: symbolFromEndpoint, direction: dirIngress}}
+		linkDir := bpffsEndpointLinksDir(bpf.CiliumPath(), ep)
 
 		if ep.RequireEgressProg() {
 			progs = append(progs, progDefinition{progName: symbolToEndpoint, direction: dirEgress})
 		} else {
-			err := removeTCFilters(ep.InterfaceName(), netlink.HANDLE_MIN_EGRESS)
+			iface, err := netlink.LinkByName(device)
 			if err != nil {
-				log.WithField("device", ep.InterfaceName()).Error(err)
+				log.WithError(err).WithField("device", device).Warn("Link does not exist")
+			}
+			if err := detachSKBProgram(iface, symbolToEndpoint, linkDir, netlink.HANDLE_MIN_EGRESS); err != nil {
+				log.WithField("device", device).Error(err)
 			}
 		}
 
 		finalize, err := replaceDatapath(ctx,
 			replaceDatapathOptions{
-				device:   ep.InterfaceName(),
+				device:   device,
 				elf:      objPath,
 				programs: progs,
-				linkDir:  bpffsEndpointLinksDir(bpf.CiliumPath(), ep),
+				linkDir:  linkDir,
+				tcx:      option.Config.EnableTCX,
 			},
 		)
 		if err != nil {
 			scopedLog := ep.Logger(subsystem).WithFields(logrus.Fields{
 				logfields.Path: objPath,
-				logfields.Veth: ep.InterfaceName(),
+				logfields.Veth: device,
 			})
 			// Don't log an error here if the context was canceled or timed out;
 			// this log message should only represent failures with respect to
@@ -582,7 +599,7 @@ func (l *loader) reloadDatapath(ctx context.Context, ep datapath.Endpoint, dirs
 
 	if ep.RequireEndpointRoute() {
 		scopedLog := ep.Logger(subsystem).WithFields(logrus.Fields{
-			logfields.Veth: ep.InterfaceName(),
+			logfields.Veth: device,
 		})
 		if ip := ep.IPv4Address(); ip.IsValid() {
 			if err := upsertEndpointRoute(ep, *iputil.AddrToIPNet(ip)); err != nil {
@@ -620,6 +637,7 @@ func (l *loader) replaceOverlayDatapath(ctx context.Context, cArgs []string, ifa
 			elf:      overlayObj,
 			programs: progs,
 			linkDir:  bpffsDeviceLinksDir(bpf.CiliumPath(), device),
+			tcx:      option.Config.EnableTCX,
 		},
 	)
 	if err != nil {

pkg/datapath/loader/loader_test.go

@@ -141,6 +141,7 @@ func TestReload(t *testing.T) {
 		elf:      objPath,
 		programs: progs,
 		linkDir:  testutils.TempBPFFS(t),
+		tcx:      true,
 	}
 	finalize, err := replaceDatapath(ctx, opts)
 	require.NoError(t, err)
@@ -289,6 +290,7 @@ func BenchmarkReplaceDatapath(b *testing.B) {
 				elf:      objPath,
 				programs: progs,
 				linkDir:  linkDir,
+				tcx:      true,
 			},
 		)
 		if err != nil {