proxy/routes: Remove fromEgressProxyRule for cilium downgrade

[ upstream commit: 53133ff ]

Although we don't install fromEgressProxyRule for now, this commit
insists on removing it to make sure further downgrade can go smoothly.

Soon We'll have another PR to install fromEgressProxyRule, and cilium
downgrade from that PR to branch tip (patch downgrade, 1.X.Y ->
1.X.{Y-1}) will be broken if we don't handle the new ip rule carefullly.

Without this patch, downgrade from higher version will leave
fromEgressProxyRule on the lower version cilium, cluster will be in a
wrong status of "having stale ip rule + not having other necessary
settings (iptables)", breaking the connectivity.

Signed-off-by: Zhichuan Liang <gray.liang@isovalent.com>
Signed-off-by: Zhichuan Liang <gray.liang@isovalent.com>

pkg/proxy/proxy.go

@@ -425,6 +425,9 @@ func (p *Proxy) ReinstallRoutingRules() error {
 			return err
 		}
 
+		if err := removeFromEgressProxyRoutesIPv4(); err != nil {
+			return err
+		}
 		if !option.Config.EnableIPSec || option.Config.TunnelingEnabled() {
 			if err := removeFromIngressProxyRoutesIPv4(); err != nil {
 				return err
@@ -451,6 +454,9 @@ func (p *Proxy) ReinstallRoutingRules() error {
 			return err
 		}
 
+		if err := removeFromEgressProxyRoutesIPv6(); err != nil {
+			return err
+		}
 		if !option.Config.EnableIPSec || option.Config.TunnelingEnabled() {
 			if err := removeFromIngressProxyRoutesIPv6(); err != nil {
 				return err

pkg/proxy/routes.go

@@ -111,7 +111,7 @@ var (
 		Protocol: linux_defaults.RTProto,
 	}
 
-	//nolint:unused // Routing rule for traffic from egress proxy.
+	// Routing rule for traffic from egress proxy.
 	fromEgressProxyRule = route.Rule{
 		Priority: linux_defaults.RulePriorityFromProxy,
 		Mark:     linux_defaults.MagicMarkEgress,
@@ -171,6 +171,13 @@ func removeStaleProxyRulesIPv4() error {
 	return removeProtoUnspecRules(netlink.FAMILY_V4)
 }
 
+func removeFromEgressProxyRoutesIPv4() error {
+	if err := route.DeleteRule(netlink.FAMILY_V4, fromEgressProxyRule); err != nil && !errors.Is(err, syscall.ENOENT) {
+		return fmt.Errorf("removing ipv4 from egress proxy routing rule: %w", err)
+	}
+	return nil
+}
+
 // installFromProxyRoutesIPv6 configures routes and rules needed to redirect ingress
 // packets from the proxy.
 func installFromProxyRoutesIPv6(ipv6 net.IP, device string) error {
@@ -247,3 +254,12 @@ func removeProtoUnspecRules(family int) error {
 	}
 	return nil
 }
+
+func removeFromEgressProxyRoutesIPv6() error {
+	if err := route.DeleteRule(netlink.FAMILY_V6, fromEgressProxyRule); err != nil {
+		if !errors.Is(err, syscall.ENOENT) && !errors.Is(err, syscall.EAFNOSUPPORT) {
+			return fmt.Errorf("removing ipv6 from egress proxy routing rule: %w", err)
+		}
+	}
+	return nil
+}