custom answer based on client namespace

[mirisu2]

Hi.
I have kubernetes cluster and I would like a clients (any pods) from ns1 asked foo.bar.baz and got 10.0.1.12 for A-record (task1).
And another clients (any pods) from ns2 asked also foo.bar.baz but now they got 10.0.2.12 for A-record (task2).
And all other clients (who also asked foo.bar.baz) (any pods) from remaining namespaces are forward to resolve to some external DNS server or something else (task3).

Ok. I code next plugin (setup.go):

package nsip

import (
	"github.com/coredns/caddy"
	"github.com/coredns/coredns/core/dnsserver"
	"github.com/coredns/coredns/plugin"
	"github.com/coredns/coredns/plugin/pkg/log"
	"strings"
)

const pluginName = "nsip"

func init() { plugin.Register(pluginName, setup) }

func setup(c *caddy.Controller) error {
	a, err := parse(c)
	if err != nil {
		log.Error(err)
		return plugin.Error(pluginName, err)
	}

	dnsserver.GetConfig(c).AddPlugin(func(next plugin.Handler) plugin.Handler {
		a.Next = next
		return a
	})

	return nil
}

func parse(c *caddy.Controller) (Record, error) {
	a := Record{}
	for c.Next() {
		r := rule{}
		args := c.RemainingArgs()
		r.zones = plugin.OriginsFromArgsOrServerBlock(args, c.ServerBlockKeys)

		for c.NextBlock() {
			p := policy{}

			p.ns = strings.ToLower(c.Val())

			remainingTokens := c.RemainingArgs()
			p.ip = remainingTokens[0]

			r.policies = append(r.policies, p)
		}
		a.Rules = append(a.Rules, r)
	}
	return a, nil
}

and nsip.go

package nsip

import (
	"context"
	"fmt"
	"github.com/coredns/coredns/plugin"
	"github.com/coredns/coredns/plugin/pkg/log"
	"github.com/coredns/coredns/request"
	"github.com/miekg/dns"
	"net"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

type Record struct {
	Next  plugin.Handler
	Rules []rule
}

type rule struct {
	zones    []string
	policies []policy
}

type policy struct {
	ns string
	ip string
}

func (a Record) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
	state := request.Request{W: w, Req: r}

	querySourceIP := state.IP()
	namespace, err := findPodNamespaceByIP(querySourceIP)
	if err != nil {
		log.Errorf("error searching for namespace: %v", err)
	}

	m := new(dns.Msg)
	m.SetReply(r)

	rr := new(dns.A)
	rr.Hdr = dns.RR_Header{Name: r.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 3600}

	ipFound := false
	for _, p := range a.Rules[0].policies {
		if p.ns == namespace {
			ipAddr := net.ParseIP(p.ip)
			if ipAddr != nil {
				rr.A = ipAddr
				m.Answer = append(m.Answer, rr)
				ipFound = true
				break
			}
		}
	}

	if !ipFound {
		return plugin.NextOrFailure(a.Name(), a.Next, ctx, w, r)
	}

	w.WriteMsg(m)

	return dns.RcodeSuccess, nil
}

func (a Record) Name() string { return "nsip" }

func findPodNamespaceByIP(ip string) (string, error) {
	config, err := rest.InClusterConfig()
	if err != nil {
		return "", fmt.Errorf("failed to get in-cluster configuration: %w", err)
	}

	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		return "", fmt.Errorf("failed to create Kubernetes client: %w", err)
	}

	pods, err := clientset.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
	if err != nil {
		return "", fmt.Errorf("failed to retrieve list of pods: %w", err)
	}

	for _, pod := range pods.Items {
		if pod.Status.PodIP == ip {
			return pod.Namespace, nil
		}
	}

	return "", fmt.Errorf("pod with IP %s not found", ip)
}

Corefile:

apiVersion: v1
data:
  Corefile: |
    bar.baz:53 {
      nsip {
        ns1 10.0.1.12
        ns2 10.0.2.12
      }
    }
    .:53 {
        errors
        health {
           lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
           pods insecure
           fallthrough in-addr.arpa ip6.arpa
           ttl 30
        }
        k8s_external example.org
        prometheus :9153
        forward . /etc/resolv.conf {
           max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system

Now, if I resolve foo.bar.baz from any pod from ns1 namespace, I got 10.0.1.12 and if I resolve foo.bar.baz from any pod from ns2 namespace, I got 10.0.2.12. Cool.

For task3 I made next changes (added forward plugin in the chain of plugins):
Corefile

...
  Corefile: |
    bar.baz:53 {
      nsip {
        ns1 10.0.1.12
        ns2 10.0.2.12
      }
      forward . /etc/resolv.conf
    }
...

And now my plugin doesn't work at all.
If I resolve foo.bar.baz from pod from ns1 - my request was forwarded.
If I resolve foo.bar.baz from pod from ns2 - my request was forwarded.
If I resolve foo.bar.baz from pod from ns3 - my request was forwarded.

What I'm expecting - the only one scenario when coredns service will use forward plugin - if coredns service got a DNS-request from ANY namespaces (except for ns1 & ns2).

What am I doing wrong? Hope someone can help.

[mirisu2]

In fact my plugin is ok. Actually, the problem was with the order of plugins in the plugin.cfg file. I just reorder plugins like this

 cat plugin.cfg 
...
nsip:github.com/mirisu2/coredns-nsip
forward:forward
...

Maybe it will be useful for someone.

[chrisohaver]

I know you have already written a custom plugin, but for what it's worth it is possible imitate the example exactly by using the built in view plugin. I don't mean to dissuade you from writing a custom plugin. A custom plugin can have advantages such as presenting a simpler and easier to read configuration (clearly evident in this case).

Using the view plugin for this would look something like the following (completely untested and unverified) configuration...

.53 {
        view view1 {
          expr metadata('kubernetes/client-namespace') == 'ns1'  || (metadata('kubernetes/client-namespace') == 'ns2' && name() != 'foo.bar.baz')
        }
        hosts {
          10.0.1.12 foo.bar.baz
          fallthrough
        }
        metadata
        errors
        health {
           lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
           pods verified
           fallthrough in-addr.arpa ip6.arpa
           ttl 30
        }
        k8s_external example.org
        prometheus :9153
        forward . /etc/resolv.conf {
           max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
}

.53 {
        hosts {
          10.0.2.12 foo.bar.baz
        }
}

[mirisu2]

Thank you for providing the example.
Maybe if the documentation for the plugin were better, I would have used it. Unfortunately, in the examples, I didn't see that it's possible to filter by namespace name in the plugin configuration. Only because of that I wrote a custom plugin.

[chrisohaver]

@mirisu2, if we extend the view plugin docs for the metadata() func ...

metadata(label string) - returns the value for the metadata matching label

to something like ...

metadata(label string) - returns the value for the metadata matching label. Metadata is a way to retrieve information from other plugins. See the metadata plugin README for more information.

... would this have helped?

[mirisu2]

I verified next configuration:

data:
  Corefile: |
    .:53 {
      view ns1 {
        expr metadata('kubernetes/client-namespace') == 'ns1' && name() == 'foo.bar.baz'
      }
      hosts {
        10.0.1.12 foo.bar.baz
        fallthrough
      }
    }
    .:53 {
      view ns2 {
        expr metadata('kubernetes/client-namespace') == 'ns2' && name() == 'foo.bar.baz'
      }
      hosts {
        10.0.2.12 foo.bar.baz
        fallthrough
      }
    }
    .:53 {
        errors
        health {
           lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
           pods insecure
           fallthrough in-addr.arpa ip6.arpa
           ttl 30
        }
        k8s_external example.org
        prometheus :9153
        forward . /etc/resolv.conf {
           max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }

Maybe I'm doing something wrong, but unfortunately it doesn't work.

I found this, but I don't understand in which place I should configure this

[chrisohaver]

In the kubernetes plugin, as in my example above.

[chrisohaver]

Also your config won’t work since metadata are not accessible across server blocks.

[chrisohaver]

I think you’d need to instantiate multiple instances of Kubernetes plugin, which is not ideal for resource usage. So a custom plugin may work better performance wise in the long run.

there has been a recent discussion on making some metadata persist across similar server blocks. But this has not been implemented yet.

[mirisu2]

The configuration like this unfortunately doesn't work:

data:
  Corefile: |
    .:53 {
      log
      view ns1 {
        expr metadata('kubernetes/client-namespace') == 'ns1' && name() == 'foo.bar.baz.'
      }
      hosts {
        10.0.1.12 foo.bar.baz
        fallthrough
      }
    }
    .:53 {
      log
      view ns2 {
        expr metadata('kubernetes/client-namespace') == 'ns2' && name() == 'foo.bar.baz.'
      }
      hosts {
        10.0.2.12 foo.bar.baz
        fallthrough
      }
    }
    .:53 {
        errors
        health {
           lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
           pods insecure
           pods verified
           fallthrough in-addr.arpa ip6.arpa
           ttl 30
        }
        k8s_external example.org
        prometheus :9153
        forward . /etc/resolv.conf {
           max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }

And the configuration like this unfortunately doesn't work too:

data:
  Corefile: |
    .:53 {
      log
      kubernetes {
        pods verified
      }
      view ns1 {
        expr metadata('kubernetes/client-namespace') == 'ns1' && name() == 'foo.bar.baz.'
      }
      hosts {
        10.0.1.12 foo.bar.baz
        fallthrough
      }
    }
    .:53 {
      log
      kubernetes {
        pods verified
      }
      view ns2 {
        expr metadata('kubernetes/client-namespace') == 'ns2' && name() == 'foo.bar.baz.'
      }
      hosts {
        10.0.2.12 foo.bar.baz
        fallthrough
      }
    }
...
    }

[chrisohaver]

Ok. Maybe it does t work then. I don’t have time to debug it right now.

[mirisu2]

ok. I just wanted to try an alternative solution.

[chrisohaver]

I just noticed you’re missing the metadata plugin.
But also metadata doesn’t persist beyond server blocks. So metadata created by Kubernetes plugin will not be usable by view plugin in another server block.