don't use dns search of the host

[pschrammel]

RKE version: v1.5.8

Docker version: (docker version,docker info preferred)
Server:
Containers: 30
Running: 20
Paused: 0
Stopped: 10
Images: 25
Server Version: 24.0.9
Storage Driver: overlay2
Backing Filesystem: extfs
Supports d_type: true
Using metacopy: false
Native Overlay Diff: true
userxattr: false
Logging Driver: json-file
Cgroup Driver: systemd
Cgroup Version: 2
Plugins:
Volume: local
Network: bridge host ipvlan macvlan null overlay
Log: awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog
Swarm: inactive
Runtimes: io.containerd.runc.v2 runc
Default Runtime: runc
Init Binary: docker-init
containerd version: e377cd56a71523140ca6ae87e30244719194a521
runc version: v1.1.12-0-g51d5e94
init version: de40ad0
Security Options:
apparmor
seccomp
Profile: builtin
cgroupns
Kernel Version: 5.15.0-105-generic
Operating System: Ubuntu 22.04.4 LTS
OSType: linux
Architecture: x86_64
CPUs: 4
Total Memory: 23.38GiB
Name: drago
ID: 22251af2-00c6-4672-83e9-7c03b099fded
Docker Root Dir: /data/var/lib/docker
Debug Mode: false
Experimental: false
Insecure Registries:
127.0.0.0/8
Live Restore Enabled: false

Operating system and kernel: (cat /etc/os-release, uname -r preferred)
cat /etc/os-release
PRETTY_NAME="Ubuntu 22.04.4 LTS"
NAME="Ubuntu"
VERSION_ID="22.04"
VERSION="22.04.4 LTS (Jammy Jellyfish)"
VERSION_CODENAME=jammy
ID=ubuntu
ID_LIKE=debian
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
UBUNTU_CODENAME=jammy

5.15.0-105-generic

Type/provider of hosts: (VirtualBox/Bare-metal/AWS/GCE/DO)

Bare-metal

cluster.yml file:

nodes:

• address: 10.1.0.180
  port: "22"
  role:
  • controlplane
  • worker
  • etcd
    hostname_override: drago
    user: peter
    docker_socket: /var/run/docker.sock
    ssh_key_path: /home/peter/.ssh/id_rsa
    ssh_cert: ""
    ssh_cert_path: ""
    labels: {}
    taints: []
    services:
    etcd:
    image: ""
    extra_args: {}
    extra_args_array: {}
    extra_binds: []
    extra_env: []
    win_extra_args: {}
    win_extra_args_array: {}
    win_extra_binds: []
    win_extra_env: []
    external_urls: []
    ca_cert: ""
    cert: ""
    key: ""
    path: ""
    uid: 0
    gid: 0
    snapshot: null
    retention: ""
    creation: ""
    backup_config: null
    kube-api:
    image: ""
    extra_args: {}
    extra_args_array: {}
    extra_binds: []
    extra_env: []
    win_extra_args: {}
    win_extra_args_array: {}
    win_extra_binds: []
    win_extra_env: []
    service_cluster_ip_range: 10.43.0.0/16
    service_node_port_range: ""
    pod_security_policy: false
    pod_security_configuration: ""
    always_pull_images: false
    secrets_encryption_config: null
    audit_log: null
    admission_configuration: null
    event_rate_limit: null
    kube-controller:
    image: ""
    extra_args: {}
    extra_args_array: {}
    extra_binds: []
    extra_env: []
    win_extra_args: {}
    win_extra_args_array: {}
    win_extra_binds: []
    win_extra_env: []
    cluster_cidr: 10.42.0.0/16
    service_cluster_ip_range: 10.43.0.0/16
    scheduler:
    image: ""
    extra_args: {}
    extra_args_array: {}
    extra_binds: []
    extra_env: []
    win_extra_args: {}
    win_extra_args_array: {}
    win_extra_binds: []
    win_extra_env: []
    kubelet:
    image: ""
    extra_args: {}
    extra_args_array: {}
    extra_binds: []
    extra_env: []
    win_extra_args: {}
    win_extra_args_array: {}
    win_extra_binds: []
    win_extra_env: []
    cluster_domain: cluster.local
    infra_container_image: ""
    cluster_dns_server: 10.43.0.10
    fail_swap_on: false
    generate_serving_certificate: false
    kubeproxy:
    image: ""
    extra_args: {}
    extra_args_array: {}
    extra_binds: []
    extra_env: []
    win_extra_args: {}
    win_extra_args_array: {}
    win_extra_binds: []
    win_extra_env: []
    network:
    plugin: canal
    mtu: 1400
    node_selector: {}
    update_strategy: null
    options:
    canal_flannel_backend_type: vxlan
    canal_autoscaler_priority_class_name: system-cluster-critical
    canal_priority_class_name: system-cluster-critical
    tolerations:
  • key: "node.kubernetes.io/unreachable"
    operator: "Exists"
    effect: "NoExecute"
    tolerationseconds: 300
  • key: "node.kubernetes.io/not-ready"
    operator: "Exists"
    effect: "NoExecute"
    tolerationseconds: 300
    update_strategy:
    strategy: RollingUpdate
    rollingUpdate:
    maxUnavailable: 6
    kxauthentication:
    strategy: x509
    sans: []
    webhook: null
    addons: ""
    addons_include: []
    system_images:
    etcd: rancher/mirrored-coreos-etcd:v3.5.10
    alpine: rancher/rke-tools:v0.1.96
    nginx_proxy: rancher/rke-tools:v0.1.96
    cert_downloader: rancher/rke-tools:v0.1.96
    kubernetes_services_sidecar: rancher/rke-tools:v0.1.96
    kubedns: rancher/mirrored-k8s-dns-kube-dns:1.22.28
    dnsmasq: rancher/mirrored-k8s-dns-dnsmasq-nanny:1.22.28
    kubedns_sidecar: rancher/mirrored-k8s-dns-sidecar:1.22.28
    kubedns_autoscaler: rancher/mirrored-cluster-proportional-autoscaler:v1.8.9
    coredns: rancher/mirrored-coredns-coredns:1.10.1
    coredns_autoscaler: rancher/mirrored-cluster-proportional-autoscaler:v1.8.9
    nodelocal: rancher/mirrored-k8s-dns-node-cache:1.22.28
    kubernetes: rancher/hyperkube:v1.28.8-rancher1
    flannel: rancher/mirrored-flannel-flannel:v0.24.2
    flannel_cni: rancher/flannel-cni:v0.3.0-rancher9
    calico_node: rancher/mirrored-calico-node:v3.27.0
    calico_cni: rancher/calico-cni:v3.27.0-rancher1
    calico_controllers: rancher/mirrored-calico-kube-controllers:v3.27.0
    calico_ctl: rancher/mirrored-calico-ctl:v3.27.0
    calico_flexvol: rancher/mirrored-calico-pod2daemon-flexvol:v3.27.0
    canal_node: rancher/mirrored-calico-node:v3.27.0
    canal_cni: rancher/calico-cni:v3.27.0-rancher1
    canal_controllers: rancher/mirrored-calico-kube-controllers:v3.27.0
    canal_flannel: rancher/mirrored-flannel-flannel:v0.24.2
    canal_flexvol: rancher/mirrored-calico-pod2daemon-flexvol:v3.27.0
    weave_node: weaveworks/weave-kube:2.8.1
    weave_cni: weaveworks/weave-npc:2.8.1
    pod_infra_container: rancher/mirrored-pause:3.7
    ingress: rancher/nginx-ingress-controller:nginx-1.9.6-rancher1
    ingress_backend: rancher/mirrored-nginx-ingress-controller-defaultbackend:1.5-rancher1
    ingress_webhook: rancher/mirrored-ingress-nginx-kube-webhook-certgen:v20231226-1a7112e06
    metrics_server: rancher/mirrored-metrics-server:v0.7.0
    windows_pod_infra_container: rancher/mirrored-pause:3.7
    aci_cni_deploy_container: noiro/cnideploy:6.0.4.1.81c2369
    aci_host_container: noiro/aci-containers-host:6.0.4.1.81c2369
    aci_opflex_container: noiro/opflex:6.0.4.1.81c2369
    aci_mcast_container: noiro/opflex:6.0.4.1.81c2369
    aci_ovs_container: noiro/openvswitch:6.0.4.1.81c2369
    aci_controller_container: noiro/aci-containers-controller:6.0.4.1.81c2369
    aci_gbp_server_container: ""
    aci_opflex_server_container: ""
    ssh_key_path: ~/.ssh/id_rsa
    ssh_cert_path: ""
    ssh_agent_auth: false
    authorization:
    mode: rbac
    options: {}
    ignore_docker_version: null
    enable_cri_dockerd: null
    kubernetes_version: ""
    private_registries: []
    ingress:
    provider: none
    options: {}
    node_selector: {}
    extra_args: {}
    dns_policy: ""
    extra_envs: []
    extra_volumes: []
    extra_volume_mounts: []
    update_strategy: null
    http_port: 0
    https_port: 0
    network_mode: ""
    tolerations: []
    default_backend: null
    default_http_backend_priority_class_name: ""
    nginx_ingress_controller_priority_class_name: ""
    default_ingress_class: null
    cluster_name: "fxnet-production"
    cloud_provider:
    name: ""
    prefix_path: ""
    win_prefix_path: ""
    addon_job_timeout: 0
    bastion_host:
    address: ""
    port: ""
    user: ""
    ssh_key: ""
    ssh_key_path: ""
    ssh_cert: ""
    ssh_cert_path: ""
    ignore_proxy_env_vars: false
    monitoring:
    provider: ""
    options: {}
    node_selector: {}
    update_strategy: null
    replicas: null
    tolerations: []
    metrics_server_priority_class_name: ""
    restore:
    restore: false
    snapshot_name: ""
    rotate_encryption_key: false
    dns:
    provider: coredns
    upstreamnameservers:
  • 1.1.1.1
  • 8.8.8.8

Steps to Reproduce:

your dhcp server should return a search domain (like fritz.box)
but as this would interfere with your k8s dns so just have a /etc/resolve.conf on your host with:
nameserver 1.1.1.1

running a docker container will reflect exactly that /etc/resolve.conf

so far so good.
next: rke up --config cluser.yml

but in a container resolve.conf

looks like

nameserver 10.43.0.10
search fxnet-tools.svc.cluster.local svc.cluster.local cluster.local fritz.box
options ndots:5

somehow the search domain of the dhcp now in the search path and things start to go terribly wrong.
any ideas? how to get rid of it? thx