From bcd37a6904ca978a02885c98f159e1142798e7b2 Mon Sep 17 00:00:00 2001 From: Gereon Vey Date: Mon, 29 Jan 2024 16:13:13 +0100 Subject: [PATCH] add kube-vip as a service load balancer (#432) * add kube-vip as a service load balancer * add molecule scenario kube-vip --------- Co-authored-by: Techno Tim --- .github/ISSUE_TEMPLATE.md | 3 ++ .github/workflows/test.yml | 1 + inventory/sample/group_vars/all.yml | 7 ++++ molecule/README.md | 2 + molecule/kube-vip/molecule.yml | 49 ++++++++++++++++++++++ molecule/kube-vip/overrides.yml | 17 ++++++++ roles/k3s_server/tasks/kube-vip.yml | 27 ++++++++++++ roles/k3s_server/tasks/main.yml | 6 +++ roles/k3s_server/templates/kubevip.yaml.j2 | 13 ++++++ roles/k3s_server/templates/vip.yaml.j2 | 2 +- roles/k3s_server_post/tasks/main.yml | 1 + 11 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 molecule/kube-vip/molecule.yml create mode 100644 molecule/kube-vip/overrides.yml create mode 100644 roles/k3s_server/tasks/kube-vip.yml create mode 100644 roles/k3s_server/templates/kubevip.yaml.j2 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index d9553dee4..ad49c1834 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -51,6 +51,9 @@ extra_agent_args: "" kube_vip_tag_version: "" +kube_vip_cloud_provider_tag_version: "" +kube_vip_lb_ip_range: "" + metal_lb_speaker_tag_version: "" metal_lb_controller_tag_version: "" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 682fa99ff..c93e9f41e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,6 +13,7 @@ jobs: - ipv6 - single_node - calico + - kube-vip fail-fast: false env: PYTHON_VERSION: "3.11" diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index 4f433af34..f6f812016 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -56,6 +56,13 @@ extra_agent_args: >- # image tag for kube-vip kube_vip_tag_version: "v0.6.4" +# tag for kube-vip-cloud-provider manifest +# kube_vip_cloud_provider_tag_version: "main" + +# kube-vip ip range for load balancer +# (uncomment to use kube-vip for services instead of MetalLB) +# kube_vip_lb_ip_range: "192.168.30.80-192.168.30.90" + # metallb type frr or native metal_lb_type: "native" diff --git a/molecule/README.md b/molecule/README.md index 40c2feebd..f8d6b7052 100644 --- a/molecule/README.md +++ b/molecule/README.md @@ -15,6 +15,8 @@ We have these scenarios: Very similar to the default scenario, but uses only a single node for all cluster functionality. - **calico**: The same as single node, but uses calico cni instead of flannel. +- **kube-vip** + The same as single node, but uses kube-vip as service loadbalancer instead of MetalLB ## How to execute diff --git a/molecule/kube-vip/molecule.yml b/molecule/kube-vip/molecule.yml new file mode 100644 index 000000000..f9487f399 --- /dev/null +++ b/molecule/kube-vip/molecule.yml @@ -0,0 +1,49 @@ +--- +dependency: + name: galaxy +driver: + name: vagrant +platforms: + - name: control1 + box: generic/ubuntu2204 + memory: 4096 + cpus: 4 + config_options: + # We currently can not use public-key based authentication on Ubuntu 22.04, + # see: https://github.com/chef/bento/issues/1405 + ssh.username: "vagrant" + ssh.password: "vagrant" + groups: + - k3s_cluster + - master + interfaces: + - network_name: private_network + ip: 192.168.30.62 +provisioner: + name: ansible + env: + ANSIBLE_VERBOSITY: 1 + playbooks: + converge: ../resources/converge.yml + side_effect: ../resources/reset.yml + verify: ../resources/verify.yml + inventory: + links: + group_vars: ../../inventory/sample/group_vars +scenario: + test_sequence: + - dependency + - cleanup + - destroy + - syntax + - create + - prepare + - converge + # idempotence is not possible with the playbook in its current form. + - verify + # We are repurposing side_effect here to test the reset playbook. + # This is why we do not run it before verify (which tests the cluster), + # but after the verify step. + - side_effect + - cleanup + - destroy diff --git a/molecule/kube-vip/overrides.yml b/molecule/kube-vip/overrides.yml new file mode 100644 index 000000000..c64b1f3ce --- /dev/null +++ b/molecule/kube-vip/overrides.yml @@ -0,0 +1,17 @@ +--- +- name: Apply overrides + hosts: all + tasks: + - name: Override host variables + ansible.builtin.set_fact: + # See: + # https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant + flannel_iface: eth1 + + # The test VMs might be a bit slow, so we give them more time to join the cluster: + retry_count: 45 + + # Make sure that our IP ranges do not collide with those of the other scenarios + apiserver_endpoint: "192.168.30.225" + # Use kube-vip instead of MetalLB + kube_vip_lb_ip_range: "192.168.30.110-192.168.30.119" diff --git a/roles/k3s_server/tasks/kube-vip.yml b/roles/k3s_server/tasks/kube-vip.yml new file mode 100644 index 000000000..d0c74afd7 --- /dev/null +++ b/roles/k3s_server/tasks/kube-vip.yml @@ -0,0 +1,27 @@ +--- +- name: Create manifests directory on first master + file: + path: /var/lib/rancher/k3s/server/manifests + state: directory + owner: root + group: root + mode: 0644 + when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] + +- name: Download vip cloud provider manifest to first master + ansible.builtin.get_url: + url: "https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/{{ kube_vip_cloud_provider_tag_version | default('main') }}/manifest/kube-vip-cloud-controller.yaml" # noqa yaml[line-length] + dest: "/var/lib/rancher/k3s/server/manifests/kube-vip-cloud-controller.yaml" + owner: root + group: root + mode: 0644 + when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] + +- name: Copy kubevip configMap manifest to first master + template: + src: "kubevip.yaml.j2" + dest: "/var/lib/rancher/k3s/server/manifests/kubevip.yaml" + owner: root + group: root + mode: 0644 + when: ansible_hostname == hostvars[groups[group_name_master | default('master')][0]]['ansible_hostname'] diff --git a/roles/k3s_server/tasks/main.yml b/roles/k3s_server/tasks/main.yml index 5a396e0b6..cc5f8239e 100644 --- a/roles/k3s_server/tasks/main.yml +++ b/roles/k3s_server/tasks/main.yml @@ -29,6 +29,12 @@ - name: Deploy metallb manifest include_tasks: metallb.yml tags: metallb + when: kube_vip_lb_ip_range is not defined + +- name: Deploy kube-vip manifest + include_tasks: kube-vip.yml + tags: kubevip + when: kube_vip_lb_ip_range is defined - name: Init cluster inside the transient k3s-init service command: diff --git a/roles/k3s_server/templates/kubevip.yaml.j2 b/roles/k3s_server/templates/kubevip.yaml.j2 new file mode 100644 index 000000000..40d8b50ed --- /dev/null +++ b/roles/k3s_server/templates/kubevip.yaml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubevip + namespace: kube-system +data: +{% if kube_vip_lb_ip_range is string %} +{# kube_vip_lb_ip_range was used in the legacy way: single string instead of a list #} +{# => transform to list with single element #} +{% set kube_vip_lb_ip_range = [kube_vip_lb_ip_range] %} +{% endif %} + range-global: {{ kube_vip_lb_ip_range | join(',') }} diff --git a/roles/k3s_server/templates/vip.yaml.j2 b/roles/k3s_server/templates/vip.yaml.j2 index 862aee642..1092557ac 100644 --- a/roles/k3s_server/templates/vip.yaml.j2 +++ b/roles/k3s_server/templates/vip.yaml.j2 @@ -43,7 +43,7 @@ spec: - name: vip_ddns value: "false" - name: svc_enable - value: "false" + value: "{{ 'true' if kube_vip_lb_ip_range is defined else 'false' }}" - name: vip_leaderelection value: "true" - name: vip_leaseduration diff --git a/roles/k3s_server_post/tasks/main.yml b/roles/k3s_server_post/tasks/main.yml index 505eebfdd..e9b98422e 100644 --- a/roles/k3s_server_post/tasks/main.yml +++ b/roles/k3s_server_post/tasks/main.yml @@ -7,6 +7,7 @@ - name: Deploy metallb pool include_tasks: metallb.yml tags: metallb + when: kube_vip_lb_ip_range is not defined - name: Remove tmp directory used for manifests file: