k8s-cluster.yml

---
# Complete Kubernetes HA Cluster Setup Playbook
# This playbook sets up a highly available Kubernetes cluster with multiple master nodes

- name: Prepare all nodes
  hosts: all
  become: yes
  vars:
    kubernetes_version: "1.28.15-1.1"
    pod_network_cidr: "10.244.0.0/16"
    service_cidr: "10.96.0.0/12"
    containerd_version: "1.7.8"

  tasks:
    - name: Wait for system to be ready
      wait_for_connection:
        timeout: 300

    - name: Update apt cache
      apt:
        update_cache: yes
        cache_valid_time: 3600

    - name: Install required packages
      apt:
        name:
          - apt-transport-https
          - ca-certificates
          - curl
          - gnupg
          - lsb-release
          - software-properties-common
        state: present

    - name: Disable swap permanently
      shell: |
        swapoff -a
        sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

    - name: Load kernel modules
      modprobe:
        name: "{{ item }}"
        state: present
      loop:
        - overlay
        - br_netfilter

    - name: Make kernel modules persistent
      copy:
        dest: /etc/modules-load.d/k8s.conf
        content: |
          overlay
          br_netfilter

    - name: Set sysctl parameters for Kubernetes
      sysctl:
        name: "{{ item.name }}"
        value: "{{ item.value }}"
        state: present
        sysctl_file: /etc/sysctl.d/k8s.conf
        reload: yes
      loop:
        - { name: net.bridge.bridge-nf-call-iptables, value: '1' }
        - { name: net.bridge.bridge-nf-call-ip6tables, value: '1' }
        - { name: net.ipv4.ip_forward, value: '1' }

    # Install containerd
    - name: Create containerd config directory
      file:
        path: /etc/containerd
        state: directory
        mode: '0755'

    - name: Download containerd
      get_url:
        url: "https://github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-amd64.tar.gz"
        dest: /tmp/containerd.tar.gz
        mode: '0644'

    - name: Extract containerd
      unarchive:
        src: /tmp/containerd.tar.gz
        dest: /usr/local
        remote_src: yes

    - name: Download containerd service file
      get_url:
        url: https://raw.githubusercontent.com/containerd/containerd/main/containerd.service
        dest: /etc/systemd/system/containerd.service
        mode: '0644'

    - name: Create containerd default config
      shell: containerd config default > /etc/containerd/config.toml
      args:
        creates: /etc/containerd/config.toml

    - name: Configure containerd to use systemd cgroup driver
      replace:
        path: /etc/containerd/config.toml
        regexp: 'SystemdCgroup = false'
        replace: 'SystemdCgroup = true'

    - name: Enable and start containerd
      systemd:
        name: containerd
        state: started
        enabled: yes
        daemon_reload: yes

    - name: Install runc
      get_url:
        url: https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.amd64
        dest: /usr/local/sbin/runc
        mode: '0755'

    - name: Create CNI plugins directory
      file:
        path: /opt/cni/bin
        state: directory
        mode: '0755'

    - name: Install CNI plugins
      unarchive:
        src: https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz
        dest: /opt/cni/bin
        remote_src: yes
        creates: /opt/cni/bin/bridge

    # Install Kubernetes packages
    - name: Add Kubernetes apt key
      apt_key:
        url: https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key
        keyring: /etc/apt/keyrings/kubernetes-apt-keyring.gpg
        state: present

    - name: Add Kubernetes apt repository
      apt_repository:
        repo: "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /"
        state: present
        filename: kubernetes

    - name: Update apt cache after adding Kubernetes repo
      apt:
        update_cache: yes

    - name: Install Kubernetes packages
      apt:
        name:
          - kubelet={{ kubernetes_version }}
          - kubeadm={{ kubernetes_version }}
          - kubectl={{ kubernetes_version }}
        state: present
        allow_downgrades: yes

    - name: Hold Kubernetes packages at current version
      dpkg_selections:
        name: "{{ item }}"
        selection: hold
      loop:
        - kubelet
        - kubeadm
        - kubectl

    - name: Enable kubelet service
      systemd:
        name: kubelet
        enabled: yes

- name: Initialize first Kubernetes master
  hosts: masters[0]
  become: yes
  vars:
    pod_network_cidr: "10.244.0.0/16"
    service_cidr: "10.96.0.0/12"
    control_plane_endpoint: "{{ hostvars[groups['masters'][0]]['ansible_host'] }}:6443"

  tasks:
    - name: Check if cluster is already initialized
      stat:
        path: /etc/kubernetes/admin.conf
      register: kubeadm_init

    - name: Initialize Kubernetes cluster on first master
      shell: |
        kubeadm init \
          --control-plane-endpoint="{{ control_plane_endpoint }}" \
          --upload-certs \
          --pod-network-cidr={{ pod_network_cidr }} \
          --service-cidr={{ service_cidr }} \
          --apiserver-advertise-address={{ ansible_host }}
      register: kubeadm_output
      when: not kubeadm_init.stat.exists

    - name: Create .kube directory for user
      file:
        path: "{{ '/root' if ansible_user == 'root' else '/home/' + ansible_user }}/.kube"
        state: directory
        owner: "{{ ansible_user }}"
        group: "{{ ansible_user }}"
        mode: '0755'

    - name: Copy admin.conf to user's kube config
      copy:
        src: /etc/kubernetes/admin.conf
        dest: "{{ '/root' if ansible_user == 'root' else '/home/' + ansible_user }}/.kube/config"
        remote_src: yes
        owner: "{{ ansible_user }}"
        group: "{{ ansible_user }}"
        mode: '0644'

    - name: Copy admin.conf to root's kube config
      copy:
        src: /etc/kubernetes/admin.conf
        dest: /root/.kube/config
        remote_src: yes
        mode: '0644'
      ignore_errors: yes

    - name: Create root .kube directory
      file:
        path: /root/.kube
        state: directory
        mode: '0755'
      when: kubeadm_output.changed

    - name: Copy admin config for root
      copy:
        src: /etc/kubernetes/admin.conf
        dest: /root/.kube/config
        remote_src: yes
        mode: '0644'
      when: kubeadm_output.changed

    - name: Set KUBECONFIG environment variable
      set_fact:
        kubeconfig_path: "{{ '/root' if ansible_user == 'root' else '/home/' + ansible_user }}/.kube/config"

    - name: Install Flannel CNI
      shell: |
        export KUBECONFIG={{ kubeconfig_path }}
        kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml
      environment:
        KUBECONFIG: "{{ kubeconfig_path }}"
      when: kubeadm_output.changed

    - name: Wait for Flannel to be ready
      shell: kubectl get pods -n kube-flannel -o jsonpath='{.items[*].status.phase}'
      environment:
        KUBECONFIG: "{{ kubeconfig_path }}"
      register: flannel_status
      until: "'Running' in flannel_status.stdout"
      retries: 30
      delay: 10
      when: kubeadm_output.changed

    - name: Upload certificates and get cert key
      shell: |
        kubeadm init phase upload-certs --upload-certs 2>&1 | grep -v "remote version" | tail -1
      register: cert_key

    - name: Get join command for control plane
      shell: kubeadm token create --print-join-command --certificate-key {{ cert_key.stdout }}
      register: join_command_control_plane
      when: cert_key.stdout is defined and cert_key.stdout != ""

    - name: Get join command for workers
      shell: kubeadm token create --print-join-command
      register: join_command_workers

    - name: Set join commands as facts
      set_fact:
        control_plane_join_command: "{{ join_command_control_plane.stdout | default('') }}"
        worker_join_command: "{{ join_command_workers.stdout | default('') }}"

- name: Join additional master nodes
  hosts: masters[1:]
  become: yes
  tasks:
    - name: Check if node is already part of cluster
      stat:
        path: /etc/kubernetes/kubelet.conf
      register: kubelet_conf

    - name: Join additional master nodes to cluster
      shell: "{{ hostvars[groups['masters'][0]]['control_plane_join_command'] }} --control-plane"
      when:
        - not kubelet_conf.stat.exists
        - hostvars[groups['masters'][0]]['control_plane_join_command'] is defined
        - hostvars[groups['masters'][0]]['control_plane_join_command'] != ''

    - name: Create .kube directory for user
      file:
        path: "{{ '/root' if ansible_user == 'root' else '/home/' + ansible_user }}/.kube"
        state: directory
        owner: "{{ ansible_user }}"
        group: "{{ ansible_user }}"
        mode: '0755'
      when: not kubelet_conf.stat.exists

    - name: Copy admin.conf to user's kube config
      copy:
        src: /etc/kubernetes/admin.conf
        dest: "{{ '/root' if ansible_user == 'root' else '/home/' + ansible_user }}/.kube/config"
        remote_src: yes
        owner: "{{ ansible_user }}"
        group: "{{ ansible_user }}"
        mode: '0644'
      when: not kubelet_conf.stat.exists

- name: Join worker nodes to cluster
  hosts: workers
  become: yes
  tasks:
    - name: Check if node is already part of cluster
      stat:
        path: /etc/kubernetes/kubelet.conf
      register: kubelet_conf

    - name: Join worker nodes to cluster
      shell: "{{ hostvars[groups['masters'][0]]['worker_join_command'] }}"
      when:
        - not kubelet_conf.stat.exists
        - hostvars[groups['masters'][0]]['worker_join_command'] is defined
        - hostvars[groups['masters'][0]]['worker_join_command'] != ''

- name: Verify cluster setup
  hosts: masters[0]
  become: yes
  tasks:
    - name: Wait for all nodes to be ready
      shell: kubectl get nodes --no-headers | grep -v " Ready" | wc -l
      register: not_ready_nodes
      until: not_ready_nodes.stdout == "0"
      retries: 30
      delay: 10

    - name: Get cluster nodes
      shell: kubectl get nodes -o wide
      register: cluster_nodes

    - name: Display cluster status
      debug:
        msg: "{{ cluster_nodes.stdout_lines }}"

    - name: Get all pods status
      shell: kubectl get pods --all-namespaces
      register: all_pods

    - name: Display pods status
      debug:
        msg: "{{ all_pods.stdout_lines }}"