Install Kubernetes and Tungsten Fabric with Kubespray on Ubuntu 16.04 LTS or Centos 7

kubespray-tf.sh

#!/bin/bash
#RUN THIS AS ROOT! this script assumes you have installed Ubuntu 16.04 LTS or Centos 7 on your nodes and can ssh to each node as root
#scp ssh keys to /root/.ssh/ and scp kubespray-tf.sh to /root/ on first master
#edit k8s_api_ip variable and master_ip_list variable and then edit the inventory.ini to match your environment
#edit pod and service cidr or k8s version in k8s-cluster.yml file below to match your needs


#set this to the IP of your first k8s master. this will eventually need to be the ip of an haproxy pointing at the master_ip_list for the K8s API
k8s_api_ip="10.9.8.21"

#set this to the IPs of your masters seperated by a comma with no spaces if there are more than one. master_ip_list="10.9.8.21,10.9.8.22,10.9.8.23"
master_ip_list="10.9.8.21"

#CENTOS ONLY set this to the gateway for your subnet
vrouter_gw="10.9.8.1"

#this is needed for floating ips
k8s_fip='''\ \ KUBERNETES_PUBLIC_FIP_POOL: "{'domain': 'default-domain', 'project': 'k8s-default', 'network': 'external', 'name': 'default'}"'''


mkdir /root/k8s-tf
cd /root/k8s-tf

#ubuntu test
if [ -f /etc/lsb-release ]; then
  #ubuntu commands
  K8S_MASTER_IP=$master_ip_list; CONTRAIL_REPO="docker.io\/opencontrailnightly"; CONTRAIL_RELEASE="latest"; mkdir -pm 777 /var/lib/contrail/kafka-logs; curl https://raw.githubusercontent.com/Juniper/contrail-kubernetes-docs/master/install/kubernetes/templates/contrail-single-step-cni-install-ubuntu.yaml | sed "s/{{ K8S_MASTER_IP }}/$K8S_MASTER_IP/g; s/{{ CONTRAIL_REPO }}/$CONTRAIL_REPO/g; s/{{ CONTRAIL_RELEASE }}/$CONTRAIL_RELEASE/g" >> /root/k8s-tf/tf.yml

  add-apt-repository ppa:ansible/ansible-2.7 -y 
  apt-get update 
  apt-get install python ansible python-pip python-netaddr git -y

else
  #centos commands
  K8S_MASTER_IP=$master_ip_list; CONTRAIL_REPO="docker.io\/opencontrailnightly"; CONTRAIL_RELEASE="latest"; mkdir -pm 777 /var/lib/contrail/kafka-logs; curl https://raw.githubusercontent.com/Juniper/contrail-kubernetes-docs/master/install/kubernetes/templates/contrail-single-step-cni-install-centos.yaml | sed "s/{{ K8S_MASTER_IP }}/$K8S_MASTER_IP/g; s/{{ CONTRAIL_REPO }}/$CONTRAIL_REPO/g; s/{{ CONTRAIL_RELEASE }}/$CONTRAIL_RELEASE/g" >> /root/k8s-tf/tf.yml
  vrouter_gw_txt="\ \ VROUTER_GATEWAY: "
  vrouter_gw_line="$vrouter_gw_txt""$vrouter_gw"
  sed -i "/  VROUTER_GATEWAY:*/c $vrouter_gw_line" /root/k8s-tf/tf.yml
  sudo yum install epel-release
  sudo yum -y update
  sudo yum -y install python python-pip pip python-netaddr git
  wget https://cbs.centos.org/kojifiles/packages/ansible/2.7.10/1.el7/noarch/ansible-2.7.10-1.el7.noarch.rpm
  sudo yum install ansible-2.7.10-1.el7.noarch.rpm
  pip install jinja2 --upgrade
fi

#kubespray inventory.ini file. edit this file to match your environment. use host.fqdn format 
cat > /root/k8s-tf/inventory.ini <<EOF 
[all]
test1.cluster.local ansible_host=10.9.8.21 etcd_member_name=etcd1
test2.cluster.local ansible_host=10.9.8.22 etcd_member_name=etcd2
test3.cluster.local ansible_host=10.9.8.23 etcd_member_name=etcd3

[kube-master]
test1.cluster.local

[etcd]
test1.cluster.local
test2.cluster.local
test3.cluster.local

[kube-node]
test2.cluster.local
test3.cluster.local

[k8s-cluster:children]
kube-master
kube-node
EOF

#kubespray k8s-custer.yml file. edit pod and service cidr or k8s version if needed
cat > /root/k8s-tf/k8s-cluster.yml <<EOF
---
# unused variables and comments have been removed
kube_config_dir: /etc/kubernetes
kube_script_dir: "{{ bin_dir }}/kubernetes-scripts"
kube_manifest_dir: "{{ kube_config_dir }}/manifests"
kube_cert_dir: "{{ kube_config_dir }}/ssl"
kube_token_dir: "{{ kube_config_dir }}/tokens"
kube_users_dir: "{{ kube_config_dir }}/users"
kube_api_anonymous_auth: true
kube_version: v1.14.1
kube_image_repo: "gcr.io/google-containers"
local_release_dir: "/tmp/releases"
retry_stagger: 5
kube_cert_group: kube-cert
kube_log_level: 2
credentials_dir: "{{ inventory_dir }}/credentials"
kube_api_pwd: "{{ lookup('password', credentials_dir + '/kube_user.creds length=15 chars=ascii_letters,digits') }}"
kube_users:
  kube:
    pass: "{{kube_api_pwd}}"
    role: admin
    groups:
      - system:masters
kube_network_plugin: cni
kube_network_plugin_multus: false
kube_service_addresses: 10.96.0.0/12
kube_pods_subnet: 10.0.0.0/12
kube_network_node_prefix: 24
kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}"
kube_apiserver_port: 6443  
kube_apiserver_insecure_port: 0  # (disabled)
kube_proxy_mode: iptables
kube_proxy_nodeport_addresses: >-
  {%- if kube_proxy_nodeport_addresses_cidr is defined -%}
  [{{ kube_proxy_nodeport_addresses_cidr }}]
  {%- else -%}
  []
  {%- endif -%}
kube_encrypt_secret_data: false
cluster_name: cluster.local
ndots: 2
dns_mode: coredns
enable_nodelocaldns: false
resolvconf_mode: docker_dns
deploy_netchecker: false
skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}"
skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}"
dns_domain: "{{ cluster_name }}"
container_manager: docker
etcd_deployment_type: docker
kubelet_deployment_type: host
helm_deployment_type: host
kubeadm_control_plane: false
kubeadm_certificate_key: "{{ lookup('password', credentials_dir + '/kubeadm_certificate_key.creds length=64 chars=hexdigits') | lower }}"
k8s_image_pull_policy: IfNotPresent
kubernetes_audit: false
dynamic_kubelet_configuration: false
default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir"
dynamic_kubelet_configuration_dir: "{{ kubelet_config_dir | default(default_kubelet_config_dir) }}"
podsecuritypolicy_enabled: false
volume_cross_zone_attachment: false
persistent_volumes_enabled: false
EOF

#git clone kubespray
git clone -b v2.10.0 https://github.com/kubernetes-sigs/kubespray.git /root/k8s-tf/kubespray

#copy config files into place
cd /root/k8s-tf/kubespray
rm -f /root/k8s-tf/kubespray/inventory/k8s-tf/inventory.ini
rm -f /root/k8s-tf/kubespray/inventory/k8s-tf/group_vars/k8s-cluster/k8s-cluster.yml
cp -r /root/k8s-tf/kubespray/inventory/sample /root/k8s-tf/kubespray/inventory/k8s-tf
cp /root/k8s-tf/inventory.ini /root/k8s-tf/kubespray/inventory/k8s-tf/inventory.ini 
cp /root/k8s-tf/k8s-cluster.yml /root/k8s-tf/kubespray/inventory/k8s-tf/group_vars/k8s-cluster/k8s-cluster.yml

#install kubernetes with kubespray
ansible-playbook -i /root/k8s-tf/kubespray/inventory/k8s-tf/inventory.ini cluster.yml --user root -vvv

cd /root/k8s-tf

#sed variables
api_txt="\ \ KUBERNETES_API_SERVER: "
k8s_api_line="$api_txt""$k8s_api_ip"
sed -i "/  KUBERNETES_API_SERVER:*/c $k8s_api_line" /root/k8s-tf/tf.yml
sed -i "/# Containers section/i $k8s_fip" /root/k8s-tf/tf.yml

#apply tf manifest
kubectl apply -f /root/k8s-tf/tf.yml

#fix coredns
kubectl get configmap -nkube-system coredns -o yaml >> /root/k8s-tf/coredns-configmap.yaml
coredns_line="\ \ \ \ \ \ \ \ forward . 10.47.255.253"
sed -i "/        forward . \/etc\/resolv.conf/c $coredns_line" /root/k8s-tf/coredns-configmap.yaml
kubectl get deployment -nkube-system coredns -o yaml >> /root/k8s-tf/coredns-deployment.yaml
sed -i "/        livenessProbe:/,+8d" /root/k8s-tf/coredns-deployment.yaml 
sed -i "/        readinessProbe:/,+8d" /root/k8s-tf/coredns-deployment.yaml
kubectl apply -f /root/k8s-tf/coredns-configmap.yaml
kubectl apply -f /root/k8s-tf/coredns-deployment.yaml