AS IS
test: cloudstack v181, k8s 1.36.1, debian 13, 6.12.86+deb13-amd64, Cilium:1.19.1, Containerd:2.3.0, CNI 1.9.1 install: cert-manager
apt install git -y
git clone https://github.com/aenix-io/etcd-operator
git clone https://github.com/cozystack/cozystack
cat <<EOF>/root/etcd-operator/charts/etcd-operator/Chart.yaml
apiVersion: v2
name: etcd-operator
type: application
version: 0.4.3 # Placeholder, the actual version will be set in the release pipeline.
appVersion: "v0.4.3" # Placeholder, the actual version will be set in the release pipeline.
EOF
cd /root/etcd-operator/charts
kubectl create ns etcd-operator
helm install etcd-operator -n etcd-operator \
--set etcdOperator.vpa.enabled=false \
--set kubeRbacProxy.vpa.enabled=false \
--set replicaCount=1 etcd-operator \
--set etcdOperator.image.repository=ghcr.io/aenix-io/etcd-operator \
--set etcdOperator.image.tag=v0.4.3 \
--set kubeRbacProxy.image.repository=quay.io/brancz/kube-rbac-proxy \
--set kubeRbacProxy.image.tag=v0.21.2
kubectl get all -n etcd-operator
cd /root/cozystack/packages/system/kamaji/charts/kamaji-crds
helm dependency build
cd /root/cozystack/packages/system/kamaji/charts/kamaji-crds
helm package .
cd /root/cozystack/packages/system/kamaji/charts/kamaji
helm repo add clastix https://clastix.github.io/charts
helm dependency build
helm package .
# Step 1: Install CRDs
kubectl create ns kamaji-system
helm install kamaji-crds -n kamaji-system /root/cozystack/packages/system/kamaji/charts/kamaji-crds/kamaji-crds-0.0.0+latest.tgz
# Step 2: Install Kamaji operator
helm install kamaji -n kamaji-system --set kamaji-etcd.deploy=false \
--set replicaCount=1 \
--set image.repository=docker.io/clastix/kamaji \
--set image.tag=26.4.5-edge \
/root/cozystack/packages/system/kamaji/charts/kamaji/kamaji-0.0.0+latest.tgz
kubectl get all -n kamaji-system
nano /root/etcd-operator/examples/manifests/etcdcluster-kamaji.yaml
### change
....
namespace: kamaji-system
on namespace: cluster1
....
---
apiVersion: etcd.aenix.io/v1alpha1
kind: EtcdCluster
metadata:
name: etcd
namespace: kamaji-system
spec:
storage:
volumeClaimTemplate:
spec:
storageClassName: managed-nfs-storage
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 4Gi
security:
tls:
peerTrustedCASecret: etcd-peer-ca-tls
peerSecret: etcd-peer-tls
serverSecret: etcd-server-tls
clientTrustedCASecret: etcd-ca-tls
clientSecret: etcd-client-tls
---
apiVersion: kamaji.clastix.io/v1alpha1
kind: DataStore
metadata:
name: cluster1
spec:
driver: etcd
endpoints:
....
kubectl create ns cluster1
cd /root/etcd-operator/examples/manifests && kubectl apply -n cluster1 -f etcdcluster-kamaji.yaml && cd ~/cozystack/packages/system/kamaji/charts/kamaji
kubectl get secret -n cluster1 etcd-client-tls -o go-template='{{index .data "tls.crt"}}'
kubectl get all -n cluster1
apt install dnsutils -y
dig etcd-2.etcd-headless.cluster1.svc.cluster.local. @10.97.128.10
kubectl rollout restart deployment.apps/kamaji -n kamaji-system
kubectl get all -n kamaji-system
export TENANT_NAMESPACE=test
export TENANT_NAME=paas
export TENANT_VERSION=v1.35.3
export TENANT_DOMAIN=cloud.local
export TENANT_SVC_CIDR=244.1.0.0/16
export TENANT_POD_CIDR=244.2.0.0/16
export TENANT_DNS_SERVICE=244.1.0.10
export TENANT_PROXY_PORT=16443
kubectl create ns ${TENANT_NAMESPACE}
kubectl create configmap tenant-audit-policy --from-file=/etc/kubernetes/policies/audit-policy.yaml -n test
cat > ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml <<EOF
apiVersion: kamaji.clastix.io/v1alpha1
kind: TenantControlPlane
metadata:
name: ${TENANT_NAME}
namespace: ${TENANT_NAMESPACE}
labels:
tenant.clastix.io: ${TENANT_NAME}
spec:
dataStore: cluster1
controlPlane:
deployment:
additionalVolumeMounts:
apiServer:
- name: audit-policy
mountPath: /etc/kubernetes/audit
readOnly: true
additionalVolumes:
- name: audit-policy
configMap:
name: tenant-audit-policy
registrySettings:
registry: harbor.iblog.pro/registry.k8s.io
replicas: 3
additionalMetadata:
labels:
tenant.clastix.io: ${TENANT_NAME}
extraArgs:
apiServer:
- --profiling=false
- --audit-log-format=json
- --audit-log-maxage=7
- --audit-log-maxbackup=10
- --audit-log-maxsize=100
- --audit-log-path=/dev/stdout
- --authorization-webhook-cache-authorized-ttl=5m0s
- --authorization-webhook-cache-unauthorized-ttl=30s
- --audit-policy-file=/etc/kubernetes/audit/audit-policy.yaml
- --profiling=false
- --request-timeout=300s
- --tls-min-version=VersionTLS13
- --service-node-port-range=30000-32767
- --authorization-mode=Node,RBAC
- --kubelet-preferred-address-types=InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP
- --default-not-ready-toleration-seconds=300
- --default-unreachable-toleration-seconds=300
- --enable-aggregator-routing=False
- --anonymous-auth=True
- --event-ttl=1h0m0s
- --kubelet-preferred-address-types=InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP
- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256
controllerManager:
- --profiling=false
scheduler:
- --profiling=false
resources: {}
service:
additionalMetadata:
labels:
tenant.clastix.io: ${TENANT_NAME}
serviceType: LoadBalancer
kubernetes:
version: ${TENANT_VERSION}
kubelet:
cgroupfs: systemd
configurationJSONPatches: [{"op": "remove", "path": "/nodeLeaseDurationSeconds"}, {"op": "remove", "path": "/address"}, {"op": "remove", "path": "/cpuCFSQuotaPeriod"}, {"op": "remove", "path": "/configMapAndSecretChangeDetectionStrategy"}, {"op": "remove", "path": "/cpuManagerPolicy"}, {"op": "remove", "path": "/iptablesMasqueradeBit"}, {"op": "remove", "path": "/port"}, {"op": "remove", "path": "/imagePullCredentialsVerificationPolicy"}, {"op": "remove", "path": "/enableControllerAttachDetach"}, {"op": "remove", "path": "/failCgroupV1"}, {"op": "remove", "path": "/seccompDefault"}, {"op": "remove", "path": "/enableDebuggingHandlers"}, {"op": "remove", "path": "/containerLogMonitorInterval"}, {"op": "remove", "path": "/failSwapOn"}, {"op": "remove", "path": "/enableDebugFlagsHandler"}, {"op": "remove", "path": "/podLogsDir"}, {"op": "remove", "path": "/volumePluginDir"}, {"op": "remove", "path": "/containerLogMaxWorkers"}, {"op": "remove", "path": "/eventBurst"}, {"op": "remove", "path": "/registryBurst"}, {"op": "remove", "path": "/memoryManagerPolicy"}, {"op": "remove", "path": "/enforceNodeAllocatable"}, {"op": "remove", "path": "/iptablesDropBit"}, {"op": "remove", "path": "/makeIPTablesUtilChains"}, {"op": "remove", "path": "/contentType"}, {"op": "remove", "path": "/eventRecordQPS"}, {"op": "remove", "path": "/mergeDefaultEvictionSettings"}, {"op": "remove", "path": "/oomScoreAdj"}, {"op": "remove", "path": "/topologyManagerPolicy"}, {"op": "remove", "path": "/topologyManagerScope"}, {"op": "remove", "path": "/enableSystemLogHandler"}, {"op": "remove", "path": "/enableServer"}, {"op": "remove", "path": "/cpuCFSQuota"}, {"op": "remove", "path": "/localStorageCapacityIsolation"}, {"op": "remove", "path": "/enableProfilingHandler"}, {"op": "add", "path": "/evictionMinimumReclaim", "value": {"imagefs.available": "2Gi", "memory.available": "0Mi", "nodefs.available": "500Mi"}}, {"op": "add", "path": "/cgroupRoot", "value": "/"}, {"op": "add", "path": "/allowedUnsafeSysctls", "value": ["net.core.somaxconn"]}, {"op": "add", "path": "/kubeletCgroups", "value": "/kubelet.slice"}, {"op": "add", "path": "/evictionSoftGracePeriod", "value": {"imagefs.available": "2m30s", "imagefs.inodesFree": "2m30s", "memory.available": "2m30s", "nodefs.available": "2m30s", "nodefs.inodesFree": "2m30s"}}, {"op": "move", "from": "/registryPullQPS", "path": "/maxParallelImagePulls"}, {"op": "add", "path": "/kubeReservedCgroup", "value": "/kubelet.slice"}, {"op": "add", "path": "/tlsCipherSuites", "value": ["TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_RSA_WITH_AES_256_GCM_SHA384", "TLS_RSA_WITH_AES_128_GCM_SHA256", "TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384", "TLS_CHACHA20_POLY1305_SHA256"]}, {"op": "add", "path": "/podsPerCore", "value": 5}, {"op": "add", "path": "/evictionSoft", "value": {"imagefs.available": "30%", "imagefs.inodesFree": "25%", "memory.available": "500Mi", "nodefs.available": "25%", "nodefs.inodesFree": "15%"}}, {"op": "add", "path": "/systemReserved", "value": {"cpu": "500m", "ephemeral-storage": "1Gi", "memory": "1000Mi"}}, {"op": "add", "path": "/evictionMaxPodGracePeriod", "value": 300}, {"op": "add", "path": "/evictionHard", "value": {"imagefs.available": "25%", "imagefs.inodesFree": "15%", "memory.available": "500Mi", "nodefs.available": "20%", "nodefs.inodesFree": "10%"}}, {"op": "add", "path": "/kubeReserved", "value": {"cpu": "500m", "ephemeral-storage": "3Gi", "memory": "500Mi"}}, {"op": "add", "path": "/tlsMinVersion", "value": "VersionTLS13"}, {"op": "replace", "path": "/streamingConnectionIdleTimeout", "value": "0s"}, {"op": "replace", "path": "/hairpinMode", "value": "hairpin-veth"}, {"op": "replace", "path": "/authentication/webhook/cacheTTL", "value": "0s"}, {"op": "replace", "path": "/nodeStatusUpdateFrequency", "value": "30s"}, {"op": "replace", "path": "/runtimeRequestTimeout", "value": "0s"}, {"op": "replace", "path": "/podPidsLimit", "value": 4096}, {"op": "replace", "path": "/nodeStatusReportFrequency", "value": "20s"}, {"op": "replace", "path": "/fileCheckFrequency", "value": "0s"}, {"op": "replace", "path": "/authorization/webhook/cacheUnauthorizedTTL", "value": "0s"}, {"op": "replace", "path": "/authorization/webhook/cacheAuthorizedTTL", "value": "0s"}, {"op": "replace", "path": "/imageGCHighThresholdPercent", "value": 55}, {"op": "replace", "path": "/containerLogMaxSize", "value": "50Mi"}, {"op": "replace", "path": "/shutdownGracePeriodCriticalPods", "value": "5s"}, {"op": "remove", "path": "/logging/format"}, {"op": "replace", "path": "/logging/flushFrequency", "value": 0}, {"op": "replace", "path": "/httpCheckFrequency", "value": "0s"}, {"op": "replace", "path": "/imageMinimumGCAge", "value": "0s"}, {"op": "replace", "path": "/volumeStatsAggPeriod", "value": "0s"}, {"op": "replace", "path": "/syncFrequency", "value": "0s"}, {"op": "replace", "path": "/shutdownGracePeriod", "value": "15s"}, {"op": "replace", "path": "/cpuManagerReconcilePeriod", "value": "0s"}, {"op": "remove", "path": "/crashLoopBackOff/maxContainerRestartPeriod"}, {"op": "replace", "path": "/imageGCLowThresholdPercent", "value": 50}, {"op": "replace", "path": "/maxPods", "value": 20}, {"op": "replace", "path": "/serializeImagePulls", "value": false}, {"op": "replace", "path": "/clusterDNS/0", "value": "10.97.128.10"}]
admissionControllers:
- ResourceQuota
- LimitRanger
networkProfile:
port: ${TENANT_PORT}
certSANs:
- ${TENANT_NAME}.${TENANT_DOMAIN}
serviceCidr: ${TENANT_SVC_CIDR}
podCidr: ${TENANT_POD_CIDR}
dnsServiceIPs:
- ${TENANT_DNS_SERVICE}
addons:
coreDNS: {}
konnectivity:
server:
image: registry.k8s.io/kas-network-proxy/proxy-server
version: v0.34.0
port: ${TENANT_PROXY_PORT}
agent:
image: registry.k8s.io/kas-network-proxy/proxy-agent
version: v0.34.0
kubeProxy: {}
EOF
kubectl -n ${TENANT_NAMESPACE} apply -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
kubectl get tenantcontrolplanes.kamaji.clastix.io --all-namespaces
kubectl get all,cm,secret -n test
kubectl get all,cm,secret,tenantcontrolplanes.kamaji.clastix.io -n test
kubectl get -n test secret/paas-admin-kubeconfig -o yaml | grep super-admin.conf: | awk '{ print $2}' | base64 -d >1
kubectl --kubeconfig=1 cluster-info
kubectl get -n test secret/paas-admin-kubeconfig -o yaml | grep super-admin.conf: | awk '{ print $2}' | base64 -d > ${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
JOIN_CMD=$(echo "sudo ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command)
echo $JOIN_CMD
root@node130:~# kubectl --kubeconfig=test-paas.kubeconfig get node,pod -A -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
node/node142 Ready <none> 71s v1.36.1 192.168.200.142 <none> Debian GNU/Linux 13 (trixie) 6.12.86+deb13-amd64 containerd://2.3.0
node/node143 Ready <none> 68s v1.36.1 192.168.200.143 <none> Debian GNU/Linux 13 (trixie) 6.12.86+deb13-amd64 containerd://2.3.0
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
kube-system pod/coredns-7d764666f9-ntw9l 0/1 ContainerCreating 0 3m46s <none> node142 <none> <none>
kube-system pod/coredns-7d764666f9-zlfc5 0/1 ContainerCreating 0 3m46s <none> node143 <none> <none>
kube-system pod/konnectivity-agent-bxft7 0/1 ContainerCreating 0 67s <none> node143 <none> <none>
kube-system pod/konnectivity-agent-p54h2 0/1 ContainerCreating 0 69s <none> node142 <none> <none>
kube-system pod/kube-proxy-6tvzm 1/1 Running 0 71s 192.168.200.142 node142 <none> <none>
kube-system pod/kube-proxy-vdqxs 1/1 Running 0 68s 192.168.200.143 node143 <none> <none>
root@node130:~#
helm install cilium oci://quay.io/cilium/charts/cilium \
--version 1.19.3 \
--namespace kube-system
root@node130:~# kubectl --kubeconfig=test-paas.kubeconfig get node,pod -A -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
node/node142 Ready <none> 5m6s v1.36.1 192.168.200.142 <none> Debian GNU/Linux 13 (trixie) 6.12.86+deb13-amd64 containerd://2.3.0
node/node143 Ready <none> 5m3s v1.36.1 192.168.200.143 <none> Debian GNU/Linux 13 (trixie) 6.12.86+deb13-amd64 containerd://2.3.0
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
kube-system pod/cilium-4b7dt 1/1 Running 0 110s 192.168.200.142 node142 <none> <none>
kube-system pod/cilium-envoy-44929 1/1 Running 0 110s 192.168.200.143 node143 <none> <none>
kube-system pod/cilium-envoy-76w89 1/1 Running 0 110s 192.168.200.142 node142 <none> <none>
kube-system pod/cilium-operator-86b4d5df4f-p5wmt 1/1 Running 0 110s 192.168.200.142 node142 <none> <none>
kube-system pod/cilium-operator-86b4d5df4f-zjrbl 1/1 Running 0 110s 192.168.200.143 node143 <none> <none>
kube-system pod/cilium-qtwlq 1/1 Running 0 110s 192.168.200.143 node143 <none> <none>
kube-system pod/coredns-7d764666f9-ntw9l 1/1 Running 0 7m41s 10.0.1.105 node142 <none> <none>
kube-system pod/coredns-7d764666f9-zlfc5 1/1 Running 0 7m41s 10.0.0.114 node143 <none> <none>
kube-system pod/konnectivity-agent-bxft7 1/1 Running 0 5m2s 10.0.0.40 node143 <none> <none>
kube-system pod/konnectivity-agent-p54h2 1/1 Running 0 5m4s 10.0.1.163 node142 <none> <none>
kube-system pod/kube-proxy-6tvzm 1/1 Running 0 5m6s 192.168.200.142 node142 <none> <none>
kube-system pod/kube-proxy-vdqxs 1/1 Running 0 5m3s 192.168.200.143 node143 <none> <none>
root@node130:~#
root@node130:~# kubectl get node,pod -n cluster1 -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
node/node130 Ready control-plane,master 25m v1.36.1 192.168.200.130 <none> Debian GNU/Linux 13 (trixie) 6.12.86+deb13-amd64 (amd64) containerd://2.3.0
node/node131 Ready control-plane,master 24m v1.36.1 192.168.200.131 <none> Debian GNU/Linux 13 (trixie) 6.12.86+deb13-amd64 (amd64) containerd://2.3.0
node/node132 Ready control-plane,master 24m v1.36.1 192.168.200.132 <none> Debian GNU/Linux 13 (trixie) 6.12.86+deb13-amd64 (amd64) containerd://2.3.0
node/node133 Ready worker 25m v1.36.1 192.168.200.133 <none> Debian GNU/Linux 13 (trixie) 6.12.86+deb13-amd64 (amd64) containerd://2.3.0
node/node140 Ready worker 25m v1.36.1 192.168.200.140 <none> Debian GNU/Linux 13 (trixie) 6.12.86+deb13-amd64 (amd64) containerd://2.3.0
node/node141 Ready worker 25m v1.36.1 192.168.200.141 <none> Debian GNU/Linux 13 (trixie) 6.12.86+deb13-amd64 (amd64) containerd://2.3.0
root@node130:~# kubectl get node,pod -n cluster1 -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod/etcd-0 1/1 Running 0 11m 10.97.4.134 node141 <none> <none>
pod/etcd-1 1/1 Running 0 6m55s 10.97.3.162 node133 <none> <none>
pod/etcd-2 1/1 Running 0 11m 10.97.1.201 node140 <none> <none>
root@node130:~# kubectl get pod -n test -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod/paas-775bd59f9c-4llkb 4/4 Running 0 7m28s 10.97.6.19 node132 <none> <none>
pod/paas-775bd59f9c-m7lnm 4/4 Running 0 9m27s 10.97.3.201 node133 <none> <none>
pod/paas-775bd59f9c-vjlk9 4/4 Running 0 9m27s 10.97.1.139 node140 <none> <none>
root@node130:~#
root@node130:~# cat /root/etcd-operator/examples/manifests/etcdcluster-kamaji.yaml
---
apiVersion: etcd.aenix.io/v1alpha1
kind: EtcdCluster
metadata:
name: etcd
namespace: cluster1
spec:
storage:
volumeClaimTemplate:
spec:
storageClassName: managed-nfs-storage
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 4Gi
security:
tls:
peerTrustedCASecret: etcd-peer-ca-tls
peerSecret: etcd-peer-tls
serverSecret: etcd-server-tls
clientTrustedCASecret: etcd-ca-tls
clientSecret: etcd-client-tls
---
apiVersion: kamaji.clastix.io/v1alpha1
kind: DataStore
metadata:
name: cluster1
spec:
driver: etcd
endpoints:
- etcd-0.etcd-headless.cluster1.svc:2379
- etcd-1.etcd-headless.cluster1.svc:2379
- etcd-2.etcd-headless.cluster1.svc:2379
tlsConfig:
certificateAuthority:
certificate:
secretReference:
keyPath: tls.crt
name: etcd-ca-tls
namespace: cluster1
privateKey:
secretReference:
keyPath: tls.key
name: etcd-ca-tls
namespace: cluster1
clientCertificate:
certificate:
secretReference:
keyPath: tls.crt
name: etcd-client-tls
namespace: cluster1
privateKey:
secretReference:
keyPath: tls.key
name: etcd-client-tls
namespace: cluster1
---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: etcd-selfsigning-issuer
namespace: cluster1
spec:
selfSigned: {}
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: etcd-peer-ca
namespace: cluster1
spec:
isCA: true
usages:
- "signing"
- "key encipherment"
- "cert sign"
commonName: etcd-peer-ca
subject:
organizations:
- ACME Inc.
organizationalUnits:
- Widgets
secretName: etcd-peer-ca-tls
privateKey:
algorithm: RSA
size: 4096
issuerRef:
name: etcd-selfsigning-issuer
kind: Issuer
group: cert-manager.io
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: etcd-ca
namespace: cluster1
spec:
isCA: true
usages:
- "signing"
- "key encipherment"
- "cert sign"
commonName: etcd-ca
subject:
organizations:
- ACME Inc.
organizationalUnits:
- Widgets
secretName: etcd-ca-tls
privateKey:
algorithm: RSA
size: 4096
issuerRef:
name: etcd-selfsigning-issuer
kind: Issuer
group: cert-manager.io
---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: etcd-peer-issuer
namespace: cluster1
spec:
ca:
secretName: etcd-peer-ca-tls
---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: etcd-issuer
namespace: cluster1
spec:
ca:
secretName: etcd-ca-tls
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: etcd-server
namespace: cluster1
spec:
secretName: etcd-server-tls
isCA: false
usages:
- "client auth"
- "server auth"
- "signing"
- "key encipherment"
dnsNames:
- etcd-0
- etcd-0.etcd-headless
- etcd-0.etcd-headless.cluster1.svc
- etcd-0.etcd-headless.cluster1.svc.cluster.local
- etcd-1
- etcd-1.etcd-headless
- etcd-1.etcd-headless.cluster1.svc
- etcd-1.etcd-headless.cluster1.svc.cluster.local
- etcd-2
- etcd-2.etcd-headless
- etcd-2.etcd-headless.cluster1.svc
- etcd-2.etcd-headless.cluster1.svc.cluster.local
- localhost
ipAddresses:
- 127.0.0.1
privateKey:
rotationPolicy: Always
algorithm: RSA
size: 4096
issuerRef:
name: etcd-issuer
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: etcd-peer
namespace: cluster1
spec:
secretName: etcd-peer-tls
isCA: false
usages:
- "server auth"
- "client auth"
- "signing"
- "key encipherment"
dnsNames:
- etcd-0
- etcd-0.etcd-headless
- etcd-0.etcd-headless.cluster1.svc
- etcd-0.etcd-headless.cluster1.svc.cluster.local
- etcd-1
- etcd-1.etcd-headless
- etcd-1.etcd-headless.cluster1.svc
- etcd-1.etcd-headless.cluster1.svc.cluster.local
- etcd-2
- etcd-2.etcd-headless
- etcd-2.etcd-headless.cluster1.svc
- etcd-2.etcd-headless.cluster1.svc.cluster.local
- localhost
- "127.0.0.1"
privateKey:
rotationPolicy: Always
algorithm: RSA
size: 4096
issuerRef:
name: etcd-peer-issuer
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: etcd-client
namespace: cluster1
spec:
commonName: root
secretName: etcd-client-tls
usages:
- "signing"
- "key encipherment"
- "client auth"
privateKey:
rotationPolicy: Always
algorithm: RSA
size: 4096
issuerRef:
name: etcd-issuer
kind: Issuer
root@node130:~#
nano /root/etcd-operator/examples/manifests/etcdcluster-kamaji.yaml
....
namespace: kamaji-system
on namespace: cluster1
....
---
apiVersion: etcd.aenix.io/v1alpha1
kind: EtcdCluster
metadata:
name: etcd
namespace: kamaji-system
spec:
storage:
volumeClaimTemplate:
spec:
storageClassName: managed-nfs-storage
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 4Gi
security:
tls:
peerTrustedCASecret: etcd-peer-ca-tls
peerSecret: etcd-peer-tls
serverSecret: etcd-server-tls
clientTrustedCASecret: etcd-ca-tls
clientSecret: etcd-client-tls
---
apiVersion: kamaji.clastix.io/v1alpha1
kind: DataStore
metadata:
name: cluster1
spec:
driver: etcd
endpoints:
....
kubectl create ns cluster1
cd /root/etcd-operator/examples/manifests && kubectl apply -n cluster1 -f etcdcluster-kamaji.yaml && cd ~/cozystack/packages/system/kamaji/charts/kamaji
kubectl get secret -n cluster1 etcd-client-tls -o go-template='{{index .data "tls.crt"}}'
kubectl get all -n cluster1
apt install dnsutils -y
dig etcd-2.etcd-headless.cluster1.svc.cluster.local. @10.97.128.10
kubectl rollout restart deployment.apps/kamaji -n kamaji-system
kubectl get all -n kamaji-system
export TENANT_NAMESPACE=cluster1
export TENANT_NAME=cluster1
export TENANT_VERSION=v1.35.3
export TENANT_DOMAIN=cloud.local
export TENANT_SVC_CIDR=244.1.0.0/16
export TENANT_POD_CIDR=244.2.0.0/16
export TENANT_DNS_SERVICE=244.1.0.10
export TENANT_PROXY_PORT=16443
kubectl create ns ${TENANT_NAMESPACE}
kubectl create configmap tenant-audit-policy --from-file=/etc/kubernetes/policies/audit-policy.yaml -n test
cat > ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml <<EOF
apiVersion: kamaji.clastix.io/v1alpha1
kind: TenantControlPlane
metadata:
name: ${TENANT_NAME}
namespace: ${TENANT_NAMESPACE}
labels:
tenant.clastix.io: ${TENANT_NAME}
spec:
dataStore: cluster1
controlPlane:
deployment:
additionalVolumeMounts:
apiServer:
- name: audit-policy
mountPath: /etc/kubernetes/audit
readOnly: true
additionalVolumes:
- name: audit-policy
configMap:
name: tenant-audit-policy
registrySettings:
registry: harbor.iblog.pro/registry.k8s.io
replicas: 3
additionalMetadata:
labels:
tenant.clastix.io: ${TENANT_NAME}
extraArgs:
apiServer:
- --profiling=false
- --audit-log-format=json
- --audit-log-maxage=7
- --audit-log-maxbackup=10
- --audit-log-maxsize=100
- --audit-log-path=/dev/stdout
- --authorization-webhook-cache-authorized-ttl=5m0s
- --authorization-webhook-cache-unauthorized-ttl=30s
- --audit-policy-file=/etc/kubernetes/audit/audit-policy.yaml
- --profiling=false
- --request-timeout=300s
- --tls-min-version=VersionTLS13
- --service-node-port-range=30000-32767
- --authorization-mode=Node,RBAC
- --kubelet-preferred-address-types=InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP
- --default-not-ready-toleration-seconds=300
- --default-unreachable-toleration-seconds=300
- --enable-aggregator-routing=False
- --anonymous-auth=True
- --event-ttl=1h0m0s
- --kubelet-preferred-address-types=InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP
- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256
controllerManager:
- --profiling=false
scheduler:
- --profiling=false
resources: {}
service:
additionalMetadata:
labels:
tenant.clastix.io: ${TENANT_NAME}
serviceType: LoadBalancer
kubernetes:
version: ${TENANT_VERSION}
kubelet:
cgroupfs: systemd
configurationJSONPatches: [{"op": "remove", "path": "/nodeLeaseDurationSeconds"}, {"op": "remove", "path": "/address"}, {"op": "remove", "path": "/cpuCFSQuotaPeriod"}, {"op": "remove", "path": "/configMapAndSecretChangeDetectionStrategy"}, {"op": "remove", "path": "/cpuManagerPolicy"}, {"op": "remove", "path": "/iptablesMasqueradeBit"}, {"op": "remove", "path": "/port"}, {"op": "remove", "path": "/imagePullCredentialsVerificationPolicy"}, {"op": "remove", "path": "/enableControllerAttachDetach"}, {"op": "remove", "path": "/failCgroupV1"}, {"op": "remove", "path": "/seccompDefault"}, {"op": "remove", "path": "/enableDebuggingHandlers"}, {"op": "remove", "path": "/containerLogMonitorInterval"}, {"op": "remove", "path": "/failSwapOn"}, {"op": "remove", "path": "/enableDebugFlagsHandler"}, {"op": "remove", "path": "/podLogsDir"}, {"op": "remove", "path": "/volumePluginDir"}, {"op": "remove", "path": "/containerLogMaxWorkers"}, {"op": "remove", "path": "/eventBurst"}, {"op": "remove", "path": "/registryBurst"}, {"op": "remove", "path": "/memoryManagerPolicy"}, {"op": "remove", "path": "/enforceNodeAllocatable"}, {"op": "remove", "path": "/iptablesDropBit"}, {"op": "remove", "path": "/makeIPTablesUtilChains"}, {"op": "remove", "path": "/contentType"}, {"op": "remove", "path": "/eventRecordQPS"}, {"op": "remove", "path": "/mergeDefaultEvictionSettings"}, {"op": "remove", "path": "/oomScoreAdj"}, {"op": "remove", "path": "/topologyManagerPolicy"}, {"op": "remove", "path": "/topologyManagerScope"}, {"op": "remove", "path": "/enableSystemLogHandler"}, {"op": "remove", "path": "/enableServer"}, {"op": "remove", "path": "/cpuCFSQuota"}, {"op": "remove", "path": "/localStorageCapacityIsolation"}, {"op": "remove", "path": "/enableProfilingHandler"}, {"op": "add", "path": "/evictionMinimumReclaim", "value": {"imagefs.available": "2Gi", "memory.available": "0Mi", "nodefs.available": "500Mi"}}, {"op": "add", "path": "/cgroupRoot", "value": "/"}, {"op": "add", "path": "/allowedUnsafeSysctls", "value": ["net.core.somaxconn"]}, {"op": "add", "path": "/kubeletCgroups", "value": "/kubelet.slice"}, {"op": "add", "path": "/evictionSoftGracePeriod", "value": {"imagefs.available": "2m30s", "imagefs.inodesFree": "2m30s", "memory.available": "2m30s", "nodefs.available": "2m30s", "nodefs.inodesFree": "2m30s"}}, {"op": "move", "from": "/registryPullQPS", "path": "/maxParallelImagePulls"}, {"op": "add", "path": "/kubeReservedCgroup", "value": "/kubelet.slice"}, {"op": "add", "path": "/tlsCipherSuites", "value": ["TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_RSA_WITH_AES_256_GCM_SHA384", "TLS_RSA_WITH_AES_128_GCM_SHA256", "TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384", "TLS_CHACHA20_POLY1305_SHA256"]}, {"op": "add", "path": "/podsPerCore", "value": 5}, {"op": "add", "path": "/evictionSoft", "value": {"imagefs.available": "30%", "imagefs.inodesFree": "25%", "memory.available": "500Mi", "nodefs.available": "25%", "nodefs.inodesFree": "15%"}}, {"op": "add", "path": "/systemReserved", "value": {"cpu": "500m", "ephemeral-storage": "1Gi", "memory": "1000Mi"}}, {"op": "add", "path": "/evictionMaxPodGracePeriod", "value": 300}, {"op": "add", "path": "/evictionHard", "value": {"imagefs.available": "25%", "imagefs.inodesFree": "15%", "memory.available": "500Mi", "nodefs.available": "20%", "nodefs.inodesFree": "10%"}}, {"op": "add", "path": "/kubeReserved", "value": {"cpu": "500m", "ephemeral-storage": "3Gi", "memory": "500Mi"}}, {"op": "add", "path": "/tlsMinVersion", "value": "VersionTLS13"}, {"op": "replace", "path": "/streamingConnectionIdleTimeout", "value": "0s"}, {"op": "replace", "path": "/hairpinMode", "value": "hairpin-veth"}, {"op": "replace", "path": "/authentication/webhook/cacheTTL", "value": "0s"}, {"op": "replace", "path": "/nodeStatusUpdateFrequency", "value": "30s"}, {"op": "replace", "path": "/runtimeRequestTimeout", "value": "0s"}, {"op": "replace", "path": "/podPidsLimit", "value": 4096}, {"op": "replace", "path": "/nodeStatusReportFrequency", "value": "20s"}, {"op": "replace", "path": "/fileCheckFrequency", "value": "0s"}, {"op": "replace", "path": "/authorization/webhook/cacheUnauthorizedTTL", "value": "0s"}, {"op": "replace", "path": "/authorization/webhook/cacheAuthorizedTTL", "value": "0s"}, {"op": "replace", "path": "/imageGCHighThresholdPercent", "value": 55}, {"op": "replace", "path": "/containerLogMaxSize", "value": "50Mi"}, {"op": "replace", "path": "/shutdownGracePeriodCriticalPods", "value": "5s"}, {"op": "remove", "path": "/logging/format"}, {"op": "replace", "path": "/logging/flushFrequency", "value": 0}, {"op": "replace", "path": "/httpCheckFrequency", "value": "0s"}, {"op": "replace", "path": "/imageMinimumGCAge", "value": "0s"}, {"op": "replace", "path": "/volumeStatsAggPeriod", "value": "0s"}, {"op": "replace", "path": "/syncFrequency", "value": "0s"}, {"op": "replace", "path": "/shutdownGracePeriod", "value": "15s"}, {"op": "replace", "path": "/cpuManagerReconcilePeriod", "value": "0s"}, {"op": "remove", "path": "/crashLoopBackOff/maxContainerRestartPeriod"}, {"op": "replace", "path": "/imageGCLowThresholdPercent", "value": 50}, {"op": "replace", "path": "/maxPods", "value": 20}, {"op": "replace", "path": "/serializeImagePulls", "value": false}, {"op": "replace", "path": "/clusterDNS/0", "value": "10.97.128.10"}]
admissionControllers:
- ResourceQuota
- LimitRanger
networkProfile:
port: ${TENANT_PORT}
certSANs:
- ${TENANT_NAME}.${TENANT_DOMAIN}
serviceCidr: ${TENANT_SVC_CIDR}
podCidr: ${TENANT_POD_CIDR}
dnsServiceIPs:
- ${TENANT_DNS_SERVICE}
addons:
coreDNS: {}
konnectivity:
server:
image: registry.k8s.io/kas-network-proxy/proxy-server
version: v0.34.0
port: ${TENANT_PROXY_PORT}
agent:
image: registry.k8s.io/kas-network-proxy/proxy-agent
version: v0.34.0
kubeProxy: {}
EOF
kubectl -n ${TENANT_NAMESPACE} apply -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
kubectl get tenantcontrolplanes.kamaji.clastix.io --all-namespaces
kubectl get all,cm,secret -n ${TENANT_NAMESPACE}
kubectl get all,cm,secret,tenantcontrolplanes.kamaji.clastix.io -n ${TENANT_NAMESPACE}
kubectl get -n ${TENANT_NAMESPACE} secret/paas-admin-kubeconfig -o yaml | grep super-admin.conf: | awk '{ print $2}' | base64 -d >1
kubectl --kubeconfig=1 cluster-info
kubectl get -n ${TENANT_NAMESPACE} secret/paas-admin-kubeconfig -o yaml | grep super-admin.conf: | awk '{ print $2}' | base64 -d > ${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
JOIN_CMD=$(echo "sudo ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command)
echo $JOIN_CMD