Skip to content

Instantly share code, notes, and snippets.

@azalio

azalio/talos_k8s_upgrade.md

Last active February 17, 2025 08:28
Show Gist options
  • Save azalio/4017548d95cc9b75ae2e8c9725fccca6 to your computer and use it in GitHub Desktop.
Save azalio/4017548d95cc9b75ae2e8c9725fccca6 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
talos_k8s_upgrade.md 
package main

import (
	"context"
	"flag"
	"fmt"
	"os"

	"github.com/siderolabs/go-kubernetes/kubernetes/upgrade"
	taloskubernetes "github.com/siderolabs/talos/pkg/cluster/kubernetes"
	"github.com/siderolabs/talos/pkg/kubernetes"
	"github.com/siderolabs/talos/pkg/machinery/client"
	"github.com/siderolabs/talos/pkg/machinery/client/config"
	"github.com/siderolabs/talos/pkg/machinery/config/encoder"
	clientgo "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

// Client represents a Kubernetes client
type Client struct {
	client *client.Client
}

// ClusterState holds the cluster state information and implements kubernetes.UpgradeProvider
type ClusterState struct {
	client *client.Client
}

// Client implements the kubernetes.UpgradeProvider interface
func (s *ClusterState) Client(nodes ...string) (*client.Client, error) {
	return s.client, nil
}

// K8sClient implements the kubernetes.UpgradeProvider interface
func (s *ClusterState) K8sClient(ctx context.Context) (*clientgo.Clientset, error) {
	return nil, nil // simplified for example
}

// Close implements the kubernetes.UpgradeProvider interface
func (s *ClusterState) Close() error {
	return nil
}

// K8sClose implements the kubernetes.UpgradeProvider interface
func (s *ClusterState) K8sClose() error {
	return nil
}

// Kubeconfig implements the kubernetes.UpgradeProvider interface
func (s *ClusterState) Kubeconfig(ctx context.Context) ([]byte, error) {
	return s.client.Kubeconfig(ctx)
}

// K8sHelper implements the kubernetes.UpgradeProvider interface
func (s *ClusterState) K8sHelper(ctx context.Context) (*kubernetes.Client, error) {
	config, err := s.K8sRestConfig(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to get kubernetes config: %w", err)
	}

	return kubernetes.NewForConfig(config)
}

func (s *ClusterState) K8sRestConfig(ctx context.Context) (*rest.Config, error) {
	kubeconfig, err := s.client.Kubeconfig(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to get kubeconfig: %w", err)
	}

	config, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig)
	if err != nil {
		return nil, fmt.Errorf("failed to parse kubeconfig: %w", err)
	}

	return config, nil
}

// UpgradeKubernetes upgrades Kubernetes to the specified version using the proper k8s upgrade functionality
func (c *Client) UpgradeKubernetes(ctx context.Context, version string) error {
	state := &ClusterState{
		client: c.client,
	}

	currentVersion := "v1.32.0"

	path, err := upgrade.NewPath(currentVersion, version)
	if err != nil {
		return fmt.Errorf("failed to create upgrade path: %w", err)
	}

	upgradeOptions := taloskubernetes.UpgradeOptions{
		Path:       path,
		EncoderOpt: encoder.WithComments(encoder.CommentsDisabled),
	}

	if err := taloskubernetes.Upgrade(ctx, state, upgradeOptions); err != nil {
		return fmt.Errorf("failed to upgrade kubernetes: %w", err)
	}

	return nil
}

func main() {
	talosConfigPath := flag.String("talosconfig", "./talosconfig", "Path to Talos config file")
	targetVersion := flag.String("version", "v1.32.1", "Target Kubernetes version")
	flag.Parse()

	// Load Talos config
	talosConfig, err := config.Open(*talosConfigPath)
	if err != nil {
		fmt.Printf("Failed to load Talos config: %v\n", err)
		os.Exit(1)
	}

	ctx := context.Background()

	// Initialize Talos client
	c, err := client.New(ctx,
		client.WithConfig(talosConfig),
		client.WithEndpoints(talosConfig.Contexts[talosConfig.Context].Endpoints...))
	if err != nil {
		fmt.Printf("Failed to create Talos client: %v\n", err)
		os.Exit(1)
	}

	client := &Client{
		client: c,
	}

	fmt.Printf("Upgrading Kubernetes to version %s\n", *targetVersion)
	err = client.UpgradeKubernetes(ctx, *targetVersion)
	if err != nil {
		fmt.Printf("Failed to upgrade Kubernetes: %v\n", err)
		os.Exit(1)
	}

	fmt.Println("Kubernetes upgrade completed successfully")
}
talosctl --talosconfig=./talosconfig  --nodes=10.10.10.10 logs kubelet  | grep kube-apiserver | tail -2
10.10.10.10: {"ts":1739736143843.821,"logger":"UnhandledError","caller":"kuberuntime/kuberuntime_manager.go:1341","msg":"Unhandled Error","err":"container &Container{Name:kube-apiserver,Image::v1.32.1,Command:[/usr/local/bin/kube-apiserver --admission-control-config-file=/system/config/kubernetes/kube-apiserver/admission-control-config.yaml --advertise-address=$(POD_IP) --allow-privileged=true --anonymous-auth=false --api-audiences=https://10.10.10.10:6443 --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/log/audit/kube/kube-apiserver.log --audit-policy-file=/system/config/kubernetes/kube-apiserver/auditpolicy.yaml --authorization-config=/system/config/kubernetes/kube-apiserver/authorization-config.yaml --bind-address=0.0.0.0 --client-ca-file=/system/secrets/kubernetes/kube-apiserver/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --encryption-provider-config=/system/secrets/kubernetes/kube-apiserver/encryptionconfig.yaml --etcd-cafile=/system/secrets/kubernetes/kube-apiserver/etcd-client-ca.crt --etcd-certfile=/system/secrets/kubernetes/kube-apiserver/etcd-client.crt --etcd-keyfile=/system/secrets/kubernetes/kube-apiserver/etcd-client.key --etcd-servers=https://localhost:2379 --kubelet-client-certificate=/system/secrets/kubernetes/kube-apiserver/apiserver-kubelet-client.crt --kubelet-client-key=/system/secrets/kubernetes/kube-apiserver/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/system/secrets/kubernetes/kube-apiserver/front-proxy-client.crt --proxy-client-key-file=/system/secrets/kubernetes/kube-apiserver/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/system/secrets/kubernetes/kube-apiserver/aggregator-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://10.10.10.10:6443 --service-account-key-file=/system/secrets/kubernetes/kube-apiserver/service-account.pub --service-account-signing-key-file=/system/secrets/kubernetes/kube-apiserver/service-account.key --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/system/secrets/kubernetes/kube-apiserver/apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --tls-min-version=VersionTLS12 --tls-private-key-file=/system/secrets/kubernetes/kube-apiserver/apiserver.key],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{EnvVar{Name:POD_IP,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:status.podIP,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{200 -3} {<nil>} 200m DecimalSI},memory: {{536870912 0} {<nil>}  BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:secrets,ReadOnly:true,MountPath:/system/secrets/kubernetes/kube-apiserver,SubPath:,MountPropagation:nil,SubPathExpr:,RecursiveReadOnly:nil,},VolumeMount{Name:config,ReadOnly:true,MountPath:/system/config/kubernetes/kube-apiserver,SubPath:,MountPropagation:nil,SubPathExpr:,RecursiveReadOnly:nil,},VolumeMount{Name:audit,ReadOnly:false,MountPath:/var/log/audit/kube,SubPath:,MountPropagation:nil,SubPathExpr:,RecursiveReadOnly:nil,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[NET_BIND_SERVICE],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:&SeccompProfile{Type:RuntimeDefault,LocalhostProfile:nil,},AppArmorProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod kube-apiserver-azalio-talos-cp1_kube-system(9b087cdb05c80aa17c1d0bf130d7e17f): InvalidImageName: Failed to apply default image tag \":v1.32.1\": couldn't parse image name \":v1.32.1\": invalid reference format"}
10.10.10.10: {"ts":1739736143845.4382,"caller":"kubelet/pod_workers.go:1301","msg":"Error syncing pod, skipping","pod":{"name":"kube-apiserver-azalio-talos-cp1","namespace":"kube-system"},"podUID":"9b087cdb05c80aa17c1d0bf130d7e17f","err":"failed to \"StartContainer\" for \"kube-apiserver\" with InvalidImageName: \"Failed to apply default image tag \\\":v1.32.1\\\": couldn't parse image name \\\":v1.32.1\\\": invalid reference format\"","errCauses":[{"error":"failed to \"StartContainer\" for \"kube-apiserver\" with InvalidImageName: \"Failed to apply default image tag \\\":v1.32.1\\\": couldn't parse image name \\\":v1.32.1\\\": invalid reference format\""}]}

Before my patch

go run upgrade_k8s.go
Upgrading Kubernetes to version v1.32.1
discovered controlplane nodes ["10.10.10.10"]
updating "kube-apiserver" to version "1.32.1"
 > "10.10.10.10": starting update
 > update kube-apiserver: v1.32.0 -> 1.32.1
 > "10.10.10.10": machine configuration patched
 > "10.10.10.10": waiting for kube-apiserver pod update
 > "10.10.10.10": kube-apiserver: waiting, config version mismatch: got "12", expected "13"
^Csignal: interrupt

After my patch

go run upgrade_k8s.go
Upgrading Kubernetes to version v1.32.1
Failed to upgrade Kubernetes: failed to upgrade kubernetes: invalid upgrade options: apiserver: invalid image reference: repository name must have at least one component
exit status 1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment