Skip to content

Instantly share code, notes, and snippets.

@areed
Created February 26, 2020 03:03
Show Gist options
  • Save areed/07be76e1b48243adfe1b75c70a806596 to your computer and use it in GitHub Desktop.
Save areed/07be76e1b48243adfe1b75c70a806596 to your computer and use it in GitHub Desktop.
#!/bin/bash
namespace=replicated-$(replicatedctl app inspect --template="{{ .ID }}" | sed 's/\r//')
function list_pods_with_shared_fs_mount() {
kubectl -n $namespace get pods -ojsonpath="{ range .items[*]}{ .metadata.name } { .spec.volumes[?(@.flexVolume.options.fsName)]}{'\n'}{ end }" | grep rook-shared-fs | awk '{ print $1 }'
}
function find_pods_with_failed_shared_fs_mount() {
while read -r pod; do
if pod_has_successful_shared_fs_mount $pod; then
continue
fi
echo "$pod"
done < <(list_pods_with_shared_fs_mount)
}
function cp_from_pod_to_shared() {
local podName="$1"
local mountPathInPod="$2"
local mountedSubPathOfSharedFS="$3"
local tmpDir="./tmp${podName}"
local dir="$tmpDir/$mountedSubPathOfSharedFS"
mkdir -p $dir
kubectl -n $namespace cp $podName:$mountPathInPod $dir
ls $tmpDir | xargs -I '{}' kubectl cp $tmpDir/'{}' shared:/fs
}
function pod_has_successful_shared_fs_mount() {
local podName="$1"
kubectl -n $namespace exec $podName df | awk '{ print $1 }' | grep -q ":6789"
}
function start_shared_pod() {
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
name: shared
spec:
containers:
- command:
- /bin/bash
- "-c"
- sleep infinity
image: quay.io/replicated/replicated-operator:stable-2.42.2
name: shared
volumeMounts:
- mountPath: /fs
name: rook-shared-fs
volumeMounts:
- mountPath: /fs
name: rook-shared-fs
volumes:
- flexVolume:
driver: ceph.rook.io/rook
fsType: ceph
options:
clusterNamespace: rook-ceph
fsName: rook-shared-fs
name: rook-shared-fs
EOF
}
function recover() {
cp_from_pod_to_shared my-app-pod-xxx /var/lib/shared backups/dump
}
# Usage: if any pods have failed mounts, edit recover function and run it to copy from the pod to
# the shared fs before that pod is deleted
find_pods_with_failed_shared_fs_mount
# start_shared_pod
# recover
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment