venkataanil

#!/usr/bin/env bash
# Bastion cleanup before redeploying an MNO cluster (Scale Lab / Performance Lab).
#
# Implements the cleanup steps documented in:
#   docs/troubleshooting.md  (pods, registry, ingress VIP / power-off, MinIO)
#   ansible/bastion-dnsmasq-clean.yml
#   ansible/bastion-minio-clean.yml
#
# Run as root on the bastion from the jetlag repo root, for example:
#   ./scripts/mno-bastion-cleanup.sh -i ansible/inventory/cloud99.local

venkataanil

/root $ date -u;fping -s -D -c 1 -u -i 150 -t 5000 -r 0 -g 198.18.2.1 198.18.2.254; date -u
Wed Sep 16 14:56:30 UTC 2026
[1789570590.12313] 198.18.2.1   : [0], 64 bytes, 5.03 ms (5.03 avg, 0% loss)
[1789570590.27096] 198.18.2.2   : [0], 64 bytes, 2.61 ms (2.61 avg, 0% loss)
[1789570590.42180] 198.18.2.3   : [0], 64 bytes, 3.19 ms (3.19 avg, 0% loss)
[1789570590.57260] 198.18.2.4   : [0], 64 bytes, 3.75 ms (3.75 avg, 0% loss)
[1789570590.72216] 198.18.2.5   : [0], 64 bytes, 3.07 ms (3.07 avg, 0% loss)
[1789570590.87181] 198.18.2.6   : [0], 64 bytes, 2.46 ms (2.46 avg, 0% loss)
[1789570591.02240] 198.18.2.7   : [0], 64 bytes, 2.81 ms (2.81 avg, 0% loss)
[1789570591.17161] 198.18.2.8   : [0], 64 bytes, 1.79 ms (1.79 avg, 0% loss)

venkataanil

/root $ fping -c 1 -i 150 -s -D -r 1 -g 198.18.1.1 198.18.1.254 
[1789566146.05452] 198.18.1.1   : [0], 64 bytes, 3.66 ms (3.66 avg, 0% loss)
[1789566146.20475] 198.18.1.2   : [0], 64 bytes, 3.64 ms (3.64 avg, 0% loss)
[1789566146.35410] 198.18.1.3   : [0], 64 bytes, 2.71 ms (2.71 avg, 0% loss)
[1789566146.50343] 198.18.1.4   : [0], 64 bytes, 1.80 ms (1.80 avg, 0% loss)
[1789566146.65475] 198.18.1.5   : [0], 64 bytes, 2.88 ms (2.88 avg, 0% loss)
[1789566146.80516] 198.18.1.6   : [0], 64 bytes, 3.02 ms (3.02 avg, 0% loss)
[1789566146.95569] 198.18.1.7   : [0], 64 bytes, 3.30 ms (3.30 avg, 0% loss)
[1789566147.10509] 198.18.1.8   : [0], 64 bytes, 2.47 ms (2.47 avg, 0% loss)
[1789566147.25610] 198.18.1.9   : [0], 64 bytes, 3.24 ms (3.24 avg, 0% loss)

venkataanil

/root $ fping -c 1 -i 200 -s -r 1 -g 198.18.1.1 198.18.1.254 
198.18.1.1   : [0], 64 bytes, 5.24 ms (5.24 avg, 0% loss)
198.18.1.2   : [0], 64 bytes, 3.87 ms (3.87 avg, 0% loss)
198.18.1.3   : [0], 64 bytes, 3.83 ms (3.83 avg, 0% loss)
198.18.1.4   : [0], 64 bytes, 3.01 ms (3.01 avg, 0% loss)
198.18.1.5   : [0], 64 bytes, 3.29 ms (3.29 avg, 0% loss)
198.18.1.6   : [0], 64 bytes, 2.72 ms (2.72 avg, 0% loss)
198.18.1.7   : [0], 64 bytes, 2.60 ms (2.60 avg, 0% loss)
198.18.1.8   : [0], 64 bytes, 2.76 ms (2.76 avg, 0% loss)
198.18.1.9   : [0], 64 bytes, 2.19 ms (2.19 avg, 0% loss)

venkataanil

# oc debug node/$n -- chroot /host  grep  "over 4096 resubmit" /var/log/openvswitch/ovs-vswitchd.log
Temporary namespace openshift-debug-td67g is created for debugging node...
Starting pod/f32-h11-000-r640-debug-cljpc ...
To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.
2026-09-08T00:39:58.600Z|07940|ofproto_dpif_xlate(handler214)|WARN|over 4096 resubmit actions on bridge br-int while processing arp,in_port=1,vlan_tci=0x0000,dl_src=40:a6:b7:0c:fd:10,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=198.18.0.24,arp_tpa=198.18.2.170,arp_op=1,arp_sha=40:a6:b7:0c:fd:10,arp_tha=00:00:00:00:00:00
2026-09-08T00:40:58.603Z|07942|ofproto_dpif_xlate(handler214)|WARN|over 4096 resubmit actions on bridge br-int while processing arp,in_port=1,vlan_tci=0x0000,dl_src=40:a6:b7:0d:3f:80,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=198.18.0.27,arp_tpa=198.18.1.97,arp_op=1,arp_sha=40:a6:b7:0d:3f:80,arp_tha=00:00:00:00:00:00
2026-09-08T00:41:58.573Z|07944|ofproto_dpif_xlate(handler214)|WARN|ove

venkataanil

# oc logs -n cudn-density-1 -c ping-test app-1-d9c8946dc-2bv6v 
ping to 198.18.1.1 succeeded
ping to 198.18.1.2 attempt 1 failed, retrying...
ping to 198.18.1.2 attempt 2 failed, retrying...
ping to 198.18.1.2 attempt 3 failed, retrying...
ping to 198.18.1.2 attempt 4 failed, retrying...
ping to 198.18.1.2 attempt 5 failed, retrying...
ping to 198.18.1.2 attempt 6 failed, retrying...
ping to 198.18.1.2 attempt 7 failed, retrying...
ping to 198.18.1.2 attempt 8 failed, retrying...

venkataanil

# oc logs -n cudn-density-0 -c ping-test app-1-75869f7c9d-wh6j6 -f
ping to 198.18.1.1 succeeded
ping to 198.18.1.2 attempt 1 failed, retrying...
ping to 198.18.1.2 attempt 2 failed, retrying...
ping to 198.18.1.2 attempt 3 failed, retrying...
ping to 198.18.1.2 attempt 4 failed, retrying...
ping to 198.18.1.2 attempt 5 failed, retrying...
ping to 198.18.1.2 attempt 6 failed, retrying...
ping to 198.18.1.2 attempt 7 failed, retrying...
ping to 198.18.1.2 attempt 8 failed, retrying...

venkataanil

# for n in `oc get nodes | grep -v NAME | awk '{print $1}'`; do oc debug node/$n -- chroot /host  grep "over 4096 resubmit" /var/log/openvswitch/ovs-vswitchd.log; done
Starting pod/f31-h10-000-r640-debug-7gm8q ...
To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.

Removing debug pod ...
error: non-zero exit code from debug container
Starting pod/f31-h11-000-r640-debug-9brkb ...
To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.
2026-09-01T04:55:42.657Z|00002|ofproto_dpif_xlate(handler210)|WARN|over 4096 resubmit actions on bridge br-int while processing arp,in_port=1,vlan_tci=0x0000,dl_src=40:a6:b7:0d:29:e0,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=198.18.0.5,arp_tpa=198.18.0.22,arp_op=1,arp_sha=40:a6:b7:0d:29:e0,arp_tha=00:00:00:00:00:00
2026-09-01T04:55:42.657Z|00003|ofproto_dpif_xlate(handler210)|WARN|over 4096 resubmit actions on bridge br-int while processing arp,in_port=1,vlan_tci=0x0000,d

venkataanil

# for n in `oc get nodes | grep -v NAME | awk '{print $1}'`; do oc debug node/$n -- chroot /host  grep -c "over 4096 resubmit" /var/log/openvswitch/ovs-vswitchd.log; done
Starting pod/f31-h10-000-r640-debug-zwkzh ...
To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.
0

Removing debug pod ...
error: non-zero exit code from debug container
Starting pod/f31-h11-000-r640-debug-9t9sl ...
To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.
4

venkataanil

#!/bin/bash
# =============================================================================
# fix-kubevirt-checksum-crd.sh
#
# Workaround for KubeVirt Issue #17858:
#   VMI stuck in "Scheduled" on Kubernetes 1.36 because the CRD declares
#   containerDiskVolume.checksum as int32, but the actual CRC32 value can
#   exceed the int32 max (2,147,483,647).
#
# What this script does: