Skip to content

Instantly share code, notes, and snippets.

@vorlon001

vorlon001/readme.md

Last active May 13, 2026 18:30
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save vorlon001/b0af8a7be56ba48e4d0c670c87acf1b5 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save vorlon001/b0af8a7be56ba48e4d0c670c87acf1b5 to your computer and use it in GitHub Desktop.
Download ZIP
Kubernetes Deploy cilium with Native routing via eBGP
Raw
readme.md

Deploy: v179, v1.36.0, Debian 13, 6.12.86+deb13-amd64, Containerd: 2.3.0, cilium: 1.19.1, CNI Plugins: 1.9.1

helm repo add cilium https://helm.cilium.io/

# https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#hybrid-dsr-and-snat-mode
# snat  dsr loadBalancer.mode

helm install cilium cloudstack/cilium --version 1.19.1  \
   --namespace kube-system \
   --set routingMode=native \
   --set kubeProxyReplacement=true \
   --set loadBalancer.mode=snat \
   --set loadBalancer.dsrDispatch=geneve \
   --set ipv4NativeRoutingCIDR=10.96.0.0/16 \
   --set ipam.operator.clusterPoolIPv4PodCIDRList="10.96.0.0/17" \
   --set ipam.operator.clusterPoolIPv4MaskSize="24" \
   --set bgpControlPlane.enabled=true \
   --set bgp.enabled=false \
   --set nodePort.enabled=true

helm upgrade cilium cloudstack/cilium --version 1.19.1  \
   --namespace kube-system \
   --set routingMode=native \
   --set kubeProxyReplacement=true \
   --set loadBalancer.mode=snat \
   --set loadBalancer.dsrDispatch=geneve \
   --set ipv4NativeRoutingCIDR=10.96.0.0/16 \
   --set ipam.operator.clusterPoolIPv4PodCIDRList="10.96.0.0/17" \
   --set ipam.operator.clusterPoolIPv4MaskSize="24" \
   --set bgpControlPlane.enabled=true \
   --set bgp.enabled=false \
   --set nodePort.enabled=true
### WORK!
helm upgrade cilium cloudstack/cilium --version 1.19.1  \
   --namespace kube-system \
   --set routingMode=native \
   --set kubeProxyReplacement=true \
   --set loadBalancer.mode=snat \
   --set loadBalancer.dsrDispatch=geneve \
   --set loadBalancer.l7.backend=envoy \
   --set nodePort.enabled=true \
   --set egressGateway.enabled=true \
   --set cni.exclusive=false \
   --set hostFirewall.enabled=false \
   --set ingressController.enabled=true \
   --set ingressController.loadbalancerMode=dedicated \
   --set bandwidthManager.enabled=true \
   --set gatewayAPI.enabled=true \
   --set bpf.masquerade=true \
   --set ipv4.enabled=true \
   --set ipv6.enabled=false \
   --set ipMasqAgent.enabled=false \
   --set ipv4NativeRoutingCIDR=10.96.0.0/16 \
   --set ipam.operator.clusterPoolIPv4PodCIDRList="10.96.0.0/17" \
   --set ipam.operator.clusterPoolIPv4MaskSize="24" \
   --set bgpControlPlane.enabled=true \
   --set bgp.enabled=false \
   --set nodePort.enabled=true

kubectl rollout restart deployment/cilium-operator -n kube-system
kubectl rollout restart daemonset cilium -n kube-system
kubectl get node,pod -n kube-system -o wide



## BAD
helm upgrade cilium cloudstack/cilium --version 1.19.1  \
   --namespace kube-system \
   --set routingMode=native \
   --set kubeProxyReplacement=true \
   --set loadBalancer.mode=hybrid \
   --set loadBalancer.dsrDispatch=geneve \
   --set ipv4NativeRoutingCIDR=10.96.0.0/16 \
   --set ipam.operator.clusterPoolIPv4PodCIDRList="10.96.0.0/17" \
   --set ipam.operator.clusterPoolIPv4MaskSize="24" \
   --set bgpControlPlane.enabled=true \
   --set bgp.enabled=false \
   --set nodePort.enabled=true

kubectl rollout restart daemonset cilium -n kube-system
Эта критическая ошибка (fatal) возникает из-за несовместимости параметров режима Kube-Proxy Replacement и конфигурации туннелирования в Cilium.Вы настроили режим NodePort в значении "hybrid", а для диспетчеризации трафика указали тип "geneve". Однако сам Cilium при этом настроен на использование другого сетевого режима (например, routingMode: native или туннеля vxlan), в то время как гибридный режим NodePort с диспетчеризацией Geneve строго требует, чтобы основным туннельным протоколом кластера тоже являлся Geneve.




cat <<EOF | kubectl apply -f -
---
apiVersion: cilium.io/v2
kind: CiliumBGPAdvertisement
metadata:
  name: onetwo
  labels:
    advertise: onetwo
spec:
spec:
  advertisements:
    - advertisementType: "PodCIDR"
EOF

cat <<EOF | kubectl apply -f -
---
apiVersion: cilium.io/v2
kind: CiliumBGPAdvertisement
metadata:
  name: onetwo
  labels:
    advertise: onetwo
spec:
spec:
  advertisements:
    - advertisementType: "PodCIDR"
      attributes:
        communities:
          standard:
            - "65530:100"
    - advertisementType: "Service"
      service:
        addresses:
#          - ClusterIP
#          - ExternalIP
          - LoadBalancerIP
#      selector:
#        matchLabels:
#          type: external
      selector:
        matchExpressions:
         - {key: somekey, operator: NotIn, values: ['never-used-value']}
      attributes:
        communities:
          standard:
            - "65530:200"
EOF

cat <<EOF | kubectl apply -f -
---
apiVersion: cilium.io/v2
kind: CiliumBGPPeerConfig
metadata:
  name: onetwo
spec:
  families:
  - advertisements:
      matchLabels:
        advertise: onetwo
    afi: ipv4
    safi: unicast
  gracefulRestart:
    enabled: true
    restartTimeSeconds: 15
  timers:
    connectRetryTimeSeconds: 120
    holdTimeSeconds: 9
    keepAliveTimeSeconds: 3
EOF

cat <<EOF | kubectl apply -f -
---
apiVersion: cilium.io/v2
kind: CiliumBGPNodeConfigOverride
metadata:
  name: node130
spec:
  bgpInstances:
    - name: gobgp
      routerID: 127.0.0.1
      localPort: 8179
      peers:
        - name: frr
          localAddress: 127.0.0.1
EOF

## cat <<EOF | kubectl apply -f -
## ---
## apiVersion: cilium.io/v2alpha1
## kind: CiliumL2AnnouncementPolicy
## metadata:
##   name: onetwo
## spec:
##   serviceSelector:
##     matchLabels:
##       color: red
##   nodeSelector:
##     matchExpressions:
##       - key: node-role.kubernetes.io/control-plane
##         operator: Exists
##   interfaces: 
##   loadBalancerIPs: true
## EOF

cat <<EOF | kubectl apply -f -
---
apiVersion: cilium.io/v2
kind: CiliumBGPClusterConfig
metadata:
  labels:
    app.kubernetes.io/instance: cilium
  name: node130
spec:
  bgpInstances:
  - name: node130
    localASN: 65666
    localPort: 8179
    peers:
    - name: node130
      peerASN: 0
      peerAddress: 127.0.0.1
      peerConfigRef:
        name: onetwo
---
apiVersion: cilium.io/v2
kind: CiliumBGPClusterConfig
metadata:
  labels:
    app.kubernetes.io/instance: cilium
  name: node131
spec:
  bgpInstances:
  - name: node131
    localASN: 65666
    localPort: 8179
    peers:
    - name: node131
      peerASN: 0
      peerAddress: 127.0.0.1
      peerConfigRef:
        name: onetwo
---
apiVersion: cilium.io/v2
kind: CiliumBGPClusterConfig
metadata:
  labels:
    app.kubernetes.io/instance: cilium
  name: node132
spec:
  bgpInstances:
  - name: node132
    localASN: 65666
    localPort: 8179
    peers:
    - name: node132
      peerASN: 0
      peerAddress: 127.0.0.1
      peerConfigRef:
        name: onetwo
---
apiVersion: cilium.io/v2
kind: CiliumBGPClusterConfig
metadata:
  labels:
    app.kubernetes.io/instance: cilium
  name: node133
spec:
  bgpInstances:
  - name: node133
    localASN: 65666
    localPort: 8179
    peers:
    - name: node133
      peerASN: 0
      peerAddress: 127.0.0.1
      peerConfigRef:
        name: onetwo
---
apiVersion: cilium.io/v2
kind: CiliumBGPClusterConfig
metadata:
  labels:
    app.kubernetes.io/instance: cilium
  name: node140
spec:
  bgpInstances:
  - name: node140
    localASN: 65666
    localPort: 8179
    peers:
    - name: node130
      peerASN: 0
      peerAddress: 127.0.0.1
      peerConfigRef:
        name: onetwo
---
apiVersion: cilium.io/v2
kind: CiliumBGPClusterConfig
metadata:
  labels:
    app.kubernetes.io/instance: cilium
  name: node141
spec:
  bgpInstances:
  - name: node141
    localASN: 65666
    localPort: 8179
    peers:
    - name: node141
      peerASN: 0
      peerAddress: 127.0.0.1
      peerConfigRef:
        name: onetwo
---
apiVersion: cilium.io/v2
kind: CiliumBGPClusterConfig
metadata:
  labels:
    app.kubernetes.io/instance: cilium
  name: node142
spec:
  bgpInstances:
  - name: node142
    localASN: 65666
    localPort: 8179
    peers:
    - name: node142
      peerASN: 0
      peerAddress: 127.0.0.1
      peerConfigRef:
        name: onetwo
---
apiVersion: cilium.io/v2
kind: CiliumBGPClusterConfig
metadata:
  labels:
    app.kubernetes.io/instance: cilium
  name: node143
spec:
  bgpInstances:
  - name: node143
    localASN: 65666
    localPort: 8179
    peers:
    - name: node143
      peerASN: 0
      peerAddress: 127.0.0.1
      peerConfigRef:
        name: onetwo
EOF

kubectl rollout restart ds cilium -n kube-system

kubectl exec -n kube-system -ti <cilium-pod-name> -- cilium bgp peers

root@node130:~# kubectl exec -n kube-system -ti cilium-k7nfj -- cilium  bpf lb list
SERVICE ADDRESS             BACKEND ADDRESS (REVNAT_ID) (SLOT)
10.97.186.62:443/TCP (0)    0.0.0.0:0 (5) (0) [ClusterIP, InternalLocal, non-routable]
10.97.128.10:9153/TCP (0)   0.0.0.0:0 (3) (0) [ClusterIP, non-routable]
10.97.128.10:9153/TCP (1)   10.96.5.20:9153/TCP (3) (1)
10.97.186.62:443/TCP (1)    192.168.200.130:4244/TCP (5) (1)
10.97.128.10:53/TCP (1)     10.96.5.20:53/TCP (1) (1)
10.97.128.10:53/TCP (0)     0.0.0.0:0 (1) (0) [ClusterIP, non-routable]
10.97.128.1:443/TCP (3)     192.168.200.132:6443/TCP (4) (3)
10.97.128.10:53/UDP (1)     10.96.5.20:53/UDP (2) (1)
10.97.128.1:443/TCP (2)     192.168.200.131:6443/TCP (4) (2)
10.97.128.1:0/ANY (0)       0.0.0.0:0 (0) (0) [ClusterIP, non-routable]
10.97.128.10:0/ANY (0)      0.0.0.0:0 (0) (0) [ClusterIP, non-routable]
10.97.128.1:443/TCP (1)     192.168.200.130:6443/TCP (4) (1)
10.97.186.62:0/ANY (0)      0.0.0.0:0 (0) (0) [ClusterIP, non-routable]
10.97.128.10:53/UDP (2)     10.96.5.244:53/UDP (2) (2)
10.97.128.1:443/TCP (0)     0.0.0.0:0 (4) (0) [ClusterIP, non-routable]
10.97.128.10:53/TCP (2)     10.96.5.244:53/TCP (1) (2)
10.97.128.10:9153/TCP (2)   10.96.5.244:9153/TCP (3) (2)
10.97.128.10:53/UDP (0)     0.0.0.0:0 (2) (0) [ClusterIP, non-routable]
..............
root@node130:~#

root@node130:~# kubectl exec -n kube-system -ti cilium-k7nfj -- cilium bgp peers
Local AS   Peer AS   Peer Address    Session       Uptime   Family         Received   Advertised
65666      65130     127.0.0.1:179   established   23m37s   ipv4/unicast   18         1
root@node130:~#

root@node130:~# ip r s
default via 192.168.200.1 dev enp1s0.200 proto static
10.96.0.0/24 nhid 57 via 192.168.202.132 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
10.96.1.0/24 nhid 58 via 192.168.202.133 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
10.96.2.0/24 nhid 61 via 192.168.202.142 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
10.96.3.0/24 nhid 62 via 192.168.202.140 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
10.96.4.0/24 nhid 60 via 192.168.202.143 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
10.96.5.0/24 nhid 59 via 192.168.202.141 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
10.96.6.0/24 via 10.96.6.113 dev cilium_host proto kernel src 10.96.6.113
10.96.6.113 dev cilium_host proto kernel scope link
10.96.7.0/24 nhid 56 via 192.168.202.131 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.10.131 nhid 56 via 192.168.202.131 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.10.132 nhid 57 via 192.168.202.132 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.10.133 nhid 58 via 192.168.202.133 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.10.140 nhid 62 via 192.168.202.140 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.10.141 nhid 59 via 192.168.202.141 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.10.142 nhid 61 via 192.168.202.142 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.10.143 nhid 60 via 192.168.202.143 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.11.131 nhid 56 via 192.168.202.131 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.11.132 nhid 57 via 192.168.202.132 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.11.133 nhid 58 via 192.168.202.133 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.11.140 nhid 62 via 192.168.202.140 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.11.141 nhid 59 via 192.168.202.141 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.11.142 nhid 61 via 192.168.202.142 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.11.143 nhid 60 via 192.168.202.143 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.12.131 nhid 56 via 192.168.202.131 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.12.132 nhid 57 via 192.168.202.132 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.12.133 nhid 58 via 192.168.202.133 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.12.140 nhid 62 via 192.168.202.140 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.12.141 nhid 59 via 192.168.202.141 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.12.142 nhid 61 via 192.168.202.142 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.12.143 nhid 60 via 192.168.202.143 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.13.131 nhid 56 via 192.168.202.131 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.13.132 nhid 57 via 192.168.202.132 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.13.133 nhid 58 via 192.168.202.133 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.13.140 nhid 62 via 192.168.202.140 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.13.141 nhid 59 via 192.168.202.141 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.13.142 nhid 61 via 192.168.202.142 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.13.143 nhid 60 via 192.168.202.143 dev enp1s0.600 proto bgp src 192.168.10.130 metric 20
192.168.200.0/24 dev enp1s0.200 proto kernel scope link src 192.168.200.130
192.168.201.0/24 dev enp1s0.400 proto kernel scope link src 192.168.201.130
192.168.202.0/24 dev enp1s0.600 proto kernel scope link src 192.168.202.130
192.168.203.0/24 dev enp1s0.800 proto kernel scope link src 192.168.203.130
root@node130:~#

root@node170:~# ip r s
default via 192.168.200.1 dev enp1s0.200 proto static
12.0.2.90 nhid 70 proto bgp src 192.168.10.170 metric 20
        nexthop via 192.168.202.130 dev enp1s0.600 weight 1
        nexthop via 192.168.202.131 dev enp1s0.600 weight 1
12.0.2.95 nhid 70 proto bgp src 192.168.10.170 metric 20
        nexthop via 192.168.202.130 dev enp1s0.600 weight 1
        nexthop via 192.168.202.131 dev enp1s0.600 weight 1
12.0.2.100 nhid 70 proto bgp src 192.168.10.170 metric 20
        nexthop via 192.168.202.130 dev enp1s0.600 weight 1
        nexthop via 192.168.202.131 dev enp1s0.600 weight 1
12.0.100.90 nhid 70 proto bgp src 192.168.10.170 metric 20
        nexthop via 192.168.202.130 dev enp1s0.600 weight 1
        nexthop via 192.168.202.131 dev enp1s0.600 weight 1
12.0.100.91 nhid 70 proto bgp src 192.168.10.170 metric 20
        nexthop via 192.168.202.130 dev enp1s0.600 weight 1
        nexthop via 192.168.202.131 dev enp1s0.600 weight 1
12.0.100.92 nhid 70 proto bgp src 192.168.10.170 metric 20
        nexthop via 192.168.202.130 dev enp1s0.600 weight 1
        nexthop via 192.168.202.131 dev enp1s0.600 weight 1
12.0.100.93 nhid 70 proto bgp src 192.168.10.170 metric 20
        nexthop via 192.168.202.130 dev enp1s0.600 weight 1
        nexthop via 192.168.202.131 dev enp1s0.600 weight 1
12.0.100.94 nhid 70 proto bgp src 192.168.10.170 metric 20
        nexthop via 192.168.202.130 dev enp1s0.600 weight 1
        nexthop via 192.168.202.131 dev enp1s0.600 weight 1
12.0.100.95 nhid 70 proto bgp src 192.168.10.170 metric 20
        nexthop via 192.168.202.130 dev enp1s0.600 weight 1
        nexthop via 192.168.202.131 dev enp1s0.600 weight 1
192.168.10.130 nhid 47 via 192.168.202.130 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.10.131 nhid 48 via 192.168.202.131 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.10.132 nhid 71 via 192.168.202.132 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.10.133 nhid 72 via 192.168.202.133 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.10.140 nhid 73 via 192.168.202.140 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.10.141 nhid 74 via 192.168.202.141 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.10.142 nhid 75 via 192.168.202.142 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.10.143 nhid 76 via 192.168.202.143 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.11.130 nhid 47 via 192.168.202.130 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.11.131 nhid 48 via 192.168.202.131 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.11.132 nhid 71 via 192.168.202.132 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.11.133 nhid 72 via 192.168.202.133 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.11.140 nhid 73 via 192.168.202.140 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.11.141 nhid 74 via 192.168.202.141 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.11.142 nhid 75 via 192.168.202.142 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.11.143 nhid 76 via 192.168.202.143 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.12.130 nhid 47 via 192.168.202.130 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.12.131 nhid 48 via 192.168.202.131 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.12.132 nhid 71 via 192.168.202.132 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.12.133 nhid 72 via 192.168.202.133 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.12.140 nhid 73 via 192.168.202.140 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.12.141 nhid 74 via 192.168.202.141 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.12.142 nhid 75 via 192.168.202.142 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.12.143 nhid 76 via 192.168.202.143 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.13.130 nhid 47 via 192.168.202.130 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.13.131 nhid 48 via 192.168.202.131 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.13.132 nhid 71 via 192.168.202.132 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.13.133 nhid 72 via 192.168.202.133 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.13.140 nhid 73 via 192.168.202.140 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.13.141 nhid 74 via 192.168.202.141 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.13.142 nhid 75 via 192.168.202.142 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.13.143 nhid 76 via 192.168.202.143 dev enp1s0.600 proto bgp src 192.168.10.170 metric 20
192.168.200.0/24 dev enp1s0.200 proto kernel scope link src 192.168.200.170
192.168.201.0/24 dev enp1s0.400 proto kernel scope link src 192.168.201.170
192.168.202.0/24 dev enp1s0.600 proto kernel scope link src 192.168.202.170
192.168.203.0/24 dev enp1s0.800 proto kernel scope link src 192.168.203.170
root@node170:~#

### on node170
cat<<EOF>>/etc/sysctl.conf
net.ipv4.fib_multipath_hash_policy=1
net.ipv4.fib_multipath_use_neigh=1
net.ipv4.fib_multipath_hash_fields=37
EOF

root@node170:~# cat /etc/frr/frr.conf
frr version 10.3
frr defaults traditional
hostname node170
log syslog informational
no ip forwarding
no ipv6 forwarding
service integrated-vtysh-config
!
ip prefix-list LocalLink seq 10 permit 192.168.10.0/24 le 32
ip prefix-list LocalLink seq 20 permit 192.168.11.0/24 le 32
ip prefix-list LocalLink seq 30 permit 192.168.12.0/24 le 32
ip prefix-list LocalLink seq 40 permit 192.168.13.0/24 le 32
ip prefix-list LocalLinkDeny seq 5 permit 192.168.200.0/24 le 32
!
route-map Local permit 10
 match ip address prefix-list LocalLink
 set community 65170:1000
exit
!
route-map Local deny 65535
 match ip address prefix-list LocalLinkDeny
exit
!
route-map zebra_to_kernel permit 15
 match ip address prefix-list LocalLink
 set src 192.168.10.170
exit
!
route-map zebra_to_kernel permit 35
 set src 192.168.10.170
exit
!
router bgp 65170
 no bgp ebgp-requires-policy
 no bgp default ipv4-unicast
 bgp bestpath as-path multipath-relax
 no bgp network import-check
 neighbor 192.168.202.130 remote-as 65130
 neighbor 192.168.202.130 update-source enp1s0.600
 neighbor 192.168.202.131 remote-as 65131
 neighbor 192.168.202.131 update-source enp1s0.600
 neighbor 192.168.202.132 remote-as 65132
 neighbor 192.168.202.132 update-source enp1s0.600
 neighbor 192.168.202.133 remote-as 65133
 neighbor 192.168.202.133 update-source enp1s0.600
 neighbor 192.168.202.140 remote-as 65140
 neighbor 192.168.202.140 update-source enp1s0.600
 neighbor 192.168.202.141 remote-as 65141
 neighbor 192.168.202.141 update-source enp1s0.600
 neighbor 192.168.202.142 remote-as 65142
 neighbor 192.168.202.142 update-source enp1s0.600
 neighbor 192.168.202.143 remote-as 65143
 neighbor 192.168.202.143 update-source enp1s0.600
 bgp allow-martian-nexthop
 !
 address-family ipv4 unicast
  redistribute connected route-map Local
  redistribute static
  neighbor 192.168.202.130 activate
  neighbor 192.168.202.130 soft-reconfiguration inbound
  neighbor 192.168.202.131 activate
  neighbor 192.168.202.131 soft-reconfiguration inbound
  neighbor 192.168.202.132 activate
  neighbor 192.168.202.132 soft-reconfiguration inbound
  neighbor 192.168.202.133 activate
  neighbor 192.168.202.133 soft-reconfiguration inbound
  neighbor 192.168.202.140 activate
  neighbor 192.168.202.140 soft-reconfiguration inbound
  neighbor 192.168.202.141 activate
  neighbor 192.168.202.141 soft-reconfiguration inbound
  neighbor 192.168.202.142 activate
  neighbor 192.168.202.142 soft-reconfiguration inbound
  neighbor 192.168.202.143 activate
  neighbor 192.168.202.143 soft-reconfiguration inbound
 exit-address-family
exit
!
ip protocol bgp route-map zebra_to_kernel
!
root@node170:~#

### V1
root@node130:~# cat /etc/frr/frr.conf
frr version 10.3
frr defaults traditional
hostname node130
log syslog informational
service integrated-vtysh-config
!
ip prefix-list LocalLink seq 10 permit 192.168.10.0/24 le 32
ip prefix-list LocalLink seq 20 permit 192.168.11.0/24 le 32
ip prefix-list LocalLink seq 30 permit 192.168.12.0/24 le 32
ip prefix-list LocalLink seq 40 permit 192.168.13.0/24 le 32
ip prefix-list LocalLinkDeny seq 5 permit 192.168.200.0/24 le 32
!
route-map ANNOUNCE_RULE_CLUSTER permit 1
 match community LocalLink
exit
!
route-map ANNOUNCE_RULE_CLUSTER permit 10
 match community PodCidr_Kubernetes
exit
!
route-map ANNOUNCE_RULE_EXTERNAL permit 1
 match community LocalLinkExternal
exit
!
route-map ANNOUNCE_RULE_EXTERNAL permit 10
 match community SvcCidr_Kubernetes
exit
!
route-map Local permit 10
 match ip address prefix-list LocalLink
 set community 65130:1000
exit
!
route-map Local deny 65535
 match ip address prefix-list LocalLinkDeny
exit
!
route-map net_cilium permit 5
 match community PodCidr_Cilium
 set as-path exclude all
 set community 65130:100
 set ip next-hop 192.168.202.130
exit
!
route-map net_cilium permit 10
 match community SvcCidr_Cilium
 set as-path exclude all
 set community 65130:200
 set ip next-hop 192.168.202.130
exit
!
!
route-map zebra_to_kernel permit 10
 match community PodCidr_Kubernetes
 set src 192.168.10.130
exit
!
route-map zebra_to_kernel permit 15
 match community LocalLink
 set src 192.168.10.130
exit
!
route-map zebra_to_kernel deny 20
 match community SvcCidr_Kubernetes
exit
!
!
router bgp 65130
 no bgp ebgp-requires-policy
 no bgp default ipv4-unicast
 no bgp network import-check
 neighbor 127.0.0.1 remote-as 65666
 neighbor 127.0.0.1 port 8179
 neighbor 127.0.0.1 update-source lo
 neighbor 192.168.202.131 remote-as 65131
 neighbor 192.168.202.131 update-source enp1s0.600
 neighbor 192.168.202.132 remote-as 65132
 neighbor 192.168.202.132 update-source enp1s0.600
 neighbor 192.168.202.133 remote-as 65133
 neighbor 192.168.202.133 update-source enp1s0.600
 neighbor 192.168.202.140 remote-as 65140
 neighbor 192.168.202.140 update-source enp1s0.600
 neighbor 192.168.202.141 remote-as 65141
 neighbor 192.168.202.141 update-source enp1s0.600
 neighbor 192.168.202.142 remote-as 65142
 neighbor 192.168.202.142 update-source enp1s0.600
 neighbor 192.168.202.143 remote-as 65143
 neighbor 192.168.202.143 update-source enp1s0.600
 neighbor 192.168.202.170 remote-as 65170
 neighbor 192.168.202.170 update-source enp1s0.600
 bgp allow-martian-nexthop
 !
 address-family ipv4 unicast
  redistribute connected route-map Local
  redistribute static
  neighbor 127.0.0.1 activate
  neighbor 127.0.0.1 soft-reconfiguration inbound
  neighbor 127.0.0.1 route-map net_cilium in
  neighbor 192.168.202.131 activate
  neighbor 192.168.202.131 soft-reconfiguration inbound
  neighbor 192.168.202.131 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.132 activate
  neighbor 192.168.202.132 soft-reconfiguration inbound
  neighbor 192.168.202.132 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.133 activate
  neighbor 192.168.202.133 soft-reconfiguration inbound
  neighbor 192.168.202.133 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.140 activate
  neighbor 192.168.202.140 soft-reconfiguration inbound
  neighbor 192.168.202.140 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.141 activate
  neighbor 192.168.202.141 soft-reconfiguration inbound
  neighbor 192.168.202.141 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.142 activate
  neighbor 192.168.202.142 soft-reconfiguration inbound
  neighbor 192.168.202.142 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.143 activate
  neighbor 192.168.202.143 soft-reconfiguration inbound
  neighbor 192.168.202.143 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.170 activate
  neighbor 192.168.202.170 soft-reconfiguration inbound
  neighbor 192.168.202.170 route-map ANNOUNCE_RULE_EXTERNAL out
 exit-address-family
exit
!
bgp community-list standard LocalLink seq 5 permit 65130:1000
bgp community-list expanded LocalLinkExternal seq 5 permit _65[0-9][0-9][0-9]:1000_
bgp community-list standard PodCidr_Cilium seq 5 permit 65530:100
bgp community-list standard PodCidr_Kubernetes seq 5 permit 65130:100
bgp community-list standard SvcCidr_Cilium seq 5 permit 65530:200
bgp community-list standard SvcCidr_Kubernetes seq 5 permit 65130:200
!
ip protocol bgp route-map zebra_to_kernel
!
root@node130:~#
### V10
node130# sh run
Building configuration...

Current configuration:
!
frr version 10.3
frr defaults traditional
hostname node130
log syslog informational
service integrated-vtysh-config
!
ip prefix-list LocalLink seq 10 permit 192.168.10.0/24 le 32
ip prefix-list LocalLink seq 20 permit 192.168.11.0/24 le 32
ip prefix-list LocalLink seq 30 permit 192.168.12.0/24 le 32
ip prefix-list LocalLink seq 40 permit 192.168.13.0/24 le 32
ip prefix-list LocalLinkDeny seq 5 permit 192.168.200.0/24 le 32
!
route-map ANNOUNCE_RULE_CLUSTER permit 1
 match community LocalLink
exit
!
route-map ANNOUNCE_RULE_CLUSTER permit 10
 match community PodCidr_Kubernetes
exit
!
route-map ANNOUNCE_RULE_CLUSTER_IN permit 10
 match community LocalLinkExternal
 set tag 1000
exit
!
route-map ANNOUNCE_RULE_CLUSTER_IN permit 20
 match community PodCidr_Kubernetes_External
 set tag 100
exit
!
route-map ANNOUNCE_RULE_CLUSTER_IN permit 100
exit
!
route-map ANNOUNCE_RULE_EXTERNAL permit 1
 match community LocalLinkExternal
exit
!
route-map ANNOUNCE_RULE_EXTERNAL permit 10
 match community SvcCidr_Kubernetes
exit
!
route-map ANNOUNCE_RULE_EXTERNAL_IN permit 10
 match community LocalLinkExternal
 set tag 1000
exit
!
route-map ANNOUNCE_RULE_EXTERNAL_IN permit 20
 match community PodCidr_Kubernetes_External
 set tag 100
exit
!
route-map ANNOUNCE_RULE_EXTERNAL_IN permit 100
exit
!
route-map Local permit 10
 match ip address prefix-list LocalLink
 set community 65130:1000
exit
!
route-map Local deny 65535
 match ip address prefix-list LocalLinkDeny
exit
!
route-map net_cilium permit 5
 match community PodCidr_Cilium
 set as-path exclude all
 set community 65130:100
 set ip next-hop 192.168.10.130
 set tag 100
exit
!
route-map net_cilium permit 10
 match community SvcCidr_Cilium
 set as-path exclude all
 set community 65130:200
 set ip next-hop 192.168.10.130
 set tag 200
exit
!
route-map zebra_to_kernel permit 10
 match tag 100
 set src 192.168.10.130
exit
!
route-map zebra_to_kernel permit 15
 match tag 1000
 set src 192.168.13.130
exit
!
router bgp 65130
 no bgp ebgp-requires-policy
 no bgp default ipv4-unicast
 no bgp network import-check
 neighbor 127.0.0.1 remote-as 65666
 neighbor 127.0.0.1 port 8179
 neighbor 127.0.0.1 update-source lo
 neighbor 192.168.202.131 remote-as 65131
 neighbor 192.168.202.131 update-source enp1s0.600
 neighbor 192.168.202.132 remote-as 65132
 neighbor 192.168.202.132 update-source enp1s0.600
 neighbor 192.168.202.133 remote-as 65133
 neighbor 192.168.202.133 update-source enp1s0.600
 neighbor 192.168.202.140 remote-as 65140
 neighbor 192.168.202.140 update-source enp1s0.600
 neighbor 192.168.202.141 remote-as 65141
 neighbor 192.168.202.141 update-source enp1s0.600
 neighbor 192.168.202.142 remote-as 65142
 neighbor 192.168.202.142 update-source enp1s0.600
 neighbor 192.168.202.143 remote-as 65143
 neighbor 192.168.202.143 update-source enp1s0.600
 neighbor 192.168.202.170 remote-as 65170
 neighbor 192.168.202.170 update-source enp1s0.600
 bgp allow-martian-nexthop
 !
 address-family ipv4 unicast
  redistribute connected route-map Local
  redistribute static
  neighbor 127.0.0.1 activate
  neighbor 127.0.0.1 soft-reconfiguration inbound
  neighbor 127.0.0.1 route-map net_cilium in
  neighbor 192.168.202.131 activate
  neighbor 192.168.202.131 soft-reconfiguration inbound
  neighbor 192.168.202.131 route-map ANNOUNCE_RULE_CLUSTER_IN in
  neighbor 192.168.202.131 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.132 activate
  neighbor 192.168.202.132 soft-reconfiguration inbound
  neighbor 192.168.202.132 route-map ANNOUNCE_RULE_CLUSTER_IN in
  neighbor 192.168.202.132 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.133 activate
  neighbor 192.168.202.133 soft-reconfiguration inbound
  neighbor 192.168.202.133 route-map ANNOUNCE_RULE_CLUSTER_IN in
  neighbor 192.168.202.133 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.140 activate
  neighbor 192.168.202.140 soft-reconfiguration inbound
  neighbor 192.168.202.140 route-map ANNOUNCE_RULE_CLUSTER_IN in
  neighbor 192.168.202.140 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.141 activate
  neighbor 192.168.202.141 soft-reconfiguration inbound
  neighbor 192.168.202.141 route-map ANNOUNCE_RULE_CLUSTER_IN in
  neighbor 192.168.202.141 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.142 activate
  neighbor 192.168.202.142 soft-reconfiguration inbound
  neighbor 192.168.202.142 route-map ANNOUNCE_RULE_CLUSTER_IN in
  neighbor 192.168.202.142 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.143 activate
  neighbor 192.168.202.143 soft-reconfiguration inbound
  neighbor 192.168.202.143 route-map ANNOUNCE_RULE_CLUSTER_IN in
  neighbor 192.168.202.143 route-map ANNOUNCE_RULE_CLUSTER out
  neighbor 192.168.202.170 activate
  neighbor 192.168.202.170 soft-reconfiguration inbound
  neighbor 192.168.202.170 route-map ANNOUNCE_RULE_EXTERNAL_IN in
  neighbor 192.168.202.170 route-map ANNOUNCE_RULE_EXTERNAL out
 exit-address-family
exit
!
bgp community-list standard LocalLink seq 5 permit 65130:1000
bgp community-list expanded LocalLinkExternal seq 5 permit _65[0-9][0-9][0-9]:1000_
bgp community-list standard PodCidr_Cilium seq 5 permit 65530:100
bgp community-list standard PodCidr_Kubernetes seq 5 permit 65130:100
bgp community-list expanded PodCidr_Kubernetes_External seq 5 permit _65[0-9][0-9][0-9]:100_
bgp community-list standard SvcCidr_Cilium seq 5 permit 65530:200
bgp community-list standard SvcCidr_Kubernetes seq 5 permit 65130:200
!
ip protocol bgp route-map zebra_to_kernel
!
end
node130#
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment