Kube-proxy generated iptables rules

README.md

Calculate the iptables generated rules

iptables_generated_rules_test.go returns the number of iptables generated rules based on the following inputs:

1. ServiceType: NodePort, ClusterIP, LoadBalancer
2. Max number of Services
3. Max number of Endpoints of Services
4. Increment: It increments in each iteration the number of services and endpoints per services until it reaches the defined maximumts

It returns, in CSV format, a table with:

• ServiceType
• Services
• Endpoint/Service
• FilterRules
• NATRules
• Latency

How to use it

1. In your kubernetes repository folder, drop the file iptables_generated_rules_test.go in pkg/proxy/iptables

2. Edit the Input variables in the file at your convinience

	// Define experiment variables
	maxServices := 1005
	maxEndpoints := 502
	increment := 500
	serviceTypes := []v1.ServiceType{v1.ServiceTypeClusterIP, v1.ServiceTypeNodePort, v1.ServiceTypeLoadBalancer}

3. Run with go test, use the -v flag:

/usr/bin/go test -timeout 100s -run ^TestNumberIptablesRules$ k8s.io/kubernetes/pkg/proxy/iptables -v                                    SIGINT   master 
=== RUN   TestNumberIptablesRules
=== RUN   TestNumberIptablesRules/ClusterIP-1-1
=== RUN   TestNumberIptablesRules/ClusterIP-501-1
=== RUN   TestNumberIptablesRules/ClusterIP-1001-1
=== RUN   TestNumberIptablesRules/ClusterIP-1-501
=== RUN   TestNumberIptablesRules/ClusterIP-501-501
=== RUN   TestNumberIptablesRules/ClusterIP-1001-501
=== RUN   TestNumberIptablesRules/NodePort-1-1
=== RUN   TestNumberIptablesRules/NodePort-501-1
=== RUN   TestNumberIptablesRules/NodePort-1001-1
=== RUN   TestNumberIptablesRules/NodePort-1-501
=== RUN   TestNumberIptablesRules/NodePort-501-501
=== RUN   TestNumberIptablesRules/NodePort-1001-501
=== RUN   TestNumberIptablesRules/LoadBalancer-1-1
=== RUN   TestNumberIptablesRules/LoadBalancer-501-1
=== RUN   TestNumberIptablesRules/LoadBalancer-1001-1
=== RUN   TestNumberIptablesRules/LoadBalancer-1-501
=== RUN   TestNumberIptablesRules/LoadBalancer-501-501
=== RUN   TestNumberIptablesRules/LoadBalancer-1001-501
ServiceType,Services,Endpoint/Service,FilterRules,NATRules,Latency
ClusterIP,1,1,5,11,1.16653ms
ClusterIP,501,1,5,2511,8.228011ms
ClusterIP,1001,1,5,5011,14.535362ms
ClusterIP,1,501,5,1511,3.95692ms
ClusterIP,501,501,5,754011,804.495785ms
ClusterIP,1001,501,5,1506511,1.480063753s
NodePort,1,1,5,13,452.31µs
NodePort,501,1,5,3513,6.81702ms
NodePort,1001,1,5,7013,24.221882ms
NodePort,1,501,5,1513,2.625701ms
NodePort,501,501,5,755013,823.539865ms
NodePort,1001,501,5,1508513,1.530764605s
LoadBalancer,1,1,5,19,473.241µs
LoadBalancer,501,1,5,6519,9.170061ms
LoadBalancer,1001,1,5,13019,20.564253ms
LoadBalancer,1,501,5,1519,2.07762ms
LoadBalancer,501,501,5,758019,841.162086ms
LoadBalancer,1001,501,5,1514519,1.681468323s

Experiment.md

• master e52b7be7d719673a1a5ae28ad3bb0d6db58d35ec

ServiceType,Services,Endpoint/Service,FilterRules,NATRules,Latency
ClusterIP,1,1,5,11,1.16653ms
ClusterIP,501,1,5,2511,8.228011ms
ClusterIP,1001,1,5,5011,14.535362ms
ClusterIP,1,501,5,1511,3.95692ms
ClusterIP,501,501,5,754011,804.495785ms
ClusterIP,1001,501,5,1506511,1.480063753s
NodePort,1,1,5,13,452.31µs
NodePort,501,1,5,3513,6.81702ms
NodePort,1001,1,5,7013,24.221882ms
NodePort,1,501,5,1513,2.625701ms
NodePort,501,501,5,755013,823.539865ms
NodePort,1001,501,5,1508513,1.530764605s
LoadBalancer,1,1,5,19,473.241µs
LoadBalancer,501,1,5,6519,9.170061ms
LoadBalancer,1001,1,5,13019,20.564253ms
LoadBalancer,1,501,5,1519,2.07762ms
LoadBalancer,501,501,5,758019,841.162086ms
LoadBalancer,1001,501,5,1514519,1.681468323s

• v1.21.1

ServiceType,Services,Endpoint/Service,FilterRules,NATRules,Latency
ClusterIP,1,1,5,11,752.4µs
ClusterIP,501,1,5,2511,9.859351ms
ClusterIP,1001,1,5,5011,18.108942ms
ClusterIP,1,501,5,1511,2.601601ms
ClusterIP,501,501,5,754011,777.795159ms
ClusterIP,1001,501,5,1506511,1.595336393s
NodePort,1,1,5,13,723.77µs
NodePort,501,1,5,3513,8.631251ms
NodePort,1001,1,5,7013,18.304563ms
NodePort,1,501,5,1513,2.68134ms
NodePort,501,501,5,755013,867.08065ms
NodePort,1001,501,5,1508513,1.915207521s
LoadBalancer,1,1,5,19,499.92µs
LoadBalancer,501,1,5,6519,10.823271ms
LoadBalancer,1001,1,5,13019,21.438213ms
LoadBalancer,1,501,5,1519,2.58859ms
LoadBalancer,501,501,5,758019,935.42395ms
LoadBalancer,1001,501,5,1514519,1.773926314s

• PR 97238

ServiceType,Services,Endpoint/Service,FilterRules,NATRules,Latency
ClusterIP,1,1,5,11,580.9µs
ClusterIP,501,1,5,2511,12.707701ms
ClusterIP,1001,1,5,5011,15.763422ms
ClusterIP,1,501,5,1511,3.58585ms
ClusterIP,501,501,5,754011,864.481299ms
ClusterIP,1001,501,5,1506511,2.35422558s
NodePort,1,1,5,13,683.31µs
NodePort,501,1,5,3513,9.816691ms
NodePort,1001,1,5,7013,21.262122ms
NodePort,1,501,5,1513,3.06024ms
NodePort,501,501,5,755013,838.829967ms
NodePort,1001,501,5,1508513,2.028615802s
LoadBalancer,1,1,5,19,617.59µs
LoadBalancer,501,1,5,6519,17.172622ms
LoadBalancer,1001,1,5,13019,32.990744ms
LoadBalancer,1,501,5,1519,5.861021ms
LoadBalancer,501,501,5,758019,910.592684ms
LoadBalancer,1001,501,5,1514519,1.718009657s

iptables_generated_rules_test.go

/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package iptables

import (
	"encoding/csv"
	"fmt"
	"net"
	"os"
	"strconv"
	"testing"
	"time"

	v1 "k8s.io/api/core/v1"
	discovery "k8s.io/api/discovery/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/component-base/metrics/testutil"
	"k8s.io/kubernetes/pkg/proxy/metrics"

	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
	iptablestest "k8s.io/kubernetes/pkg/util/iptables/testing"

	utilnet "k8s.io/utils/net"
	utilpointer "k8s.io/utils/pointer"
)

func TestNumberIptablesRules(t *testing.T) {

	// Define experiment variables
	maxServices := 1005
	maxEndpoints := 502
	increment := 500
	serviceTypes := []v1.ServiceType{v1.ServiceTypeClusterIP, v1.ServiceTypeNodePort, v1.ServiceTypeLoadBalancer}

	// Write as CSV, we set the headers first
	records := [][]string{
		{"ServiceType", "Services", "Endpoint/Service", "FilterRules", "NATRules", "Latency"},
	}

	for _, svcType := range serviceTypes {
		for j := 1; j < maxEndpoints; j += increment {
			for i := 1; i < maxServices; i += increment {

				t.Run(fmt.Sprintf("%s-%d-%d", svcType, i, j), func(t *testing.T) {
					ipt := iptablestest.NewFake()
					fp := NewFakeProxier(ipt)
					metrics.RegisterMetrics()
					// t.Logf("Generate %d Services Type %s with %d endpoints each", i, svcType, j)
					svcs, eps := generateServiceEndpoints(svcType, i, j)

					// t.Logf("Updating kube-proxy")
					makeServiceMap(fp, svcs...)
					populateEndpointSlices(fp, eps...)

					// t.Logf("Syncing kube-proxy")
					start := time.Now()
					fp.syncProxyRules()
					elapsed := time.Since(start)
					// t.Logf("kube-proxy sync rules latency: %v", elapsed)

					// t.Logf("Gathering kube-proxy metrics")
					nFilterRules, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableFilter)))
					if err != nil {
						t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err)
					}

					nNatRules, err := testutil.GetGaugeMetricValue(metrics.IptablesRulesTotal.WithLabelValues(string(utiliptables.TableNAT)))
					if err != nil {
						t.Errorf("failed to get %s value, err: %v", metrics.IptablesRulesTotal.Name, err)
					}

					// t.Logf("kube-proxy iptables rules: Filter %.0f NAT %.0f on %v", nFilterRules, nNatRules, elapsed)
					result := []string{string(svcType), strconv.Itoa(i), strconv.Itoa(j), strconv.FormatFloat(nFilterRules, 'f', -1, 64), strconv.FormatFloat(nNatRules, 'f', -1, 64), elapsed.String()}
					records = append(records, result)

					// clear metrics
					metrics.IptablesRulesTotal.Delete(map[string]string{"table": string(utiliptables.TableNAT)})
					metrics.IptablesRulesTotal.Delete(map[string]string{"table": string(utiliptables.TableFilter)})
				})
			}
		}
	}

	// t.Logf("EXPERIMENT OUTPUT %v", records)
	w := csv.NewWriter(os.Stdout)
	w.WriteAll(records) // calls Flush internally

	if err := w.Error(); err != nil {
		t.Fatalf("error writing csv: %v", err)
	}
}

// generateServiceEndpoints generate
func generateServiceEndpoints(svcType v1.ServiceType, nServices, nEndpoints int) ([]*v1.Service, []*discovery.EndpointSlice) {
	services := make([]*v1.Service, nServices)
	endpoints := make([]*discovery.EndpointSlice, nServices)

	// base parameters
	basePort := 80
	base := utilnet.BigForIP(net.ParseIP("10.0.0.1"))
	// generate a base endpoint object
	baseEp := utilnet.BigForIP(net.ParseIP("172.16.0.1"))
	epPort := 8080

	tcpProtocol := v1.ProtocolTCP
	ep := &discovery.EndpointSlice{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "ep",
			Namespace: "namespace",
		},
		AddressType: discovery.AddressTypeIPv4,
		Endpoints:   []discovery.Endpoint{},
		Ports: []discovery.EndpointPort{{
			Name:     utilpointer.StringPtr(fmt.Sprintf("%d", epPort)),
			Port:     utilpointer.Int32(int32(epPort)),
			Protocol: &tcpProtocol,
		}},
	}
	addresses := []string{}
	for j := 0; j < nEndpoints; j++ {
		ipEp := utilnet.AddIPOffset(baseEp, j)
		addresses = append(addresses, ipEp.String())
	}
	ep.Endpoints = []discovery.Endpoint{{
		Addresses: addresses,
	}}
	// Create the Services and associate and endpoint object to each one
	for i := 0; i < nServices; i++ {
		ip := utilnet.AddIPOffset(base, i)
		svcName := fmt.Sprintf("svc%d", i)
		services[i] = &v1.Service{
			ObjectMeta: metav1.ObjectMeta{
				Name:      svcName,
				Namespace: "namespace",
			},
			Spec: v1.ServiceSpec{
				ClusterIP: ip.String(),
				Ports: []v1.ServicePort{{
					Name:       fmt.Sprintf("%d", epPort),
					Protocol:   v1.ProtocolTCP,
					Port:       int32(basePort + i),
					TargetPort: intstr.FromInt(epPort),
				}},
				Type: svcType,
			},
		}

		if svcType == v1.ServiceTypeNodePort || svcType == v1.ServiceTypeLoadBalancer {
			services[i].Spec.Ports[0].NodePort = int32(30000 + i)

		}
		if svcType == v1.ServiceTypeLoadBalancer {
			services[i].Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
				IP: "1.2.3.4",
			}}
			services[i].Spec.ExternalIPs = []string{"1.2.3.4"}
			services[i].Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
			services[i].Spec.HealthCheckNodePort = int32(32000 + i)
		}

		endpoints[i] = ep.DeepCopy()
		endpoints[i].Name = svcName
		endpoints[i].ObjectMeta.Labels = map[string]string{
			discovery.LabelServiceName: svcName,
		}

	}

	return services, endpoints
}