Kernel tuning for dedicated linux server. /etc/sysctl.d/60-sysctl.conf

60-sysctl.conf

################################################################################
# /etc/sysctl.d/60-sysctl.conf
# Performance-Optimized Kernel Tuning for Web + DB Servers
# Apply with: sysctl --system
# Verify with: sysctl -a | grep <param>
# Based on: DigitalOcean "Tune Up: Optimizing Linux Performance" (Sep 2025)
#           + Google BBR quick-start guide + Linux kernel docs
#
# ⚠️  BEFORE APPLYING:
#     1. Benchmark current state: sysctl -a > sysctl.before
#     2. Apply new config: sysctl --system
#     3. Verify BBR is active: sysctl net.ipv4.tcp_congestion_control
#     4. Verify fq qdisc: tc qdisc show dev eth0
#     5. Load BBR module: modprobe tcp_bbr
#     6. Profile with: perf, bpftrace, iostat (see DO article)
################################################################################

################################################################################
# MEMORY MANAGEMENT
################################################################################

# Swappiness: Controls kernel's tendency to swap (0-100).
# Default: 60. Value 10 favors keeping pages in memory.
# Impact: 10-30%+ gain for DB workloads by preventing unnecessary swapping.
vm.swappiness = 10

# Dirty page management — controls write behavior for I/O performance.
# These values ensure more frequent, smaller writes instead of large I/O spikes.
# Impact: 5-15% improvement in write-heavy workloads.

# Percentage of total memory where background writeback starts
vm.dirty_background_ratio = 5

# Maximum percentage of memory holding dirty pages before forced synchronous writes
vm.dirty_ratio = 15

# Time in centiseconds dirty data can stay in memory before being written (15s)
# Default is 30s. Reducing prevents large writeback spikes.
vm.dirty_expire_centisecs = 1500

# Time in centiseconds between background writeback cycles (2.5s)
# More frequent cycles = smoother I/O, better for concurrent DB/web workloads
vm.dirty_writeback_centisecs = 250

# Memory overcommit behavior for database workloads.
# Value 0 = heuristic overcommit (kernel default)
# Value 1 = always overcommit — essential for PostgreSQL/MySQL large buffer allocation
# Value 2 = strict overcommit — use if memory is constrained (see overcommit_ratio)
# ⚠️  Monitor OOM killer logs; switch to 2 if OOM kills occur.
vm.overcommit_memory = 1

# Percentage of RAM available for overcommit (only used when overcommit_memory=2)
# 200% = 2× physical RAM. Ignored when overcommit_memory=1 but kept for safety.
vm.overcommit_ratio = 200

# VFS cache pressure — controls reclaiming of directory/inode cache.
# Default: 100. Lower value preserves dentry/inode cache.
# Impact: 5-10% filesystem performance improvement for metadata-heavy workloads.
vm.vfs_cache_pressure = 75

# Minimum free RAM threshold (128MB) to prevent system freeze under memory pressure.
# Critical stability parameter.
# ⚠️  Scale with RAM: 128MB for ≤32GB, 256MB for 64GB+, 512MB for 128GB+.
vm.min_free_kbytes = 131072

################################################################################
# NETWORK STACK — CONGESTION CONTROL & QDISC
################################################################################

# TCP Congestion Control: BBR (Bottleneck Bandwidth and RTT)
# Impact: 5-40% throughput improvement, reduced latency on modern networks.
# Significantly outperforms Cubic/Reno on high-BDP and lossy networks.
# Requires: modprobe tcp_bbr && echo tcp_bbr >> /etc/modules-load.d/bbr.conf
net.ipv4.tcp_congestion_control = bbr

# ⚠️  CRITICAL: Default queuing discipline — use 'fq' with BBR
# CHANGED FROM: fq_codel → fq
# Reason: BBR relies on pacing. fq_codel does NOT implement pacing. On kernel
# <4.20 this breaks BBR; on 4.20+ BBR falls back to inefficient internal pacing
# using high-resolution timers. Google BBR docs explicitly recommend fq.
# fq also provides per-flow pacing which is critical for highly-loaded servers.
# Ref: https://github.com/google/bbr/blob/master/Documentation/bbr-quick-start.md
# Ref: DigitalOcean article — Networking Stack Optimization section
net.core.default_qdisc = fq

################################################################################
# NETWORK STACK — TCP BUFFERS
################################################################################

# TCP buffer sizes (min, default, max in bytes).
# Impact: 15-50% throughput improvement on high-BDP networks.
# Max 32MB buffers for 1Gbps+ connections and cross-region deployments.
# These values match the DO article recommendation for high-bandwidth tuning.
net.ipv4.tcp_rmem = 4096 87380 33554432
net.ipv4.tcp_wmem = 4096 65536 33554432

# Core socket buffer limits — must be ≥ TCP max buffer sizes.
# Required for the large TCP buffers above to take effect.
net.core.rmem_max = 33554432
net.core.wmem_max = 33554432

# Default socket buffer sizes (separate from TCP-specific above).
# Ensures non-TCP sockets also benefit from adequate buffer space.
net.core.rmem_default = 87380
net.core.wmem_default = 65536

# Optimize TCP buffer sizing for autotuning.
# Default: 1. Keep enabled to allow kernel to auto-tune buffers within rmem/wmem range.
net.ipv4.tcp_moderate_rcvbuf = 1

################################################################################
# NETWORK STACK — CONNECTION QUEUES (NEW — from DO article recommendations)
################################################################################

# Maximum listen backlog — upper limit for SYN_RECV queue across all sockets.
# Default: 128 (too low for high-concurrency web servers).
# Impact: Prevents connection drops under SYN flood / high connection rates.
# ⚠️  Must be ≥ net.core.somaxconn
net.ipv4.tcp_max_syn_backlog = 8192

# Maximum number of connections queued for accept() across all sockets.
# Default: 4096 (kernel 5.x+). Set to match tcp_max_syn_backlog for consistency.
# Impact: Prevents "connection refused" under heavy load.
# Nginx/HAProxy listen directive should reference this value.
net.core.somaxconn = 8192

# Maximum number of packets queued on the per-CPU input side before dropping.
# Default: 1000. Increase for high PPS (packets per second) workloads.
# Impact: Prevents packet drops on NIC → kernel handoff under heavy traffic.
net.core.netdev_max_backlog = 8192

# Maximum number of packets queued in the receive path per socket.
# Default: varies. Increase for burst-heavy workloads.
net.core.optmem_max = 65536

################################################################################
# NETWORK STACK — CONNECTION LIFECYCLE (NEW — from DO article recommendations)
################################################################################

# Disable TCP slow start after idle periods.
# Default: 1 (enabled). Setting to 0 prevents throughput collapse when
# connections resume after idle periods.
# Impact: 10-20% improvement for spiky web traffic patterns.
net.ipv4.tcp_slow_start_after_idle = 0

# Allow reuse of TIME_WAIT sockets for new connections.
# Default: 0. Setting to 1 enables safe reuse from protocol perspective.
# Impact: Significant for short-lived connections (HTTP, API servers).
# ⚠️  Only safe on client-side connections; less critical for server-side.
net.ipv4.tcp_tw_reuse = 1

# Time (seconds) TCP sockets stay in FIN-WAIT-2 state.
# Default: 60. Reducing to 15 frees socket resources faster.
# Impact: Important for high-connection-churn workloads.
net.ipv4.tcp_fin_timeout = 15

# Disable TCP metrics caching across connections.
# Default: 1. Disabling prevents stale metrics from affecting new connections.
# Recommended when using BBR, which maintains its own model.
net.ipv4.tcp_no_metrics_save = 1

# TCP keepalive — reduce detection of dead connections.
# Default: tcp_keepalive_time=7200 (2h), tcp_keepalive_intvl=75, tcp_keepalive_probes=9
# These values detect dead connections in ~30s instead of ~11 minutes.
# Impact: Frees socket resources faster for DB connection pools and long-lived sessions.
net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_keepalive_intvl = 10
net.ipv4.tcp_keepalive_probes = 3

# TCP Fast Open — reduces connection latency by sending data in SYN packet.
# Bit flags: 1 = client-side enabled, 2 = server-side enabled, 3 = both.
# Impact: Reduces round-trip latency for repeated connections by 1 RTT.
# ⚠️  Ensure application support (Nginx ≥1.11.7, PostgreSQL, etc.)
net.ipv4.tcp_fastopen = 3

# Path MTU Discovery — probe for optimal MTU.
# Default: 0 (disabled). Value 1 enables basic probing, 2 enables advanced.
# Impact: Avoids fragmentation, improves throughput on diverse network paths.
net.ipv4.tcp_mtu_probing = 1

# SYN cookies — protect against SYN flood attacks.
# Default: 1. Value 1 enables when SYN backlog overflows (recommended).
# Set to 2 to always use (lower performance but maximum protection).
net.ipv4.tcp_syncookies = 1

# Enable selective acknowledgments — critical for high-latency connections.
# Default: 1 on most distros. Explicitly set for safety.
net.ipv4.tcp_sack = 1

# Enable window scaling — required for bandwidth-delay product >64KB.
# Default: 1. Explicitly set.
net.ipv4.tcp_window_scaling = 1

# Maximum number of TIME_WAIT sockets simultaneously held by the system.
# Default: varies. Not normally a problem with tcp_tw_reuse=1.
# Only set if you hit limits; otherwise let the kernel auto-manage.
# net.ipv4.tcp_max_tw_buckets = 32768

# Local port range for outgoing connections.
# Default: 32768 60999. Widen for high-outbound-connection workloads.
# ⚠️  Only relevant if this server also makes outbound connections (API calls, etc.)
net.ipv4.ip_local_port_range = 1024 65535

# Maximum number of connections that can be in SYN_RECV state per socket.
# This is per-listener; tcp_max_syn_backlog above is system-wide.
# Default: derived from tcp_max_syn_backlog. Usually no need to set separately.

################################################################################
# SECURITY-AWARE NETWORK SETTINGS
################################################################################

# Protect against SYN flood attacks
net.ipv4.tcp_syncookies = 1

# Ignore ICMP redirects — prevent MITM attacks
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0

# Do not send ICMP redirects
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0

# Log Martian packets (packets with impossible addresses)
# Useful for debugging; disable in production if log volume is excessive.
# net.ipv4.conf.all.log_martians = 1

################################################################################
# FILESYSTEM (NEW — complements memory section)
################################################################################

# Maximum number of open file descriptors system-wide.
# Default: varies (often 795037). Set high for web + DB workloads.
# ⚠️  Also set in /etc/security/limits.conf: * soft nofile 1048576
# ⚠️  Also set in systemd service: LimitNOFILE=1048576
fs.file-max = 1048576

# Maximum number of inotify watches.
# Default: 8192. Increase if running container workloads or file watchers.
fs.inotify.max_user_watches = 524288

# Maximum number of inotify instances per user.
fs.inotify.max_user_instances = 512

61-sysctl-old-os.conf

################################################################################
# /etc/sysctl.d/61-sysctl-old-os.conf
# Performance-Critical Kernel Optimizations for Web + DB Server
# Target:  CentOS 6 / RHEL 6 (kernel 2.6.32)
# Workload: High-concurrency web services with database workloads
# Apply:   sysctl --system
# Verify:  sysctl -a | grep <param>
#
# ⚠️  KERNEL 2.6.32 LIMITATIONS:
#     - NO BBR congestion control (only cubic/reno available)
#     - NO fq_codel qdisc (default is pfifo_fast)
#     - NO TCP Fast Open
#     - dmesg_restrict / kptr_restrict not available (< 2.6.37/2.6.38)
#     - Consider upgrading kernel if possible — 2.6.32 is EOL since 2020
#
# ⚠️  BEFORE APPLYING:
#     1. Backup: sysctl -a > /root/sysctl.before.$(date +%F)
#     2. Apply:  sysctl --system
#     3. Verify: sysctl --system 2>&1 (check for errors)
#     4. Test:   Load production traffic gradually, monitor dmesg for OOM
################################################################################

########################
# MEMORY MANAGEMENT
########################

# Swappiness (0-100) — controls kernel's tendency to swap.
# Default: 60. Value 10 keeps hot DB pages in RAM.
# Impact: 10-30% for database workloads by preventing swap-induced latency spikes.
vm.swappiness = 10

# VFS cache pressure — controls reclaiming of dentry/inode caches.
# Default: 100. Lower value preserves directory/inode cache.
# CHANGED FROM: 50 → 75. Value 50 is too aggressive — can starve page cache
# under memory pressure, causing swap storms on DB + web combined workloads.
# 75 is a better balance: preserves metadata cache while leaving room for pages.
vm.vfs_cache_pressure = 75

# Dirty page management — controls writeback behavior for I/O performance.
# These values smooth I/O patterns and prevent write storms.

# Maximum % of RAM with dirty pages before forced synchronous writes.
# Impact: Balances write coalescing vs responsiveness.
vm.dirty_ratio = 15

# Background writeback threshold — starts at 5% of RAM.
# Impact: Prevents write storms, smooths I/O patterns.
vm.dirty_background_ratio = 5

# NEW: Dirty data expires after 30 seconds (3000 centiseconds).
# Default: 3000 (30s). Explicitly set — old dirty data is written back.
# Prevents large accumulation of stale dirty pages.
vm.dirty_expire_centisecs = 3000

# NEW: Background writeback runs every 5 seconds (500 centiseconds).
# Default: 500 (5s). More frequent = smoother I/O for DB workloads.
# Do NOT set below 250 on kernel 2.6.32 — can cause excessive I/O thrashing.
vm.dirty_writeback_centisecs = 500

# Minimum free memory reserve (64MB) — prevents low-memory deadlocks.
# Impact: System stability under memory pressure.
# ⚠️  Scale with RAM: 64MB for ≤16GB, 128MB for 32GB, 256MB for 64GB+.
vm.min_free_kbytes = 65536

# Memory overcommit mode:
#   0 = heuristic (default)
#   1 = always overcommit — PostgreSQL/MariaDB need this for fork() operations
#   2 = strict — uses overcommit_ratio
# ⚠️  Mode 1 ignores overcommit_ratio entirely. Monitor OOM killer in dmesg.
vm.overcommit_memory = 1

# Overcommit ratio — percentage of physical RAM allowed for overcommit.
# ONLY effective when overcommit_memory = 2. Shown here for reference if you
# switch to strict mode. Default: 50. Value 100 = physical RAM + swap.
# CHANGED: Added explicit note that this is inert with overcommit_memory=1.
vm.overcommit_ratio = 100

########################
# SHARED MEMORY (Database Performance)
########################
# ⚠️  SCALE THESE TO YOUR SYSTEM RAM:
#     shmmax = desired shared memory in bytes (e.g., 16GB = 17179869184)
#     shmall = shmmax / PAGE_SIZE  (PAGE_SIZE is typically 4096 on x86_64)
#     For 8GB shared memory:  shmmax = 8589934592, shmall = 2097152
#     For 16GB shared memory: shmmax = 17179869184, shmall = 4194304

# Maximum shared memory segment size (16GB)
# Enables PostgreSQL shared_buffers, MySQL InnoDB buffer_pool
kernel.shmmax = 17179869184

# Total shared memory pages (16GB / 4096 = 4,194,304 pages)
# Controls aggregate shared memory for multiple DB instances
kernel.shmall = 4194304

# Maximum shared memory segments system-wide
# 4096 supports multiple DB instances or applications
kernel.shmmni = 4096

########################
# NETWORK CORE (Connection Handling)
########################

# Maximum pending connections in listen backlog.
# Default: 128. Raised to 4096 for high-traffic web servers.
# ⚠️  Must be ≥ your application's listen() backlog (e.g., Nginx `backlog` param).
net.core.somaxconn = 4096

# CHANGED FROM: 5000 → 8192 (next power-of-2).
# Kernel 2.6.32 internally rounds to power-of-2; explicit power-of-2 avoids confusion.
# Impact: Reduces packet drops under high PPS (10-20% on 10Gbps+).
net.core.netdev_max_backlog = 8192

# Maximum socket receive buffer — 16MB.
# Must be ≥ tcp_rmem max value below.
net.core.rmem_max = 16777216

# Maximum socket send buffer — 16MB.
# Must be ≥ tcp_wmem max value below.
net.core.wmem_max = 16777216

# NEW: Default socket receive buffer size.
# Saves per-socket setup overhead; kernel auto-tunes from here.
net.core.rmem_default = 87380

# NEW: Default socket send buffer size.
net.core.wmem_default = 65536

# NEW: Maximum ancillary buffer per socket (64KB).
# Default: 20480. Raised for sendmsg/recvmsg with control messages.
net.core.optmem_max = 65536

########################
# TCP/IP STACK (Throughput & Concurrency)
########################

# SYN backlog queue — defends against SYN floods + handles bursts.
# Default: 1024. 8192 for high-traffic web.
# ⚠️  Only effective when tcp_syncookies = 0 or backlog exceeds somaxconn.
net.ipv4.tcp_max_syn_backlog = 8192

# Reuse TIME_WAIT sockets for new connections.
# Safe on server-side. Dramatically reduces port exhaustion.
# ⚠️  Do NOT enable tcp_tw_recycle — it breaks behind NAT/LOAD BALANCERS
#     (tcp_tw_recycle is tempting but dangerous in kernel 2.6.32 with L7 proxies).
net.ipv4.tcp_tw_reuse = 1

# TIME_WAIT socket bucket limit.
# Default: varies. 262144 supports high connection turnover rates.
net.ipv4.tcp_max_tw_buckets = 262144

# FIN timeout — seconds sockets linger in FIN-WAIT-2.
# Default: 60. CHANGED FROM: 25 → 15.
# 25s is still too conservative for web+DB workloads with high connection churn.
# 15s frees resources faster; safe for internal DB connections and HTTP keepalive.
net.ipv4.tcp_fin_timeout = 15

# TCP buffer auto-tuning (min, default, max in bytes).
# Kernel auto-tunes within these ranges based on BDP.
# 16MB max is appropriate for kernel 2.6.32 era (1-10Gbps links).
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216

# Enable RFC 1323 window scaling — essential for high-speed networks.
net.ipv4.tcp_window_scaling = 1

# Enable Selective ACKs (SACK) — 5-15% improvement on lossy networks.
net.ipv4.tcp_sack = 1

# NEW: Enable Forward ACK (FACK) — improves SACK-based recovery.
# Available in 2.6.32. Works with tcp_sack for better loss recovery.
net.ipv4.tcp_fack = 1

# NEW: Enable DSACK (Duplicate SACK) — helps sender detect reordering.
# Available in 2.6.32. Complements SACK for better spurious retransmit handling.
net.ipv4.tcp_dsack = 1

# NEW: Enable TCP timestamps — required for RTT estimation accuracy.
# Default: 1 on most distros. Explicitly set; needed for PAWS (Protection
# Against Wrapped Sequence numbers) on high-speed connections.
net.ipv4.tcp_timestamps = 1

# Disable slow start after idle periods.
# Maintains congestion window after pauses — critical for DB connection pools
# and keepalive connections that go idle between bursts.
net.ipv4.tcp_slow_start_after_idle = 0

# NEW: Disable TCP metrics caching between connections.
# Default: 0 (saves metrics). Setting to 1 prevents stale RTT/cwnd metrics
# from previous connections from polluting new ones.
# Especially important on servers with diverse client populations.
net.ipv4.tcp_no_metrics_save = 1

# NEW: Enable TCP memory auto-tuning moderation.
# Default: 1. Explicitly set — allows kernel to moderate receive buffer growth.
net.ipv4.tcp_moderate_rcvbuf = 1

########################
# TCP KEEPALIVE (Dead Connection Detection)
# NEW SECTION — defaults are too conservative for production DB/web servers.
# Default: keepalive_time=7200 (2h), keepalive_intvl=75, keepalive_probes=9
# These values detect dead connections in ~45s instead of ~11 minutes.
########################

# Seconds before first keepalive probe (5 minutes).
net.ipv4.tcp_keepalive_time = 300

# Seconds between keepalive retransmissions.
net.ipv4.tcp_keepalive_intvl = 15

# Number of unacknowledged probes before declaring dead.
net.ipv4.tcp_keepalive_probes = 3

########################
# LOCAL PORT RANGE
# NEW — widen for high-outbound-connection workloads (API calls, proxy, etc.)
# Default: 32768 60999. Widened to increase available ephemeral ports.
########################
net.ipv4.ip_local_port_range = 1024 65535

########################
# CONNECTION TRACKING (Firewall/NAT)
# ⚠️  REQUIRES: modprobe nf_conntrack
#     If module is not loaded, these will error silently or fail.
#     Add to /etc/sysconfig/iptables-config or modprobe.d if needed.
########################

# Maximum tracked connections — prevent conntrack table exhaustion.
# Default: depends on RAM. ~65536 typical on 4GB system.
# ⚠️  Each entry uses ~300 bytes. 262144 × 300 = ~75MB RAM when full.
net.netfilter.nf_conntrack_max = 262144

# Established connection timeout — 2 hours (7200s).
# Default: 432000 (5 days!) — far too long for production.
# 7200s balances memory usage vs connection state preservation.
net.netfilter.nf_conntrack_tcp_timeout_established = 7200

########################
# FILE SYSTEM (Concurrency Limits)
########################

# Maximum open file descriptors system-wide.
# Critical for Nginx, Apache, Tomcat with many concurrent connections.
# ⚠️  Also configure per-process limits:
#     /etc/security/limits.conf:  * soft nofile 1048576
#                                 * hard nofile 2097152
fs.file-max = 2097152

# Maximum async I/O operations.
# Essential for PostgreSQL, MySQL async I/O subsystem.
# ⚠️  Do NOT set above 1048576 on 2.6.32 — kernel may ignore it.
fs.aio-max-nr = 1048576

########################
# SECURITY PARAMETERS
########################

# Address space layout randomization (ASLR).
# Available since kernel 2.6.12.
kernel.randomize_va_space = 2

# NOTE: kernel.dmesg_restrict and kernel.kptr_restrict are NOT available
# in kernel 2.6.32 (added in 2.6.37 and 2.6.38 respectively).
# REMOVED from original config to prevent sysctl errors.
# WORKAROUND: Restrict dmesg via file permissions:
#   chmod 750 /bin/dmesg
#   And add to /etc/rc.local:  chmod 750 /bin/dmesg

# SYN cookies — DDoS resistance.
# When enabled (1), activates only when SYN backlog overflows.
# Do NOT set to 2 (always) — it breaks SYN backlog functionality.
net.ipv4.tcp_syncookies = 1

# SYN/SYNACK retry limits — prevents resource exhaustion from half-open attacks.
# 2 retries = ~6 seconds before dropping (reasonable for legitimate clients).
net.ipv4.tcp_syn_retries = 2
net.ipv4.tcp_synack_retries = 2

# TCP TIME-WAIT assassination hazard protection (RFC 1337).
# Prevents old duplicate segments from causing errors.
net.ipv4.tcp_rfc1337 = 1

# ARP cache limits — prevents neighbor table overflow.
# gc_thresh1 = minimum entries before garbage collection starts
# gc_thresh2 = soft maximum (GC runs more aggressively above this)
# gc_thresh3 = hard maximum (new entries are dropped above this)
net.ipv4.neigh.default.gc_thresh1 = 512
net.ipv4.neigh.default.gc_thresh2 = 1024
net.ipv4.neigh.default.gc_thresh3 = 2048

########################
# NETWORK SECURITY (IPv4)
########################

# Reverse path filtering — anti-spoofing (RFC 2827).
# ⚠️  Value 1 (strict) can cause issues with asymmetric routing.
#     Use value 2 (loose mode) if behind a load balancer or multi-homed.
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1

# Disable ICMP redirects — prevent MITM attacks.
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0

# Disable source routing — prevents source-based routing attacks.
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0

# Log martian packets (packets with impossible source addresses).
# ⚠️  CHANGED FROM: 1 → 0 on default interface.
#     Reason: On kernel 2.6.32 with heavy traffic, martian logging can
#     generate excessive syslog I/O and consume disk space.
#     Enable only when debugging network issues.
net.ipv4.conf.all.log_martians = 1
net.ipv4.conf.default.log_martians = 0

# Broadcast ping protection — prevents Smurf attacks.
net.ipv4.icmp_echo_ignore_broadcasts = 1

# Bogus ICMP error response protection — prevents log spam from broken routers.
net.ipv4.icmp_ignore_bogus_error_responses = 1

########################
# NETWORK SECURITY (IPv6)
# NOTE: If IPv6 is disabled on this system, these will error.
# Comment out the entire block if IPv6 is disabled
# (check: ls /proc/sys/net/ipv6/ — if empty, IPv6 is disabled).
########################

# Disable IPv6 redirects and source routing
net.ipv6.conf.all.accept_redirects = 0
net.ipv6.conf.default.accept_redirects = 0
net.ipv6.conf.all.accept_source_route = 0
net.ipv6.conf.default.accept_source_route = 0

80-k8s.conf

################################################################################
# /etc/sysctl.d/80-k8s.conf
# Performance-Critical Kernel Optimizations for Kubernetes Nodes
#
# Target:  Production K8s clusters (kernel 5.4+, tested on 5.15/6.x)
# CNI:     Calico / Cilium / Flannel (adjust rp_filter for your CNI)
# Proxy:   kube-proxy nftables mode
# Runtime: containerd (adjust inotify for your pod density)
#
# Apply:   sysctl --system
# Verify:  sysctl -a | grep <param>
#
# ⚠️  PREREQUISITES (must be done BEFORE applying):
#     1. modprobe br_netfilter && echo br_netfilter >> /etc/modules-load.d/k8s.conf
#     2. modprobe tcp_bbr     && echo tcp_bbr     >> /etc/modules-load.d/k8s.conf
#     3. modprobe overlay     && echo overlay      >> /etc/modules-load.d/k8s.conf
#     4. Disable swap: swapoff -a && sed -i '/swap/d' /etc/fstab
#     5. sysctl --system
#
# ⚠️  INTERACTION WITH KUBELET:
#     - kubelet has its own resource reservations (--system-reserved,
#       --kube-reserved, --eviction-hard) which complement these settings.
#     - Some sysctl can be set per-pod via securityContext.sysctls
#       (e.g., net.ipv4.tcp_keepalive_time for specific workloads).
#     - K8s 1.25+ prefers cgroup v2; verify with: stat -fc %T /sys/fs/cgroup
#
# Reference: DigitalOcean "Tune Up: Optimizing Linux Performance" (Sep 2025)
#            Google BBR quick-start, Linux kernel IP sysctl documentation
################################################################################

########################
# MEMORY MANAGEMENT
########################

# CHANGED FROM: 10 → 0
# Kubelet REQUIRES swap disabled. swappiness=0 ensures the kernel never swaps
# even if swap partitions accidentally exist. K8s ≤1.27 refuses to start with
# swap enabled; 1.28+ requires --fail-swap-on=false.
# If you explicitly enable K8s swap support (1.28+, alpha), revert to 10.
vm.swappiness = 0

# Memory overcommit mode 1 (always overcommit).
# Essential for container runtimes and kubelet. Containers often allocate
# large virtual address spaces they never fully use (JVM, Go runtime, etc.).
# Mode 1 prevents fork()/mmap() failures in container startup paths.
# ⚠️  Monitor OOM kills: journalctl -k | grep -i oom
vm.overcommit_memory = 1

# Overcommit ratio — ONLY effective when overcommit_memory = 2.
# Shown for reference; inert with mode 1.
vm.overcommit_ratio = 100

# Dirty page writeback thresholds.
# Container workloads are bursty — these values prevent write storms while
# keeping I/O smooth across many pods writing simultaneously.

# Background writeback starts at 5% of RAM.
vm.dirty_background_ratio = 5

# Forced synchronous writes at 10% of RAM — aggressive for responsiveness.
# ⚠️  If pods run heavy DB workloads (PostgreSQL/MySQL in pods), consider 15.
vm.dirty_ratio = 10

# Dirty data expires after 30s (3000 centiseconds).
# Default: 3000. Explicitly set.
vm.dirty_expire_centisecs = 3000

# Background writeback cycle every 5 seconds.
# ⚠️  Lower values (e.g., 250/2.5s) smooth I/O further but increase background
#     I/O activity. 500 is a good balance for multi-tenant K8s nodes.
vm.dirty_writeback_centisecs = 500

# CHANGED FROM: 50 → 75
# VFS cache pressure — controls dentry/inode cache reclaim rate.
# Value 50 is too aggressive on K8s nodes: starves page cache for
# application data while hoarding container layer metadata.
# 75 preserves overlayfs metadata cache while leaving room for page cache.
vm.vfs_cache_pressure = 75

# Minimum free memory reserve (128MB).
# Prevents OOM deadlocks, critical for kubelet stability.
# ⚠️  Scale with RAM: 128MB for ≤32GB, 256MB for 64GB, 512MB for 128GB+.
# ⚠️  Complement with kubelet --system-reserved=memory=500Mi and
#     --eviction-hard=memory.available<500Mi
vm.min_free_kbytes = 131072

########################
# SHARED MEMORY (Database-in-Pod Support)
########################

# Maximum shared memory segment: 64GB.
# Enables large PostgreSQL shared_buffers / MySQL buffer_pool when run in pods.
# ⚠️  Adjust to your largest DB pod's shared memory requirement.
#     For no DB pods: these can stay at system defaults.
kernel.shmmax = 68719476736

# Total shared memory pages: 64GB / 4096 = 16,777,216 pages.
kernel.shmall = 16777216

# Maximum shared memory segments.
kernel.shmmni = 4096

########################
# KUBERNETES NETWORKING (Required)
########################

# IPv4 forwarding — required for pod-to-pod and pod-to-external communication.
# Without this, K8s networking (CNI) will not function.
net.ipv4.ip_forward = 1

# Bridge netfilter — required for kube-proxy iptables/nftables mode and
# K8s NetworkPolicies to apply to bridge traffic (pod-to-pod on same node).
# ⚠️  REQUIRES: modprobe br_netfilter (see prerequisites above)
# ⚠️  Verify: ls /proc/sys/net/bridge/ (if missing, module not loaded)
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1

# ARPTables bridge call — only needed for specific CNI plugins.
# Comment out if your CNI doesn't require it (most don't).
# net.bridge.bridge-nf-call-arptables = 1

# IPv6 forwarding — required for dual-stack K8s clusters.
# Comment out entirely if running IPv4-only (single-stack).
net.ipv6.conf.all.forwarding = 1
net.ipv6.conf.default.forwarding = 1

########################
# ARP CACHE (Cluster-Scale)
########################

# ARP cache thresholds — prevent neighbor table overflow in large clusters.
# gc_thresh1 = minimum entries before GC starts
# gc_thresh2 = soft maximum (aggressive GC above this)
# gc_thresh3 = hard maximum (new entries are dropped above this)
# These values support 500+ node clusters with heavy inter-node traffic.
net.ipv4.neigh.default.gc_thresh1 = 2048
net.ipv4.neigh.default.gc_thresh2 = 4096
net.ipv4.neigh.default.gc_thresh3 = 8192

########################
# TCP CONNECTION HANDLING
########################

# SYN backlog — handles connection bursts during pod autoscaling events.
net.ipv4.tcp_max_syn_backlog = 8192

# CHANGED FROM: 65535 → 32768
# Listen backlog maximum. 65535 is the kernel max but wastes memory.
# 32768 is more than sufficient for any single K8s node; match your
# Ingress controller / service mesh (Envoy) listen() backlog.
net.core.somaxconn = 32768

# Reuse TIME_WAIT sockets — safe for server-side, critical for service mesh.
net.ipv4.tcp_tw_reuse = 1

# CHANGED FROM: 1440000 → 524288
# Maximum TIME_WAIT buckets. 1.44M × ~200 bytes = ~288MB of kernel memory.
# 524288 (~100MB) is sufficient for clusters with proper tcp_tw_reuse=1
# and tcp_fin_timeout=15. Increase only if you see kernel warnings in dmesg.
net.ipv4.tcp_max_tw_buckets = 524288

# CHANGED FROM: 30 → 15
# FIN-WAIT-2 timeout. Pods are ephemeral with high connection churn.
# 15s frees socket resources fast; standard for production K8s.
net.ipv4.tcp_fin_timeout = 15

# Maximum orphaned sockets — sockets with no associated file descriptor.
# Limits kernel memory for orphan sockets during connection storms.
# ⚠️  If exceeded, kernel logs warnings. Increase only if you see them.
net.ipv4.tcp_max_orphans = 262144

# Disable slow start after idle — maintains cwnd for gRPC, WebSocket,
# and service-mesh sidecar connections that have idle periods.
net.ipv4.tcp_slow_start_after_idle = 0

# Disable TCP metrics caching — prevents stale RTT/cwnd from previous
# connections polluting new pod connections.
net.ipv4.tcp_no_metrics_save = 1

# TCP Fast Open — reduces connection latency by 1 RTT for repeated connections.
# Bit flags: 1=client, 2=server, 3=both.
# ⚠️  Verify application support: Envoy 1.17+, Nginx Ingress 1.11.7+
net.ipv4.tcp_fastopen = 3

########################
# TCP KEEPALIVE (Dead Connection Detection)
# NEW — Critical for K8s where pods can die without clean TCP close.
# Without keepalive, connections to dead pods hang until TCP timeout (13+ min).
# These values detect dead endpoints in ~45 seconds.
########################

net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp_keepalive_probes = 3

########################
# TCP CONGESTION & THROUGHPUT
########################

# BBR congestion control — superior for inter-node, inter-zone, and
# cross-region traffic in cloud environments.
# ⚠️  REQUIRES: modprobe tcp_bbr (see prerequisites)
# ⚠️  Verify: sysctl net.ipv4.tcp_available_congestion_control
net.ipv4.tcp_congestion_control = bbr

# Fair Queuing scheduler — implements per-flow pacing required by BBR.
# ⚠️  Must be 'fq' for BBR (not fq_codel — lacks pacing).
net.core.default_qdisc = fq

# TCP window scaling (RFC 1323) — essential for >64KB windows.
net.ipv4.tcp_window_scaling = 1

# Selective ACKs (SACK) — faster loss recovery on lossy networks.
net.ipv4.tcp_sack = 1

# NEW: Forward ACK — complements SACK for better loss recovery.
net.ipv4.tcp_fack = 1

# NEW: TCP timestamps — required for accurate RTT estimation and PAWS.
# Essential for BBR's bandwidth estimation.
net.ipv4.tcp_timestamps = 1

# TCP autocorking — coalesces small packets to reduce overhead.
# Beneficial for RPC-heavy service mesh traffic.
net.ipv4.tcp_autocorking = 1

# NEW: Enable TCP moderate receive buffer auto-tuning.
net.ipv4.tcp_moderate_rcvbuf = 1

# Path MTU Discovery — probing mode prevents fragmentation in
# overlay networks (VXLAN, IP-in-IP, WireGuard).
net.ipv4.tcp_mtu_probing = 1

########################
# NETWORK BUFFERS
########################

# CHANGED FROM: 30000 → 32768 (power-of-2).
# Per-CPU packet backlog before dropping. Handles bursty 10Gbps+ traffic.
net.core.netdev_max_backlog = 32768

# NAPI polling budget — packets processed per NAPI cycle.
# 600 is aggressive; increases throughput but may increase latency.
# ⚠️  On kernel 5.10+, netdev_budget_usecs takes precedence over netdev_budget.
net.core.netdev_budget = 600

# NAPI time budget — 5ms per cycle. Balances throughput vs latency.
net.core.netdev_budget_usecs = 5000

# Maximum socket buffers — must be ≥ tcp_rmem/tcp_wmem max values.
# 16MB max supports high-BDP paths (10Gbps × 50ms RTT = ~62.5MB;
# auto-tuning will stay within these limits).
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216

# Default socket buffer sizes — reduces per-socket setup overhead.
net.core.rmem_default = 87380
net.core.wmem_default = 65536

# Ancillary buffer per socket — sendmsg/recvmsg control messages.
net.core.optmem_max = 65536

# TCP buffer auto-tuning ranges (min, default, max).
# Kernel auto-tunes within these bounds based on bandwidth-delay product.
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216

# Ephemeral port range — widened for high-outbound-connection workloads.
# ⚠️  K8s NodePort range (default 30000-32767) must fall within this.
#     If you use non-default NodePort range, adjust accordingly.
net.ipv4.ip_local_port_range = 1024 65535

########################
# CONNECTION TRACKING (conntrack)
# ⚠️  REQUIRES: modprobe nf_conntrack
# ⚠️  kube-proxy nftables mode uses LESS conntrack than iptables mode.
#     If using nftables mode exclusively, these can be more conservative.
# ⚠️  Each entry uses ~300 bytes. 2M × 300 = ~600MB RAM when table is full.
#     Ensure node has sufficient RAM beyond pod requests.
# ⚠️  Alternative: Cilium with eBPF bypasses conntrack entirely for
#     pod-to-pod traffic, reducing these settings' importance.
########################

# Maximum tracked connections — supports large clusters (1000+ pods per node).
net.netfilter.nf_conntrack_max = 2097152

# CHANGED FROM: 86400 (24h) → 7200 (2h)
# Pods are ephemeral — entries for deleted pods waste table space.
# 2h is sufficient for any legitimate long-lived connection in K8s.
# If pods maintain genuinely long-lived external connections, increase to 28800 (8h).
net.netfilter.nf_conntrack_tcp_timeout_established = 7200

########################
# FILE SYSTEM & LIMITS
########################

# Maximum open file descriptors system-wide.
# Kubelet, containerd, and high-concurrency sidecars need many FDs.
# ⚠️  Also set per-process limits in container runtime config:
#     containerd: /etc/containerd/config.toml  > [plugins."io.containerd.grpc.v1.cri".container]
#     rlimits: { nofile: 1048576 }
fs.file-max = 2097152

# Inotify watches — critical for kubelet (pod status), containerd (layer events),
# and sidecars (Filebeat, Fluentd, etc.).
# Default: 8192. 524288 supports ~500 pods with active file watchers.
fs.inotify.max_user_watches = 524288

# NEW: Inotify instances per user — default is 128.
# Kubelet + containerd + 20 sidecar pods can easily exceed 128.
# 8192 provides headroom for dense nodes.
fs.inotify.max_user_instances = 8192

# Maximum async I/O operations — for DB-in-pod workloads.
fs.aio-max-nr = 1048576

# NEW: Allow mount detach while still in use.
# Prevents "device busy" errors when pods are destroyed and volumes are
# unmounted. Critical for smooth pod termination with PVCs.
# Available on kernel 4.x+.
# ⚠️  Check: sysctl fs.may_detach_mounts (may not exist on all distros)
# fs.may_detach_mounts = 1

########################
# SECURITY PARAMETERS
########################

# ASLR — full randomization.
kernel.randomize_va_space = 2

# Restrict dmesg to root — prevents container escape via kernel log info.
kernel.dmesg_restrict = 1

# Restrict /proc/<pid>/maps — prevents container info leaks.
kernel.kptr_restrict = 2

# CHANGED FROM: 1 → 2 (maximum restriction).
# ptrace scope:
#   1 = no ptrace across user boundaries (blocks cross-pod debugging)
#   2 = admin-only ptrace (requires CAP_SYS_PTRACE)
# ⚠️  Value 2 prevents `kubectl debug` and sidecar `strace` entirely.
#     Use value 1 if you need pod-level debugging tools in production.
kernel.yama.ptrace_scope = 1

# Core dumps — pipe to null rather than writing to disk.
# ⚠️  This prevents post-mortem debugging of crashes in pods.
#     For dev/staging clusters, use a path like /var/crash/core.%e.%p.%t
#     and configure containerd to handle core dumps per-container.
kernel.core_pattern = |/bin/false

# SYN cookies — DDoS resistance. Activates only when SYN backlog overflows.
net.ipv4.tcp_syncookies = 1

# SYN retry limits — prevents half-open connection resource exhaustion.
net.ipv4.tcp_syn_retries = 2
net.ipv4.tcp_synack_retries = 2

# TCP TIME-WAIT assassination protection (RFC 1337).
net.ipv4.tcp_rfc1337 = 1

########################
# NETWORK SECURITY (IPv4)
########################

# CHANGED FROM: 1 (strict) → 2 (loose)
# Reverse path filtering. Strict mode (1) drops packets from "unexpected"
# interfaces, which BREAKS overlay network traffic (VXLAN, IP-in-IP, WireGuard)
# that arrives on the overlay interface with a source IP from the underlay.
# Loose mode (2) only checks that the source IP is routable.
# ⚠️  CNI-specific notes:
#     - Calico with IP-in-IP: REQUIRES loose mode (2)
#     - Cilium VXLAN: works with both, 2 is safer
#     - Flannel VXLAN: works with both, 2 is safer
#     - AWS VPC CNI: can use strict (1) — no overlay
net.ipv4.conf.all.rp_filter = 2
net.ipv4.conf.default.rp_filter = 2

# Disable ICMP redirects — prevent MITM.
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0

# Disable source routing.
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0

# CHANGED FROM: 1 → 0
# Martian logging — overlay/encapsulated traffic constantly triggers false
# positives in K8s (VXLAN outer headers, service CIDRs, etc.).
# Enable only when actively debugging network issues.
net.ipv4.conf.all.log_martians = 0
net.ipv4.conf.default.log_martians = 0

# Broadcast ping protection.
net.ipv4.icmp_echo_ignore_broadcasts = 1

# Bogus ICMP error protection.
net.ipv4.icmp_ignore_bogus_error_responses = 1

########################
# NETWORK SECURITY (IPv6)
# Comment out entire block if running IPv4-only single-stack.
########################

net.ipv6.conf.all.accept_redirects = 0
net.ipv6.conf.default.accept_redirects = 0
net.ipv6.conf.all.accept_source_route = 0
net.ipv6.conf.default.accept_source_route = 0

80-pve.conf

################################################################################
# /etc/sysctl.d/80-pve.conf
# Performance-Critical Proxmox VE Host Tuning
#
# Target:  PVE 9.x on Debian 13 (kernel 6.x)
# Workload: Mixed KVM VMs + LXC containers (database, web, network-intensive)
# Hardware: 64GB+ RAM, multi-core CPU, high-speed storage (NVMe/SSD)
#
# Apply:   sysctl --system
# Verify:  sysctl -a | grep <param>
#
# ⚠️  PREREQUISITES:
#     1. modprobe tcp_bbr     && echo tcp_bbr     >> /etc/modules-load.d/pve.conf
#     2. modprobe br_netfilter && echo br_netfilter >> /etc/modules-load.d/pve.conf
#     3. sysctl --system
#
# ⚠️  INTERACTION WITH PVE:
#     - PVE applies some defaults via /etc/sysctl.d/pve.conf* — check for conflicts.
#     - PVE Datacenter > Options allows some sysctl per-cluster.
#     - VM balloon driver interacts with vm.swappiness — keep swap low.
#     - ZFS ARC competes with VM memory — see ZFS section below.
#     - Ceph OSD hosts have ADDITIONAL requirements (see Ceph section).
#
# ⚠️  ALWAYS benchmark before and after:
#     sysctl -a > /root/sysctl.before.$(date +%F)
#     fio --name=test --ioengine=libaio --rw=randrw --bs=4k --numjobs=4 \
#         --size=1G --runtime=60 --time_based --group_reporting
################################################################################

########################
# MEMORY MANAGEMENT
########################

# Swappiness — controls kernel tendency to swap.
# Default: 60. Value 10 strongly prefers keeping pages in RAM.
# On PVE: prevents swap thrashing that stalls VMs under memory pressure.
# ⚠️  If using KSM (Kernel Same-page Merging) or balloon drivers,
#     value 1 may be even better to avoid swapping merged pages.
# ⚠️  If using ZFS, the ARC is reclaimable; swappiness matters less
#     because ARC pages can be freed without swap.
vm.swappiness = 10

# CHANGED FROM: 50 → 80
# VFS cache pressure — controls dentry/inode cache vs page cache reclaim.
# On PVE: PAGE CACHE is critical — it holds VM disk image data (qcow2/raw).
# Value 50 hoards metadata at the expense of VM page cache, causing more
# disk reads for VM data. 80 allows the kernel to balance metadata + data.
# Keep below 100 to preserve overlay filesystem cache for containers.
vm.vfs_cache_pressure = 80

# Dirty page management — prevents I/O spikes that stall VMs.
# These are CRITICAL on PVE: large dirty writeback pauses cause VM freezes.

# Background writeback starts at 5% of RAM (3.2GB on 64GB system).
vm.dirty_background_ratio = 5

# Forced synchronous writes at 10% of RAM (6.4GB on 64GB system).
# Keeps write pauses short enough that VMs don't experience I/O freezes.
# ⚠️  On ZFS: ZFS has its own ARC writeback; these values are less impactful.
# ⚠️  On Ceph: Ceph writes go via network; these affect local journal only.
vm.dirty_ratio = 10

# NEW: Dirty data expires after 30s (3000 centiseconds).
# Prevents stale dirty pages from accumulating for long periods.
vm.dirty_expire_centisecs = 3000

# NEW: Background writeback runs every 5 seconds.
# Smoother writeback prevents burst I/O that stalls VMs.
# ⚠️  On systems with many VMs doing writes, consider 250 (2.5s) for
#     even smoother I/O, at the cost of slightly higher background I/O.
vm.dirty_writeback_centisecs = 500

# Minimum free memory reserve (512MB for 64GB+ systems).
# Prevents OOM deadlocks under memory pressure — critical for PVE host stability.
# ⚠️  Scale with RAM:
#     128MB for ≤16GB, 256MB for 32GB, 512MB for 64GB, 1GB for 128GB+
# ⚠️  Complement with VM balloon drivers and PVE memory overcommit settings.
vm.min_free_kbytes = 524288

# Memory overcommit mode 1 (always overcommit).
# Essential for KVM (QEMU overcommits for VM RAM) and LXC containers.
# Without this, VM startup can fail on memory allocation.
# ⚠️  Monitor OOM: dmesg | grep -i "out of memory"
vm.overcommit_memory = 1

# Maximum memory map count — required for Elasticsearch, databases in containers.
# Default: 65530. 262144 supports heavy mmap workloads.
vm.max_map_count = 262144

# NEW: Proactive memory compaction for transparent huge pages.
# Helps KVM VMs get contiguous 2MB pages, reducing page faults.
# Available on kernel 5.x+. Default: 20. Value 20 is proactive.
# ⚠️  Only effective if transparent hugepages are enabled.
# vm.compaction_proactiveness = 20

# Huge pages — set ONLY if using static hugepage-backed VMs.
# 0 = let kernel auto-manage via transparent hugepages.
# To reserve 1GB of 2MB hugepages: 524288 (524288 × 2MB = 1GB)
# ⚠️  These pages are locked and NOT available for other use.
#     Only set if you have VMs configured with hugepages=2M/1G in PVE config.
vm.nr_hugepages = 0

########################
# SHARED MEMORY (Database VMs/Containers)
########################

# Maximum shared memory segment: 64GB.
# Enables large PostgreSQL/MySQL shared buffers inside VMs and containers.
kernel.shmmax = 68719476736

# CHANGED FROM: 4294967296 (16TB!) → 16777216 (64GB / 4096 = 16,777,216 pages).
# Original value was 4 BILLION pages = 16TB, which is incorrect for a 64GB system.
# This caused excessive kernel memory allocation for page table tracking.
kernel.shmall = 16777216

# Maximum shared memory segments.
kernel.shmmni = 4096

# Semaphore limits (SEMMSL SEMMNS SEMOPM SEMMNI).
# CHANGED FROM: 128 → 256 SEMMNI (max semaphore arrays).
# 128 is too low for multiple PostgreSQL instances across VMs/containers.
# 250 = max semaphores per array
# 32000 = max semaphores system-wide
# 100 = max ops per semop call
# 256 = max semaphore arrays (supports ~8 PostgreSQL instances)
kernel.sem = 250 32000 100 256

########################
# SYSTEM RESOURCE LIMITS
########################

# Maximum open file descriptors.
# Critical for QEMU processes (each VM opens many disk/image files).
fs.file-max = 2097152

# Maximum PID value — allows up to 4M concurrent processes/threads.
# Each PVE VM spawns multiple QEMU threads; dense hosts need high PID space.
# ⚠️  Default (32768) is too low for 20+ VMs with VCPU threads.
kernel.pid_max = 4194304

# CHANGED FROM: 524288 → 262144
# Maximum kernel threads. 524K wastes kernel memory for thread structures.
# 262K supports 20+ VMs with many VCPUs each plus host threads.
kernel.threads-max = 262144

# Inotify limits — critical for LXC containers and monitoring agents.
# Default: max_user_instances=128, max_user_watches=8192.
fs.inotify.max_user_instances = 512
fs.inotify.max_user_watches = 524288

# Maximum async I/O operations — for database VMs/containers.
fs.aio-max-nr = 1048576

########################
# PROXMOX VE REQUIRED FUNCTIONALITY
########################

# IPv4 forwarding — required for VM/container networking and NAT.
net.ipv4.ip_forward = 1

# IPv6 forwarding — required for dual-stack VM/container networking.
net.ipv6.conf.all.forwarding = 1

# Bridge netfilter — REQUIRED for PVE datacenter firewall on VM bridges.
# Without this, PVE firewall rules are silently ignored.
# ⚠️  REQUIRES: modprobe br_netfilter
# ⚠️  Verify: ls /proc/sys/net/bridge/
# ⚠️  If NOT using PVE firewall, set to 0 for ~5-10% network performance gain
#     (bridge netfilter adds per-packet overhead).
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-arptables = 1

########################
# ARP CACHE (VM/Container Density)
########################

# ARP thresholds — prevent neighbor table overflow on dense PVE hosts.
# Each VM/container adds entries; 50+ VMs with multiple NICs can exhaust defaults.
net.ipv4.neigh.default.gc_thresh1 = 2048
net.ipv4.neigh.default.gc_thresh2 = 4096
net.ipv4.neigh.default.gc_thresh3 = 8192

########################
# TCP CONNECTION HANDLING
########################

# SYN backlog — handles connection bursts during VM boot storms or scaling events.
net.ipv4.tcp_max_syn_backlog = 8192

# Listen backlog maximum — accommodates PVE web GUI + API + VM services.
net.core.somaxconn = 8192

# Reuse TIME_WAIT sockets — safe for server-side, critical for backup/restore.
net.ipv4.tcp_tw_reuse = 1

# FIN-WAIT-2 timeout — faster cleanup for short-lived API/backup connections.
net.ipv4.tcp_fin_timeout = 15

# Disable slow start after idle — maintains throughput for VM migration,
# storage replication, and backup connections with idle periods.
net.ipv4.tcp_slow_start_after_idle = 0

# Disable TCP metrics caching — prevents stale RTT/cwnd from affecting
# diverse VM/container traffic patterns.
net.ipv4.tcp_no_metrics_save = 1

# TCP Fast Open — reduces connection latency by 1 RTT for repeated connections.
# Bit flags: 1=client, 2=server, 3=both.
# CHANGED: Removed tcp_fastopen_blackhole_timeout_sec=0 (see below).
net.ipv4.tcp_fastopen = 3

# ⚠️  REMOVED: tcp_fastopen_blackhole_timeout_sec = 0
#     Setting to 0 disables blackhole detection. If middleboxes (firewalls,
#     routers, VPNs) silently drop TFO packets, connectivity breaks permanently
#     with no fallback to regular TCP handshake. Keep kernel default (3600).

########################
# TCP KEEPALIVE (Dead Connection Detection)
# NEW — PVE hosts have long-lived connections (storage replication, migration,
# cluster corosync, PVE proxy) that must detect dead peers quickly.
# Default: 7200/75/9 = 11+ minutes to detect dead connection.
# These values: 300/15/3 = ~45 seconds.
########################

net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp_keepalive_probes = 3

########################
# TCP CONGESTION & THROUGHPUT
########################

# BBR congestion control — superior for VM migration, storage replication,
# and cross-datacenter traffic over high-latency/high-BDP links.
# ⚠️  REQUIRES: modprobe tcp_bbr
net.ipv4.tcp_congestion_control = bbr

# Fair Queuing — implements per-flow pacing required by BBR.
# ⚠️  Must be 'fq' for BBR to function correctly (not fq_codel).
net.core.default_qdisc = fq

# TCP window scaling — essential for >64KB windows on high-BDP paths.
net.ipv4.tcp_window_scaling = 1

# Selective ACKs — faster loss recovery, critical for migration/replication.
net.ipv4.tcp_sack = 1

# TCP timestamps — required for BBR's RTT estimation and PAWS.
net.ipv4.tcp_timestamps = 1

# Reduce per-connection latency for interactive VM protocols (VNC, SPICE, SSH).
# Sends partial segments more aggressively instead of buffering.
net.ipv4.tcp_notsent_lowat = 16384

# Path MTU Discovery — prevents fragmentation in overlay networks and tunnels.
net.ipv4.tcp_mtu_probing = 1

# Enable TCP moderate receive buffer auto-tuning.
net.ipv4.tcp_moderate_rcvbuf = 1

########################
# NETWORK BUFFERS
########################

# Interface packet backlog — handles bursty traffic from multiple VMs.
net.core.netdev_max_backlog = 16384

# Maximum socket buffers — must be ≥ tcp_rmem/tcp_wmem max values.
# 32MB supports VM migration and large backup transfers over high-BDP links.
net.core.rmem_max = 33554432
net.core.wmem_max = 33554432

# Default socket buffer sizes.
net.core.rmem_default = 87380
net.core.wmem_default = 65536

# Ancillary buffer per socket.
net.core.optmem_max = 65536

# TCP buffer auto-tuning ranges (min, default, max in bytes).
# 32MB max for VM migration, storage replication, and backup transfers.
net.ipv4.tcp_rmem = 4096 87380 33554432
net.ipv4.tcp_wmem = 4096 65536 33554432

# CHANGED FROM: 65536 131072 262144 (pages)
# TCP memory limits (low, pressure, max in PAGES).
# Removed explicit setting — kernel auto-tunes based on system RAM.
# Hardcoding can cause unexpected throttling under memory pressure.
# Uncomment ONLY if you see "TCP: out of memory" in dmesg:
# net.ipv4.tcp_mem = 786432 1048576 1572864

# Ephemeral port range — widened for PVE backup/restore and VM outbound traffic.
net.ipv4.ip_local_port_range = 1024 65535

########################
# CONNECTION TRACKING
# ⚠️  REQUIRES: modprobe nf_conntrack
# ⚠️  If NOT using PVE firewall, conntrack is less critical.
#     Each entry uses ~300 bytes. 1M × 300 = ~300MB RAM when table is full.
########################

# Maximum tracked connections — supports many VMs/containers.
net.netfilter.nf_conntrack_max = 1048576

# CHANGED FROM: 43200 (12h) → 7200 (2h)
# VMs and containers are dynamic; stale entries waste conntrack for 12 hours.
# 2h is sufficient for any legitimate long-lived connection.
# PVE corosync uses UDP, not TCP — not affected by this timeout.
net.netfilter.nf_conntrack_tcp_timeout_established = 7200

# Optimized timeouts for non-established states.
net.netfilter.nf_conntrack_tcp_timeout_close_wait = 60
net.netfilter.nf_conntrack_tcp_timeout_fin_wait = 120
net.netfilter.nf_conntrack_tcp_timeout_time_wait = 120
net.netfilter.nf_conntrack_udp_timeout = 30
net.netfilter.nf_conntrack_udp_timeout_stream = 120
net.netfilter.nf_conntrack_generic_timeout = 120
net.netfilter.nf_conntrack_icmp_timeout = 30

########################
# BPF JIT
########################

# Enable BPF JIT compiler — significant performance gain for eBPF tools.
# Used by: Cilium CNI (if PVE hosts run K8s), bpftrace, Falco, etc.
net.core.bpf_jit_enable = 1

# CHANGED FROM: 2 → 1
# BPF JIT hardening:
#   0 = none
#   1 = hardening for UNPRIVILEGED programs only (recommended)
#   2 = hardening for ALL programs (adds 5-10% overhead to every BPF program)
# On PVE, all BPF is typically root-only. Value 1 is sufficient.
net.core.bpf_jit_harden = 1

########################
# SECURITY
########################

# ASLR — full address space layout randomization.
kernel.randomize_va_space = 2

# Restrict kernel pointer exposure.
kernel.kptr_restrict = 2

# Restrict dmesg to root — prevents VM/container info leaks via kernel log.
kernel.dmesg_restrict = 1

# Restrict perf events — prevents container/VM side-channel attacks via PMU.
kernel.perf_event_paranoid = 2

# Ptrace scope — restricts cross-process debugging.
# ⚠️  Value 1 allows same-user ptrace. Value 2 requires CAP_SYS_PTRACE.
#     If using 'strace' inside LXC containers for debugging, use value 1.
kernel.yama.ptrace_scope = 1

# CHANGED FROM: /dev/null → |/bin/false
# Core dump handling. |/bin/false causes the kernel to skip core generation
# entirely (pipe target exits immediately). Writing to /dev/null as a file
# path still generates the entire core in memory before discarding it.
# ⚠️  For debugging production crashes, use a real path:
#     kernel.core_pattern = /var/crash/core.%e.%p.%s.%t
kernel.core_pattern = |/bin/false

# SYN cookies — DDoS resistance. Activates only on SYN backlog overflow.
net.ipv4.tcp_syncookies = 1

# SYN retry limits — prevents half-open connection resource exhaustion.
net.ipv4.tcp_syn_retries = 2
net.ipv4.tcp_synack_retries = 2

# TCP TIME-WAIT assassination protection (RFC 1337).
net.ipv4.tcp_rfc1337 = 1

# CHANGED FROM: 1 (strict) → 2 (loose)
# Reverse path filtering. Strict mode drops packets from "unexpected"
# interfaces — breaks PVE bridges, VLANs, SDN, and multi-homed VMs.
# Loose mode only verifies the source IP is globally routable.
# ⚠️  If NOT using PVE SDN/VLANs and have simple single-bridge topology,
#     strict (1) is acceptable.
net.ipv4.conf.all.rp_filter = 2
net.ipv4.conf.default.rp_filter = 2

# Disable ICMP redirects — prevent MITM.
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0

# Disable source routing.
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0

# Martian logging — disabled by default on PVE.
# Bridge traffic and VXLAN overlays trigger constant false positives.
# Enable ONLY when debugging network issues.
net.ipv4.conf.all.log_martians = 0
net.ipv4.conf.default.log_martians = 0

# Broadcast ping protection.
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.icmp_ignore_bogus_error_responses = 1

# IPv6 security.
net.ipv6.conf.all.accept_redirects = 0
net.ipv6.conf.default.accept_redirects = 0
net.ipv6.conf.all.accept_source_route = 0
net.ipv6.conf.default.accept_source_route = 0

########################
# WATCHDOG & DEBUGGING
########################

# Disable NMI watchdog — saves ~0.5-1% CPU on heavily loaded systems.
# ⚠️  ONLY disable if you have external monitoring (PVE HA, watchdog agent).
#     NMI watchdog detects hard lockups; disabling reduces debuggability.
#     If using PVE HA with hardware watchdog, this is safe.
kernel.nmi_watchdog = 0

########################
# ZFS STORAGE BACKEND
# ⚠️  ZFS on Linux uses MODULE PARAMETERS, NOT sysctl!
#     Do NOT use sysctl or /etc/sysctl.d/ for ZFS tuning.
#     Instead, create /etc/modprobe.d/zfs.conf with these options:
#
#   # Limit ARC to 16GB on a 64GB system (leave RAM for VMs)
#   options zfs zfs_arc_max=17179869184
#   options zfs zfs_arc_min=2147483648
#
#   # Tune ZIO scheduler for SSD/NVMe
#   options zfs zfs_vdev_scheduler=none
#
#   # Disable ZFS prefetch for database workloads (NVMe)
#   # options zfs zfs_prefetch_disable=1
#
# ⚠️  KEY RULE: ARC max should be RAM minus VM/container memory reservations
#     minus ~4GB for host. Example: 64GB - 48GB VMs - 4GB host = 12GB ARC.
#
# ⚠️  Apply with: modprobe -r zfs && modprobe zfs
#     Or reboot after changing /etc/modprobe.d/zfs.conf
########################

########################
# CEPH STORAGE BACKEND (Hyperconverged PVE)
# If running Ceph OSDs on PVE nodes, add these to /etc/sysctl.d/ceph.conf:
#
#   # Increase PID space for OSD processes
#   kernel.pid_max = 4194304
#
#   # File descriptors for OSDs (each OSD opens many PG files)
#   fs.file-max = 4194304
#
#   # Allow more open files per process
#   # (also set in ceph.conf: max_open_files)
#
#   # Larger dirty ratios for OSD write-back
#   vm.dirty_background_ratio = 10
#   vm.dirty_ratio = 20
#
#   # Network buffer tuning for OSD replication traffic
#   net.core.rmem_max = 134217728
#   net.core.wmem_max = 134217728
#   net.ipv4.tcp_rmem = 4096 87380 134217728
#   net.ipv4.tcp_wmem = 4096 65536 134217728
#
#   # Increase conntrack for OSD heartbeat + replication
#   net.netfilter.nf_conntrack_max = 2097152
#
#   # Disable transparent hugepages for Ceph (causes latency spikes)
#   # (via /sys/kernel/mm/transparent_hugepage/enabled = never)
#
#   # Ceph uses messenger_v2 which benefits from large socket buffers.
#   # See Ceph documentation for detailed OSD tuning.
########################