nginx: Log complete request/response with all headers

log-http-headers.md

1. switch nginx image to openresty/openresty

2. add the following to server/location (/etc/nginx/conf.d/default.conf)

   set $req_header "";
   set $resp_header "";
   header_filter_by_lua_block{ 
      local h = ngx.req.get_headers();
      for k, v in pairs(h) do
         ngx.var.req_header = ngx.var.req_header .. k.."="..v.." ";
      end
      local rh = ngx.resp.get_headers();
      for k, v in pairs(rh) do
         ngx.var.resp_header = ngx.var.resp_header .. k.."="..v.." ";
      end
   }

   lua_need_request_body on;
   set $resp_body "";
   body_filter_by_lua_block {
      local resp_body = string.sub(ngx.arg[1], 1, 1000)
      ngx.ctx.buffered = (ngx.ctx.buffered or "") .. resp_body
      if ngx.arg[2] then
         ngx.var.resp_body = ngx.ctx.buffered
      end
   }

   # access_log  /dev/stdout log_req_resp;
   access_log   /var/log/nginx-access.log log_req_resp;

3. add the following to http (/usr/local/openresty/nginx/conf/nginx.conf)

   log_format log_req_resp '$remote_addr - $remote_user [$time_local] '
      '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" '
      '$request_time req_header:"$req_header" resp_header:"$resp_header" '
      'req_body:"$request_body" resp_body:"$resp_body"';   

individual headers

request headers:$http_<header> 
 sent headers: $sent_http_<header>

@mvillafuertem

Map your nginx.conf into the container, is this what you need?

docker run -d -v /your-path/nginx.conf:/etc/nginx.conf openresty/openresty

This isn't working, Its not recognizing the log format "nginx: [emerg] unknown log format "log_req_resp" in /etc/nginx/conf.d/libre.conf:42"
I have confirmed that my openresty container can see the config. here is its nginx.conf:

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log notice;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    
    log_format log_req_resp '$remote_addr - $remote_user [$time_local] '
                            '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" '
                            '$request_time req_header:"$req_header" resp_header:"$resp_header" '
                            'req_body:"$request_body" resp_body:"$resp_body"'; 

    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;
    
    resolver  127.0.0.11 valid=30s;

    include /etc/nginx/conf.d/*.conf;
}

and here is the server conf:

server {
    listen 80;
    server_name libre.*;

    return 301 https://$server_name$request_uri;
}

server {
    listen 443 ssl http2;
    server_name libre.*;

    include /etc/nginx/snippets/ssl.conf;
    include /etc/nginx/snippets/authelia-location.conf;

    set $upstream http://librenms:8000;

    location / {
        set $req_header "";
        set $resp_header "";
        header_filter_by_lua_block{ 
            local h = ngx.req.get_headers();
            for k, v in pairs(h) do
                ngx.var.req_header = ngx.var.req_header .. k.."="..v.." ";
            end
            local rh = ngx.resp.get_headers();
            for k, v in pairs(rh) do
                ngx.var.resp_header = ngx.var.resp_header .. k.."="..v.." ";
            end
        }

        lua_need_request_body on;
        set $resp_body "";
        body_filter_by_lua_block {
            local resp_body = string.sub(ngx.arg[1], 1, 1000)
            ngx.ctx.buffered = (ngx.ctx.buffered or "") .. resp_body
            if ngx.arg[2] then
                ngx.var.resp_body = ngx.ctx.buffered
            end
        }

        access_log  /dev/stdout log_req_resp;
        include /etc/nginx/snippets/proxy.conf;
        include /etc/nginx/snippets/authelia-authrequest.conf;
        proxy_pass $upstream; 
    }
}

What am I doing wrong?

Logging of responce header is not working with the Lua code, because error from nginx:
2025/02/03 15:23:35 [error] 6#6: *1 failed to run header_filter_by_lua*: header_filter_by_lua(nginx.conf:33):9: attempt to concatenate local 'v' (a table value) stack traceback:         header_filter_by_lua(nginx.conf:33):9: in main chunk while reading response header from upstream, client: 127.0.0.1, server: _, request: "GET / HTTP/1.1", upstream: "https://10.211.81.171:8443/", host: "127.0.0.1:8080" 127.0.0.1 - - [03/Feb/2025:15:23:35 +0000] "GET / HTTP/1.1" 302 0 "-" "curl/7.88.1" "-" req_header: "user-agent=curl/7.88.1 accept=*/* host=127.0.0.1:8080 " req_body: "-" resp_header: "" resp_body: ""
It's need in formating, because in value may be array or hash. Here is solution: https://stackoverflow.com/a/66010794/7672941

It worked like a charm!

getting req_body and resp_body like a charm

using openresty saves my life. Also I did the enhancement such as making the encoded json as single liner json so it will be readable for human.
in my case I am using kubernetes and intercepting my backend by using openresty sidecar.

apiVersion: v1
kind: ConfigMap
metadata:
  name: test-collection-service-nginx-config
  labels:
    app: test-collection-service
    app.kubernetes.io/name: test-collection-service
    app.kubernetes.io/instance: test-collection-service
data:
  nginx.conf: |
    # Use nobody instead of nginx user
    user nobody;
    worker_processes auto;

    error_log /dev/stderr debug;
    pid /var/run/nginx.pid;

    # File descriptor limits - increase for high traffic
    worker_rlimit_nofile 65535;

    events {
        worker_connections 4096;
        multi_accept on;
        use epoll;
    }

    http {
        include /usr/local/openresty/nginx/conf/mime.types;
        default_type application/octet-stream;
        
        # Generate unique request ID for tracking
        map $remote_addr$request_uri$time_iso8601 $custom_request_id {
            default $hostname-$pid-$connection-$connection_requests;
        }
        
        # Set pod and node name
        map $hostname $pod_name {
            default "$hostname";
        }
        
        map $hostname $node_name {
            default "unknown";
        }
        
        # Configure global Lua settings
        lua_need_request_body on;
        
        # Enhanced log format that includes headers and body
        log_format log_req_resp escape=json
          '{'
            '"time":"$time_iso8601",'
            '"remote_addr":"$remote_addr",'
            '"request":"$request",'
            '"status":"$status",'
            '"body_bytes_sent":"$body_bytes_sent",'
            '"request_time":"$request_time",'
            '"user_agent":"$http_user_agent",'
            '"http_host":"$http_host",'
            '"method":"$request_method",'
            '"uri":"$request_uri",'
            '"content_type":"$http_content_type",'
            '"content_length":"$http_content_length",'
            '"pod_name":"$pod_name",'
            '"node_name":"$node_name",'
            '"req_header":"$req_header",'
            '"resp_header":"$resp_header",'
            '"req_body":$req_body,'
            '"resp_body":$resp_body,'
            '"request_id":"$custom_request_id"'
          '}';
        
        # Optimize for performance
        sendfile on;
        tcp_nopush on;
        tcp_nodelay on;
        keepalive_timeout 65;
        
        # Use fixed paths for temp directories
        client_body_temp_path /tmp/nginx/client_body_temp 1 2;
        proxy_temp_path /tmp/nginx/proxy_temp 1 2;
        fastcgi_temp_path /tmp/nginx/fastcgi_temp 1 2;
        uwsgi_temp_path /tmp/nginx/uwsgi_temp 1 2;
        scgi_temp_path /tmp/nginx/scgi_temp 1 2;
        
        server {
            listen 9000;
            
            # Set variables for request and response headers/body
            # These must be in server or location context, not http
            set $req_header "";
            set $resp_header "";
            set $resp_body "";
            set $req_body "";
            
            # Set up access log to use our custom format
            access_log /dev/stdout log_req_resp;
            error_log /dev/stderr debug;
            
            # Configure request body handling
            client_body_buffer_size 5m;
            client_max_body_size 100m;
            client_body_timeout 300s;
            client_header_timeout 60s;
            
            # Proxy settings
            proxy_connect_timeout 60s;
            proxy_send_timeout 300s;
            proxy_read_timeout 300s;
            
            # API endpoints
            location ~ ^/collection/ {
                # Add unique request ID to all requests
                add_header X-Custom-Request-ID $custom_request_id;
                
                # Define Lua blocks to capture headers and body
                access_by_lua_block {
                    -- Capture the request body
                    ngx.req.read_body()
                    local req_body = ngx.req.get_body_data()
                    
                    if req_body then
                        -- Limit to 1000 characters
                        req_body = string.sub(req_body, 1, 1000)
                        
                        -- Remove all newlines, carriage returns and excess whitespace
                        req_body = req_body:gsub("\r", "")
                        req_body = req_body:gsub("\n", "")
                        
                        -- Compact JSON by removing unnecessary whitespace
                        req_body = req_body:gsub("%s+", " ")
                        req_body = req_body:gsub("%s*([{}%[%],:]*)%s*", "%1")
                        
                        -- Try to handle it as JSON if it looks like JSON
                        local is_json = false
                        if req_body:match("^%s*{") or req_body:match("^%s*%[") then
                            is_json = true
                        end
                        
                        if is_json then
                            -- For JSON, just pass it through as-is
                            ngx.var.req_body = req_body
                        else
                            -- For non-JSON, quote it
                            ngx.var.req_body = [["]] .. req_body:gsub([["]], [[\\"]]) .. [["]]
                        end
                    else
                        ngx.var.req_body = '""'
                    end
                }
                
                header_filter_by_lua_block { 
                    local h = ngx.req.get_headers()
                    local req_header_str = ""
                    for k, v in pairs(h) do
                        -- Handle both string and table values
                        if type(v) == "table" then
                            v = table.concat(v, ",")
                        end
                        req_header_str = req_header_str .. k .. "=" .. v .. " "
                    end
                    ngx.var.req_header = req_header_str
                    
                    local rh = ngx.resp.get_headers()
                    local resp_header_str = ""
                    for k, v in pairs(rh) do
                        -- Handle both string and table values
                        if type(v) == "table" then
                            v = table.concat(v, ",")
                        end
                        resp_header_str = resp_header_str .. k .. "=" .. v .. " "
                    end
                    ngx.var.resp_header = resp_header_str
                }
                
                body_filter_by_lua_block {
                    local resp_body = string.sub(ngx.arg[1], 1, 1000)
                    ngx.ctx.buffered = (ngx.ctx.buffered or "") .. resp_body
                    if ngx.arg[2] then
                        -- Grab the complete response body
                        local body = ngx.ctx.buffered
                        
                        -- Remove all newlines, carriage returns and excess whitespace
                        body = body:gsub("\r", "")
                        body = body:gsub("\n", "")
                        
                        -- Try to determine if response is JSON
                        local is_json = false
                        if body:match("^%s*{") or body:match("^%s*%[") then
                            is_json = true
                            
                            -- If it's JSON, compact it by removing unnecessary whitespace
                            body = body:gsub("%s+", " ")
                            body = body:gsub("%s*([{}%[%],:]*)%s*", "%1")
                        end
                        
                        if is_json then
                            -- For JSON, just pass it through as-is
                            ngx.var.resp_body = body
                        else
                            -- For non-JSON, quote it
                            ngx.var.resp_body = [["]] .. body:gsub([["]], [[\\"]]) .. [["]]
                        end
                    end
                }
                
                # Proxy all requests to the FastAPI service
                proxy_pass http://127.0.0.1:8000;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Proto $scheme;
                
                # Pass additional identifiers
                proxy_set_header X-Pod-Name $pod_name;
                proxy_set_header X-Node-Name $node_name;
                proxy_set_header X-Custom-Request-ID $custom_request_id;
                
                proxy_buffering off;
                proxy_set_header X-Maximum-Upload-Size "100MB";
            }
            
            # Health check endpoint
            location = /collection/health-check {
                access_log off;
                proxy_pass http://127.0.0.1:8000;
                proxy_set_header Host $host;
            }
        }
    }

Result: