分析 nginx 的 access log

parse_nginx_access_log.py

#!/usr/bin/env python
# encoding: utf-8
import re

LOG = '''\
192.168.20.16 - - [28/Sep/2016:15:55:43 +0800] "GET /v2/dae-rasha/manifests/dc26e9e9873baf1af7dc0169496f4de373e4fc01__1d374a6a93125b5bddc57b0b1c34243a5cdcd9586c19ad0b432133b7d81ef123 HTTP/1.1" 200 2008 "-" "docker/1.10.3 go/go1.5.3 git-commit/20f81dd kernel/3.18.10-gentoo os/linux arch/amd64" "-" 0.011
192.168.20.16 - - [28/Sep/2016:15:55:43 +0800] "GET /v2/dae-lunar/manifests/3137109cad460d22d6585ebbe90f3f31b6db5c77__9046a5f27c39cb2f8c110f7c6599f945648e55acf4979c3e6af42f92ad67143a HTTP/1.1" 200 2007 "-" "docker/1.10.3 go/go1.5.3 git-commit/20f81dd kernel/3.18.10-gentoo os/linux arch/amd64" "-" 0.011
192.168.20.16 - - [28/Sep/2016:15:55:43 +0800] "GET /v2/dae-lunar/blobs/sha256:fb781d214197779f45d002c20ff3f5084b817389889c6f9057ca02e9ef34dc85 HTTP/1.1" 200 2839 "-" "docker/1.10.3 go/go1.5.3 git-commit/20f81dd kernel/3.18.10-gentoo os/linux arch/amd64" "-" 0.010
192.168.20.16 - - [28/Sep/2016:15:55:43 +0800] "GET /v2/dae-rasha/blobs/sha256:b80ae88db42a9c6de2d813f1452b2cdd164faead57249d81c12953748c866a40 HTTP/1.1" 200 2838 "-" "docker/1.10.3 go/go1.5.3 git-commit/20f81dd kernel/3.18.10-gentoo os/linux arch/amd64" "-" 0.013
192.168.20.16 - - [28/Sep/2016:15:55:43 +0800] "GET /v2/dae-lunar/blobs/sha256:370d791318a0c1f2e993a61f82d1aa4a175c62cf7da61407308f3b85e5b2d3a1 HTTP/1.1" 200 3057 "-" "docker/1.10.3 go/go1.5.3 git-commit/20f81dd kernel/3.18.10-gentoo os/linux arch/amd64" "-" 0.014
192.168.20.16 - - [28/Sep/2016:15:55:43 +0800] "GET /v2/dae-lunar/blobs/sha256:562d11d419e17a1ce0a0ec8c3897b107e69e7e36afa8af88f9b88afb5fd2f7ca HTTP/1.1" 200 6085179 "-" "docker/1.10.3 go/go1.5.3 git-commit/20f81dd kernel/3.18.10-gentoo os/linux arch/amd64" "-" 0.156
192.168.20.16 - - [28/Sep/2016:15:55:43 +0800] "GET /v2/dae-rasha/blobs/sha256:7438787cde8725fc586a57c89edc6fdc6df4525f890a73faef521016d032ac5e HTTP/1.1" 200 12916033 "-" "docker/1.10.3 go/go1.5.3 git-commit/20f81dd kernel/3.18.10-gentoo os/linux arch/amd64" "-" 0.294
192.168.1.101 - - [28/Sep/2016:15:56:32 +0800] "GET /v2/ HTTP/1.1" 200 244 "-" "docker/1.10.3 go/go1.5.3 git-commit/20f81dd kernel/3.18.10-gentoo os/linux arch/amd64" "-" 0.002
192.168.1.101 - - [28/Sep/2016:15:56:33 +0800] "GET /v2/dae-sweetleaf/manifests/41a75f4063f01d71aa3594266dd470399ed707cf__4098d2e18680b3d3dc85a6ba3c826ea3d27fbfa99d0519e834f6f0e4712d7101 HTTP/1.1" 200 2008 "-" "docker/1.10.3 go/go1.5.3 git-commit/20f81dd kernel/3.18.10-gentoo os/linux arch/amd64" "-" 0.011
192.168.1.101 - - [28/Sep/2016:15:56:33 +0800] "GET /v2/dae-sweetleaf/blobs/sha256:3c9b005ca74dd11e5facc7c862743e56070f2a9dfdd7915023ee052fc1165fd9 HTTP/1.1" 200 2860 "-" "docker/1.10.3 go/go1.5.3 git-commit/20f81dd kernel/3.18.10-gentoo os/linux arch/amd64" "-" 0.008
'''

RE_ACCESS_LOG = re.compile(r'''
    (?P<ip_address>\d+.\d+.\d+.\d+)\s-\s-\s    # IP address
    (?P<datetime>\[(.+)\])\s                   # datetime
    "
        (?P<method>\w+)\s                      # method
        (?P<path>.+)\s                         # path
        (?P<protocol>\w+/.+)                   # protocol
    "\s
    (?P<status>\d+)\s                          # status
    (?P<bandwidth>\d+)\s                       # bandwidth
    (?P<referrer>"([^"]+)")\s                  # referrer
    (?P<user_agent>"([^"]+)")                  # user agent
''', re.VERBOSE)


for line in LOG.splitlines():
    res = RE_ACCESS_LOG.match(line)
    print 'ip_address\t', res.group('ip_address')
    print 'datetime\t', res.group('datetime')
    print 'method\t\t', res.group('method')
    print 'path\t\t', res.group('path')
    print 'protocol\t', res.group('protocol')
    print 'status\t\t', res.group('status')
    print 'bandwidth\t', res.group('bandwidth')
    print 'referrer\t', res.group('referrer')
    print 'user_agent\t', res.group('user_agent')
    print '-' * 20