nginx log grok

ELK学习

grok 工具

http://grokdebug.herokuapp.com/
https://github.com/logstash-plugins/logstash-patterns-core/blob/master/patterns/grok-patterns

nginx access log

log_format  main  '$remote_addr - $remote_user [$time_local] "$host" "$request" $request_time '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
message:
    112.80.254.148 - - [18/Jul/2017:15:27:01 +0800] "www.9douyu.com" "GET /resources/image/20150520/555c0cc341be5.jpg HTTP/1.1" 0.000 301 178 "www.9douyu.com" "Mozilla/5.0 (Windows NT 5.1; rv:8.0.1) Gecko/20100101 Firefox/8.0.1" "-"
grok:
    %{IP:clientip} - (?:%{USERNAME:[@metadata][http_user]}|-) \[%{HTTPDATE:timestamp}\] \"%{HOSTNAME:http_host}\" \"(?:%{WORD:verb} %{NOTSPACE:request}(?:HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:request_time} %{NUMBER:response} (?:%{NUMBER:bytes}|-) \"%{DATA:referrer}\" \"%{DATA:agent}\" \"(?:%{IP:http_x_forwarded_for}|-)\"

nginx error log

message:
    2017/06/24 04:24:49 [error] 16#16: *1442460 upstream timed out (110: Operation timed out) while reading response header from upstream, client: 101.201.208.151, server: res.9douyu.com, request: "POST /api/checkCard HTTP/1.0", upstream: "http://172.20.0.1:21001/api/checkCard", host: "tc-api.9douyu.com"
grok:
    (?<timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:severity}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(?:, client: (?<client>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server})(?:, request: %{QS:request})?(?:, upstream: \"%{URI:upstream}\")?(?:, host: %{QS:host})
example:
    (?<timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:severity}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(?:, client: (?<client>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server})(?:, request: %{QS:request})?(?:, upstream: \"%{URI:upstream}\")?(?:, host: %{QS:host})?(?:, referrer: \"%{URI:referrer}\"