准备说明

根据业务情况，会出现ELK解析多种格式的日志需求，这时需要在logstash的配置文件中配置grok规则解析日志文件，grok解析建议使用在线工具测试。

在线Grok解析工具地址：Grok Debugger

在线测试样例：

Grok Debugger
Grok Debugger

Grok的语句需要写在ELK的logstash中的配置文件中，如下图：

Logstash文件配置
Logstash文件配置

异常日志

1
2
3
4
5
2018-11-09 23:01:18.766  [ERROR]  com.ailk.rpc.server.handler.ServerHandler - 调用com.ailk.search.server.SearchServer.search时发生错误！
java.lang.reflect.InvocationTargetException
	at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
	at java.lang.reflect.Method.invoke(Method.java:597)
grok解析

%{TIMESTAMP_ISO8601:log_time} [%{DATA:log_level}] %{GREEDYDATA:message}

1
2
3
4
5
6
7
8
9
10
filebeat配置
filebeat:
   prospectors:
     -
       paths:
         - /home/elk/logs/*.log
       type: log
       multiline.pattern: '^\['
       multiline.negate: true
       multiline.match: after
解析结果
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
{
"log_time": [
  [
    "2018-11-09 23:01:18.766"
  ]
],
"YEAR": [
  [
    "2018"
  ]
],
"MONTHNUM": [
  [
    "11"
  ]
],
"MONTHDAY": [
  [
    "09"
  ]
],
"HOUR": [
  [
    "23",
    null
  ]
],
"MINUTE": [
  [
    "01",
    null
  ]
],
"SECOND": [
  [
    "18.766"
  ]
],
"ISO8601_TIMEZONE": [
  [
    null
  ]
],
"log_level": [
  [
    "ERROR"
  ]
],
"message": [
  [
    " com.ailk.rpc.server.handler.ServerHandler - 调用com.ailk.search.server.SearchServer.search时发生错误！"
  ]
]

业务报文日志

1
2
3
4
5
6
7
8
9
<operation_in>请求报文：service_name接口名，sysfunc_id功能号，operator_id 操作员id，organ_id 机构号，request_seq 请求流水
2018-11-12 15:03:41.388 639211542011357848  [DEBUG]  com.base.core.aop.http.HttpClient.send(HttpClient.java:128) - 
reqid:b7fb8f90ddeb11e83d622c02b34132f7;AOP 发送信息: <?xml version="1.0" encoding="GBK"?><operation_in<service_name>BSM_SaleSystemLogin</service_name>     
<sysfunc_id>91008027</sysfunc_id><request_type>1002</request_type><verify_code>304147201506190000000040</verify_code><operator_id>9991445</operator_id>              
 <organ_id>9999997</organ_id><request_time>20181112150341</request_time><request_seq>154200622111</request_seq><request_source>304147</request_source><request_target></request_target><msg_version>0100</msg_version><cont_version>0100</cont_version><access_token></access_token><content><request><msisdn>13666945211</msisdn><password>871221</password><portal_id>101704</portal_id><login_type>34</login_type><machine_mac>0000</machine_mac><machine_ip>120.33.230.198, 10.46.161.182, </machine_ip><machine_cpu></machine_cpu><machine_system_ver>12.0.1</machine_system_ver><machine_totalmemory></machine_totalmemory><machine_usablememory></machine_usablememory><machine_ie_ver></machine_ie_ver></request></content></operation_in>
<operation_out>
<operation_out><service_name>BSM_SaleSystemLogin</service_name><request_type>1002</request_type><sysfunc_id>91008027</sysfunc_id>
<request_seq>154200622111</request_seq><response_time>20181112150342</response_time><response_seq>471860579309</response_seq><request_source>304147</request_source><response><resp_type>0</resp_type><resp_code>0000</resp_code><resp_desc/></response><content><response><base_info><verifycode>173616671275425657328820</verifycode><operator_id>132394</operator_id><row><msisdn>13666945211</msisdn><role_id>6100004</role_id><owning_mode>1</owning_mode><status>1</status><inure_time>20170623145448</inure_time><expire_time>30000101000000</expire_time><request_source>0</request_source><modify_time>20170623145448</modify_time><modify_operator_id>4020205</modify_operator_id><modify_content>创建手机号码与角色对应关系
grok解析

%{TIMESTAMP_ISO8601:log_time} %{DATA:serial_number} [%{DATA:log_level}] %{GREEDYDATA:message}%{DATA:service_name} %{DATA:sysfunc_id}%{DATA:other}%{DATA:organ_id}%{DATA:request_time}%{DATA:request_seq}%{DATA:other}

解析结果
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
{
  "log_time": [
    [
      "2018-11-12 15:03:41.388"
    ]
  ],
  "YEAR": [
    [
      "2018"
    ]
  ],
  "MONTHNUM": [
    [
      "11"
    ]
  ],
  "MONTHDAY": [
    [
      "12"
    ]
  ],
  "HOUR": [
    [
      "15",
      null
    ]
  ],
  "MINUTE": [
    [
      "03",
      null
    ]
  ],
  "SECOND": [
    [
      "41.388"
    ]
  ],
  "ISO8601_TIMEZONE": [
    [
      null
    ]
  ],
  "serial_number": [
    [
      "639211542011357848 "
    ]
  ],
  "log_level": [
    [
      "DEBUG"
    ]
  ],
  "message": [
    [
      " com.base.core.aop.http.HttpClient.send(HttpClient.java:128) - reqid:b7fb8f90ddeb11e83d622c02b34132f7;AOP 发送信息: <?xml version="1.0" encoding="GBK"?>   <operation_in"
    ]
  ],
  "service_name": [
    [
      "BSM_SaleSystemLogin"
    ]
  ],
  "sysfunc_id": [
    [
      "91008027"
    ]
  ],
  "other": [
    [
      "1002</request_type><verify_code>304147201506190000000040</verify_code><operator_id>9991445",
      "304147</request_source><request_target></request_target><msg_version>0100</msg_version><cont_version>0100</cont_version><access_token></access_token><content><request><msisdn>13666945211</msisdn><password>871221</password><portal_id>101704</portal_id><login_type>34</login_type><machine_mac>0000</machine_mac><machine_ip>120.33.230.198, 10.46.161.182, </machine_ip><machine_cpu></machine_cpu><machine_system_ver>12.0.1</machine_system_ver><machine_totalmemory></machine_totalmemory><machine_usablememory></machine_usablememory><machine_ie_ver></machine_ie_ver></request></content></operation_in>"
    ]
  ],
  "organ_id": [
    [
      "9999997"
    ]
  ],
  "request_time": [
    [
      "20181112150341"
    ]
  ],
  "request_seq": [
    [
      "154200622111"
    ]
  ]
}

nginx的access.log一条请求就是一条交易量

1
10.48.224.3 - - [12/Nov/2018:14:26:50 +0800] "POST /o2o_usercenter_svc/remote/bsspInvokeService?req_sid=9c5d5600e64311e808a886c802c592cb&syslogid=null HTTP/1.1" 200 234 "-" "Java/1.7.0_21"
grok解析

%{IPORHOST:ip} - %{DATA:data} [%{HTTPDATE:timestamp}] \”%{WORD:method} %{DATA:nginx_access_url} HTTP/%{NUMBER:ngnix_access_http_version}\” %{NUMBER:nginx_access_response_code} %{NUMBER:nginx_access_body_sent_bytes} \”%{DATA:nginx_access_referrer]}\” \”%{DATA:nginx_access_agent}\”

解析结果
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
{
  {
  "ip": [
    [
      "10.48.224.3"
    ]
  ],
  "HOSTNAME": [
    [
      "10.48.224.3"
    ]
  ],
  "IP": [
    [
      null
    ]
  ],
  "IPV6": [
    [
      null
    ]
  ],
  "IPV4": [
    [
      null
    ]
  ],
  "data": [
    [
      "-"
    ]
  ],
  "timestamp": [
    [
      "12/Nov/2018:14:26:50 +0800"
    ]
  ],
  "MONTHDAY": [
    [
      "12"
    ]
  ],
  "MONTH": [
    [
      "Nov"
    ]
  ],
  "YEAR": [
    [
      "2018"
    ]
  ],
  "TIME": [
    [
      "14:26:50"
    ]
  ],
  "HOUR": [
    [
      "14"
    ]
  ],
  "MINUTE": [
    [
      "26"
    ]
  ],
  "SECOND": [
    [
      "50"
    ]
  ],
  "INT": [
    [
      "+0800"
    ]
  ],
  "method": [
    [
      "POST"
    ]
  ],
  "nginx_access_url": [
    [
      "/o2o_usercenter_svc/remote/bsspInvokeService?req_sid=9c5d5600e64311e808a886c802c592cb&syslogid=null"
    ]
  ],
  "ngnix_access_http_version": [
    [
      "1.1"
    ]
  ],
  "BASE10NUM": [
    [
      "1.1",
      "200",
      "234"
    ]
  ],
  "nginx_access_response_code": [
    [
      "200"
    ]
  ],
  "nginx_access_body_sent_bytes": [
    [
      "234"
    ]
  ],
  "nginx_access_referrer]": [
    [
      "-"
    ]
  ],
  "nginx_access_agent": [
    [
      "Java/1.7.0_21"
    ]
  ]
}

nginx error日志解析

1
2018/11/01 23:30:39 [error] 15105#0: *397937824 connect() failed (111: Connection refused) while connecting to upstream, client: 10.48.224.3, server: 127.0.0.1, request: "POST /o2o_usercenter_svc/remote/sysUserInfoService?req_sid=1612e430ddeb11e83d622c02b34132f7&syslogid=null HTTP/1.1", upstream: "http://127.0.0.1:8082/o2o_usercenter_svc/remote/sysUserInfoService?req_sid=1612e430ddeb11e83d622c02b34132f7&syslogid=null", host: "10.46.148.155:9090"
grok解析

(?%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) [%{LOGLEVEL:severity}] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(?:, client: (?%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server}?)(?:, request: %{QS:request})?(?:, upstream: (?\”%{URI}\”|%{QS}))?(?:, host: %{QS:request_host})?(?:, referrer: \”%{URI:referrer}\”)?

解析结果
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
{
  "timestamp": [
    [
      "2018/11/01 23:30:39"
    ]
  ],
  "YEAR": [
    [
      "2018"
    ]
  ],
  "MONTHNUM": [
    [
      "11"
    ]
  ],
  "MONTHDAY": [
    [
      "01"
    ]
  ],
  "TIME": [
    [
      "23:30:39"
    ]
  ],
  "HOUR": [
    [
      "23"
    ]
  ],
  "MINUTE": [
    [
      "30"
    ]
  ],
  "SECOND": [
    [
      "39"
    ]
  ],
  "severity": [
    [
      "error"
    ]
  ],
  "pid": [
    [
      "15105"
    ]
  ],
  "NUMBER": [
    [
      "0"
    ]
  ],
  "BASE10NUM": [
    [
      "0"
    ]
  ],
  "errormessage": [
    [
      "*397937824 connect() failed (111: Connection refused) while connecting to upstream"
    ]
  ],
  "remote_addr": [
    [
      "10.48.224.3"
    ]
  ],
  "IP": [
    [
      "10.48.224.3",
      null,
      null,
      null
    ]
  ],
  "IPV6": [
    [
      null,
      null,
      null,
      null
    ]
  ],
  "IPV4": [
    [
      "10.48.224.3",
      null,
      null,
      null
    ]
  ],
  "HOSTNAME": [
    [
      null,
      "127.0.0.1",
      "127.0.0.1",
      null
    ]
  ],
  "server": [
    [
      "127.0.0.1"
    ]
  ],
  "request": [
    [
      ""POST /o2o_usercenter_svc/remote/sysUserInfoService?req_sid=1612e430ddeb11e83d622c02b34132f7&syslogid=null HTTP/1.1""
    ]
  ],
  "QUOTEDSTRING": [
    [
      ""POST /o2o_usercenter_svc/remote/sysUserInfoService?req_sid=1612e430ddeb11e83d622c02b34132f7&syslogid=null HTTP/1.1"",
      null,
      ""10.46.148.155:9090""
    ]
  ],
  "upstream": [
    [
      ""http://127.0.0.1:8082/o2o_usercenter_svc/remote/sysUserInfoService?req_sid=1612e430ddeb11e83d622c02b34132f7&syslogid=null""
    ]
  ],
  "URI": [
    [
      "http://127.0.0.1:8082/o2o_usercenter_svc/remote/sysUserInfoService?req_sid=1612e430ddeb11e83d622c02b34132f7&syslogid=null"
    ]
  ],
  "URIPROTO": [
    [
      "http",
      null
    ]
  ],
  "USER": [
    [
      null,
      null
    ]
  ],
  "USERNAME": [
    [
      null,
      null
    ]
  ],
  "URIHOST": [
    [
      "127.0.0.1:8082",
      null
    ]
  ],
  "IPORHOST": [
    [
      "127.0.0.1",
      null
    ]
  ],
  "port": [
    [
      "8082",
      null
    ]
  ],
  "URIPATHPARAM": [
    [
      "/o2o_usercenter_svc/remote/sysUserInfoService?req_sid=1612e430ddeb11e83d622c02b34132f7&syslogid=null",
      null
    ]
  ],
  "URIPATH": [
    [
      "/o2o_usercenter_svc/remote/sysUserInfoService",
      null
    ]
  ],
  "URIPARAM": [
    [
      "?req_sid=1612e430ddeb11e83d622c02b34132f7&syslogid=null",
      null
    ]
  ],
  "QS": [
    [
      null
    ]
  ],
  "fire_wall_ip": [
    [
      ""10.46.148.155:9090""
    ]
  ],
  "referrer": [
    [
      null
    ]
  ]
}