1. FINDOLOGIC GmbH
Boring Logfiles → Sexy Dashboards

2. Boring Logfiles
178.236.7.219 - - [27/Oct/2014:06:43:13 +0100] "GET
/ps/amazon.de/index.php?shopkey=DEADBEEF&shopurl=https
%3A%2F%2Fwww.amazon.de
%2F&multishop_id=0&userip=173.194.112.2&referer=https
%3A%2F%2Fwww.amazon.de%2FTechnik
%2FFernseher&count=0&group[]=2&service=search&query=le
d HTTP/1.1" 200 12363 "-" "-"

3. Sexy Dashboards

4. Stack
● Elasticsearch
● Logstash
● Kibana

5. Infrastruktur
App 1 App 2 App 3
Logstash 1 Logstash 2 Logstash 3
ElasticSearch
Kibana

6. Logstash Architektur
Input Filter Output
● Grok
● Search / Replace
● Split
● GeoIP
● DNS
● URL-Decode
● ...
● ElasticSearch
● Redis
● CouchDB
● ...
● Pipe
● Socket
● File
● ...

7. Logstash stand-alone
● Logs zentralisieren
● In DB speichern
● Durchsuchbar machen

8. Setup
● Logstash herunterladen, extrahieren
● Elasticsearch installieren
● fertig

9. Georg?
● CTO bei FINDOLOGIC GmbH
● http://www.findologic.com
● g.sorst@findologic.com
● @piefke_schorsch

10. Hands on
● Grok
● kv
● Query
● Endpoint
● GeoIP

11. Logstash.conf
input {
file {
path => "/home/georg/coding/logstash-apache/access.log"
start_position => beginning
}
#pipe {
# command => "ssh frontend1.findologic.com tail -f /var/log/apache2/access.log"
#}
}
filter {
grok {
match => { "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}" }
}
# split the query string: query=test&count=0&first=10 → request_query: test, request_count: 0, request_first: 10
kv {
field_split => "&?"
source => "request"
prefix => "request_"
}
# urldecode does not decode the + into whitespace so do this manually
mutate {
gsub => [
"request_query", "+", " "
]
}
# URL-decode the query so it is easier to read. We don't really about the other request_* parameters
urldecode {
field => "request_query"
}
# extract the script file, eg. index.php
mutate {
gsub => [
# Remove the query string including the ? : /path/index.php?key=value -> /path/index.php
"request", "?.*$", "",
# Remove everything up to the last / : /path/index.php -> index.php
"request", "^.*/", ""
]
}
# Default to index.php if no script file is given
if [request] == "" {
mutate {
replace => [ "request", "index.php" ]
}
}
# If the IP is syntactically valid resolve it geographically
if [request_userip] =~ "^d+.d+.d+.d+$" {
geoip {
source => "request_userip"
target => "client_geoip"
}
}
}
output {
elasticsearch {
host => localhost
}
#stdout { codec => rubydebug }
}

12. Kibana Dashboard
{
"title": "Logstash Search",
"services": {
"query": {
"list": {
"0": {
"query": "*",
"alias": "",
"color": "#7EB26D",
"id": 0,
"pin": false,
"type": "lucene",
"enable": true
},
"1": {
"id": 1,
"color": "#EAB839",
"alias": "Suche",
"pin": false,
"type": "lucene",
"enable": true,
"query": "request:index.php"
},
"2": {
"id": 2,
"color": "#6ED0E0",
"alias": "Autocomplete",
"pin": false,
"type": "lucene",
"enable": true,
"query": "request:autocomplete.php"
},
"3": {
"id": 3,
"color": "#EF843C",
"alias": "",
"pin": false,
"type": "lucene",
"enable": true,
"query": "client_geoip.country_code2:DE OR client_geoip.country_code2:AT OR client_geoip.country_code2:CH"
}
},
"ids": [
0,
1,
2,
3
]
},
"filter": {
"list": {
"0": {
"type": "time",
"field": "@timestamp",
"from": "now-24h",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
},
"1": {
"type": "time",
"from": "2014-10-27T18:04:33.357Z",
"to": "2014-10-27T19:18:47.459Z",
"field": "@timestamp",
"mandate": "must",
"active": true,
"alias": "",
"id": 1
}
},
"ids": [
0,
1
]
}
},
"rows": [
{
"title": "Graph",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "histogram",
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"auto_int": true,
"resolution": 100,
"interval": "30s",
"fill": 3,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": true,
"stack": true,
"points": false,
"lines": false,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"queries": {
"mode": "selected",
"ids": [
1,
2
]
},
"title": "Events over time",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": true
},
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"pointradius": 5,
"show_query": true,
"legend_counts": true,
"zerofill": true,
"derivative": false
}
],
"notice": false
},
{
"title": "",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 6,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "request_query",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "pie",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": ""
},
{
"error": false,
"span": 6,
"editable": true,
"type": "bettermap",
"loadingEditor": false,
"field": "client_geoip.location",
"size": 1000,
"spyable": true,
"tooltip": "_id",
"queries": {
"mode": "selected",
"ids": [
3
]
}
}
],
"notice": false
},
{
"title": "Events",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"title": "All events",
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"@timestamp",
"desc"
],
"style": {
"font-size": "9pt"
},
"overflow": "min-height",
"fields": [],
"localTime": true,
"timeField": "@timestamp",
"highlight": [],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3
]
},
"field_list": true,
"status": "Stable",
"trimFactor": 300,
"normTimes": true,
"all_fields": false
}
],
"notice": false
}
],
"editable": true,
"failover": false,
"index": {
"interval": "day",
"pattern": "[logstash-]YYYY.MM.DD",
"default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED",
"warm_fields": true
},
"style": "light",
"panel_hints": true,
"pulldowns": [
{
"type": "query",
"collapse": false,
"notice": false,
"query": "*",
"pinned": true,
"history": [
"client_geoip.country_code2:DE OR client_geoip.country_code2:AT OR client_geoip.country_code2:CH",
"request:autocomplete.php",
"request:index.php",
"*"
],
"remember": 10,
"enable": true
},
{
"type": "filtering",
"collapse": false,
"notice": true,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"now": false,
"filter_id": 0,
"enable": true
}
],
"loader": {
"save_gist": false,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"refresh": false
}
Vergrößern zum Anzeigen