Proatividade na análise de logs com Elasticsearch...

Proatividade na análise de logs com

Elasticsearch, Logstash e Kibana

Olá!Leonardo Comelli | @leocomelli

64.242.88.10 - - [07/Mar/2004:16:05:49 -0800] "GET /twiki/bin/edit/Main/Double_bounce_sender?topicparent=Main.ConfigurationVariables HTTP/1.1" 401 1284664.242.88.10 - - [07/Mar/2004:16:06:51 -0800] "GET /twiki/bin/rdiff/TWiki/NewUserTemplate?rev1=1.3&rev2=1.2 HTTP/1.1" 200 452364.242.88.10 - - [07/Mar/2004:16:10:02 -0800] "GET /mailman/listinfo/hsdivision HTTP/1.1" 200 629164.242.88.10 - - [07/Mar/2004:16:11:58 -0800] "GET /twiki/bin/view/TWiki/WikiSyntax HTTP/1.1" 200 735264.242.88.10 - - [07/Mar/2004:16:20:55 -0800] "GET /twiki/bin/view/Main/DCCAndPostFix HTTP/1.1" 200 525364.242.88.10 - - [07/Mar/2004:16:23:12 -0800] "GET /twiki/bin/oops/TWiki/AppendixFileSystem?template=oopsmore¶m1=1.12¶m2=1.12 HTTP/1.1" 200 1138264.242.88.10 - - [07/Mar/2004:16:24:16 -0800] "GET /twiki/bin/view/Main/PeterThoeny HTTP/1.1" 200 492464.242.88.10 - - [07/Mar/2004:16:29:16 -0800] "GET /twiki/bin/edit/Main/Header_checks?topicparent=Main.ConfigurationVariables HTTP/1.1" 401 1285164.242.88.10 - - [07/Mar/2004:16:30:29 -0800] "GET /twiki/bin/attach/Main/OfficeLocations HTTP/1.1" 401 1285164.242.88.10 - - [07/Mar/2004:16:31:48 -0800] "GET /twiki/bin/view/TWiki/WebTopicEditTemplate HTTP/1.1" 200 373264.242.88.10 - - [07/Mar/2004:16:32:50 -0800] "GET /twiki/bin/view/Main/WebChanges HTTP/1.1" 200 4052064.242.88.10 - - [07/Mar/2004:16:33:53 -0800] "GET /twiki/bin/edit/Main/Smtpd_etrn_restrictions?topicparent=Main.ConfigurationVariables HTTP/1.1" 401 1285164.242.88.10 - - [07/Mar/2004:16:35:19 -0800] "GET /mailman/listinfo/business HTTP/1.1" 200 637964.242.88.10 - - [07/Mar/2004:16:36:22 -0800] "GET /twiki/bin/rdiff/Main/WebIndex?rev1=1.2&rev2=1.1 HTTP/1.1" 200 4637364.242.88.10 - - [07/Mar/2004:16:37:27 -0800] "GET /twiki/bin/view/TWiki/DontNotify HTTP/1.1" 200 414064.242.88.10 - - [07/Mar/2004:16:39:24 -0800] "GET /twiki/bin/view/Main/TokyoOffice HTTP/1.1" 200 385364.242.88.10 - - [07/Mar/2004:16:43:54 -0800] "GET /twiki/bin/view/Main/MikeMannix HTTP/1.1" 200 368664.242.88.10 - - [07/Mar/2004:16:45:56 -0800] "GET /twiki/bin/attach/Main/PostfixCommands HTTP/1.1" 401 1284664.242.88.10 - - [07/Mar/2004:16:47:12 -0800] "GET /robots.txt HTTP/1.1" 200 6864.242.88.10 - - [07/Mar/2004:16:47:46 -0800] "GET /twiki/bin/rdiff/Know/ReadmeFirst?rev1=1.5&rev2=1.4 HTTP/1.1" 200 572464.242.88.10 - - [07/Mar/2004:16:49:04 -0800] "GET /twiki/bin/view/Main/TWikiGroups?rev=1.2 HTTP/1.1" 200 516264.242.88.10 - - [07/Mar/2004:16:50:54 -0800] "GET /twiki/bin/rdiff/Main/ConfigurationVariables HTTP/1.1" 200 5967964.242.88.10 - - [07/Mar/2004:16:52:35 -0800] "GET /twiki/bin/edit/Main/Flush_service_name?topicparent=Main.ConfigurationVariables HTTP/1.1" 401 1285164.242.88.10 - - [07/Mar/2004:16:53:46 -0800] "GET /twiki/bin/rdiff/TWiki/TWikiRegistration HTTP/1.1" 200 3439564.242.88.10 - - [07/Mar/2004:16:54:55 -0800] "GET /twiki/bin/rdiff/Main/NicholasLee HTTP/1.1" 200 723564.242.88.10 - - [07/Mar/2004:16:56:39 -0800] "GET /twiki/bin/view/Sandbox/WebHome?rev=1.6 HTTP/1.1" 200 854564.242.88.10 - - [07/Mar/2004:16:58:54 -0800] "GET /mailman/listinfo/administration HTTP/1.1" 200 6459lordgun.org - - [07/Mar/2004:17:01:53 -0800] "GET /razor.html HTTP/1.1" 200 286964.242.88.10 - - [07/Mar/2004:17:09:01 -0800] "GET /twiki/bin/search/Main/SearchResult?scope=text®ex=on&search=Joris%20*Benschop[^A-Za-z] HTTP/1.1" 200 428464.242.88.10 - - [07/Mar/2004:17:10:20 -0800] "GET /twiki/bin/oops/TWiki/TextFormattingRules?template=oopsmore¶m1=1.37¶m2=1.37 HTTP/1.1" 200 1140064.242.88.10 - - [07/Mar/2004:17:13:50 -0800] "GET /twiki/bin/edit/TWiki/DefaultPlugin?t=1078688936 HTTP/1.1" 401 1284664.242.88.10 - - [07/Mar/2004:17:16:00 -0800] "GET /twiki/bin/search/Main/?scope=topic®ex=on&search=^g HTTP/1.1" 200 367564.242.88.10 - - [07/Mar/2004:17:17:27 -0800] "GET /twiki/bin/search/TWiki/?scope=topic®ex=on&search=^d HTTP/1.1" 200 5773lj1036.inktomisearch.com - - [07/Mar/2004:17:18:36 -0800] "GET /robots.txt HTTP/1.0" 200 68lj1090.inktomisearch.com - - [07/Mar/2004:17:18:41 -0800] "GET /twiki/bin/view/Main/LondonOffice HTTP/1.0" 200 386064.242.88.10 - - [07/Mar/2004:17:21:44 -0800] "GET /twiki/bin/attach/TWiki/TablePlugin HTTP/1.1" 401 1284664.242.88.10 - - [07/Mar/2004:17:22:49 -0800] "GET /twiki/bin/view/TWiki/ManagingWebs?rev=1.22 HTTP/1.1" 200 931064.242.88.10 - - [07/Mar/2004:17:23:54 -0800] "GET /twiki/bin/statistics/Main HTTP/1.1" 200 80864.242.88.10 - - [07/Mar/2004:17:26:30 -0800] "GET /twiki/bin/view/TWiki/WikiCulture HTTP/1.1" 200 593564.242.88.10 - - [07/Mar/2004:17:27:37 -0800] "GET /twiki/bin/edit/Main/WebSearch?t=1078669682 HTTP/1.1" 401 1284664.242.88.10 - - [07/Mar/2004:17:28:45 -0800] "GET /twiki/bin/oops/TWiki/ResetPassword?template=oopsmore¶m1=1.4¶m2=1.4 HTTP/1.1" 200 1128164.242.88.10 - - [07/Mar/2004:17:29:59 -0800] "GET /twiki/bin/view/TWiki/ManagingWebs?skin=print HTTP/1.1" 200 880664.242.88.10 - - [07/Mar/2004:17:31:39 -0800] "GET /twiki/bin/edit/Main/UvscanAndPostFix?topicparent=Main.WebHome HTTP/1.1" 401 1284664.242.88.10 - - [07/Mar/2004:17:35:35 -0800] "GET /twiki/bin/view/TWiki/KlausWriessnegger HTTP/1.1" 200 384864.242.88.10 - - [07/Mar/2004:17:39:39 -0800] "GET /twiki/bin/view/Main/SpamAssassin HTTP/1.1" 200 408164.242.88.10 - - [07/Mar/2004:17:42:15 -0800] "GET /twiki/bin/oops/TWiki/RichardDonkin?template=oopsmore¶m1=1.2¶m2=1.2 HTTP/1.1" 200 1128164.242.88.10 - - [07/Mar/2004:17:46:17 -0800] "GET /twiki/bin/rdiff/TWiki/AlWilliams?rev1=1.3&rev2=1.2 HTTP/1.1" 200 448564.242.88.10 - - [07/Mar/2004:17:47:43 -0800] "GET /twiki/bin/rdiff/TWiki/AlWilliams?rev1=1.2&rev2=1.1 HTTP/1.1" 200 523464.242.88.10 - - [07/Mar/2004:17:50:44 -0800] "GET /twiki/bin/view/TWiki/SvenDowideit HTTP/1.1" 200 3616

log

log$ cat access.log | grep 401

log$ cat access.log | grep 404

tornando seu log útil…

200.164.237.13 - - [27/Aug/2015:12:37:38 -0300] "GET / HTTP/1.1" 200 763 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/601.1.39 (KHTML, like Gecko) Version/9.0 Safari/601.1.39"

LOGSTASH

LOGSTASH

ARMAZENAR

COLETAR DADOS DO LOG

MANIPULAR ENRIQUECER

LOGSTASH

ARMAZENAR



input

LOGSTASH

ARMAZENAR



input

Filter

LOGSTASH

ARMAZENAR



input

Filter

output

LOGSTASHinput { stdin{}}

filter { mutate { add_field => {“_type” => “test” } }}

output { stdout { codec => rubydebug }}

LOGSTASH

$ echo "qconrio 2015" | ./logstash/bin/logstash -f sample.conf

Logstash startup completed{ "message" => "qconrio 2015", "@version" => "1", "@timestamp" => "2015-08-24T03:41:13.956Z", "host" => "241191a9debd", "_type" => "meudoc"}Logstash shutdown completed

LOGSTASH

input filter output

file

syslog

log4j

date

grok

geoip

S3

kafkaES

https://goo.gl/AbhrMihttps://goo.gl/2ofebshttps://goo.gl/oo7fMr

https://goo.gl/AbhrMi

https://goo.gl/2ofebs

https://goo.gl/oo7fMr

ELASTICSEARCH

ELASTICSEARCH

DADOS E ANÁLISE EM TEMPO REAL ALTA DISPONIBILIDADE

MULTI-TENANCY FULL TEXT SEARCH

ORIENTADO A DOCUMENTOS SCHEMA FREE

RESTFUL API PERSISTÊNCIA POR OPERAÇÃO

ELASTICSEARCH

BD Relacional Elasticsearchdatabase indextable type

row documentcolumn fieldschema mappingpartition shard

ELASTICSEARCH

$ curl -X PUT http://localhost:9200/qcon/talk/1 -d ‘{ “name" : “Proatividade na analise de log com ELK”, “date" : “2015-08-27T16:45:00”, “city" : “Rio de Janeiro”}’

adicionar endpoint indice tipo id

documento

ELASTICSEARCH

$ curl -X GET http://localhost:9200/qcon/talk/1

obter endpoint indice tipo id

ELASTICSEARCHLOGSTASH

ELASTICSEARCH

input { file{ path => “/var/log/apache2/access.log" }}

filter { grok { match => { "message" => "%{COMBINEDAPACHELOG}" } }}

output { elasticsearch { host => localhost }}

LOGSTASH

ELASTICSEARCH

$ curl -X GET http://localhost:9200/logstash-*/_count

obter endpoint indice action

logstash-%{+YYYY.MM.dd}

http://qcon.leo.sh

ELASTICSEARCH{

"_index" : "logstash-2015.08.25", "_type" : "logs", "_id" : "AU9ik9_koi5WviutsXW2", "_score" : 1.0, “_source":{

"message":"186.194.65.168 - - [25/Aug/2015:01:58:21 +0000] \"GET /icons/ubuntu-logo.png HTTP/1.1\" 200 3688 \"http://qcon.leo.sh/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/601.1.39 (KHTML, like Gecko) Version/9.0 Safari/601.1.39\”",“@version":"1",“@timestamp":"2015-08-25T01:58:21.000Z",“host":"ip-172-31-31-206",“path":"/var/log/apache2/access.log",“clientip":"186.194.65.168",“ident":"-",“auth":"-","timestamp":"25/Aug/2015:01:58:21 +0000”,“verb":"GET",“request":"/icons/ubuntu-logo.png",“httpversion":"1.1",“response":"200",“bytes":"3688",“referrer":"\"http://qcon.leo.sh/\"","agent":"\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/601.1.39 (KHTML, like Gecko) Version/9.0 Safari/601.1.39\""}

}

um pouco mais de dados…

input { ...}

filter { ... geoip { source => "clientip" target => "geoip" database => "/opt/logstash/GeoLiteCity.dat" add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] }

mutate { convert => [ "[geoip][coordinates]", "float"] }}

output { ...}

ELASTICSEARCH LOGSTASH

ELASTICSEARCH

{ "_index" : "logstash-2015.08.25", "_type" : "logs", "_id" : "AU9ik9_koi5WviutsXW2", "_score" : 1.0, “_source”:{

...“geoip":{“ip":"186.194.65.168",“country_code2":"BR",“country_code3":"BRA",“country_name":"Brazil",“continent_code":"SA",“latitude":-23.547699999999992,“longitude":-46.63579999999999,“location":[-46.63579999999999,-23.547699999999992],“coordinates”:[-46.63579999999999,-23.547699999999992]

} }

KIBANA

Dashboard personalizados

Interface flexíveis

Exportar dados com facilidade

Análises sofisticadas

KIBANA

ambiente inicial

ambiente atual

ORGANIZE OS LOGS

VERIFIQUE O QUE É RELEVANTE

ENRIQUEÇA AS INFORMAÇÕES

FAÇA A ANÁLISE

centralização não é tudo!

obrigado.@leocomelli

Proatividade na análise de logs com Elasticsearch...

Documents

Transcript of Proatividade na análise de logs com Elasticsearch...