Skip to content
This repository has been archived by the owner on Jun 8, 2023. It is now read-only.

Commit

Permalink
Migration of dashboards, template, and pipelines to 7.x
Browse files Browse the repository at this point in the history
  • Loading branch information
Andrew Selden committed Dec 7, 2019
1 parent e6b2e61 commit df9b3a5
Show file tree
Hide file tree
Showing 7 changed files with 250 additions and 2,098 deletions.
2 changes: 1 addition & 1 deletion aws/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ dependencies {
'com.amazonaws:aws-lambda-java-core:1.1.0',
'com.amazonaws:aws-lambda-java-events:1.1.0',
'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.3.1',
'org.apache.logging.log4j:log4j-core:2.11.2'
'org.apache.logging.log4j:log4j-core:2.11.2',
)
}

Expand Down
84 changes: 83 additions & 1 deletion conf/cloudflare-index-template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,39 @@
],
"mappings": {
"properties": {
"observer.vendor": {
"type": "keyword"
},
"observer.type": {
"type": "keyword"
},
"ecs.version": {
"type": "keyword"
},
"event.dataset": {
"type": "keyword"
},
"cloudflare.waf.flags": {
"type": "keyword"
},
"cloudflare.firewall.matches.actions" : {
"type": "keyword"
},
"cloudflare.firewall.matches.rule_ids" : {
"type": "keyword"
},
"cloudflare.firewall.matches.sources" : {
"type": "keyword"
},
"cloudflare.edge.colo.code" : {
"type": "keyword"
},
"cloudflare.client.ssl.protocol": {
"type": "keyword"
},
"cloudflare.client.request.protocol" : {
"type": "keyword"
},
"cloudflare.waf.rule.id": {
"type": "keyword"
},
Expand Down Expand Up @@ -59,12 +86,21 @@
},
"region_code": {
"type": "keyword"
},
"continent_name": {
"type": "keyword"
},
"region_iso_code": {
"type": "keyword"
}
}
},
"cloudflare.origin.response.time": {
"type": "long"
},
"cloudflare.origin.ip": {
"type": "ip"
},
"@version": {
"type": "keyword"
},
Expand Down Expand Up @@ -98,6 +134,51 @@
"user_agent.original": {
"type": "keyword"
},
"user_agent.device.name": {
"type": "keyword"
},
"user_agent.name": {
"type": "keyword"
},
"user_agent.os.full": {
"type": "keyword"
},
"user_agent.os.kernel": {
"type": "keyword"
},
"user_agent.os.platform": {
"type": "keyword"
},
"user_agent.os.name": {
"type": "keyword"
},
"user_agent.os.version": {
"type": "keyword"
},
"user_agent.version": {
"type": "keyword"
},
"user_agent.major": {
"type": "keyword"
},
"user_agent.minor": {
"type": "keyword"
},
"user_agent.patch": {
"type": "keyword"
},
"user_agent.build": {
"type": "keyword"
},
"user_agent.os_name": {
"type": "keyword"
},
"user_agent.os_major": {
"type": "keyword"
},
"user_agent.os_minor": {
"type": "keyword"
},
"cloudflare.edge.response.compression_ratio": {
"type": "float"
},
Expand Down Expand Up @@ -258,7 +339,8 @@
"settings": {
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
"number_of_replicas": "1",
"mapping.ignore_malformed": true
}
}
}
18 changes: 16 additions & 2 deletions conf/cloudflare-ingest-pipeline-daily.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,9 @@
}
},
{
"remove" : {
"rename": {
"field": "ClientRequestProtocol",
"target_field": "cloudflare.client.request.protocol",
"ignore_missing": true
}
},
Expand Down Expand Up @@ -445,13 +446,25 @@
"ignore_missing": true
}
},
{
"remove" : {
"if" : "ctx.OriginResponseHTTPExpires == ''",
"field" : "OriginResponseHTTPExpires"
}
},
{
"rename": {
"field": "OriginResponseHTTPExpires",
"target_field": "cloudflare.origin.response.http.expires",
"ignore_missing": true
}
},
{
"remove" : {
"if" : "ctx.OriginResponseHTTPLastModified == ''",
"field" : "OriginResponseHTTPLastModified"
}
},
{
"rename": {
"field": "OriginResponseHTTPLastModified",
Expand Down Expand Up @@ -579,4 +592,5 @@
}
}
]
}
}

24 changes: 19 additions & 5 deletions conf/cloudflare-ingest-pipeline-weekly.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,9 @@
}
},
{
"remove": {
"rename": {
"field": "ClientRequestProtocol",
"target_field": "cloudflare.client.request.protocol",
"ignore_missing": true
}
},
Expand Down Expand Up @@ -229,8 +230,8 @@
}
},
{
"user_agent": {
"field": "ClientRequestUserAgent",
"user_agent" : {
"field" : "ClientRequestUserAgent",
"target_field": "user_agent",
"ecs": true,
"ignore_missing": true
Expand All @@ -243,7 +244,7 @@
}
},
{
"remove": {
"remove" : {
"field": "ClientRequestUserAgent",
"ignore_missing": true
}
Expand Down Expand Up @@ -445,13 +446,25 @@
"ignore_missing": true
}
},
{
"remove" : {
"if" : "ctx.OriginResponseHTTPExpires == ''",
"field" : "OriginResponseHTTPExpires"
}
},
{
"rename": {
"field": "OriginResponseHTTPExpires",
"target_field": "cloudflare.origin.response.http.expires",
"ignore_missing": true
}
},
{
"remove" : {
"if" : "ctx.OriginResponseHTTPLastModified == ''",
"field" : "OriginResponseHTTPLastModified"
}
},
{
"rename": {
"field": "OriginResponseHTTPLastModified",
Expand Down Expand Up @@ -579,4 +592,5 @@
}
}
]
}
}

43 changes: 43 additions & 0 deletions conf/logstash-cloudflare.conf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
input {
stdin {}
}

filter {
json {
source => "message"
}
geoip {
source => "ClientIP"
target => "geoip"
}
mutate {
rename => ["geoip", "ClientIP" ]
}

date {
match => [ "EdgeStartTimestamp", "yyyy-MM-dd'T'HH:mm:ssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ" ]
timezone => "UTC"
target => "@timestamp"
}
date {
match => [ "EdgeStartTimestamp", "yyyy-MM-dd'T'HH:mm:ssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ" ]
timezone => "UTC"
target => "EdgeStartTimestamp"
}
useragent {
source => "ClientRequestUserAgent"
target => "UserAgent"
}
}

output {
stdout {
codec => rubydebug
}
elasticsearch {
hosts => [" < YOUR ELASTICSEARCH HOSTS GO HERE > "]
index => "cloudflare-%{+YYYY.MM.dd}"
user => "elastic"
password => "< YOUR ELASTICSEARCH PASSWORD GOES HERE >"
}
}

0 comments on commit df9b3a5

Please sign in to comment.