diff --git a/S3-API-Audit-log.md b/S3-API-Audit-log.md index 63dec74..8adab8e 100644 --- a/S3-API-Audit-log.md +++ b/S3-API-Audit-log.md @@ -26,3 +26,55 @@ {"requester":"bennu","host_id":"api-698ccd9645-g8fht","status":200,"time":1639395992,"operation":"REST.GET.OBJECT","remote_ip":"10.106.70.45","signature_version":"SigV4","bucket":"bennu-files","user_agent":"Python/3.8 aiohttp/3.6.2","key":"/2021/12/13/69f82cd8-ff31-476d-aa53-5e1e2109b84c","request_id":"570ceb8d3b8c31d51070910a78b26045","host_header":"bennu-files.s3-proxy.svc","error_code":""} ``` +### How to ingest log ? + +#### Fluent + +#### Logstash + +logstash.conf: +``` + filter { + if [tags][0] and [tags][0] =~ /s3.access/ { + ruby { + code => 'event.set("environment", ((event.get("tags").first).split(".")).first)' + add_field => { "[@metadata][input_type]" => "s3.access" } + remove_field => [ host, "@timestamp", "@version", port, tags ] + } + } + if ![environment] or [environment] == "" { + mutate { + replace => { "environment" => "unknown" } + } + } + } + input { + tcp { + codec => fluent + port => 24224 + } + } + output { + if [@metadata][input_type] == "s3.access" { + clickhouse { + headers => ["Authorization", "Basic ${CLICKHOUSE_BASIC_AUTH}"] + http_hosts => ["${CLICKHOUSE_URL}", "${CLICKHOUSE_URL}"] + table => "${CLICKHOUSE_TABLE}" + flush_size => 1000 + pool_max => 1000 + idle_flush_time => 5 + backoff_time => 3 + request_tolerance => 5 + automatic_retries => 1 + save_on_failure => true + save_dir => "${CLICKHOUSE_SAVE_DIR}" + date_time_input_format => "best_effort" + skip_unknown => "1" + id => "clickhouse" + } + } + } +``` + + +