From c0882c33fc2f37513f9d423c9e0f1244db16d096 Mon Sep 17 00:00:00 2001 From: Tristan Deloche Date: Thu, 28 Jan 2021 23:43:32 +0000 Subject: [PATCH] Fix and extend docker-compose and grafana dashboard setup --- CHANGELOG.md | 3 + docker/docker-compose.yml | 4 +- .../grafana/dashboards/mangadex_at_home.json | 194 +++++++++++++----- src/main/kotlin/mdnet/ServerManager.kt | 38 ++-- 4 files changed, 174 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f26955c..24c9975 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- [2021-01-28] Add distinct timerange and total egress stats to dashboard [@_tde9] +- [2021-01-28] Add average cache hitrate panel [@_tde9] ### Changed - [2021-01-29] Add HikariCP connection pool [@carbotaniuman]. @@ -15,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed ### Fixed +- [2021-01-28] Fix client reqs dashboard panel [@_tde9] ### Security diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4b28166..9fff097 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,6 +11,7 @@ services: - ./settings.yaml:/mangahome/settings.yaml:ro - ./data/cache/:/mangahome/data/ environment: + # If your client is intended to do more than ~100rqps, it is recommended replacing -XX:+UseG1GC with -XX:+UseShenandoahGC JAVA_TOOL_OPTIONS: "-Xms1G -Xmx1G -XX:+UseG1GC -Xss512K" privileged: true command: [ @@ -68,6 +69,3 @@ services: options: max-size: "20m" max-file: "2" - -networks: - mangadex-at-home: { } diff --git a/docker/grafana/dashboards/mangadex_at_home.json b/docker/grafana/dashboards/mangadex_at_home.json index 34a22ff..ed16bbf 100644 --- a/docker/grafana/dashboards/mangadex_at_home.json +++ b/docker/grafana/dashboards/mangadex_at_home.json @@ -17,7 +17,7 @@ "editable": false, "gnetId": null, "graphTooltip": 1, - "iteration": 1611371909784, + "iteration": 1611971739196, "links": [], "panels": [ { @@ -468,7 +468,7 @@ "overrides": [] }, "gridPos": { - "h": 2, + "h": 3, "w": 5, "x": 0, "y": 8 @@ -542,7 +542,7 @@ "overrides": [] }, "gridPos": { - "h": 2, + "h": 3, "w": 2, "x": 5, "y": 8 @@ -612,7 +612,7 @@ "overrides": [] }, "gridPos": { - "h": 2, + "h": 3, "w": 2, "x": 7, "y": 8 @@ -682,7 +682,7 @@ "overrides": [] }, "gridPos": { - "h": 2, + "h": 3, "w": 2, "x": 9, "y": 8 @@ -752,7 +752,7 @@ "overrides": [] }, "gridPos": { - "h": 2, + "h": 3, "w": 2, "x": 11, "y": 8 @@ -800,7 +800,6 @@ "filterable": false }, "mappings": [], - "max": 0.2, "min": 0, "thresholds": { "mode": "absolute", @@ -824,7 +823,7 @@ "overrides": [] }, "gridPos": { - "h": 2, + "h": 3, "w": 4, "x": 13, "y": 8 @@ -854,9 +853,9 @@ "displayAliasType": "Warning / Critical", "displayType": "Regular", "displayValueWithAlias": "Never", - "expr": "max(rate(jvm_gc_pause_seconds_sum[$itvl]) > 0 / rate(jvm_gc_pause_seconds_count[$itvl]))", + "expr": "sum(rate(jvm_gc_pause_seconds_sum[$itvl])) / sum(rate(process_uptime_seconds[$itvl]))", "interval": "", - "legendFormat": "{{ kubernetes_pod_name }}", + "legendFormat": "", "refId": "A", "units": "none", "valueHandler": "Number Threshold" @@ -864,7 +863,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Time spend in GC pause", + "title": "Time lost to GC", "transparent": true, "type": "stat" }, @@ -894,7 +893,7 @@ "overrides": [] }, "gridPos": { - "h": 2, + "h": 3, "w": 3, "x": 17, "y": 8 @@ -905,8 +904,8 @@ "options": { "colorMode": "value", "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", + "justifyMode": "center", + "orientation": "vertical", "reduceOptions": { "calcs": [ "lastNotNull" @@ -914,7 +913,7 @@ "fields": "", "values": false }, - "textMode": "auto" + "textMode": "value_and_name" }, "pluginVersion": "7.3.7", "targets": [ @@ -927,15 +926,29 @@ "expr": "sum(increase(http_server_request_latency_seconds_count{status=~\"2..\", path=~\"(.*)data/{chapterHash}/{fileName}\"}[$__range]))", "instant": false, "interval": "", - "legendFormat": "", + "legendFormat": "Over timerange", "refId": "B", "units": "none", "valueHandler": "Number Threshold" + }, + { + "aggregation": "Last", + "decimals": 2, + "displayAliasType": "Warning / Critical", + "displayType": "Regular", + "displayValueWithAlias": "Never", + "expr": "sum(increase(http_server_request_latency_seconds_count{status=~\"2..\", path=~\"(.*)data/{chapterHash}/{fileName}\"}[1y]))", + "instant": false, + "interval": "", + "legendFormat": "Total (1y)", + "refId": "A", + "units": "none", + "valueHandler": "Number Threshold" } ], "timeFrom": null, "timeShift": null, - "title": "Images served", + "title": "Served images", "transparent": true, "type": "stat" }, @@ -965,7 +978,7 @@ "overrides": [] }, "gridPos": { - "h": 2, + "h": 3, "w": 4, "x": 20, "y": 8 @@ -977,7 +990,7 @@ "colorMode": "value", "graphMode": "none", "justifyMode": "auto", - "orientation": "horizontal", + "orientation": "vertical", "reduceOptions": { "calcs": [ "lastNotNull" @@ -985,7 +998,7 @@ "fields": "", "values": false }, - "textMode": "auto" + "textMode": "value_and_name" }, "pluginVersion": "7.3.7", "targets": [ @@ -998,15 +1011,29 @@ "expr": "sum(increase(client_sent_bytes_total[$__range]))", "instant": true, "interval": "", - "legendFormat": "", + "legendFormat": "Over timerange", "refId": "B", "units": "none", "valueHandler": "Number Threshold" + }, + { + "aggregation": "Last", + "decimals": 2, + "displayAliasType": "Warning / Critical", + "displayType": "Regular", + "displayValueWithAlias": "Never", + "expr": "sum(increase(client_sent_bytes_total[1y]))", + "instant": true, + "interval": "", + "legendFormat": "Total (1y)", + "refId": "A", + "units": "none", + "valueHandler": "Number Threshold" } ], "timeFrom": null, "timeShift": null, - "title": "Bytes sent", + "title": "Sent data", "transparent": true, "type": "stat" }, @@ -1017,7 +1044,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 12, "panels": [], @@ -1059,7 +1086,7 @@ "h": 14, "w": 8, "x": 0, - "y": 11 + "y": 12 }, "hiddenSeries": false, "id": 2, @@ -1167,6 +1194,7 @@ "Duration": "super-light-green", "IllegalStateException": "yellow", "Success": "green", + "TTFB": "super-light-blue", "Upstream failures": "dark-red" }, "bars": true, @@ -1185,7 +1213,7 @@ "h": 8, "w": 8, "x": 8, - "y": 11 + "y": 12 }, "hiddenSeries": false, "id": 13, @@ -1216,12 +1244,15 @@ "renderer": "flot", "seriesOverrides": [ { + "$$hashKey": "object:158", "alias": "Success", "color": "#37872D" }, { + "$$hashKey": "object:159", "alias": "TTFB", "bars": false, + "color": "#C0D8FF", "fill": 0, "lines": true, "linewidth": 2, @@ -1234,7 +1265,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(http_client_request_count_total{status=~\"2..\"}[$itvl]))", + "expr": "sum(rate(http_client_request_latency_seconds_count{status=~\"2..\"}[$itvl]))", "hide": false, "instant": false, "interval": "$itvl", @@ -1270,6 +1301,7 @@ }, "yaxes": [ { + "$$hashKey": "object:182", "format": "reqps", "label": null, "logBase": 1, @@ -1278,6 +1310,7 @@ "show": true }, { + "$$hashKey": "object:183", "format": "s", "label": null, "logBase": 1, @@ -1291,6 +1324,77 @@ "alignLevel": null } }, + { + "cacheTimeout": null, + "datasource": "Prometheus", + "description": "Average cache hitrate of the current timerange\n\nSplits timerange in blocks of [$itvl], and averages their averages", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "#EAB839", + "value": 0.75 + }, + { + "color": "dark-green", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 54, + "interval": null, + "links": [], + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.7", + "targets": [ + { + "expr": "sum(rate(http_server_request_latency_seconds_count{path=~\"(.*)data/{chapterHash}/{fileName}\", cache=\"HIT\"}[$itvl])) / sum(rate(http_server_request_latency_seconds_count{path=~\"(.*)data/{chapterHash}/{fileName}\"}[$itvl]))", + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg hitrate over timerange", + "type": "stat" + }, { "aliasColors": { "ClientAbortException": "dark-red", @@ -1312,10 +1416,10 @@ "fill": 10, "fillGradient": 0, "gridPos": { - "h": 14, + "h": 10, "w": 8, "x": 16, - "y": 11 + "y": 16 }, "hiddenSeries": false, "id": 3, @@ -1325,6 +1429,7 @@ "current": true, "max": true, "min": true, + "rightSide": false, "show": true, "sort": "current", "sortDesc": true, @@ -1332,7 +1437,7 @@ "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 0, "nullPointMode": "null", "options": { "alertThreshold": true @@ -1374,6 +1479,7 @@ }, "yaxes": [ { + "$$hashKey": "object:314", "format": "reqps", "label": null, "logBase": 1, @@ -1382,12 +1488,13 @@ "show": true }, { - "format": "short", - "label": null, + "$$hashKey": "object:315", + "format": "percentunit", + "label": "hitrate", "logBase": 1, - "max": null, - "min": null, - "show": true + "max": "100", + "min": "0", + "show": false } ], "yaxis": { @@ -1421,7 +1528,7 @@ "h": 6, "w": 8, "x": 8, - "y": 19 + "y": 20 }, "hiddenSeries": false, "id": 49, @@ -1506,7 +1613,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 26 }, "id": 53, "panels": [ @@ -1531,7 +1638,7 @@ "h": 15, "w": 11, "x": 0, - "y": 26 + "y": 27 }, "hideEmpty": true, "hideZero": true, @@ -1609,7 +1716,7 @@ "h": 15, "w": 13, "x": 11, - "y": 26 + "y": 27 }, "hiddenSeries": false, "id": 41, @@ -1734,11 +1841,6 @@ "text": "3m", "value": "3m" }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, { "selected": false, "text": "10m", @@ -1785,7 +1887,7 @@ "value": "30d" } ], - "query": "2m,3m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "query": "2m,3m,10m,30m,1h,6h,12h,1d,7d,14d,30d", "queryValue": "", "refresh": 2, "skipUrlSync": false, @@ -1801,5 +1903,5 @@ "timezone": "", "title": "MangaDex@Home - Personal client dashboard", "uid": "a7sZAw2Mk", - "version": 27 -} \ No newline at end of file + "version": 4 +} diff --git a/src/main/kotlin/mdnet/ServerManager.kt b/src/main/kotlin/mdnet/ServerManager.kt index cc4cccf..40ac3ce 100644 --- a/src/main/kotlin/mdnet/ServerManager.kt +++ b/src/main/kotlin/mdnet/ServerManager.kt @@ -27,7 +27,8 @@ import mdnet.logging.info import mdnet.logging.warn import mdnet.metrics.DefaultMicrometerMetrics import mdnet.server.getServer -import mdnet.settings.* +import mdnet.settings.ClientSettings +import mdnet.settings.RemoteSettings import org.apache.hc.client5.http.config.RequestConfig import org.apache.hc.client5.http.cookie.StandardCookieSpec import org.apache.hc.client5.http.impl.classic.HttpClients @@ -36,6 +37,9 @@ import org.apache.hc.core5.util.TimeValue import org.apache.hc.core5.util.Timeout import org.http4k.client.ApacheClient import org.http4k.core.BodyMode +import org.http4k.core.then +import org.http4k.filter.ClientFilters +import org.http4k.filter.MicrometerMetrics import org.http4k.server.Http4kServer import org.slf4j.LoggerFactory import java.util.concurrent.CountDownLatch @@ -73,23 +77,25 @@ class ServerManager( .setMaxConnTotal(500) .setMaxConnPerRoute(500) .build() - private val apache = ApacheClient( - responseBodyMode = BodyMode.Stream, - client = HttpClients.custom() - .disableConnectionState() - .setDefaultRequestConfig( - RequestConfig.custom() - .setCookieSpec(StandardCookieSpec.IGNORE) - .setConnectTimeout(Timeout.ofSeconds(2)) - .setResponseTimeout(Timeout.ofSeconds(2)) - .setConnectionRequestTimeout(Timeout.ofSeconds(1)) + private val apache = ClientFilters.MicrometerMetrics.RequestCounter(registry) + .then(ClientFilters.MicrometerMetrics.RequestTimer(registry)) + .then( + ApacheClient( + responseBodyMode = BodyMode.Stream, + client = HttpClients.custom() + .disableConnectionState() + .setDefaultRequestConfig( + RequestConfig.custom() + .setCookieSpec(StandardCookieSpec.IGNORE) + .setConnectTimeout(Timeout.ofSeconds(2)) + .setResponseTimeout(Timeout.ofSeconds(2)) + .setConnectionRequestTimeout(Timeout.ofSeconds(1)) + .build() + ) + .setConnectionManager(connectionManager) .build() ) - .setConnectionManager( - connectionManager - ) - .build() - ) + ) // state that must only be accessed from the thread on the executor private var state: State