Fix and extend docker-compose and grafana dashboard setup

CHANGELOG.md

@@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Added
+- [2021-01-28] Add distinct timerange and total egress stats to dashboard [@_tde9]
+- [2021-01-28] Add average cache hitrate panel [@_tde9]
 
 ### Changed
 - [2021-01-29] Add HikariCP connection pool [@carbotaniuman].
@@ -15,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed
 
 ### Fixed
+- [2021-01-28] Fix client reqs dashboard panel [@_tde9]
 
 ### Security
 

docker/docker-compose.yml

@@ -11,6 +11,7 @@ services:
       - ./settings.yaml:/mangahome/settings.yaml:ro
       - ./data/cache/:/mangahome/data/
     environment:
+      # If your client is intended to do more than ~100rqps, it is recommended replacing -XX:+UseG1GC with -XX:+UseShenandoahGC
       JAVA_TOOL_OPTIONS: "-Xms1G -Xmx1G -XX:+UseG1GC -Xss512K"
     privileged: true
     command: [
@@ -68,6 +69,3 @@ services:
       options:
         max-size: "20m"
         max-file: "2"
-
-networks:
-  mangadex-at-home: { }

docker/grafana/dashboards/mangadex_at_home.json

@@ -17,7 +17,7 @@
   "editable": false,
   "gnetId": null,
   "graphTooltip": 1,
-  "iteration": 1611371909784,
+  "iteration": 1611971739196,
   "links": [],
   "panels": [
     {
@@ -468,7 +468,7 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 2,
+        "h": 3,
         "w": 5,
         "x": 0,
         "y": 8
@@ -542,7 +542,7 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 2,
+        "h": 3,
         "w": 2,
         "x": 5,
         "y": 8
@@ -612,7 +612,7 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 2,
+        "h": 3,
         "w": 2,
         "x": 7,
         "y": 8
@@ -682,7 +682,7 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 2,
+        "h": 3,
         "w": 2,
         "x": 9,
         "y": 8
@@ -752,7 +752,7 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 2,
+        "h": 3,
         "w": 2,
         "x": 11,
         "y": 8
@@ -800,7 +800,6 @@
             "filterable": false
           },
           "mappings": [],
-          "max": 0.2,
           "min": 0,
           "thresholds": {
             "mode": "absolute",
@@ -824,7 +823,7 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 2,
+        "h": 3,
         "w": 4,
         "x": 13,
         "y": 8
@@ -854,9 +853,9 @@
           "displayAliasType": "Warning / Critical",
           "displayType": "Regular",
           "displayValueWithAlias": "Never",
-          "expr": "max(rate(jvm_gc_pause_seconds_sum[$itvl]) > 0 / rate(jvm_gc_pause_seconds_count[$itvl]))",
+          "expr": "sum(rate(jvm_gc_pause_seconds_sum[$itvl])) / sum(rate(process_uptime_seconds[$itvl]))",
           "interval": "",
-          "legendFormat": "{{ kubernetes_pod_name }}",
+          "legendFormat": "",
           "refId": "A",
           "units": "none",
           "valueHandler": "Number Threshold"
@@ -864,7 +863,7 @@
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Time spend in GC pause",
+      "title": "Time lost to GC",
       "transparent": true,
       "type": "stat"
     },
@@ -894,7 +893,7 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 2,
+        "h": 3,
         "w": 3,
         "x": 17,
         "y": 8
@@ -905,8 +904,8 @@
       "options": {
         "colorMode": "value",
         "graphMode": "none",
-        "justifyMode": "auto",
+        "justifyMode": "center",
-        "orientation": "horizontal",
+        "orientation": "vertical",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -914,7 +913,7 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "textMode": "value_and_name"
       },
       "pluginVersion": "7.3.7",
       "targets": [
@@ -927,15 +926,29 @@
           "expr": "sum(increase(http_server_request_latency_seconds_count{status=~\"2..\", path=~\"(.*)data/{chapterHash}/{fileName}\"}[$__range]))",
           "instant": false,
           "interval": "",
-          "legendFormat": "",
+          "legendFormat": "Over timerange",
           "refId": "B",
           "units": "none",
           "valueHandler": "Number Threshold"
+        },
+        {
+          "aggregation": "Last",
+          "decimals": 2,
+          "displayAliasType": "Warning / Critical",
+          "displayType": "Regular",
+          "displayValueWithAlias": "Never",
+          "expr": "sum(increase(http_server_request_latency_seconds_count{status=~\"2..\", path=~\"(.*)data/{chapterHash}/{fileName}\"}[1y]))",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "Total (1y)",
+          "refId": "A",
+          "units": "none",
+          "valueHandler": "Number Threshold"
         }
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Images served",
+      "title": "Served images",
       "transparent": true,
       "type": "stat"
     },
@@ -965,7 +978,7 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 2,
+        "h": 3,
         "w": 4,
         "x": 20,
         "y": 8
@@ -977,7 +990,7 @@
         "colorMode": "value",
         "graphMode": "none",
         "justifyMode": "auto",
-        "orientation": "horizontal",
+        "orientation": "vertical",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -985,7 +998,7 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "textMode": "value_and_name"
       },
       "pluginVersion": "7.3.7",
       "targets": [
@@ -998,15 +1011,29 @@
           "expr": "sum(increase(client_sent_bytes_total[$__range]))",
           "instant": true,
           "interval": "",
-          "legendFormat": "",
+          "legendFormat": "Over timerange",
           "refId": "B",
           "units": "none",
           "valueHandler": "Number Threshold"
+        },
+        {
+          "aggregation": "Last",
+          "decimals": 2,
+          "displayAliasType": "Warning / Critical",
+          "displayType": "Regular",
+          "displayValueWithAlias": "Never",
+          "expr": "sum(increase(client_sent_bytes_total[1y]))",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Total (1y)",
+          "refId": "A",
+          "units": "none",
+          "valueHandler": "Number Threshold"
         }
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Bytes sent",
+      "title": "Sent data",
       "transparent": true,
       "type": "stat"
     },
@@ -1017,7 +1044,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 10
+        "y": 11
       },
       "id": 12,
       "panels": [],
@@ -1059,7 +1086,7 @@
         "h": 14,
         "w": 8,
         "x": 0,
-        "y": 11
+        "y": 12
       },
       "hiddenSeries": false,
       "id": 2,
@@ -1167,6 +1194,7 @@
         "Duration": "super-light-green",
         "IllegalStateException": "yellow",
         "Success": "green",
+        "TTFB": "super-light-blue",
         "Upstream failures": "dark-red"
       },
       "bars": true,
@@ -1185,7 +1213,7 @@
         "h": 8,
         "w": 8,
         "x": 8,
-        "y": 11
+        "y": 12
       },
       "hiddenSeries": false,
       "id": 13,
@@ -1216,12 +1244,15 @@
       "renderer": "flot",
       "seriesOverrides": [
         {
+          "$$hashKey": "object:158",
           "alias": "Success",
           "color": "#37872D"
         },
         {
+          "$$hashKey": "object:159",
           "alias": "TTFB",
           "bars": false,
+          "color": "#C0D8FF",
           "fill": 0,
           "lines": true,
           "linewidth": 2,
@@ -1234,7 +1265,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(http_client_request_count_total{status=~\"2..\"}[$itvl]))",
+          "expr": "sum(rate(http_client_request_latency_seconds_count{status=~\"2..\"}[$itvl]))",
           "hide": false,
           "instant": false,
           "interval": "$itvl",
@@ -1270,6 +1301,7 @@
       },
       "yaxes": [
         {
+          "$$hashKey": "object:182",
           "format": "reqps",
           "label": null,
           "logBase": 1,
@@ -1278,6 +1310,7 @@
           "show": true
         },
         {
+          "$$hashKey": "object:183",
           "format": "s",
           "label": null,
           "logBase": 1,
@@ -1291,6 +1324,77 @@
         "alignLevel": null
       }
     },
+    {
+      "cacheTimeout": null,
+      "datasource": "Prometheus",
+      "description": "Average cache hitrate of the current timerange\n\nSplits timerange in blocks of [$itvl], and averages their averages",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": null,
+            "filterable": false
+          },
+          "mappings": [],
+          "max": 1,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "dark-red",
+                "value": null
+              },
+              {
+                "color": "#EAB839",
+                "value": 0.75
+              },
+              {
+                "color": "dark-green",
+                "value": 0.9
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 8,
+        "x": 16,
+        "y": 12
+      },
+      "id": 54,
+      "interval": null,
+      "links": [],
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.3.7",
+      "targets": [
+        {
+          "expr": "sum(rate(http_server_request_latency_seconds_count{path=~\"(.*)data/{chapterHash}/{fileName}\", cache=\"HIT\"}[$itvl])) / sum(rate(http_server_request_latency_seconds_count{path=~\"(.*)data/{chapterHash}/{fileName}\"}[$itvl]))",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "B"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Avg hitrate over timerange",
+      "type": "stat"
+    },
     {
       "aliasColors": {
         "ClientAbortException": "dark-red",
@@ -1312,10 +1416,10 @@
       "fill": 10,
       "fillGradient": 0,
       "gridPos": {
-        "h": 14,
+        "h": 10,
         "w": 8,
         "x": 16,
-        "y": 11
+        "y": 16
       },
       "hiddenSeries": false,
       "id": 3,
@@ -1325,6 +1429,7 @@
         "current": true,
         "max": true,
         "min": true,
+        "rightSide": false,
         "show": true,
         "sort": "current",
         "sortDesc": true,
@@ -1332,7 +1437,7 @@
         "values": true
       },
       "lines": true,
-      "linewidth": 1,
+      "linewidth": 0,
       "nullPointMode": "null",
       "options": {
         "alertThreshold": true
@@ -1374,6 +1479,7 @@
       },
       "yaxes": [
         {
+          "$$hashKey": "object:314",
           "format": "reqps",
           "label": null,
           "logBase": 1,
@@ -1382,12 +1488,13 @@
           "show": true
         },
         {
-          "format": "short",
+          "$$hashKey": "object:315",
-          "label": null,
+          "format": "percentunit",
+          "label": "hitrate",
           "logBase": 1,
-          "max": null,
+          "max": "100",
-          "min": null,
+          "min": "0",
-          "show": true
+          "show": false
         }
       ],
       "yaxis": {
@@ -1421,7 +1528,7 @@
         "h": 6,
         "w": 8,
         "x": 8,
-        "y": 19
+        "y": 20
       },
       "hiddenSeries": false,
       "id": 49,
@@ -1506,7 +1613,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 25
+        "y": 26
       },
       "id": 53,
       "panels": [
@@ -1531,7 +1638,7 @@
             "h": 15,
             "w": 11,
             "x": 0,
-            "y": 26
+            "y": 27
           },
           "hideEmpty": true,
           "hideZero": true,
@@ -1609,7 +1716,7 @@
             "h": 15,
             "w": 13,
             "x": 11,
-            "y": 26
+            "y": 27
           },
           "hiddenSeries": false,
           "id": 41,
@@ -1734,11 +1841,6 @@
             "text": "3m",
             "value": "3m"
           },
-          {
-            "selected": false,
-            "text": "5m",
-            "value": "5m"
-          },
           {
             "selected": false,
             "text": "10m",
@@ -1785,7 +1887,7 @@
             "value": "30d"
           }
         ],
-        "query": "2m,3m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
+        "query": "2m,3m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
         "queryValue": "",
         "refresh": 2,
         "skipUrlSync": false,
@@ -1801,5 +1903,5 @@
   "timezone": "",
   "title": "MangaDex@Home - Personal client dashboard",
   "uid": "a7sZAw2Mk",
-  "version": 27
+  "version": 4
 }

src/main/kotlin/mdnet/ServerManager.kt

@@ -27,7 +27,8 @@ import mdnet.logging.info
 import mdnet.logging.warn
 import mdnet.metrics.DefaultMicrometerMetrics
 import mdnet.server.getServer
-import mdnet.settings.*
+import mdnet.settings.ClientSettings
+import mdnet.settings.RemoteSettings
 import org.apache.hc.client5.http.config.RequestConfig
 import org.apache.hc.client5.http.cookie.StandardCookieSpec
 import org.apache.hc.client5.http.impl.classic.HttpClients
@@ -36,6 +37,9 @@ import org.apache.hc.core5.util.TimeValue
 import org.apache.hc.core5.util.Timeout
 import org.http4k.client.ApacheClient
 import org.http4k.core.BodyMode
+import org.http4k.core.then
+import org.http4k.filter.ClientFilters
+import org.http4k.filter.MicrometerMetrics
 import org.http4k.server.Http4kServer
 import org.slf4j.LoggerFactory
 import java.util.concurrent.CountDownLatch
@@ -73,23 +77,25 @@ class ServerManager(
         .setMaxConnTotal(500)
         .setMaxConnPerRoute(500)
         .build()
-    private val apache = ApacheClient(
+    private val apache = ClientFilters.MicrometerMetrics.RequestCounter(registry)
-        responseBodyMode = BodyMode.Stream,
+        .then(ClientFilters.MicrometerMetrics.RequestTimer(registry))
-        client = HttpClients.custom()
+        .then(
-            .disableConnectionState()
+            ApacheClient(
-            .setDefaultRequestConfig(
+                responseBodyMode = BodyMode.Stream,
-                RequestConfig.custom()
+                client = HttpClients.custom()
-                    .setCookieSpec(StandardCookieSpec.IGNORE)
+                    .disableConnectionState()
-                    .setConnectTimeout(Timeout.ofSeconds(2))
+                    .setDefaultRequestConfig(
-                    .setResponseTimeout(Timeout.ofSeconds(2))
+                        RequestConfig.custom()
-                    .setConnectionRequestTimeout(Timeout.ofSeconds(1))
+                            .setCookieSpec(StandardCookieSpec.IGNORE)
+                            .setConnectTimeout(Timeout.ofSeconds(2))
+                            .setResponseTimeout(Timeout.ofSeconds(2))
+                            .setConnectionRequestTimeout(Timeout.ofSeconds(1))
+                            .build()
+                    )
+                    .setConnectionManager(connectionManager)
                     .build()
             )
-            .setConnectionManager(
+        )
-                connectionManager
-            )
-            .build()
-    )
 
     // state that must only be accessed from the thread on the executor
     private var state: State