gitlab-com-mangadex-pub/mangadex_at_home
1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00
Code Releases Activity

Fix and extend docker-compose and grafana dashboard setup

Browse source
This commit is contained in:
Tristan Deloche 2021-01-28 23:43:32 +00:00
parent 5abf9dcc4c
commit c0882c33fc
No known key found for this signature in database
GPG key ID: B6BDB16331573E59
4 changed files with 174 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- [2021-01-28] Add distinct timerange and total egress stats to dashboard [@_tde9]
- [2021-01-28] Add average cache hitrate panel [@_tde9]
### Changed
- [2021-01-29] Add HikariCP connection pool [@carbotaniuman].
@ -15,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed
### Fixed
- [2021-01-28] Fix client reqs dashboard panel [@_tde9]
### Security

4
docker/docker-compose.yml
View file

@ -11,6 +11,7 @@ services:
- ./settings.yaml:/mangahome/settings.yaml:ro
- ./data/cache/:/mangahome/data/
environment:
# If your client is intended to do more than ~100rqps, it is recommended replacing -XX:+UseG1GC with -XX:+UseShenandoahGC
JAVA_TOOL_OPTIONS: "-Xms1G -Xmx1G -XX:+UseG1GC -Xss512K"
privileged: true
command: [
@ -68,6 +69,3 @@ services:
options:
max-size: "20m"
max-file: "2"
networks:
mangadex-at-home: { }

192
docker/grafana/dashboards/mangadex_at_home.json
View file

@ -17,7 +17,7 @@
"editable": false,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1611371909784,
"iteration": 1611971739196,
"links": [],
"panels": [
{
@ -468,7 +468,7 @@
"overrides": []
},
"gridPos": {
"h": 2,
"h": 3,
"w": 5,
"x": 0,
"y": 8
@ -542,7 +542,7 @@
"overrides": []
},
"gridPos": {
"h": 2,
"h": 3,
"w": 2,
"x": 5,
"y": 8
@ -612,7 +612,7 @@
"overrides": []
},
"gridPos": {
"h": 2,
"h": 3,
"w": 2,
"x": 7,
"y": 8
@ -682,7 +682,7 @@
"overrides": []
},
"gridPos": {
"h": 2,
"h": 3,
"w": 2,
"x": 9,
"y": 8
@ -752,7 +752,7 @@
"overrides": []
},
"gridPos": {
"h": 2,
"h": 3,
"w": 2,
"x": 11,
"y": 8
@ -800,7 +800,6 @@
"filterable": false
},
"mappings": [],
"max": 0.2,
"min": 0,
"thresholds": {
"mode": "absolute",
@ -824,7 +823,7 @@
"overrides": []
},
"gridPos": {
"h": 2,
"h": 3,
"w": 4,
"x": 13,
"y": 8
@ -854,9 +853,9 @@
"displayAliasType": "Warning / Critical",
"displayType": "Regular",
"displayValueWithAlias": "Never",
"expr": "max(rate(jvm_gc_pause_seconds_sum[$itvl]) > 0 / rate(jvm_gc_pause_seconds_count[$itvl]))",
"expr": "sum(rate(jvm_gc_pause_seconds_sum[$itvl])) / sum(rate(process_uptime_seconds[$itvl]))",
"interval": "",
"legendFormat": "{{ kubernetes_pod_name }}",
"legendFormat": "",
"refId": "A",
"units": "none",
"valueHandler": "Number Threshold"
@ -864,7 +863,7 @@
],
"timeFrom": null,
"timeShift": null,
"title": "Time spend in GC pause",
"title": "Time lost to GC",
"transparent": true,
"type": "stat"
},
@ -894,7 +893,7 @@
"overrides": []
},
"gridPos": {
"h": 2,
"h": 3,
"w": 3,
"x": 17,
"y": 8
@ -905,8 +904,8 @@
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"justifyMode": "center",
"orientation": "vertical",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -914,7 +913,7 @@
"fields": "",
"values": false
},
"textMode": "auto"
"textMode": "value_and_name"
},
"pluginVersion": "7.3.7",
"targets": [
@ -927,15 +926,29 @@
"expr": "sum(increase(http_server_request_latency_seconds_count{status=~\"2..\", path=~\"(.*)data/{chapterHash}/{fileName}\"}[$__range]))",
"instant": false,
"interval": "",
"legendFormat": "",
"legendFormat": "Over timerange",
"refId": "B",
"units": "none",
"valueHandler": "Number Threshold"
},
{
"aggregation": "Last",
"decimals": 2,
"displayAliasType": "Warning / Critical",
"displayType": "Regular",
"displayValueWithAlias": "Never",
"expr": "sum(increase(http_server_request_latency_seconds_count{status=~\"2..\", path=~\"(.*)data/{chapterHash}/{fileName}\"}[1y]))",
"instant": false,
"interval": "",
"legendFormat": "Total (1y)",
"refId": "A",
"units": "none",
"valueHandler": "Number Threshold"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Images served",
"title": "Served images",
"transparent": true,
"type": "stat"
},
@ -965,7 +978,7 @@
"overrides": []
},
"gridPos": {
"h": 2,
"h": 3,
"w": 4,
"x": 20,
"y": 8
@ -977,7 +990,7 @@
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"orientation": "vertical",
"reduceOptions": {
"calcs": [
"lastNotNull"
@ -985,7 +998,7 @@
"fields": "",
"values": false
},
"textMode": "auto"
"textMode": "value_and_name"
},
"pluginVersion": "7.3.7",
"targets": [
@ -998,15 +1011,29 @@
"expr": "sum(increase(client_sent_bytes_total[$__range]))",
"instant": true,
"interval": "",
"legendFormat": "",
"legendFormat": "Over timerange",
"refId": "B",
"units": "none",
"valueHandler": "Number Threshold"
},
{
"aggregation": "Last",
"decimals": 2,
"displayAliasType": "Warning / Critical",
"displayType": "Regular",
"displayValueWithAlias": "Never",
"expr": "sum(increase(client_sent_bytes_total[1y]))",
"instant": true,
"interval": "",
"legendFormat": "Total (1y)",
"refId": "A",
"units": "none",
"valueHandler": "Number Threshold"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Bytes sent",
"title": "Sent data",
"transparent": true,
"type": "stat"
},
@ -1017,7 +1044,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 10
"y": 11
},
"id": 12,
"panels": [],
@ -1059,7 +1086,7 @@
"h": 14,
"w": 8,
"x": 0,
"y": 11
"y": 12
},
"hiddenSeries": false,
"id": 2,
@ -1167,6 +1194,7 @@
"Duration": "super-light-green",
"IllegalStateException": "yellow",
"Success": "green",
"TTFB": "super-light-blue",
"Upstream failures": "dark-red"
},
"bars": true,
@ -1185,7 +1213,7 @@
"h": 8,
"w": 8,
"x": 8,
"y": 11
"y": 12
},
"hiddenSeries": false,
"id": 13,
@ -1216,12 +1244,15 @@
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:158",
"alias": "Success",
"color": "#37872D"
},
{
"$$hashKey": "object:159",
"alias": "TTFB",
"bars": false,
"color": "#C0D8FF",
"fill": 0,
"lines": true,
"linewidth": 2,
@ -1234,7 +1265,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(http_client_request_count_total{status=~\"2..\"}[$itvl]))",
"expr": "sum(rate(http_client_request_latency_seconds_count{status=~\"2..\"}[$itvl]))",
"hide": false,
"instant": false,
"interval": "$itvl",
@ -1270,6 +1301,7 @@
},
"yaxes": [
{
"$$hashKey": "object:182",
"format": "reqps",
"label": null,
"logBase": 1,
@ -1278,6 +1310,7 @@
"show": true
},
{
"$$hashKey": "object:183",
"format": "s",
"label": null,
"logBase": 1,
@ -1291,6 +1324,77 @@
"alignLevel": null
}
},
{
"cacheTimeout": null,
"datasource": "Prometheus",
"description": "Average cache hitrate of the current timerange\n\nSplits timerange in blocks of [$itvl], and averages their averages",
"fieldConfig": {
"defaults": {
"custom": {
"align": null,
"filterable": false
},
"mappings": [],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "dark-red",
"value": null
},
{
"color": "#EAB839",
"value": 0.75
},
{
"color": "dark-green",
"value": 0.9
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 8,
"x": 16,
"y": 12
},
"id": 54,
"interval": null,
"links": [],
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "7.3.7",
"targets": [
{
"expr": "sum(rate(http_server_request_latency_seconds_count{path=~\"(.*)data/{chapterHash}/{fileName}\", cache=\"HIT\"}[$itvl])) / sum(rate(http_server_request_latency_seconds_count{path=~\"(.*)data/{chapterHash}/{fileName}\"}[$itvl]))",
"interval": "",
"legendFormat": "",
"refId": "B"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Avg hitrate over timerange",
"type": "stat"
},
{
"aliasColors": {
"ClientAbortException": "dark-red",
@ -1312,10 +1416,10 @@
"fill": 10,
"fillGradient": 0,
"gridPos": {
"h": 14,
"h": 10,
"w": 8,
"x": 16,
"y": 11
"y": 16
},
"hiddenSeries": false,
"id": 3,
@ -1325,6 +1429,7 @@
"current": true,
"max": true,
"min": true,
"rightSide": false,
"show": true,
"sort": "current",
"sortDesc": true,
@ -1332,7 +1437,7 @@
"values": true
},
"lines": true,
"linewidth": 1,
"linewidth": 0,
"nullPointMode": "null",
"options": {
"alertThreshold": true
@ -1374,6 +1479,7 @@
},
"yaxes": [
{
"$$hashKey": "object:314",
"format": "reqps",
"label": null,
"logBase": 1,
@ -1382,12 +1488,13 @@
"show": true
},
{
"format": "short",
"label": null,
"$$hashKey": "object:315",
"format": "percentunit",
"label": "hitrate",
"logBase": 1,
"max": null,
"min": null,
"show": true
"max": "100",
"min": "0",
"show": false
}
],
"yaxis": {
@ -1421,7 +1528,7 @@
"h": 6,
"w": 8,
"x": 8,
"y": 19
"y": 20
},
"hiddenSeries": false,
"id": 49,
@ -1506,7 +1613,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 25
"y": 26
},
"id": 53,
"panels": [
@ -1531,7 +1638,7 @@
"h": 15,
"w": 11,
"x": 0,
"y": 26
"y": 27
},
"hideEmpty": true,
"hideZero": true,
@ -1609,7 +1716,7 @@
"h": 15,
"w": 13,
"x": 11,
"y": 26
"y": 27
},
"hiddenSeries": false,
"id": 41,
@ -1734,11 +1841,6 @@
"text": "3m",
"value": "3m"
},
{
"selected": false,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "10m",
@ -1785,7 +1887,7 @@
"value": "30d"
}
],
"query": "2m,3m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
"query": "2m,3m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
"queryValue": "",
"refresh": 2,
"skipUrlSync": false,
@ -1801,5 +1903,5 @@
"timezone": "",
"title": "MangaDex@Home - Personal client dashboard",
"uid": "a7sZAw2Mk",
"version": 27
"version": 4
}

38
src/main/kotlin/mdnet/ServerManager.kt
View file

@ -27,7 +27,8 @@ import mdnet.logging.info
import mdnet.logging.warn
import mdnet.metrics.DefaultMicrometerMetrics
import mdnet.server.getServer
import mdnet.settings.*
import mdnet.settings.ClientSettings
import mdnet.settings.RemoteSettings
import org.apache.hc.client5.http.config.RequestConfig
import org.apache.hc.client5.http.cookie.StandardCookieSpec
import org.apache.hc.client5.http.impl.classic.HttpClients
@ -36,6 +37,9 @@ import org.apache.hc.core5.util.TimeValue
import org.apache.hc.core5.util.Timeout
import org.http4k.client.ApacheClient
import org.http4k.core.BodyMode
import org.http4k.core.then
import org.http4k.filter.ClientFilters
import org.http4k.filter.MicrometerMetrics
import org.http4k.server.Http4kServer
import org.slf4j.LoggerFactory
import java.util.concurrent.CountDownLatch
@ -73,23 +77,25 @@ class ServerManager(
.setMaxConnTotal(500)
.setMaxConnPerRoute(500)
.build()
private val apache = ApacheClient(
responseBodyMode = BodyMode.Stream,
client = HttpClients.custom()
.disableConnectionState()
.setDefaultRequestConfig(
RequestConfig.custom()
.setCookieSpec(StandardCookieSpec.IGNORE)
.setConnectTimeout(Timeout.ofSeconds(2))
.setResponseTimeout(Timeout.ofSeconds(2))
.setConnectionRequestTimeout(Timeout.ofSeconds(1))
private val apache = ClientFilters.MicrometerMetrics.RequestCounter(registry)
.then(ClientFilters.MicrometerMetrics.RequestTimer(registry))
.then(
ApacheClient(
responseBodyMode = BodyMode.Stream,
client = HttpClients.custom()
.disableConnectionState()
.setDefaultRequestConfig(
RequestConfig.custom()
.setCookieSpec(StandardCookieSpec.IGNORE)
.setConnectTimeout(Timeout.ofSeconds(2))
.setResponseTimeout(Timeout.ofSeconds(2))
.setConnectionRequestTimeout(Timeout.ofSeconds(1))
.build()
)
.setConnectionManager(connectionManager)
.build()
)
.setConnectionManager(
connectionManager
)
.build()
)
)
// state that must only be accessed from the thread on the executor
private var state: State