gitlab-com-mangadex-pub/mangadex_at_home
1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00
Code Releases Activity

Merge branch 'metrics' into 'master'

Browse source 
Update metrics

See merge request mangadex-pub/mangadex_at_home!86
This commit is contained in:
carbotaniuman 2021-05-21 15:23:49 +00:00
parent 49c19e7879 b0dfc9ff12
commit 5b154a6b20
8 changed files with 369 additions and 194 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

528
docker/grafana/dashboards/mangadex_at_home.json
View file

@ -14,10 +14,10 @@
} }
] ]
}, },
"editable": false, "editable": true,
"gnetId": null, "gnetId": null,
"graphTooltip": 1, "graphTooltip": 1,
"iteration": 1612974883967, "iteration": 1617066952719,
"links": [], "links": [],
"panels": [ "panels": [
{ {
@ -115,7 +115,7 @@
}, },
"gridPos": { "gridPos": {
"h": 8, "h": 8,
"w": 8, "w": 6,
"x": 0, "x": 0,
"y": 1 "y": 1
}, },
@ -250,8 +250,8 @@
}, },
"gridPos": { "gridPos": {
"h": 8, "h": 8,
"w": 8, "w": 6,
"x": 8, "x": 6,
"y": 1 "y": 1
}, },
"id": 19, "id": 19,
@ -310,6 +310,167 @@
"title": "RAM", "title": "RAM",
"type": "timeseries" "type": "timeseries"
}, },
{
"cacheTimeout": null,
"datasource": "Prometheus",
"description": "Cache size allocated, does not include metadata or other overhead, but does somewhat account for block sizes.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 30,
"gradientMode": "hue",
"hideFrom": {
"graph": false,
"legend": false,
"tooltip": false
},
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Max"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "dark-red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Used"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "dark-green",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Max"
},
"properties": [
{
"id": "custom.fillOpacity",
"value": 0
},
{
"id": "custom.lineWidth",
"value": 2
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 6,
"x": 12,
"y": 1
},
"id": 57,
"interval": null,
"links": [],
"options": {
"graph": {},
"legend": {
"calcs": [
"min",
"max",
"mean",
"last"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltipOptions": {
"mode": "multi"
}
},
"pluginVersion": "7.4.0",
"targets": [
{
"aggregation": "Last",
"decimals": 2,
"displayAliasType": "Warning / Critical",
"displayType": "Regular",
"displayValueWithAlias": "Never",
"expr": "sum(cache_used_bytes)",
"hide": false,
"interval": "",
"legendFormat": "Used",
"refId": "A",
"units": "none",
"valueHandler": "Number Threshold"
},
{
"aggregation": "Last",
"decimals": 2,
"displayAliasType": "Warning / Critical",
"displayType": "Regular",
"displayValueWithAlias": "Never",
"expr": "sum(cache_max_bytes)",
"hide": false,
"instant": false,
"interval": "",
"legendFormat": "Max",
"refId": "B",
"units": "none",
"valueHandler": "Number Threshold"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Cache",
"type": "timeseries"
},
{ {
"cacheTimeout": null, "cacheTimeout": null,
"datasource": "Prometheus", "datasource": "Prometheus",
@ -408,8 +569,8 @@
}, },
"gridPos": { "gridPos": {
"h": 8, "h": 8,
"w": 8, "w": 6,
"x": 16, "x": 18,
"y": 1 "y": 1
}, },
"id": 20, "id": 20,
@ -1792,7 +1953,7 @@
"type": "timeseries" "type": "timeseries"
}, },
{ {
"collapsed": true, "collapsed": false,
"datasource": null, "datasource": null,
"gridPos": { "gridPos": {
"h": 1, "h": 1,
@ -1801,181 +1962,180 @@
"y": 27 "y": 27
}, },
"id": 53, "id": 53,
"panels": [ "panels": [],
{
"circleMaxSize": "10",
"circleMinSize": "1",
"colors": [
"#73BF69",
"#FADE2A",
"#C4162A"
],
"datasource": "Prometheus",
"decimals": 0,
"esMetric": "Count",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"gridPos": {
"h": 15,
"w": 11,
"x": 0,
"y": 28
},
"hideEmpty": true,
"hideZero": true,
"id": 39,
"initialZoom": "2",
"locationData": "countries",
"mapCenter": "custom",
"mapCenterLatitude": "30",
"mapCenterLongitude": "20",
"maxDataPoints": 1,
"mouseWheelZoom": false,
"pluginVersion": "7.3.4",
"showLegend": true,
"stickyLabels": false,
"tableQueryOptions": {
"geohashField": "geohash",
"labelField": "country",
"latitudeField": "latitude",
"longitudeField": "longitude",
"metricField": "metric",
"queryType": "geohash"
},
"targets": [
{
"expr": "sum(increase(requests_country_counts_total[$__range])) by (country)",
"format": "time_series",
"instant": true,
"interval": "",
"legendFormat": "{{ country }}",
"refId": "A"
}
],
"thresholds": "1000,10000",
"timeFrom": null,
"timeShift": null,
"title": "Origin of requests over timerange",
"type": "grafana-worldmap-panel",
"unitPlural": "",
"unitSingle": "",
"unitSingular": "",
"valueName": "current"
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {},
"thresholds": {
"mode": "absolute",
"steps": []
}
},
"overrides": []
},
"fill": 8,
"fillGradient": 0,
"gridPos": {
"h": 15,
"w": 13,
"x": 11,
"y": 28
},
"hiddenSeries": false,
"id": 41,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"sort": "avg",
"sortDesc": true,
"total": false,
"values": true
},
"lines": false,
"linewidth": 2,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": false
},
"percentage": false,
"pluginVersion": "7.4.0",
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "topk(5, sum(rate(requests_country_counts_total[$itvl])) by (country))",
"interval": "$itvl",
"legendFormat": "{{ country }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Country spread [$itvl]",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:104",
"format": "reqps",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"$$hashKey": "object:105",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "GeoIP", "title": "GeoIP",
"type": "row" "type": "row"
},
{
"circleMaxSize": "10",
"circleMinSize": "1",
"colors": [
"#73BF69",
"#FADE2A",
"#C4162A"
],
"datasource": "Prometheus",
"decimals": 0,
"esMetric": "Count",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"gridPos": {
"h": 15,
"w": 11,
"x": 0,
"y": 28
},
"hideEmpty": true,
"hideZero": true,
"id": 39,
"initialZoom": "2",
"locationData": "countries",
"mapCenter": "custom",
"mapCenterLatitude": "30",
"mapCenterLongitude": "20",
"maxDataPoints": 1,
"mouseWheelZoom": false,
"pluginVersion": "7.3.4",
"showLegend": true,
"stickyLabels": false,
"tableQueryOptions": {
"geohashField": "geohash",
"labelField": "country",
"latitudeField": "latitude",
"longitudeField": "longitude",
"metricField": "metric",
"queryType": "geohash"
},
"targets": [
{
"expr": "sum(increase(requests_country_counts_total[$__range])) by (country)",
"format": "time_series",
"instant": true,
"interval": "",
"legendFormat": "{{ country }}",
"refId": "A"
}
],
"thresholds": "1000,10000",
"timeFrom": null,
"timeShift": null,
"title": "Origin of requests over timerange",
"type": "grafana-worldmap-panel",
"unitPlural": "",
"unitSingle": "",
"unitSingular": "",
"valueName": "current"
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {},
"thresholds": {
"mode": "absolute",
"steps": []
}
},
"overrides": []
},
"fill": 8,
"fillGradient": 0,
"gridPos": {
"h": 15,
"w": 13,
"x": 11,
"y": 28
},
"hiddenSeries": false,
"id": 41,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"sort": "avg",
"sortDesc": true,
"total": false,
"values": true
},
"lines": false,
"linewidth": 2,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": false
},
"percentage": false,
"pluginVersion": "7.4.0",
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "topk(5, sum(rate(requests_country_counts_total[$itvl])) by (country))",
"interval": "$itvl",
"legendFormat": "{{ country }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Country spread [$itvl]",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:104",
"format": "reqps",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"$$hashKey": "object:105",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
} }
], ],
"refresh": "30s", "refresh": "30s",
@ -2069,12 +2229,12 @@
] ]
}, },
"time": { "time": {
"from": "now-30m", "from": "now-5m",
"to": "now" "to": "now"
}, },
"timepicker": {}, "timepicker": {},
"timezone": "", "timezone": "",
"title": "MangaDex@Home - Personal client dashboard", "title": "MangaDex@Home - Personal client dashboard",
"uid": "a7sZAw2Mk", "uid": "a7sZAw2Mk",
"version": 4 "version": 2
} }

16
src/main/kotlin/mdnet/ServerManager.kt
View file

@ -18,6 +18,8 @@ along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/ */
package mdnet package mdnet
import io.micrometer.core.instrument.Gauge
import io.micrometer.core.instrument.binder.BaseUnits
import io.micrometer.prometheus.PrometheusConfig import io.micrometer.prometheus.PrometheusConfig
import io.micrometer.prometheus.PrometheusMeterRegistry import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.cache.ImageStorage import mdnet.cache.ImageStorage
@ -105,7 +107,19 @@ class ServerManager(
fun start() { fun start() {
LOGGER.info { "Image server starting" } LOGGER.info { "Image server starting" }
DefaultMicrometerMetrics(registry, storage.cacheDirectory) DefaultMicrometerMetrics(registry)
Gauge.builder(
"cache.used",
storage,
{ it.size.toDouble() }
).baseUnit(BaseUnits.BYTES).register(registry)
Gauge.builder(
"cache.max",
storage,
{ it.maxSize.toDouble() }
).baseUnit(BaseUnits.BYTES).register(registry)
loginAndStartServer() loginAndStartServer()
var lastBytesSent = statistics.bytesSent.get() var lastBytesSent = statistics.bytesSent.get()

2
src/main/kotlin/mdnet/cache/ImageStorage.kt vendored
View file

@ -55,7 +55,7 @@ data class Image(val data: ImageMetadata, val stream: InputStream)
*/ */
class ImageStorage( class ImageStorage(
var maxSize: Long, var maxSize: Long,
val cacheDirectory: Path, private val cacheDirectory: Path,
private val database: Database, private val database: Database,
autoPrune: Boolean = true autoPrune: Boolean = true
) { ) {

2
src/main/kotlin/mdnet/data/Token.kt
View file

@ -23,4 +23,4 @@ import com.fasterxml.jackson.databind.annotation.JsonNaming
import java.time.OffsetDateTime import java.time.OffsetDateTime
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class) @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
data class Token(val expires: OffsetDateTime, val hash: String, val clientId: String? = null, val ip: String? = null) data class Token(val expires: OffsetDateTime, val hash: String, val clientId: String)

5
src/main/kotlin/mdnet/metrics/DefaultMicrometerMetrics.kt
View file

@ -19,7 +19,6 @@ along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
package mdnet.metrics package mdnet.metrics
import io.micrometer.core.instrument.Tag import io.micrometer.core.instrument.Tag
import io.micrometer.core.instrument.binder.jvm.DiskSpaceMetrics
import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics
import io.micrometer.core.instrument.binder.jvm.JvmHeapPressureMetrics import io.micrometer.core.instrument.binder.jvm.JvmHeapPressureMetrics
import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics
@ -30,9 +29,8 @@ import io.micrometer.core.instrument.binder.system.ProcessorMetrics
import io.micrometer.core.instrument.binder.system.UptimeMetrics import io.micrometer.core.instrument.binder.system.UptimeMetrics
import io.micrometer.prometheus.PrometheusMeterRegistry import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.BuildInfo import mdnet.BuildInfo
import java.nio.file.Path
class DefaultMicrometerMetrics(registry: PrometheusMeterRegistry, cacheDirectory: Path) { class DefaultMicrometerMetrics(registry: PrometheusMeterRegistry) {
init { init {
UptimeMetrics( UptimeMetrics(
mutableListOf( mutableListOf(
@ -47,6 +45,5 @@ class DefaultMicrometerMetrics(registry: PrometheusMeterRegistry, cacheDirectory
JvmHeapPressureMetrics().bindTo(registry) JvmHeapPressureMetrics().bindTo(registry)
FileDescriptorMetrics().bindTo(registry) FileDescriptorMetrics().bindTo(registry)
LogbackMetrics().bindTo(registry) LogbackMetrics().bindTo(registry)
DiskSpaceMetrics(cacheDirectory.toFile()).bindTo(registry)
} }
} }

2
src/main/kotlin/mdnet/server/ImageHandler.kt
View file

@ -44,7 +44,7 @@ class ImageServer(
registry: PrometheusMeterRegistry registry: PrometheusMeterRegistry
) { ) {
private val executor = Executors.newCachedThreadPool() private val executor = Executors.newCachedThreadPool()
private val cacheLookupTimer = Timer.builder("cache_lookup") private val cacheLookupTimer = Timer.builder("cache.lookup")
.publishPercentiles(0.5, 0.75, 0.9, 0.99) .publishPercentiles(0.5, 0.75, 0.9, 0.99)
.register(registry) .register(registry)

5
src/main/kotlin/mdnet/server/ImageServer.kt
View file

@ -22,6 +22,7 @@ import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig
import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry
import io.github.resilience4j.micrometer.tagged.TaggedCircuitBreakerMetrics import io.github.resilience4j.micrometer.tagged.TaggedCircuitBreakerMetrics
import io.micrometer.core.instrument.FunctionCounter import io.micrometer.core.instrument.FunctionCounter
import io.micrometer.core.instrument.binder.BaseUnits
import io.micrometer.prometheus.PrometheusMeterRegistry import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.cache.ImageStorage import mdnet.cache.ImageStorage
import mdnet.data.Statistics import mdnet.data.Statistics
@ -98,10 +99,10 @@ fun getServer(
) )
FunctionCounter.builder( FunctionCounter.builder(
"client_sent_bytes", "client.sent",
statistics, statistics,
{ it.bytesSent.get().toDouble() } { it.bytesSent.get().toDouble() }
).register(registry) ).baseUnit(BaseUnits.BYTES).register(registry)
val verifier = TokenVerifier( val verifier = TokenVerifier(
tokenKey = remoteSettings.tokenKey, tokenKey = remoteSettings.tokenKey,

3
src/main/kotlin/mdnet/settings/PingResult.kt
View file

@ -36,6 +36,7 @@ data class RemoteSettings(
val imageServer: Uri, val imageServer: Uri,
val latestBuild: Int, val latestBuild: Int,
val url: Uri, val url: Uri,
val clientId: String,
@field:Secret val tokenKey: ByteArray, @field:Secret val tokenKey: ByteArray,
val compromised: Boolean, val compromised: Boolean,
val paused: Boolean, val paused: Boolean,
@ -51,6 +52,7 @@ data class RemoteSettings(
if (imageServer != other.imageServer) return false if (imageServer != other.imageServer) return false
if (latestBuild != other.latestBuild) return false if (latestBuild != other.latestBuild) return false
if (url != other.url) return false if (url != other.url) return false
if (clientId != other.clientId) return false
if (!tokenKey.contentEquals(other.tokenKey)) return false if (!tokenKey.contentEquals(other.tokenKey)) return false
if (compromised != other.compromised) return false if (compromised != other.compromised) return false
if (paused != other.paused) return false if (paused != other.paused) return false
@ -64,6 +66,7 @@ data class RemoteSettings(
var result = imageServer.hashCode() var result = imageServer.hashCode()
result = 31 * result + latestBuild result = 31 * result + latestBuild
result = 31 * result + url.hashCode() result = 31 * result + url.hashCode()
result = 31 * result + clientId.hashCode()
result = 31 * result + tokenKey.contentHashCode() result = 31 * result + tokenKey.contentHashCode()
result = 31 * result + compromised.hashCode() result = 31 * result + compromised.hashCode()
result = 31 * result + paused.hashCode() result = 31 * result + paused.hashCode()