Merge branch 'metrics' into 'master'

Update metrics

See merge request mangadex-pub/mangadex_at_home!86

docker/grafana/dashboards/mangadex_at_home.json

@@ -14,10 +14,10 @@
       }
     ]
   },
-  "editable": false,
+  "editable": true,
   "gnetId": null,
   "graphTooltip": 1,
-  "iteration": 1612974883967,
+  "iteration": 1617066952719,
   "links": [],
   "panels": [
     {
@@ -115,7 +115,7 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 8,
+        "w": 6,
         "x": 0,
         "y": 1
       },
@@ -250,8 +250,8 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 8,
+        "w": 6,
-        "x": 8,
+        "x": 6,
         "y": 1
       },
       "id": 19,
@@ -310,6 +310,167 @@
       "title": "RAM",
       "type": "timeseries"
     },
+    {
+      "cacheTimeout": null,
+      "datasource": "Prometheus",
+      "description": "Cache size allocated, does not include metadata or other overhead, but does somewhat account for block sizes.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 30,
+            "gradientMode": "hue",
+            "hideFrom": {
+              "graph": false,
+              "legend": false,
+              "tooltip": false
+            },
+            "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Max"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "dark-red",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Used"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "dark-green",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Max"
+            },
+            "properties": [
+              {
+                "id": "custom.fillOpacity",
+                "value": 0
+              },
+              {
+                "id": "custom.lineWidth",
+                "value": 2
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 12,
+        "y": 1
+      },
+      "id": 57,
+      "interval": null,
+      "links": [],
+      "options": {
+        "graph": {},
+        "legend": {
+          "calcs": [
+            "min",
+            "max",
+            "mean",
+            "last"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltipOptions": {
+          "mode": "multi"
+        }
+      },
+      "pluginVersion": "7.4.0",
+      "targets": [
+        {
+          "aggregation": "Last",
+          "decimals": 2,
+          "displayAliasType": "Warning / Critical",
+          "displayType": "Regular",
+          "displayValueWithAlias": "Never",
+          "expr": "sum(cache_used_bytes)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Used",
+          "refId": "A",
+          "units": "none",
+          "valueHandler": "Number Threshold"
+        },
+        {
+          "aggregation": "Last",
+          "decimals": 2,
+          "displayAliasType": "Warning / Critical",
+          "displayType": "Regular",
+          "displayValueWithAlias": "Never",
+          "expr": "sum(cache_max_bytes)",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "legendFormat": "Max",
+          "refId": "B",
+          "units": "none",
+          "valueHandler": "Number Threshold"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Cache",
+      "type": "timeseries"
+    },
     {
       "cacheTimeout": null,
       "datasource": "Prometheus",
@@ -408,8 +569,8 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 8,
+        "w": 6,
-        "x": 16,
+        "x": 18,
         "y": 1
       },
       "id": 20,
@@ -1792,7 +1953,7 @@
       "type": "timeseries"
     },
     {
-      "collapsed": true,
+      "collapsed": false,
       "datasource": null,
       "gridPos": {
         "h": 1,
@@ -1801,7 +1962,10 @@
         "y": 27
       },
       "id": 53,
-      "panels": [
+      "panels": [],
+      "title": "GeoIP",
+      "type": "row"
+    },
     {
       "circleMaxSize": "10",
       "circleMinSize": "1",
@@ -1974,10 +2138,6 @@
       }
     }
   ],
-      "title": "GeoIP",
-      "type": "row"
-    }
-  ],
   "refresh": "30s",
   "schemaVersion": 27,
   "style": "dark",
@@ -2069,12 +2229,12 @@
     ]
   },
   "time": {
-    "from": "now-30m",
+    "from": "now-5m",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "",
   "title": "MangaDex@Home - Personal client dashboard",
   "uid": "a7sZAw2Mk",
-  "version": 4
+  "version": 2
 }

src/main/kotlin/mdnet/ServerManager.kt

@@ -18,6 +18,8 @@ along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
 */
 package mdnet
 
+import io.micrometer.core.instrument.Gauge
+import io.micrometer.core.instrument.binder.BaseUnits
 import io.micrometer.prometheus.PrometheusConfig
 import io.micrometer.prometheus.PrometheusMeterRegistry
 import mdnet.cache.ImageStorage
@@ -105,7 +107,19 @@ class ServerManager(
 
     fun start() {
         LOGGER.info { "Image server starting" }
-        DefaultMicrometerMetrics(registry, storage.cacheDirectory)
+        DefaultMicrometerMetrics(registry)
+        Gauge.builder(
+            "cache.used",
+            storage,
+            { it.size.toDouble() }
+        ).baseUnit(BaseUnits.BYTES).register(registry)
+
+        Gauge.builder(
+            "cache.max",
+            storage,
+            { it.maxSize.toDouble() }
+        ).baseUnit(BaseUnits.BYTES).register(registry)
+
         loginAndStartServer()
 
         var lastBytesSent = statistics.bytesSent.get()

src/main/kotlin/mdnet/cache/ImageStorage.kt

@@ -55,7 +55,7 @@ data class Image(val data: ImageMetadata, val stream: InputStream)
  */
 class ImageStorage(
     var maxSize: Long,
-    val cacheDirectory: Path,
+    private val cacheDirectory: Path,
     private val database: Database,
     autoPrune: Boolean = true
 ) {

src/main/kotlin/mdnet/data/Token.kt

@@ -23,4 +23,4 @@ import com.fasterxml.jackson.databind.annotation.JsonNaming
 import java.time.OffsetDateTime
 
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
-data class Token(val expires: OffsetDateTime, val hash: String, val clientId: String? = null, val ip: String? = null)
+data class Token(val expires: OffsetDateTime, val hash: String, val clientId: String)

src/main/kotlin/mdnet/metrics/DefaultMicrometerMetrics.kt

@@ -19,7 +19,6 @@ along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
 package mdnet.metrics
 
 import io.micrometer.core.instrument.Tag
-import io.micrometer.core.instrument.binder.jvm.DiskSpaceMetrics
 import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics
 import io.micrometer.core.instrument.binder.jvm.JvmHeapPressureMetrics
 import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics
@@ -30,9 +29,8 @@ import io.micrometer.core.instrument.binder.system.ProcessorMetrics
 import io.micrometer.core.instrument.binder.system.UptimeMetrics
 import io.micrometer.prometheus.PrometheusMeterRegistry
 import mdnet.BuildInfo
-import java.nio.file.Path
 
-class DefaultMicrometerMetrics(registry: PrometheusMeterRegistry, cacheDirectory: Path) {
+class DefaultMicrometerMetrics(registry: PrometheusMeterRegistry) {
     init {
         UptimeMetrics(
             mutableListOf(
@@ -47,6 +45,5 @@ class DefaultMicrometerMetrics(registry: PrometheusMeterRegistry, cacheDirectory
         JvmHeapPressureMetrics().bindTo(registry)
         FileDescriptorMetrics().bindTo(registry)
         LogbackMetrics().bindTo(registry)
-        DiskSpaceMetrics(cacheDirectory.toFile()).bindTo(registry)
     }
 }

src/main/kotlin/mdnet/server/ImageHandler.kt

@@ -44,7 +44,7 @@ class ImageServer(
     registry: PrometheusMeterRegistry
 ) {
     private val executor = Executors.newCachedThreadPool()
-    private val cacheLookupTimer = Timer.builder("cache_lookup")
+    private val cacheLookupTimer = Timer.builder("cache.lookup")
         .publishPercentiles(0.5, 0.75, 0.9, 0.99)
         .register(registry)
 

src/main/kotlin/mdnet/server/ImageServer.kt

@@ -22,6 +22,7 @@ import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig
 import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry
 import io.github.resilience4j.micrometer.tagged.TaggedCircuitBreakerMetrics
 import io.micrometer.core.instrument.FunctionCounter
+import io.micrometer.core.instrument.binder.BaseUnits
 import io.micrometer.prometheus.PrometheusMeterRegistry
 import mdnet.cache.ImageStorage
 import mdnet.data.Statistics
@@ -98,10 +99,10 @@ fun getServer(
     )
 
     FunctionCounter.builder(
-        "client_sent_bytes",
+        "client.sent",
         statistics,
         { it.bytesSent.get().toDouble() }
-    ).register(registry)
+    ).baseUnit(BaseUnits.BYTES).register(registry)
 
     val verifier = TokenVerifier(
         tokenKey = remoteSettings.tokenKey,

src/main/kotlin/mdnet/settings/PingResult.kt

@@ -36,6 +36,7 @@ data class RemoteSettings(
     val imageServer: Uri,
     val latestBuild: Int,
     val url: Uri,
+    val clientId: String,
     @field:Secret val tokenKey: ByteArray,
     val compromised: Boolean,
     val paused: Boolean,
@@ -51,6 +52,7 @@ data class RemoteSettings(
         if (imageServer != other.imageServer) return false
         if (latestBuild != other.latestBuild) return false
         if (url != other.url) return false
+        if (clientId != other.clientId) return false
         if (!tokenKey.contentEquals(other.tokenKey)) return false
         if (compromised != other.compromised) return false
         if (paused != other.paused) return false
@@ -64,6 +66,7 @@ data class RemoteSettings(
         var result = imageServer.hashCode()
         result = 31 * result + latestBuild
         result = 31 * result + url.hashCode()
+        result = 31 * result + clientId.hashCode()
         result = 31 * result + tokenKey.contentHashCode()
         result = 31 * result + compromised.hashCode()
         result = 31 * result + paused.hashCode()