Bump threads, sanitize uri, hide some errors, change logs some more

src/main/kotlin/mdnet/base/Application.kt

@@ -48,8 +48,8 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
             .setSocketTimeout(3000)
             .setConnectionRequestTimeout(3000)
             .build())
-        .setMaxConnTotal(10)
+        .setMaxConnTotal(75)
-        .setMaxConnPerRoute(10)
+        .setMaxConnPerRoute(75)
         .build())
 
     val app = { dataSaver: Boolean ->
@@ -57,8 +57,14 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
             val chapterHash = Path.of("chapterHash")(request)
             val fileName = Path.of("fileName")(request)
 
-            if (LOGGER.isTraceEnabled) {
+            val sanitizedUri = if (dataSaver) {
-                LOGGER.trace("Request for ${request.uri} received")
+                "/data-saver"
+            } else {
+                "/data"
+            } + "/$chapterHash/$fileName"
+
+            if (LOGGER.isInfoEnabled) {
+                LOGGER.info("Request for $sanitizedUri received")
             }
 
             val rc4Bytes = if (dataSaver) {
@@ -80,16 +86,18 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
                         listOf("public", MaxAgeTtl(Constants.MAX_AGE_CACHE).toHeaderValue()).joinToString(", ")
                     )
                     .header("Timing-Allow-Origin", "https://mangadex.org")
-                    .also {
+                    .let {
                         if (length != null) {
-                            it.body(input, length.toLong())
+                            it.body(input, length.toLong()).header("Content-Length", length)
-                            it.header("Content-Length", length)
                         } else {
-                            it.body(input)
+                            it.body(input).header("Transfer-Encoding", "chunked")
                         }
-
+                    }
+                    .let {
                         if (lastModified != null) {
                             it.header("Last-Modified", lastModified)
+                        } else {
+                            it
                         }
                     }
 
@@ -99,8 +107,8 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
 
                 // our files never change, so it's safe to use the browser cache
                 if (request.header("If-Modified-Since") != null) {
-                    if (LOGGER.isTraceEnabled) {
+                    if (LOGGER.isInfoEnabled) {
-                        LOGGER.trace("Request for ${request.uri} cached by browser")
+                        LOGGER.info("Request for $sanitizedUri cached by browser")
                     }
 
                     val lastModified = snapshot.getString(2)
@@ -109,8 +117,8 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
                     Response(Status.NOT_MODIFIED)
                         .header("Last-Modified", lastModified)
                 } else {
-                    if (LOGGER.isTraceEnabled) {
+                    if (LOGGER.isInfoEnabled) {
-                        LOGGER.trace("Request for ${request.uri} hit cache")
+                        LOGGER.info("Request for $sanitizedUri hit cache")
                     }
 
                     respondWithImage(
@@ -120,20 +128,20 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
                 }
             } else {
                 statistics.get().cacheMisses.incrementAndGet()
-                if (LOGGER.isTraceEnabled) {
+                if (LOGGER.isInfoEnabled) {
-                    LOGGER.trace("Request for ${request.uri} missed cache")
+                    LOGGER.info("Request for $sanitizedUri missed cache")
                 }
-                val mdResponse = client(Request(Method.GET, "${serverSettings.imageServer}${request.uri}"))
+                val mdResponse = client(Request(Method.GET, "${serverSettings.imageServer}$sanitizedUri"))
 
                 if (mdResponse.status != Status.OK) {
                     if (LOGGER.isTraceEnabled) {
-                        LOGGER.trace("Upstream query for ${request.uri} errored with status {}", mdResponse.status)
+                        LOGGER.trace("Upstream query for $sanitizedUri errored with status {}", mdResponse.status)
                     }
                     mdResponse.close()
                     Response(mdResponse.status)
                 } else {
                     if (LOGGER.isTraceEnabled) {
-                        LOGGER.trace("Upstream query for ${request.uri} succeeded")
+                        LOGGER.trace("Upstream query for $sanitizedUri succeeded")
                     }
 
                     val contentType = mdResponse.header("Content-Type")!!
@@ -146,7 +154,7 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
                     // concurrently so we skip the cache process
                     if (editor != null && contentLength != null && lastModified != null) {
                         if (LOGGER.isTraceEnabled) {
-                            LOGGER.trace("Request for ${request.uri} is being cached and served")
+                            LOGGER.trace("Request for $sanitizedUri is being cached and served")
                         }
                         editor.setString(1, contentType)
                         editor.setString(2, lastModified)
@@ -159,13 +167,13 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
                             // check that tee gets closed and for exceptions in this lambda
                             if (editor.getLength(0) == contentLength.toLong()) {
                                 if (LOGGER.isTraceEnabled) {
-                                    LOGGER.trace("Cache download ${request.uri} committed")
+                                    LOGGER.trace("Cache download $sanitizedUri committed")
                                 }
 
                                 editor.commit()
                             } else {
                                 if (LOGGER.isTraceEnabled) {
-                                    LOGGER.trace("Cache download ${request.uri} aborted")
+                                    LOGGER.trace("Cache download $sanitizedUri aborted")
                                 }
 
                                 editor.abort()
@@ -176,7 +184,7 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
                         editor?.abort()
 
                         if (LOGGER.isTraceEnabled) {
-                            LOGGER.trace("Request for ${request.uri} is being served")
+                            LOGGER.trace("Request for $sanitizedUri is being served")
                         }
 
                         respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified)

src/main/kotlin/mdnet/base/Netty.kt

@@ -24,6 +24,7 @@ import org.http4k.server.Http4kChannelHandler
 import org.http4k.server.Http4kServer
 import org.http4k.server.ServerConfig
 import org.slf4j.LoggerFactory
+import java.io.IOException
 import java.net.InetSocketAddress
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicReference
@@ -42,6 +43,7 @@ class Netty(private val tls: ServerSettings.TlsCert, private val clientSettings:
             workerGroup, 1024 * clientSettings.maxBurstRateKibPerSecond, 0, 50) {
             override fun doAccounting(counter: TrafficCounter) {
                 stats.get().bytesSent.getAndAdd(counter.cumulativeWrittenBytes())
+                counter.resetCumulativeTime()
             }
         }
 
@@ -68,6 +70,10 @@ class Netty(private val tls: ServerSettings.TlsCert, private val clientSettings:
                                         if (LOGGER.isTraceEnabled) {
                                             LOGGER.trace("Ignored invalid SSL connection")
                                         }
+                                    } else if (cause is IOException && cause.message?.contains("peer") == true) {
+                                        if (LOGGER.isTraceEnabled) {
+                                            LOGGER.trace("User (downloader) closed the connection")
+                                        }
                                     } else {
                                         ctx.fireExceptionCaught(cause)
                                     }

src/main/resources/logback.xml

@@ -23,9 +23,12 @@
     </appender>
 
     <root level="TRACE">
-        <appender-ref ref="STDOUT"/>
         <appender-ref ref="ASYNC"/>
     </root>
 
+    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
+    </root>
+
     <logger name="io.netty" level="INFO"/>
 </configuration>