Merge branch 'next' into 'master'

Next

See merge request mangadex-pub/mangadex_at_home!82

CHANGELOG.md

@@ -14,15 +14,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed
 
 ### Fixed
-- [2021-03-11] Fixed Access-Control-Expose-Headers typo [@lflare].
 
 ### Security
 
+## [2.0.0] - 2021-03-11
+### Changed
+- [2021-03-11] Switch back to HTTP/1.1 [@carbotaniuman].
+- [2021-03-11] Tune connection pool [@carbotaniuman].
+
+### Fixed
+- [2021-03-11] Fixed Access-Control-Expose-Headers typo [@lflare].
+- [2021-03-11] Throw exceptions less frequently [@carbotaniuman].
+- [2021-03-11] Fix various Netty issues [@carbotaniuman].
+- [2021-03-11] Don't log IOUring and Epoll errors [@carbotaniuman].
+- [2021-03-11] Ignore IPV6 when pinging [@carbotaniuman].
+
 ## [2.0.0-rc14] - 2021-03-02
 ### Changed
-- [2021-02-10] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9].
+- [2021-03-02] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9].
-- [2021-02-10] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9].
+- [2021-03-02] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9].
-- [2021-02-10] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9].
+- [2021-03-02] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9].
 - [2021-02-25] Use HTTP/2 to download when possible [@carbotaniuman].
 
 ### Fixed
@@ -376,7 +387,8 @@ This release contains many breaking changes! Of note are the changes to the cach
 ### Fixed
 - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
 
-[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...HEAD
+[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...HEAD
+[2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0
 [2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14
 [2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13
 [2.0.0-rc12]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc11...2.0.0-rc12

build.gradle

@@ -25,39 +25,38 @@ configurations {
 dependencies {
     compileOnly group: "dev.afanasev", name: "sekret-annotation", version: "0.0.7"
 
-    implementation group: "commons-io", name: "commons-io", version: "2.7"
+    implementation group: "commons-io", name: "commons-io", version: "2.8.0"
     implementation group: "org.apache.commons", name: "commons-compress", version: "1.20"
     implementation group: "ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha4"
 
     implementation group: "io.micrometer", name: "micrometer-registry-prometheus", version: "1.6.2"
     implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0"
 
-    implementation group: "org.http4k", name: "http4k-bom", version: "4.3.5.4"
-
     implementation platform(group: "org.http4k", name: "http4k-bom", version: "4.3.5.4")
+    implementation platform(group: "com.fasterxml.jackson", name: "jackson-bom", version: "2.12.1")
+    implementation platform(group: "io.netty", name: "netty-bom", version: "4.1.60.Final")
 
     implementation group: "org.http4k", name: "http4k-core"
     implementation group: "org.http4k", name: "http4k-resilience4j"
     implementation group: "io.github.resilience4j", name: "resilience4j-micrometer", version: "1.6.1"
     implementation group: "org.http4k", name: "http4k-format-jackson"
-    implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1"
+    implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml"
-    implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1"
+    implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310"
     implementation group: "org.http4k", name: "http4k-client-okhttp"
     implementation group: "org.http4k", name: "http4k-metrics-micrometer"
     implementation group: "org.http4k", name: "http4k-server-netty"
-    implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64"
+    implementation group: "io.netty", name: "netty-transport-native-epoll", classifier: "linux-x86_64"
     implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.3.Final", classifier: "linux-x86_64"
     testImplementation group: "org.http4k", name: "http4k-testing-kotest"
-    runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final"
+    runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.36.Final"
 
-    implementation group: 'com.zaxxer', name: 'HikariCP', version: '4.0.1'
+    implementation group: "com.zaxxer", name: "HikariCP", version: "4.0.2"
-    implementation group: "com.h2database", name: "h2", version: "1.4.200"
+    implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.34.0"
-    implementation group: 'org.xerial', name: 'sqlite-jdbc', version: '3.34.0'
     implementation "org.ktorm:ktorm-core:$ktorm_version"
     implementation "org.ktorm:ktorm-jackson:$ktorm_version"
 
-    implementation "info.picocli:picocli:4.5.0"
+    implementation "info.picocli:picocli:$picocli_version"
-    kapt "info.picocli:picocli-codegen:4.5.0"
+    kapt "info.picocli:picocli-codegen:$picocli_version"
 
     testImplementation "io.kotest:kotest-runner-junit5:$kotest_version"
     testImplementation "io.kotest:kotest-assertions-core:$kotest_version"

gradle.properties

@@ -1,3 +1,4 @@
 http_4k_version=4.3.0.0
 kotest_version=4.4.1
-ktorm_version=3.2.0
+ktorm_version=3.3.0
+picocli_version=4.6.1

src/main/java/mdnet/cache/CachingInputStream.java

@@ -30,6 +30,7 @@ public class CachingInputStream extends ProxyInputStream {
 	private final OutputStream cache;
 	private final ExecutorService executor;
 	private final Runnable onClose;
+	private boolean eofReached = false;
 
 	public CachingInputStream(InputStream response, ExecutorService executor, OutputStream cache, Runnable onClose) {
 		super(response);
@@ -40,7 +41,7 @@ public class CachingInputStream extends ProxyInputStream {
 
 	@Override
 	public void close() throws IOException {
-		if (read() == EOF) {
+		if (eofReached) {
 			try {
 				in.close();
 			} catch (IOException ignored) {
@@ -50,9 +51,7 @@ public class CachingInputStream extends ProxyInputStream {
 			} catch (IOException ignored) {
 			}
 			onClose.run();
-
+		} else {
-			return;
-		}
 			executor.submit(() -> {
 				try {
 					IOUtils.copy(in, cache);
@@ -70,6 +69,7 @@ public class CachingInputStream extends ProxyInputStream {
 				}
 			});
 		}
+	}
 
 	@Override
 	public int read() throws IOException {
@@ -80,6 +80,8 @@ public class CachingInputStream extends ProxyInputStream {
 			} catch (IOException ignored) {
 				// don't let write failures affect the image loading
 			}
+		} else {
+			eofReached = true;
 		}
 		return ch;
 	}
@@ -93,6 +95,8 @@ public class CachingInputStream extends ProxyInputStream {
 			} catch (IOException ignored) {
 				// don't let write failures affect the image loading
 			}
+		} else {
+			eofReached = true;
 		}
 		return n;
 	}
@@ -106,6 +110,8 @@ public class CachingInputStream extends ProxyInputStream {
 			} catch (IOException ignored) {
 				// don't let write failures affect the image loading
 			}
+		} else {
+			eofReached = true;
 		}
 		return n;
 	}

src/main/kotlin/mdnet/BackendApi.kt

@@ -23,15 +23,17 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule
 import mdnet.ServerHandlerJackson.auto
 import mdnet.logging.info
 import mdnet.settings.*
-import org.http4k.core.Body
+import okhttp3.Dns
-import org.http4k.core.HttpHandler
+import okhttp3.OkHttpClient
-import org.http4k.core.Method
+import org.http4k.client.OkHttp
-import org.http4k.core.Request
+import org.http4k.core.*
 import org.http4k.format.ConfigurableJackson
 import org.http4k.format.asConfigurable
 import org.http4k.format.withStandardMappings
 import org.http4k.lens.LensFailure
 import org.slf4j.LoggerFactory
+import java.net.Inet4Address
+import java.net.InetAddress
 
 object ServerHandlerJackson : ConfigurableJackson(
     KotlinModule()
@@ -41,7 +43,16 @@ object ServerHandlerJackson : ConfigurableJackson(
         .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
 )
 
-class BackendApi(private val settings: ClientSettings, private val client: HttpHandler) {
+class BackendApi(private val settings: ClientSettings) {
+    private val client = OkHttp(
+        client = OkHttpClient.Builder()
+            .dns(object : Dns {
+                override fun lookup(hostname: String): List<InetAddress> {
+                    return Dns.SYSTEM.lookup(hostname).filterIsInstance<Inet4Address>()
+                }
+            })
+            .build()
+    )
     private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS
 
     fun logoutFromControl(): Boolean {

src/main/kotlin/mdnet/Constants.kt

@@ -21,7 +21,7 @@ package mdnet
 import java.time.Duration
 
 object Constants {
-    const val CLIENT_BUILD = 29
+    const val CLIENT_BUILD = 30
 
     @JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
 

src/main/kotlin/mdnet/Main.kt

@@ -97,8 +97,6 @@ class Main : Runnable {
             throw IllegalArgumentException("Cache folder $cacheFolder must be a directory")
         }
 
-        migrate(databaseFolder)
-
         val client = MangaDexClient(settingsFile, databaseFolder, cacheFolder)
         val hook = Thread {
             client.shutdown()

src/main/kotlin/mdnet/MangaDexClient.kt

@@ -198,7 +198,7 @@ class MangaDexClient(private val settingsFile: File, databaseFolder: Path, cache
     }
 
     private fun validateSettings(settings: ClientSettings) {
-        if (settings.maxCacheSizeInMebibytes < 20480) {
+        if (settings.maxCacheSizeInMebibytes < 40960) {
             throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 20480 MiB (20 GiB)")
         }
 

src/main/kotlin/mdnet/Migrator.kt

@@ -1,75 +0,0 @@
-/*
-Mangadex@Home
-Copyright (c) 2020, MangaDex Network
-This file is part of MangaDex@Home.
-
-MangaDex@Home is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-MangaDex@Home is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
-*/
-package mdnet
-
-import mdnet.cache.DbImage
-import mdnet.cache.INIT_TABLE
-import org.ktorm.database.Database
-import org.ktorm.dsl.*
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.Paths
-
-fun main() {
-    migrate(Paths.get("./"))
-}
-
-fun migrate(path: Path) {
-    val h2file = path.resolve("metadata.mv.db")
-    if (!Files.exists(h2file)) {
-        return
-    }
-
-    println("Migrating database - this may take a long time")
-
-    Class.forName("org.sqlite.JDBC")
-
-    val sqliteDb = path.resolve("metadata.db")
-    Files.deleteIfExists(sqliteDb)
-
-    val sqlite = Database.connect("jdbc:sqlite:$sqliteDb")
-    sqlite.useConnection { conn ->
-        conn.prepareStatement(INIT_TABLE).use {
-            it.execute()
-        }
-    }
-
-    val db = path.resolve("metadata")
-
-    val h2 = Database.connect("jdbc:h2:$db")
-    h2.useConnection { conn ->
-        conn.prepareStatement(INIT_TABLE).use {
-            it.execute()
-        }
-    }
-
-    h2.from(DbImage).select().asIterable().chunked(1000).forEach { list ->
-        sqlite.batchInsert(DbImage) {
-            for (data in list) {
-                item {
-                    set(DbImage.id, data[DbImage.id])
-                    set(DbImage.accessed, data[DbImage.accessed])
-                    set(DbImage.size, data[DbImage.size])
-                }
-            }
-        }
-    }
-
-    Files.move(h2file, path.resolve("metadata.mv.db.old"))
-}

src/main/kotlin/mdnet/ServerManager.kt

@@ -20,7 +20,6 @@ package mdnet
 
 import io.micrometer.prometheus.PrometheusConfig
 import io.micrometer.prometheus.PrometheusMeterRegistry
-import io.netty.util.internal.SystemPropertyUtil
 import mdnet.cache.ImageStorage
 import mdnet.data.Statistics
 import mdnet.logging.error
@@ -30,6 +29,7 @@ import mdnet.metrics.DefaultMicrometerMetrics
 import mdnet.server.getServer
 import mdnet.settings.ClientSettings
 import mdnet.settings.RemoteSettings
+import okhttp3.ConnectionPool
 import okhttp3.OkHttpClient
 import okhttp3.Protocol
 import org.http4k.client.OkHttp
@@ -78,17 +78,17 @@ class ServerManager(
             OkHttp(
                 bodyMode = BodyMode.Stream,
                 client = OkHttpClient.Builder()
-                    .callTimeout(Duration.ofSeconds(30))
+                    .connectTimeout(Duration.ofSeconds(2))
-                    .connectTimeout(Duration.ofSeconds(1))
+                    .connectionPool(
-                    .writeTimeout(Duration.ofSeconds(5))
+                        ConnectionPool(
-                    .readTimeout(Duration.ofSeconds(5))
+                            maxIdleConnections = 100,
-                    .let {
+                            keepAliveDuration = 1,
-                        if (SystemPropertyUtil.get("no-client-http2").toBoolean()) {
+                            timeUnit = TimeUnit.MINUTES
-                            it.protocols(listOf(Protocol.HTTP_1_1))
+                        )
-                        } else {
+                    )
-                            it
+                    .writeTimeout(Duration.ofSeconds(10))
-                        }
+                    .readTimeout(Duration.ofSeconds(10))
-                    }
+                    .protocols(listOf(Protocol.HTTP_1_1))
                     .build()
             )
         )
@@ -100,7 +100,7 @@ class ServerManager(
 
     init {
         state = Uninitialized
-        backendApi = BackendApi(settings, OkHttp())
+        backendApi = BackendApi(settings)
     }
 
     fun start() {

src/main/kotlin/mdnet/netty/ApplicationNetty.kt

@@ -64,6 +64,7 @@ import java.security.PrivateKey
 import java.security.cert.CertificateFactory
 import java.security.cert.X509Certificate
 import java.util.Locale
+import java.util.concurrent.TimeUnit
 import javax.net.ssl.SSLException
 
 sealed class NettyTransport(threads: Int) {
@@ -76,12 +77,6 @@ sealed class NettyTransport(threads: Int) {
         }
     )
 
-    fun shutdownGracefully() {
-        bossGroup.shutdownGracefully().sync()
-        workerGroup.shutdownGracefully().sync()
-        executor.shutdownGracefully().sync()
-    }
-
     private class NioTransport(threads: Int) : NettyTransport(threads) {
         override val bossGroup = NioEventLoopGroup(1)
         override val workerGroup = NioEventLoopGroup(8)
@@ -108,7 +103,7 @@ sealed class NettyTransport(threads: Int) {
             val name = SystemPropertyUtil.get("os.name").toLowerCase(Locale.UK).trim { it <= ' ' }
 
             val threadsToUse = if (threads == 0) defaultNumThreads() else threads
-            LOGGER.info { "Choosing a transport using $threadsToUse threads" }
+            LOGGER.info { "Choosing a transport with $threadsToUse threads" }
 
             if (name.startsWith("linux")) {
                 if (!SystemPropertyUtil.get("no-iouring").toBoolean()) {
@@ -116,7 +111,7 @@ sealed class NettyTransport(threads: Int) {
                         LOGGER.info { "Using IOUring transport" }
                         return IOUringTransport(threadsToUse)
                     } else {
-                        LOGGER.info(IOUring.unavailabilityCause()) {
+                        LOGGER.info {
                             "IOUring transport not available (this may be normal)"
                         }
                     }
@@ -127,7 +122,7 @@ sealed class NettyTransport(threads: Int) {
                         LOGGER.info { "Using Epoll transport" }
                         return EpollTransport(threadsToUse)
                     } else {
-                        LOGGER.info(Epoll.unavailabilityCause()) {
+                        LOGGER.info {
                             "Epoll transport not available (this may be normal)"
                         }
                     }
@@ -145,7 +140,7 @@ class Netty(
     private val serverSettings: ServerSettings,
     private val statistics: Statistics
 ) : ServerConfig {
-    override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer {
+    override fun toServer(http: HttpHandler): Http4kServer = object : Http4kServer {
         private val transport = NettyTransport.bestForPlatform(serverSettings.threads)
 
         private lateinit var channel: Channel
@@ -190,7 +185,7 @@ class Netty(
                         )
 
                         ch.pipeline().addLast("streamer", ChunkedWriteHandler())
-                        ch.pipeline().addLast(transport.executor, "handler", Http4kChannelHandler(httpHandler))
+                        ch.pipeline().addLast(transport.executor, "handler", Http4kChannelHandler(http))
 
                         ch.pipeline().addLast(
                             "exceptions",
@@ -216,10 +211,14 @@ class Netty(
 
         override fun stop() = apply {
             channel.close().sync()
-            transport.shutdownGracefully()
+            transport.run {
+                bossGroup.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).sync()
+                workerGroup.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).sync()
+                executor.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).sync()
+            }
         }
 
-        override fun port(): Int = serverSettings.port
+        override fun port(): Int = (channel.localAddress() as InetSocketAddress).port
     }
 
     companion object {

src/main/kotlin/mdnet/server/ImageServer.kt

@@ -105,9 +105,6 @@ fun getServer(
 
     val verifier = TokenVerifier(
         tokenKey = remoteSettings.tokenKey,
-        shouldVerify = { chapter, _ ->
-            !remoteSettings.disableTokens && !(chapter == "1b682e7b24ae7dbdc5064eeeb8e8e353" || chapter == "8172a46adc798f4f4ace6663322a383e")
-        }
     )
 
     return timeRequest()

src/main/kotlin/mdnet/server/TokenVerifier.kt

@@ -37,15 +37,13 @@ import org.slf4j.LoggerFactory
 import java.time.OffsetDateTime
 import java.util.Base64
 
-class TokenVerifier(tokenKey: ByteArray, private val shouldVerify: (String, String) -> Boolean) : Filter {
+class TokenVerifier(tokenKey: ByteArray) : Filter {
     private val box = TweetNaclFast.SecretBox(tokenKey)
 
     override fun invoke(next: HttpHandler): HttpHandler {
         return then@{
             val chapterHash = Path.of("chapterHash")(it)
-            val fileName = Path.of("fileName")(it)
 
-            if (shouldVerify(chapterHash, fileName)) {
             val cleanedUri = it.uri.path.replaceBefore("/data", "/{token}")
 
             val tokenArr = try {
@@ -87,7 +85,6 @@ class TokenVerifier(tokenKey: ByteArray, private val shouldVerify: (String, Stri
                 LOGGER.info { "Request for $cleanedUri rejected for inapplicable token" }
                 return@then Response(Status.FORBIDDEN).body("Token is inapplicable for the image")
             }
-            }
 
             return@then next(it)
         }

src/test/kotlin/mdnet/cache/ImageStorageTest.kt

@@ -42,7 +42,7 @@ class ImageStorageTest : FreeSpec() {
         val imageStorage = ImageStorage(
             maxSize = 5,
             cacheDirectory = tempdir().toPath(),
-            database = Database.connect("jdbc:h2:${tempfile()}"),
+            database = Database.connect("jdbc:sqlite:${tempfile()}"),
             autoPrune = false,
         )
 
@@ -160,7 +160,7 @@ class ImageStorageSlowTest : FreeSpec() {
         val imageStorage = ImageStorage(
             maxSize = 4097,
             cacheDirectory = tempdir().toPath(),
-            database = Database.connect("jdbc:h2:${tempfile()}"),
+            database = Database.connect("jdbc:sqlite:${tempfile()}"),
         )
 
         "autoPrune" - {

src/test/kotlin/mdnet/server/ImageServerTest.kt

@@ -112,7 +112,7 @@ class ImageServerTest : FreeSpec() {
                 val storage = ImageStorage(
                     maxSize = 100000,
                     cacheDirectory = tempdir().toPath(),
-                    database = Database.connect("jdbc:h2:${tempfile()}"),
+                    database = Database.connect("jdbc:sqlite:${tempfile()}"),
                     autoPrune = false,
                 )
 
@@ -157,7 +157,7 @@ class ImageServerTest : FreeSpec() {
             val storage = ImageStorage(
                 maxSize = 100000,
                 cacheDirectory = tempdir().toPath(),
-                database = Database.connect("jdbc:h2:${tempfile()}"),
+                database = Database.connect("jdbc:sqlite:${tempfile()}"),
                 autoPrune = false,
             )
 

src/test/kotlin/mdnet/server/TokenVerifierTest.kt

@@ -31,9 +31,7 @@ class TokenVerifierTest : FreeSpec() {
         val clientKeys = TweetNaclFast.Box.keyPair()
         val box = TweetNaclFast.Box(clientKeys.publicKey, remoteKeys.secretKey)
 
-        val backend = TokenVerifier(box.before()) { _, _ ->
+        val backend = TokenVerifier(box.before()).then {
-            true
-        }.then {
             Response(Status.OK)
         }