Use HTTP/2

CHANGELOG.md

@@ -8,19 +8,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 ### Changed
-- [2020-02-10] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9]
-- [2020-02-10] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9]
-- [2020-02-10] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9]
 
 ### Deprecated
 
 ### Removed
 
 ### Fixed
-- [2020-02-21] Fix pipeline [@_tde9]
 
 ### Security
 
+## [2.0.0-rc14] - 2021-03-02
+### Changed
+- [2020-02-10] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9].
+- [2020-02-10] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9].
+- [2020-02-10] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9].
+- [2020-02-25] Use HTTP/2 to download when possible [@carbotaniuman].
+
+### Fixed
+- [2020-02-21] Fix pipeline [@_tde9].
+
 ## [2.0.0-rc13] - 2021-02-19
 ### Changed
 - [2021-02-19] Back to sqlite we go [@carbotaniuman].
@@ -369,7 +375,8 @@ This release contains many breaking changes! Of note are the changes to the cach
 ### Fixed
 - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
 
-[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...HEAD
+[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...HEAD
+[2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14
 [2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13
 [2.0.0-rc12]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc11...2.0.0-rc12
 [2.0.0-rc11]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc10...2.0.0-rc11

build.gradle

@@ -1,7 +1,7 @@
 plugins {
     id "jacoco"
     id "java"
-    id "org.jetbrains.kotlin.jvm" version "1.4.20"
+    id "org.jetbrains.kotlin.jvm" version "1.4.30"
     id "org.jetbrains.kotlin.kapt" version "1.4.0"
     id "application"
     id "com.github.johnrengelman.shadow" version "5.2.0"
@@ -32,18 +32,22 @@ dependencies {
     implementation group: "io.micrometer", name: "micrometer-registry-prometheus", version: "1.6.2"
     implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0"
 
-    implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version"
+    implementation group: "org.http4k", name: "http4k-bom", version: "4.3.5.4"
-    implementation group: "org.http4k", name: "http4k-resilience4j", version: "$http_4k_version"
+
+    implementation platform(group: "org.http4k", name: "http4k-bom", version: "4.3.5.4")
+
+    implementation group: "org.http4k", name: "http4k-core"
+    implementation group: "org.http4k", name: "http4k-resilience4j"
     implementation group: "io.github.resilience4j", name: "resilience4j-micrometer", version: "1.6.1"
-    implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version"
+    implementation group: "org.http4k", name: "http4k-format-jackson"
     implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1"
     implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1"
-    implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version"
+    implementation group: "org.http4k", name: "http4k-client-okhttp"
-    implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version"
+    implementation group: "org.http4k", name: "http4k-metrics-micrometer"
-    implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
+    implementation group: "org.http4k", name: "http4k-server-netty"
     implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64"
     implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.3.Final", classifier: "linux-x86_64"
-    testImplementation group: "org.http4k", name: "http4k-testing-kotest", version: "$http_4k_version"
+    testImplementation group: "org.http4k", name: "http4k-testing-kotest"
     runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final"
 
     implementation group: 'com.zaxxer', name: 'HikariCP', version: '4.0.1'

gradle.properties

@@ -1,4 +1,3 @@
-http_4k_version=4.1.0.0
+http_4k_version=4.3.0.0
-exposed_version=0.26.2
+kotest_version=4.4.1
-kotest_version=4.4.0.RC1
 ktorm_version=3.2.0

src/main/kotlin/mdnet/BackendApi.kt

@@ -23,7 +23,6 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule
 import mdnet.ServerHandlerJackson.auto
 import mdnet.logging.info
 import mdnet.settings.*
-import org.http4k.client.ApacheClient
 import org.http4k.core.Body
 import org.http4k.core.HttpHandler
 import org.http4k.core.Method
@@ -42,9 +41,8 @@ object ServerHandlerJackson : ConfigurableJackson(
         .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
 )
 
-class BackendApi(private val settings: ClientSettings) {
+class BackendApi(private val settings: ClientSettings, private val client: HttpHandler) {
     private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS
-    private val client = ApacheClient()
 
     fun logoutFromControl(): Boolean {
         val serverSettings = settings.serverSettings

src/main/kotlin/mdnet/Constants.kt

@@ -21,7 +21,7 @@ package mdnet
 import java.time.Duration
 
 object Constants {
-    const val CLIENT_BUILD = 28
+    const val CLIENT_BUILD = 29
 
     @JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
 

src/main/kotlin/mdnet/Main.kt

@@ -77,6 +77,10 @@ class Main : Runnable {
                 |if you are not, manually move update your --database args!
                 |note: the database file itself should be named metadata.{extension}
                 |where {extension} can be `.db` or `.mv.db`
+                |
+                |If this is your first time seeing this message, please check out the support
+                |channel as things HAVE changed. Failure to do so WILL require
+                |a cache wipe.
             """.trimMargin()
             )
             println()

src/main/kotlin/mdnet/ServerManager.kt

@@ -20,6 +20,7 @@ package mdnet
 
 import io.micrometer.prometheus.PrometheusConfig
 import io.micrometer.prometheus.PrometheusMeterRegistry
+import io.netty.util.internal.SystemPropertyUtil
 import mdnet.cache.ImageStorage
 import mdnet.data.Statistics
 import mdnet.logging.error
@@ -29,19 +30,16 @@ import mdnet.metrics.DefaultMicrometerMetrics
 import mdnet.server.getServer
 import mdnet.settings.ClientSettings
 import mdnet.settings.RemoteSettings
-import org.apache.hc.client5.http.config.RequestConfig
+import okhttp3.OkHttpClient
-import org.apache.hc.client5.http.cookie.StandardCookieSpec
+import okhttp3.Protocol
-import org.apache.hc.client5.http.impl.classic.HttpClients
+import org.http4k.client.OkHttp
-import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
-import org.apache.hc.core5.util.TimeValue
-import org.apache.hc.core5.util.Timeout
-import org.http4k.client.ApacheClient
 import org.http4k.core.BodyMode
 import org.http4k.core.then
 import org.http4k.filter.ClientFilters
 import org.http4k.filter.MicrometerMetrics
 import org.http4k.server.Http4kServer
 import org.slf4j.LoggerFactory
+import java.time.Duration
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
@@ -73,26 +71,24 @@ class ServerManager(
     private val executor = Executors.newSingleThreadScheduledExecutor()
     private val registry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
     private val statistics = Statistics()
-    private val connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
+
-        .setMaxConnTotal(500)
+    private val okhttp = ClientFilters.MicrometerMetrics.RequestCounter(registry)
-        .setMaxConnPerRoute(500)
-        .build()
-    private val apache = ClientFilters.MicrometerMetrics.RequestCounter(registry)
         .then(ClientFilters.MicrometerMetrics.RequestTimer(registry))
         .then(
-            ApacheClient(
+            OkHttp(
-                responseBodyMode = BodyMode.Stream,
+                bodyMode = BodyMode.Stream,
-                client = HttpClients.custom()
+                client = OkHttpClient.Builder()
-                    .disableConnectionState()
+                    .callTimeout(Duration.ofSeconds(30))
-                    .setDefaultRequestConfig(
+                    .connectTimeout(Duration.ofSeconds(1))
-                        RequestConfig.custom()
+                    .writeTimeout(Duration.ofSeconds(5))
-                            .setCookieSpec(StandardCookieSpec.IGNORE)
+                    .readTimeout(Duration.ofSeconds(5))
-                            .setConnectTimeout(Timeout.ofSeconds(2))
+                    .let {
-                            .setResponseTimeout(Timeout.ofSeconds(2))
+                        if (SystemPropertyUtil.get("no-client-http2").toBoolean()) {
-                            .setConnectionRequestTimeout(Timeout.ofSeconds(1))
+                            it.protocols(listOf(Protocol.HTTP_1_1))
-                            .build()
+                        } else {
-                    )
+                            it
-                    .setConnectionManager(connectionManager)
+                        }
+                    }
                     .build()
             )
         )
@@ -104,7 +100,7 @@ class ServerManager(
 
     init {
         state = Uninitialized
-        backendApi = BackendApi(settings)
+        backendApi = BackendApi(settings, OkHttp())
     }
 
     fun start() {
@@ -205,20 +201,6 @@ class ServerManager(
             45, 45, TimeUnit.SECONDS
         )
 
-        executor.scheduleWithFixedDelay(
-            {
-                try {
-                    LOGGER.info { "Closing old Apache HTTP connections" }
-
-                    connectionManager.closeExpired()
-                    connectionManager.closeIdle(TimeValue.ofSeconds(30))
-                } catch (e: Exception) {
-                    LOGGER.warn(e) { "Old Apache HTTP connection closer failed" }
-                }
-            },
-            45, 45, TimeUnit.SECONDS
-        )
-
         LOGGER.info { "Image server has started" }
     }
 
@@ -271,7 +253,7 @@ class ServerManager(
             settings.metricsSettings,
             statistics,
             registry,
-            apache,
+            okhttp,
         ).start()
 
         this.state = Running(server, remoteSettings)

src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt

@@ -25,8 +25,7 @@ import io.micrometer.prometheus.PrometheusMeterRegistry
 import mdnet.logging.debug
 import mdnet.logging.warn
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
-import org.apache.commons.io.IOUtils
+import org.http4k.client.OkHttp
-import org.http4k.client.ApacheClient
 import org.http4k.core.Filter
 import org.http4k.core.HttpHandler
 import org.http4k.core.Method
@@ -37,7 +36,6 @@ import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import java.net.InetAddress
 import java.net.UnknownHostException
-import java.nio.file.Files
 
 class GeoIpMetricsFilter(
     private val databaseReader: DatabaseReader?,
@@ -92,7 +90,8 @@ class GeoIpMetricsFilterBuilder(
     private val license: String,
     private val registry: PrometheusMeterRegistry,
 ) {
-    val client = ApacheClient()
+    private val client = OkHttp()
+
     fun build(): GeoIpMetricsFilter {
         return if (enableGeoIp) {
             LOGGER.info("GeoIp initialising")
@@ -105,32 +104,27 @@ class GeoIpMetricsFilterBuilder(
     }
 
     private fun initDatabase(): DatabaseReader {
-        val databaseFileDir = Files.createTempDirectory("mangadex-geoip")
-        val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb")
-
         val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license)
         val response = client(Request(Method.GET, geoIpDatabaseUri))
         if (response.status != Status.OK) {
             throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})")
         }
 
-        response.use {
+        return response.use { data ->
-            val archiveStream = TarArchiveInputStream(it.body.gunzippedStream().stream)
+            TarArchiveInputStream(data.body.gunzippedStream().stream).use {
-            var entry = archiveStream.nextTarEntry
+                var entry = it.nextTarEntry
-            while (!entry.name.endsWith(".mmdb")) {
+                while (!entry.name.endsWith(".mmdb")) {
-                LOGGER.debug { "Skipped non-database file: ${entry.name}" }
+                    LOGGER.debug { "Skipped non-database file: ${entry.name}" }
-                entry = archiveStream.nextTarEntry
+                    entry = it.nextTarEntry
+                }
+
+                // reads only the current entry to its end
+                DatabaseReader
+                    .Builder(it)
+                    .withCache(CHMCache())
+                    .build()
             }
-
-            // reads only the current entry to its end
-            val dbBytes = IOUtils.toByteArray(archiveStream)
-            Files.write(databaseFile, dbBytes)
         }
-
-        return DatabaseReader
-            .Builder(databaseFile.toFile())
-            .withCache(CHMCache())
-            .build()
     }
 
     companion object {