Revert "Switch from apache to okhttp"

This reverts commit a4a28e1b

CHANGELOG.md

@@ -20,8 +20,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [2.0.0-rc6] - 2021-01-27
 ### Fixed
 - [2021-01-27] Upped max threadpool size [@carbotaniuman].
-- [2021-01-27] Switch to OkHttp and hopefully fix a class of bugs [@carbotaniuman].
-- [2021-01-27] Add ability for nodes to specify external ip [@carbotaniuman].
 
 ## [2.0.0-rc5] - 2021-01-27
 ### Changed

build.gradle

@@ -36,7 +36,7 @@ dependencies {
     implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version"
     implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1"
     implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1"
-    implementation group: "org.http4k", name: "http4k-client-okhttp", version: "$http_4k_version"
+    implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version"
     implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version"
     implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
     implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64"

src/main/kotlin/mdnet/BackendApi.kt

@@ -23,10 +23,13 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule
 import mdnet.ServerHandlerJackson.auto
 import mdnet.logging.info
 import mdnet.settings.ClientSettings
-import mdnet.settings.LogoutRequest
 import mdnet.settings.RemoteSettings
-import mdnet.settings.SettingsRequest
+import org.apache.hc.client5.http.impl.DefaultSchemePortResolver
-import org.http4k.client.OkHttp
+import org.apache.hc.client5.http.impl.classic.HttpClients
+import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner
+import org.apache.hc.core5.http.HttpHost
+import org.apache.hc.core5.http.protocol.HttpContext
+import org.http4k.client.ApacheClient
 import org.http4k.core.Body
 import org.http4k.core.Method
 import org.http4k.core.Request
@@ -34,7 +37,7 @@ import org.http4k.format.ConfigurableJackson
 import org.http4k.format.asConfigurable
 import org.http4k.format.withStandardMappings
 import org.slf4j.LoggerFactory
-import java.lang.RuntimeException
+import java.net.InetAddress
 
 object ServerHandlerJackson : ConfigurableJackson(
     KotlinModule()
@@ -46,44 +49,57 @@ object ServerHandlerJackson : ConfigurableJackson(
 
 class BackendApi(private val settings: ClientSettings) {
     private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS
-    private val client = OkHttp()
+    private val client = ApacheClient(
+        client = HttpClients.custom()
+            .setRoutePlanner(
+                object : DefaultRoutePlanner(DefaultSchemePortResolver()) {
+                    override fun determineLocalAddress(firstHop: HttpHost?, context: HttpContext?): InetAddress {
+                        return InetAddress.getByName(settings.serverSettings.hostname)
+                    }
+                }
+            )
+            .build()
+    )
 
     fun logoutFromControl(): Boolean {
-        val serverSettings = settings.serverSettings
-
         LOGGER.info { "Disconnecting from the control server" }
-
+        val params = mapOf<String, Any>(
-        val request = LOGOUT_REQUEST_LENS(
+            "secret" to settings.serverSettings.secret
-            LogoutRequest(serverSettings.secret),
-            Request(Method.POST, serverAddress + "stop")
         )
 
+        val request = STRING_ANY_MAP_LENS(params, Request(Method.POST, serverAddress + "stop"))
         val response = client(request)
 
         return response.status.successful
     }
 
-    private fun getPingParams(tlsCreatedAt: String? = null): SettingsRequest {
+    private fun getPingParams(tlsCreatedAt: String? = null): Map<String, Any> {
         val serverSettings = settings.serverSettings
-        return SettingsRequest(
+        return mapOf(
-            secret = serverSettings.secret,
+            "secret" to serverSettings.secret,
-            port = if (serverSettings.externalPort != 0) {
+            "port" to let {
-                serverSettings.externalPort
+                if (serverSettings.externalPort != 0) {
-            } else {
+                    serverSettings.externalPort
-                serverSettings.port
+                } else {
+                    serverSettings.port
+                }
             },
-            buildVersion = Constants.CLIENT_BUILD,
+            "disk_space" to settings.maxCacheSizeInMebibytes * 1024 * 1024,
-            diskSpace = settings.maxCacheSizeInMebibytes * 1024 * 1024,
+            "network_speed" to serverSettings.externalMaxKilobitsPerSecond * 1000 / 8,
-            networkSpeed = serverSettings.externalMaxKilobitsPerSecond * 1000 / 8,
+            "build_version" to Constants.CLIENT_BUILD
-            ipAddress = serverSettings.externalIp,
+        ).let {
-            tlsCreatedAt = tlsCreatedAt,
+            if (tlsCreatedAt != null) {
-        )
+                it.plus("tls_created_at" to tlsCreatedAt)
+            } else {
+                it
+            }
+        }
     }
 
-    fun loginToControl(): RemoteSettings {
+    fun loginToControl(): RemoteSettings? {
         LOGGER.info { "Connecting to the control server" }
 
-        val request = SETTINGS_REQUEST_LENS(
+        val request = STRING_ANY_MAP_LENS(
             getPingParams(null),
             Request(
                 Method.POST,
@@ -95,14 +111,14 @@ class BackendApi(private val settings: ClientSettings) {
         return if (response.status.successful) {
             SERVER_SETTINGS_LENS(response)
         } else {
-            throw RuntimeException(response.bodyString())
+            null
         }
     }
 
     fun pingControl(old: RemoteSettings): RemoteSettings? {
         LOGGER.info { "Pinging the control server" }
 
-        val request = SETTINGS_REQUEST_LENS(
+        val request = STRING_ANY_MAP_LENS(
             getPingParams(old.tls!!.createdAt),
             Request(
                 Method.POST,
@@ -120,8 +136,7 @@ class BackendApi(private val settings: ClientSettings) {
 
     companion object {
         private val LOGGER = LoggerFactory.getLogger(BackendApi::class.java)
-        private val SETTINGS_REQUEST_LENS = Body.auto<SettingsRequest>().toLens()
+        private val STRING_ANY_MAP_LENS = Body.auto<Map<String, Any>>().toLens()
-        private val LOGOUT_REQUEST_LENS = Body.auto<LogoutRequest>().toLens()
         private val SERVER_SETTINGS_LENS = Body.auto<RemoteSettings>().toLens()
         private const val SERVER_ADDRESS = "https://api.mangadex.network/"
     }

src/main/kotlin/mdnet/ServerManager.kt

@@ -209,6 +209,7 @@ class ServerManager(
         this.state as Uninitialized
 
         val remoteSettings = backendApi.loginToControl()
+            ?: throw RuntimeException("Failed to get a login response from server")
         LOGGER.info { "Server settings received: $remoteSettings" }
         warnBasedOnSettings(remoteSettings)
 

src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt

@@ -26,7 +26,6 @@ import mdnet.logging.debug
 import mdnet.logging.warn
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
 import org.apache.commons.io.IOUtils
-import org.http4k.client.OkHttp
 import org.http4k.core.Filter
 import org.http4k.core.HttpHandler
 import org.http4k.core.Method
@@ -91,6 +90,7 @@ class GeoIpMetricsFilterBuilder(
     private val enableGeoIp: Boolean,
     private val license: String,
     private val registry: PrometheusMeterRegistry,
+    private val client: HttpHandler
 ) {
     fun build(): GeoIpMetricsFilter {
         return if (enableGeoIp) {
@@ -108,8 +108,6 @@ class GeoIpMetricsFilterBuilder(
         val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb")
 
         val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license)
-
-        val client = OkHttp()
         val response = client(Request(Method.GET, geoIpDatabaseUri))
         if (response.status != Status.OK) {
             throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})")

src/main/kotlin/mdnet/server/ImageServer.kt

@@ -44,8 +44,12 @@ import mdnet.security.TweetNaclFast
 import mdnet.settings.MetricsSettings
 import mdnet.settings.RemoteSettings
 import mdnet.settings.ServerSettings
-import okhttp3.OkHttpClient
+import org.apache.hc.client5.http.config.RequestConfig
-import org.http4k.client.OkHttp
+import org.apache.hc.client5.http.cookie.StandardCookieSpec
+import org.apache.hc.client5.http.impl.classic.HttpClients
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
+import org.apache.hc.core5.util.Timeout
+import org.http4k.client.ApacheClient
 import org.http4k.core.*
 import org.http4k.filter.CachingFilters
 import org.http4k.filter.ClientFilters
@@ -62,7 +66,6 @@ import java.io.BufferedInputStream
 import java.io.BufferedOutputStream
 import java.io.InputStream
 import java.time.Clock
-import java.time.Duration
 import java.time.OffsetDateTime
 import java.util.*
 import java.util.concurrent.Executors
@@ -232,20 +235,31 @@ fun getServer(
     statistics: Statistics,
     registry: PrometheusMeterRegistry,
 ): Http4kServer {
-    val okHttpClient = OkHttp(
+    val apache = ApacheClient(
-        bodyMode = BodyMode.Stream,
+        responseBodyMode = BodyMode.Stream,
-        client = OkHttpClient.Builder()
+        client = HttpClients.custom()
-            .followRedirects(false)
+            .disableConnectionState()
-            .connectTimeout(Duration.ofSeconds(2))
+            .setDefaultRequestConfig(
-            .readTimeout(Duration.ofSeconds(5))
+                RequestConfig.custom()
-            .writeTimeout(Duration.ofSeconds(5))
+                    .setCookieSpec(StandardCookieSpec.IGNORE)
+                    .setConnectTimeout(Timeout.ofSeconds(2))
+                    .setResponseTimeout(Timeout.ofSeconds(2))
+                    .setConnectionRequestTimeout(Timeout.ofSeconds(1))
+                    .build()
+            )
+            .setConnectionManager(
+                PoolingHttpClientConnectionManagerBuilder.create()
+                    .setMaxConnTotal(500)
+                    .setMaxConnPerRoute(500)
+                    .build()
+            )
             .build()
     )
 
     val client =
         ClientFilters.SetBaseUriFrom(remoteSettings.imageServer)
             .then(ClientFilters.MicrometerMetrics.RequestTimer(registry))
-            .then(okHttpClient)
+            .then(apache)
 
     val imageServer = ImageServer(
         storage = storage,
@@ -297,7 +311,7 @@ fun getServer(
             ).withFilter(
                 ServerFilters.MicrometerMetrics.RequestTimer(registry, labeler = PostTransactionLabeler())
             ).withFilter(
-                GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry).build()
+                GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry, apache).build()
             )
         )
         .asServer(Netty(remoteSettings.tls!!, serverSettings, statistics))

src/main/kotlin/mdnet/settings/ClientSettings.kt

@@ -35,7 +35,6 @@ data class ClientSettings(
 data class ServerSettings(
     @field:Secret val secret: String,
     val externalPort: Int = 0,
-    val externalIp: String? = null,
     val gracefulShutdownWaitSeconds: Int = 60,
     val hostname: String = "0.0.0.0",
     val maxKilobitsPerSecond: Long = 0,

src/main/kotlin/mdnet/settings/SettingsRequest.kt

@@ -1,39 +0,0 @@
-/*
-Mangadex@Home
-Copyright (c) 2020, MangaDex Network
-This file is part of MangaDex@Home.
-
-MangaDex@Home is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-MangaDex@Home is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
-*/
-package mdnet.settings
-
-import com.fasterxml.jackson.databind.PropertyNamingStrategies
-import com.fasterxml.jackson.databind.annotation.JsonNaming
-import dev.afanasev.sekret.Secret
-
-@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
-data class SettingsRequest(
-    @field:Secret val secret: String,
-    val ipAddress: String?,
-    val port: Int,
-    val diskSpace: Long,
-    val networkSpeed: Long,
-    val buildVersion: Int,
-    val tlsCreatedAt: String?,
-)
-
-@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
-data class LogoutRequest(
-    @field:Secret val secret: String,
-)