diff --git a/CHANGELOG.md b/CHANGELOG.md index 93a7cf5..a6bb66c 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [2.0.0-rc6] - 2021-01-27 ### Fixed - [2021-01-27] Upped max threadpool size [@carbotaniuman]. -- [2021-01-27] Switch to OkHttp and hopefully fix a class of bugs [@carbotaniuman]. -- [2021-01-27] Add ability for nodes to specify external ip [@carbotaniuman]. ## [2.0.0-rc5] - 2021-01-27 ### Changed diff --git a/build.gradle b/build.gradle index 11836ee..edba11c 100644 --- a/build.gradle +++ b/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version" implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1" implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1" - implementation group: "org.http4k", name: "http4k-client-okhttp", version: "$http_4k_version" + implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64" diff --git a/src/main/kotlin/mdnet/BackendApi.kt b/src/main/kotlin/mdnet/BackendApi.kt index 35e73f3..6d30b02 100644 --- a/src/main/kotlin/mdnet/BackendApi.kt +++ b/src/main/kotlin/mdnet/BackendApi.kt @@ -23,10 +23,13 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule import mdnet.ServerHandlerJackson.auto import mdnet.logging.info import mdnet.settings.ClientSettings -import mdnet.settings.LogoutRequest import mdnet.settings.RemoteSettings -import mdnet.settings.SettingsRequest -import org.http4k.client.OkHttp +import org.apache.hc.client5.http.impl.DefaultSchemePortResolver +import org.apache.hc.client5.http.impl.classic.HttpClients +import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner +import org.apache.hc.core5.http.HttpHost +import org.apache.hc.core5.http.protocol.HttpContext +import org.http4k.client.ApacheClient import org.http4k.core.Body import org.http4k.core.Method import org.http4k.core.Request @@ -34,7 +37,7 @@ import org.http4k.format.ConfigurableJackson import org.http4k.format.asConfigurable import org.http4k.format.withStandardMappings import org.slf4j.LoggerFactory -import java.lang.RuntimeException +import java.net.InetAddress object ServerHandlerJackson : ConfigurableJackson( KotlinModule() @@ -46,44 +49,57 @@ object ServerHandlerJackson : ConfigurableJackson( class BackendApi(private val settings: ClientSettings) { private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS - private val client = OkHttp() + private val client = ApacheClient( + client = HttpClients.custom() + .setRoutePlanner( + object : DefaultRoutePlanner(DefaultSchemePortResolver()) { + override fun determineLocalAddress(firstHop: HttpHost?, context: HttpContext?): InetAddress { + return InetAddress.getByName(settings.serverSettings.hostname) + } + } + ) + .build() + ) fun logoutFromControl(): Boolean { - val serverSettings = settings.serverSettings - LOGGER.info { "Disconnecting from the control server" } - - val request = LOGOUT_REQUEST_LENS( - LogoutRequest(serverSettings.secret), - Request(Method.POST, serverAddress + "stop") + val params = mapOf( + "secret" to settings.serverSettings.secret ) + val request = STRING_ANY_MAP_LENS(params, Request(Method.POST, serverAddress + "stop")) val response = client(request) return response.status.successful } - private fun getPingParams(tlsCreatedAt: String? = null): SettingsRequest { + private fun getPingParams(tlsCreatedAt: String? = null): Map { val serverSettings = settings.serverSettings - return SettingsRequest( - secret = serverSettings.secret, - port = if (serverSettings.externalPort != 0) { - serverSettings.externalPort - } else { - serverSettings.port + return mapOf( + "secret" to serverSettings.secret, + "port" to let { + if (serverSettings.externalPort != 0) { + serverSettings.externalPort + } else { + serverSettings.port + } }, - buildVersion = Constants.CLIENT_BUILD, - diskSpace = settings.maxCacheSizeInMebibytes * 1024 * 1024, - networkSpeed = serverSettings.externalMaxKilobitsPerSecond * 1000 / 8, - ipAddress = serverSettings.externalIp, - tlsCreatedAt = tlsCreatedAt, - ) + "disk_space" to settings.maxCacheSizeInMebibytes * 1024 * 1024, + "network_speed" to serverSettings.externalMaxKilobitsPerSecond * 1000 / 8, + "build_version" to Constants.CLIENT_BUILD + ).let { + if (tlsCreatedAt != null) { + it.plus("tls_created_at" to tlsCreatedAt) + } else { + it + } + } } - fun loginToControl(): RemoteSettings { + fun loginToControl(): RemoteSettings? { LOGGER.info { "Connecting to the control server" } - val request = SETTINGS_REQUEST_LENS( + val request = STRING_ANY_MAP_LENS( getPingParams(null), Request( Method.POST, @@ -95,14 +111,14 @@ class BackendApi(private val settings: ClientSettings) { return if (response.status.successful) { SERVER_SETTINGS_LENS(response) } else { - throw RuntimeException(response.bodyString()) + null } } fun pingControl(old: RemoteSettings): RemoteSettings? { LOGGER.info { "Pinging the control server" } - val request = SETTINGS_REQUEST_LENS( + val request = STRING_ANY_MAP_LENS( getPingParams(old.tls!!.createdAt), Request( Method.POST, @@ -120,8 +136,7 @@ class BackendApi(private val settings: ClientSettings) { companion object { private val LOGGER = LoggerFactory.getLogger(BackendApi::class.java) - private val SETTINGS_REQUEST_LENS = Body.auto().toLens() - private val LOGOUT_REQUEST_LENS = Body.auto().toLens() + private val STRING_ANY_MAP_LENS = Body.auto>().toLens() private val SERVER_SETTINGS_LENS = Body.auto().toLens() private const val SERVER_ADDRESS = "https://api.mangadex.network/" } diff --git a/src/main/kotlin/mdnet/ServerManager.kt b/src/main/kotlin/mdnet/ServerManager.kt index 1a749e9..4faef83 100644 --- a/src/main/kotlin/mdnet/ServerManager.kt +++ b/src/main/kotlin/mdnet/ServerManager.kt @@ -209,6 +209,7 @@ class ServerManager( this.state as Uninitialized val remoteSettings = backendApi.loginToControl() + ?: throw RuntimeException("Failed to get a login response from server") LOGGER.info { "Server settings received: $remoteSettings" } warnBasedOnSettings(remoteSettings) diff --git a/src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt b/src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt index 09ef17d..a72324c 100644 --- a/src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt +++ b/src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt @@ -26,7 +26,6 @@ import mdnet.logging.debug import mdnet.logging.warn import org.apache.commons.compress.archivers.tar.TarArchiveInputStream import org.apache.commons.io.IOUtils -import org.http4k.client.OkHttp import org.http4k.core.Filter import org.http4k.core.HttpHandler import org.http4k.core.Method @@ -91,6 +90,7 @@ class GeoIpMetricsFilterBuilder( private val enableGeoIp: Boolean, private val license: String, private val registry: PrometheusMeterRegistry, + private val client: HttpHandler ) { fun build(): GeoIpMetricsFilter { return if (enableGeoIp) { @@ -108,8 +108,6 @@ class GeoIpMetricsFilterBuilder( val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb") val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license) - - val client = OkHttp() val response = client(Request(Method.GET, geoIpDatabaseUri)) if (response.status != Status.OK) { throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})") diff --git a/src/main/kotlin/mdnet/server/ImageServer.kt b/src/main/kotlin/mdnet/server/ImageServer.kt index ce0a8df..d4d95e7 100644 --- a/src/main/kotlin/mdnet/server/ImageServer.kt +++ b/src/main/kotlin/mdnet/server/ImageServer.kt @@ -44,8 +44,12 @@ import mdnet.security.TweetNaclFast import mdnet.settings.MetricsSettings import mdnet.settings.RemoteSettings import mdnet.settings.ServerSettings -import okhttp3.OkHttpClient -import org.http4k.client.OkHttp +import org.apache.hc.client5.http.config.RequestConfig +import org.apache.hc.client5.http.cookie.StandardCookieSpec +import org.apache.hc.client5.http.impl.classic.HttpClients +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder +import org.apache.hc.core5.util.Timeout +import org.http4k.client.ApacheClient import org.http4k.core.* import org.http4k.filter.CachingFilters import org.http4k.filter.ClientFilters @@ -62,7 +66,6 @@ import java.io.BufferedInputStream import java.io.BufferedOutputStream import java.io.InputStream import java.time.Clock -import java.time.Duration import java.time.OffsetDateTime import java.util.* import java.util.concurrent.Executors @@ -232,20 +235,31 @@ fun getServer( statistics: Statistics, registry: PrometheusMeterRegistry, ): Http4kServer { - val okHttpClient = OkHttp( - bodyMode = BodyMode.Stream, - client = OkHttpClient.Builder() - .followRedirects(false) - .connectTimeout(Duration.ofSeconds(2)) - .readTimeout(Duration.ofSeconds(5)) - .writeTimeout(Duration.ofSeconds(5)) + val apache = ApacheClient( + responseBodyMode = BodyMode.Stream, + client = HttpClients.custom() + .disableConnectionState() + .setDefaultRequestConfig( + RequestConfig.custom() + .setCookieSpec(StandardCookieSpec.IGNORE) + .setConnectTimeout(Timeout.ofSeconds(2)) + .setResponseTimeout(Timeout.ofSeconds(2)) + .setConnectionRequestTimeout(Timeout.ofSeconds(1)) + .build() + ) + .setConnectionManager( + PoolingHttpClientConnectionManagerBuilder.create() + .setMaxConnTotal(500) + .setMaxConnPerRoute(500) + .build() + ) .build() ) val client = ClientFilters.SetBaseUriFrom(remoteSettings.imageServer) .then(ClientFilters.MicrometerMetrics.RequestTimer(registry)) - .then(okHttpClient) + .then(apache) val imageServer = ImageServer( storage = storage, @@ -297,7 +311,7 @@ fun getServer( ).withFilter( ServerFilters.MicrometerMetrics.RequestTimer(registry, labeler = PostTransactionLabeler()) ).withFilter( - GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry).build() + GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry, apache).build() ) ) .asServer(Netty(remoteSettings.tls!!, serverSettings, statistics)) diff --git a/src/main/kotlin/mdnet/settings/ClientSettings.kt b/src/main/kotlin/mdnet/settings/ClientSettings.kt index 1e9ae4b..ffb7701 100644 --- a/src/main/kotlin/mdnet/settings/ClientSettings.kt +++ b/src/main/kotlin/mdnet/settings/ClientSettings.kt @@ -35,7 +35,6 @@ data class ClientSettings( data class ServerSettings( @field:Secret val secret: String, val externalPort: Int = 0, - val externalIp: String? = null, val gracefulShutdownWaitSeconds: Int = 60, val hostname: String = "0.0.0.0", val maxKilobitsPerSecond: Long = 0, diff --git a/src/main/kotlin/mdnet/settings/SettingsRequest.kt b/src/main/kotlin/mdnet/settings/SettingsRequest.kt deleted file mode 100644 index 9a88046..0000000 --- a/src/main/kotlin/mdnet/settings/SettingsRequest.kt +++ /dev/null @@ -1,39 +0,0 @@ -/* -Mangadex@Home -Copyright (c) 2020, MangaDex Network -This file is part of MangaDex@Home. - -MangaDex@Home is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -MangaDex@Home is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this MangaDex@Home. If not, see . -*/ -package mdnet.settings - -import com.fasterxml.jackson.databind.PropertyNamingStrategies -import com.fasterxml.jackson.databind.annotation.JsonNaming -import dev.afanasev.sekret.Secret - -@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class) -data class SettingsRequest( - @field:Secret val secret: String, - val ipAddress: String?, - val port: Int, - val diskSpace: Long, - val networkSpeed: Long, - val buildVersion: Int, - val tlsCreatedAt: String?, -) - -@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class) -data class LogoutRequest( - @field:Secret val secret: String, -)