1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00

Revert "Switch from apache to okhttp"

This reverts commit a4a28e1b
This commit is contained in:
carbotaniuman 2021-01-27 20:12:47 -06:00
parent a4a28e1b87
commit 49e9523a7c
8 changed files with 74 additions and 88 deletions

View file

@ -20,8 +20,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [2.0.0-rc6] - 2021-01-27 ## [2.0.0-rc6] - 2021-01-27
### Fixed ### Fixed
- [2021-01-27] Upped max threadpool size [@carbotaniuman]. - [2021-01-27] Upped max threadpool size [@carbotaniuman].
- [2021-01-27] Switch to OkHttp and hopefully fix a class of bugs [@carbotaniuman].
- [2021-01-27] Add ability for nodes to specify external ip [@carbotaniuman].
## [2.0.0-rc5] - 2021-01-27 ## [2.0.0-rc5] - 2021-01-27
### Changed ### Changed

View file

@ -36,7 +36,7 @@ dependencies {
implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version"
implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1" implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1"
implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1" implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1"
implementation group: "org.http4k", name: "http4k-client-okhttp", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64" implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64"

View file

@ -23,10 +23,13 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule
import mdnet.ServerHandlerJackson.auto import mdnet.ServerHandlerJackson.auto
import mdnet.logging.info import mdnet.logging.info
import mdnet.settings.ClientSettings import mdnet.settings.ClientSettings
import mdnet.settings.LogoutRequest
import mdnet.settings.RemoteSettings import mdnet.settings.RemoteSettings
import mdnet.settings.SettingsRequest import org.apache.hc.client5.http.impl.DefaultSchemePortResolver
import org.http4k.client.OkHttp import org.apache.hc.client5.http.impl.classic.HttpClients
import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner
import org.apache.hc.core5.http.HttpHost
import org.apache.hc.core5.http.protocol.HttpContext
import org.http4k.client.ApacheClient
import org.http4k.core.Body import org.http4k.core.Body
import org.http4k.core.Method import org.http4k.core.Method
import org.http4k.core.Request import org.http4k.core.Request
@ -34,7 +37,7 @@ import org.http4k.format.ConfigurableJackson
import org.http4k.format.asConfigurable import org.http4k.format.asConfigurable
import org.http4k.format.withStandardMappings import org.http4k.format.withStandardMappings
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import java.lang.RuntimeException import java.net.InetAddress
object ServerHandlerJackson : ConfigurableJackson( object ServerHandlerJackson : ConfigurableJackson(
KotlinModule() KotlinModule()
@ -46,44 +49,57 @@ object ServerHandlerJackson : ConfigurableJackson(
class BackendApi(private val settings: ClientSettings) { class BackendApi(private val settings: ClientSettings) {
private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS
private val client = OkHttp() private val client = ApacheClient(
client = HttpClients.custom()
.setRoutePlanner(
object : DefaultRoutePlanner(DefaultSchemePortResolver()) {
override fun determineLocalAddress(firstHop: HttpHost?, context: HttpContext?): InetAddress {
return InetAddress.getByName(settings.serverSettings.hostname)
}
}
)
.build()
)
fun logoutFromControl(): Boolean { fun logoutFromControl(): Boolean {
val serverSettings = settings.serverSettings
LOGGER.info { "Disconnecting from the control server" } LOGGER.info { "Disconnecting from the control server" }
val params = mapOf<String, Any>(
val request = LOGOUT_REQUEST_LENS( "secret" to settings.serverSettings.secret
LogoutRequest(serverSettings.secret),
Request(Method.POST, serverAddress + "stop")
) )
val request = STRING_ANY_MAP_LENS(params, Request(Method.POST, serverAddress + "stop"))
val response = client(request) val response = client(request)
return response.status.successful return response.status.successful
} }
private fun getPingParams(tlsCreatedAt: String? = null): SettingsRequest { private fun getPingParams(tlsCreatedAt: String? = null): Map<String, Any> {
val serverSettings = settings.serverSettings val serverSettings = settings.serverSettings
return SettingsRequest( return mapOf(
secret = serverSettings.secret, "secret" to serverSettings.secret,
port = if (serverSettings.externalPort != 0) { "port" to let {
serverSettings.externalPort if (serverSettings.externalPort != 0) {
} else { serverSettings.externalPort
serverSettings.port } else {
serverSettings.port
}
}, },
buildVersion = Constants.CLIENT_BUILD, "disk_space" to settings.maxCacheSizeInMebibytes * 1024 * 1024,
diskSpace = settings.maxCacheSizeInMebibytes * 1024 * 1024, "network_speed" to serverSettings.externalMaxKilobitsPerSecond * 1000 / 8,
networkSpeed = serverSettings.externalMaxKilobitsPerSecond * 1000 / 8, "build_version" to Constants.CLIENT_BUILD
ipAddress = serverSettings.externalIp, ).let {
tlsCreatedAt = tlsCreatedAt, if (tlsCreatedAt != null) {
) it.plus("tls_created_at" to tlsCreatedAt)
} else {
it
}
}
} }
fun loginToControl(): RemoteSettings { fun loginToControl(): RemoteSettings? {
LOGGER.info { "Connecting to the control server" } LOGGER.info { "Connecting to the control server" }
val request = SETTINGS_REQUEST_LENS( val request = STRING_ANY_MAP_LENS(
getPingParams(null), getPingParams(null),
Request( Request(
Method.POST, Method.POST,
@ -95,14 +111,14 @@ class BackendApi(private val settings: ClientSettings) {
return if (response.status.successful) { return if (response.status.successful) {
SERVER_SETTINGS_LENS(response) SERVER_SETTINGS_LENS(response)
} else { } else {
throw RuntimeException(response.bodyString()) null
} }
} }
fun pingControl(old: RemoteSettings): RemoteSettings? { fun pingControl(old: RemoteSettings): RemoteSettings? {
LOGGER.info { "Pinging the control server" } LOGGER.info { "Pinging the control server" }
val request = SETTINGS_REQUEST_LENS( val request = STRING_ANY_MAP_LENS(
getPingParams(old.tls!!.createdAt), getPingParams(old.tls!!.createdAt),
Request( Request(
Method.POST, Method.POST,
@ -120,8 +136,7 @@ class BackendApi(private val settings: ClientSettings) {
companion object { companion object {
private val LOGGER = LoggerFactory.getLogger(BackendApi::class.java) private val LOGGER = LoggerFactory.getLogger(BackendApi::class.java)
private val SETTINGS_REQUEST_LENS = Body.auto<SettingsRequest>().toLens() private val STRING_ANY_MAP_LENS = Body.auto<Map<String, Any>>().toLens()
private val LOGOUT_REQUEST_LENS = Body.auto<LogoutRequest>().toLens()
private val SERVER_SETTINGS_LENS = Body.auto<RemoteSettings>().toLens() private val SERVER_SETTINGS_LENS = Body.auto<RemoteSettings>().toLens()
private const val SERVER_ADDRESS = "https://api.mangadex.network/" private const val SERVER_ADDRESS = "https://api.mangadex.network/"
} }

View file

@ -209,6 +209,7 @@ class ServerManager(
this.state as Uninitialized this.state as Uninitialized
val remoteSettings = backendApi.loginToControl() val remoteSettings = backendApi.loginToControl()
?: throw RuntimeException("Failed to get a login response from server")
LOGGER.info { "Server settings received: $remoteSettings" } LOGGER.info { "Server settings received: $remoteSettings" }
warnBasedOnSettings(remoteSettings) warnBasedOnSettings(remoteSettings)

View file

@ -26,7 +26,6 @@ import mdnet.logging.debug
import mdnet.logging.warn import mdnet.logging.warn
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.io.IOUtils import org.apache.commons.io.IOUtils
import org.http4k.client.OkHttp
import org.http4k.core.Filter import org.http4k.core.Filter
import org.http4k.core.HttpHandler import org.http4k.core.HttpHandler
import org.http4k.core.Method import org.http4k.core.Method
@ -91,6 +90,7 @@ class GeoIpMetricsFilterBuilder(
private val enableGeoIp: Boolean, private val enableGeoIp: Boolean,
private val license: String, private val license: String,
private val registry: PrometheusMeterRegistry, private val registry: PrometheusMeterRegistry,
private val client: HttpHandler
) { ) {
fun build(): GeoIpMetricsFilter { fun build(): GeoIpMetricsFilter {
return if (enableGeoIp) { return if (enableGeoIp) {
@ -108,8 +108,6 @@ class GeoIpMetricsFilterBuilder(
val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb") val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb")
val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license) val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license)
val client = OkHttp()
val response = client(Request(Method.GET, geoIpDatabaseUri)) val response = client(Request(Method.GET, geoIpDatabaseUri))
if (response.status != Status.OK) { if (response.status != Status.OK) {
throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})") throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})")

View file

@ -44,8 +44,12 @@ import mdnet.security.TweetNaclFast
import mdnet.settings.MetricsSettings import mdnet.settings.MetricsSettings
import mdnet.settings.RemoteSettings import mdnet.settings.RemoteSettings
import mdnet.settings.ServerSettings import mdnet.settings.ServerSettings
import okhttp3.OkHttpClient import org.apache.hc.client5.http.config.RequestConfig
import org.http4k.client.OkHttp import org.apache.hc.client5.http.cookie.StandardCookieSpec
import org.apache.hc.client5.http.impl.classic.HttpClients
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
import org.apache.hc.core5.util.Timeout
import org.http4k.client.ApacheClient
import org.http4k.core.* import org.http4k.core.*
import org.http4k.filter.CachingFilters import org.http4k.filter.CachingFilters
import org.http4k.filter.ClientFilters import org.http4k.filter.ClientFilters
@ -62,7 +66,6 @@ import java.io.BufferedInputStream
import java.io.BufferedOutputStream import java.io.BufferedOutputStream
import java.io.InputStream import java.io.InputStream
import java.time.Clock import java.time.Clock
import java.time.Duration
import java.time.OffsetDateTime import java.time.OffsetDateTime
import java.util.* import java.util.*
import java.util.concurrent.Executors import java.util.concurrent.Executors
@ -232,20 +235,31 @@ fun getServer(
statistics: Statistics, statistics: Statistics,
registry: PrometheusMeterRegistry, registry: PrometheusMeterRegistry,
): Http4kServer { ): Http4kServer {
val okHttpClient = OkHttp( val apache = ApacheClient(
bodyMode = BodyMode.Stream, responseBodyMode = BodyMode.Stream,
client = OkHttpClient.Builder() client = HttpClients.custom()
.followRedirects(false) .disableConnectionState()
.connectTimeout(Duration.ofSeconds(2)) .setDefaultRequestConfig(
.readTimeout(Duration.ofSeconds(5)) RequestConfig.custom()
.writeTimeout(Duration.ofSeconds(5)) .setCookieSpec(StandardCookieSpec.IGNORE)
.setConnectTimeout(Timeout.ofSeconds(2))
.setResponseTimeout(Timeout.ofSeconds(2))
.setConnectionRequestTimeout(Timeout.ofSeconds(1))
.build()
)
.setConnectionManager(
PoolingHttpClientConnectionManagerBuilder.create()
.setMaxConnTotal(500)
.setMaxConnPerRoute(500)
.build()
)
.build() .build()
) )
val client = val client =
ClientFilters.SetBaseUriFrom(remoteSettings.imageServer) ClientFilters.SetBaseUriFrom(remoteSettings.imageServer)
.then(ClientFilters.MicrometerMetrics.RequestTimer(registry)) .then(ClientFilters.MicrometerMetrics.RequestTimer(registry))
.then(okHttpClient) .then(apache)
val imageServer = ImageServer( val imageServer = ImageServer(
storage = storage, storage = storage,
@ -297,7 +311,7 @@ fun getServer(
).withFilter( ).withFilter(
ServerFilters.MicrometerMetrics.RequestTimer(registry, labeler = PostTransactionLabeler()) ServerFilters.MicrometerMetrics.RequestTimer(registry, labeler = PostTransactionLabeler())
).withFilter( ).withFilter(
GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry).build() GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry, apache).build()
) )
) )
.asServer(Netty(remoteSettings.tls!!, serverSettings, statistics)) .asServer(Netty(remoteSettings.tls!!, serverSettings, statistics))

View file

@ -35,7 +35,6 @@ data class ClientSettings(
data class ServerSettings( data class ServerSettings(
@field:Secret val secret: String, @field:Secret val secret: String,
val externalPort: Int = 0, val externalPort: Int = 0,
val externalIp: String? = null,
val gracefulShutdownWaitSeconds: Int = 60, val gracefulShutdownWaitSeconds: Int = 60,
val hostname: String = "0.0.0.0", val hostname: String = "0.0.0.0",
val maxKilobitsPerSecond: Long = 0, val maxKilobitsPerSecond: Long = 0,

View file

@ -1,39 +0,0 @@
/*
Mangadex@Home
Copyright (c) 2020, MangaDex Network
This file is part of MangaDex@Home.
MangaDex@Home is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MangaDex@Home is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet.settings
import com.fasterxml.jackson.databind.PropertyNamingStrategies
import com.fasterxml.jackson.databind.annotation.JsonNaming
import dev.afanasev.sekret.Secret
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
data class SettingsRequest(
@field:Secret val secret: String,
val ipAddress: String?,
val port: Int,
val diskSpace: Long,
val networkSpeed: Long,
val buildVersion: Int,
val tlsCreatedAt: String?,
)
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
data class LogoutRequest(
@field:Secret val secret: String,
)