1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00

Switch from apache to okhttp

This commit is contained in:
carbotaniuman 2021-01-27 17:03:49 -06:00
parent 0f51785b21
commit a4a28e1b87
8 changed files with 88 additions and 74 deletions

View file

@ -20,6 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [2.0.0-rc6] - 2021-01-27 ## [2.0.0-rc6] - 2021-01-27
### Fixed ### Fixed
- [2021-01-27] Upped max threadpool size [@carbotaniuman]. - [2021-01-27] Upped max threadpool size [@carbotaniuman].
- [2021-01-27] Switch to OkHttp and hopefully fix a class of bugs [@carbotaniuman].
- [2021-01-27] Add ability for nodes to specify external ip [@carbotaniuman].
## [2.0.0-rc5] - 2021-01-27 ## [2.0.0-rc5] - 2021-01-27
### Changed ### Changed

View file

@ -36,7 +36,7 @@ dependencies {
implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version"
implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1" implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1"
implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1" implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1"
implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-client-okhttp", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64" implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64"

View file

@ -23,13 +23,10 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule
import mdnet.ServerHandlerJackson.auto import mdnet.ServerHandlerJackson.auto
import mdnet.logging.info import mdnet.logging.info
import mdnet.settings.ClientSettings import mdnet.settings.ClientSettings
import mdnet.settings.LogoutRequest
import mdnet.settings.RemoteSettings import mdnet.settings.RemoteSettings
import org.apache.hc.client5.http.impl.DefaultSchemePortResolver import mdnet.settings.SettingsRequest
import org.apache.hc.client5.http.impl.classic.HttpClients import org.http4k.client.OkHttp
import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner
import org.apache.hc.core5.http.HttpHost
import org.apache.hc.core5.http.protocol.HttpContext
import org.http4k.client.ApacheClient
import org.http4k.core.Body import org.http4k.core.Body
import org.http4k.core.Method import org.http4k.core.Method
import org.http4k.core.Request import org.http4k.core.Request
@ -37,7 +34,7 @@ import org.http4k.format.ConfigurableJackson
import org.http4k.format.asConfigurable import org.http4k.format.asConfigurable
import org.http4k.format.withStandardMappings import org.http4k.format.withStandardMappings
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import java.net.InetAddress import java.lang.RuntimeException
object ServerHandlerJackson : ConfigurableJackson( object ServerHandlerJackson : ConfigurableJackson(
KotlinModule() KotlinModule()
@ -49,57 +46,44 @@ object ServerHandlerJackson : ConfigurableJackson(
class BackendApi(private val settings: ClientSettings) { class BackendApi(private val settings: ClientSettings) {
private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS
private val client = ApacheClient( private val client = OkHttp()
client = HttpClients.custom()
.setRoutePlanner(
object : DefaultRoutePlanner(DefaultSchemePortResolver()) {
override fun determineLocalAddress(firstHop: HttpHost?, context: HttpContext?): InetAddress {
return InetAddress.getByName(settings.serverSettings.hostname)
}
}
)
.build()
)
fun logoutFromControl(): Boolean { fun logoutFromControl(): Boolean {
val serverSettings = settings.serverSettings
LOGGER.info { "Disconnecting from the control server" } LOGGER.info { "Disconnecting from the control server" }
val params = mapOf<String, Any>(
"secret" to settings.serverSettings.secret val request = LOGOUT_REQUEST_LENS(
LogoutRequest(serverSettings.secret),
Request(Method.POST, serverAddress + "stop")
) )
val request = STRING_ANY_MAP_LENS(params, Request(Method.POST, serverAddress + "stop"))
val response = client(request) val response = client(request)
return response.status.successful return response.status.successful
} }
private fun getPingParams(tlsCreatedAt: String? = null): Map<String, Any> { private fun getPingParams(tlsCreatedAt: String? = null): SettingsRequest {
val serverSettings = settings.serverSettings val serverSettings = settings.serverSettings
return mapOf( return SettingsRequest(
"secret" to serverSettings.secret, secret = serverSettings.secret,
"port" to let { port = if (serverSettings.externalPort != 0) {
if (serverSettings.externalPort != 0) { serverSettings.externalPort
serverSettings.externalPort
} else {
serverSettings.port
}
},
"disk_space" to settings.maxCacheSizeInMebibytes * 1024 * 1024,
"network_speed" to serverSettings.externalMaxKilobitsPerSecond * 1000 / 8,
"build_version" to Constants.CLIENT_BUILD
).let {
if (tlsCreatedAt != null) {
it.plus("tls_created_at" to tlsCreatedAt)
} else { } else {
it serverSettings.port
} },
} buildVersion = Constants.CLIENT_BUILD,
diskSpace = settings.maxCacheSizeInMebibytes * 1024 * 1024,
networkSpeed = serverSettings.externalMaxKilobitsPerSecond * 1000 / 8,
ipAddress = serverSettings.externalIp,
tlsCreatedAt = tlsCreatedAt,
)
} }
fun loginToControl(): RemoteSettings? { fun loginToControl(): RemoteSettings {
LOGGER.info { "Connecting to the control server" } LOGGER.info { "Connecting to the control server" }
val request = STRING_ANY_MAP_LENS( val request = SETTINGS_REQUEST_LENS(
getPingParams(null), getPingParams(null),
Request( Request(
Method.POST, Method.POST,
@ -111,14 +95,14 @@ class BackendApi(private val settings: ClientSettings) {
return if (response.status.successful) { return if (response.status.successful) {
SERVER_SETTINGS_LENS(response) SERVER_SETTINGS_LENS(response)
} else { } else {
null throw RuntimeException(response.bodyString())
} }
} }
fun pingControl(old: RemoteSettings): RemoteSettings? { fun pingControl(old: RemoteSettings): RemoteSettings? {
LOGGER.info { "Pinging the control server" } LOGGER.info { "Pinging the control server" }
val request = STRING_ANY_MAP_LENS( val request = SETTINGS_REQUEST_LENS(
getPingParams(old.tls!!.createdAt), getPingParams(old.tls!!.createdAt),
Request( Request(
Method.POST, Method.POST,
@ -136,7 +120,8 @@ class BackendApi(private val settings: ClientSettings) {
companion object { companion object {
private val LOGGER = LoggerFactory.getLogger(BackendApi::class.java) private val LOGGER = LoggerFactory.getLogger(BackendApi::class.java)
private val STRING_ANY_MAP_LENS = Body.auto<Map<String, Any>>().toLens() private val SETTINGS_REQUEST_LENS = Body.auto<SettingsRequest>().toLens()
private val LOGOUT_REQUEST_LENS = Body.auto<LogoutRequest>().toLens()
private val SERVER_SETTINGS_LENS = Body.auto<RemoteSettings>().toLens() private val SERVER_SETTINGS_LENS = Body.auto<RemoteSettings>().toLens()
private const val SERVER_ADDRESS = "https://api.mangadex.network/" private const val SERVER_ADDRESS = "https://api.mangadex.network/"
} }

View file

@ -209,7 +209,6 @@ class ServerManager(
this.state as Uninitialized this.state as Uninitialized
val remoteSettings = backendApi.loginToControl() val remoteSettings = backendApi.loginToControl()
?: throw RuntimeException("Failed to get a login response from server")
LOGGER.info { "Server settings received: $remoteSettings" } LOGGER.info { "Server settings received: $remoteSettings" }
warnBasedOnSettings(remoteSettings) warnBasedOnSettings(remoteSettings)

View file

@ -26,6 +26,7 @@ import mdnet.logging.debug
import mdnet.logging.warn import mdnet.logging.warn
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.io.IOUtils import org.apache.commons.io.IOUtils
import org.http4k.client.OkHttp
import org.http4k.core.Filter import org.http4k.core.Filter
import org.http4k.core.HttpHandler import org.http4k.core.HttpHandler
import org.http4k.core.Method import org.http4k.core.Method
@ -90,7 +91,6 @@ class GeoIpMetricsFilterBuilder(
private val enableGeoIp: Boolean, private val enableGeoIp: Boolean,
private val license: String, private val license: String,
private val registry: PrometheusMeterRegistry, private val registry: PrometheusMeterRegistry,
private val client: HttpHandler
) { ) {
fun build(): GeoIpMetricsFilter { fun build(): GeoIpMetricsFilter {
return if (enableGeoIp) { return if (enableGeoIp) {
@ -108,6 +108,8 @@ class GeoIpMetricsFilterBuilder(
val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb") val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb")
val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license) val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license)
val client = OkHttp()
val response = client(Request(Method.GET, geoIpDatabaseUri)) val response = client(Request(Method.GET, geoIpDatabaseUri))
if (response.status != Status.OK) { if (response.status != Status.OK) {
throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})") throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})")

View file

@ -44,12 +44,8 @@ import mdnet.security.TweetNaclFast
import mdnet.settings.MetricsSettings import mdnet.settings.MetricsSettings
import mdnet.settings.RemoteSettings import mdnet.settings.RemoteSettings
import mdnet.settings.ServerSettings import mdnet.settings.ServerSettings
import org.apache.hc.client5.http.config.RequestConfig import okhttp3.OkHttpClient
import org.apache.hc.client5.http.cookie.StandardCookieSpec import org.http4k.client.OkHttp
import org.apache.hc.client5.http.impl.classic.HttpClients
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
import org.apache.hc.core5.util.Timeout
import org.http4k.client.ApacheClient
import org.http4k.core.* import org.http4k.core.*
import org.http4k.filter.CachingFilters import org.http4k.filter.CachingFilters
import org.http4k.filter.ClientFilters import org.http4k.filter.ClientFilters
@ -66,6 +62,7 @@ import java.io.BufferedInputStream
import java.io.BufferedOutputStream import java.io.BufferedOutputStream
import java.io.InputStream import java.io.InputStream
import java.time.Clock import java.time.Clock
import java.time.Duration
import java.time.OffsetDateTime import java.time.OffsetDateTime
import java.util.* import java.util.*
import java.util.concurrent.Executors import java.util.concurrent.Executors
@ -235,31 +232,20 @@ fun getServer(
statistics: Statistics, statistics: Statistics,
registry: PrometheusMeterRegistry, registry: PrometheusMeterRegistry,
): Http4kServer { ): Http4kServer {
val apache = ApacheClient( val okHttpClient = OkHttp(
responseBodyMode = BodyMode.Stream, bodyMode = BodyMode.Stream,
client = HttpClients.custom() client = OkHttpClient.Builder()
.disableConnectionState() .followRedirects(false)
.setDefaultRequestConfig( .connectTimeout(Duration.ofSeconds(2))
RequestConfig.custom() .readTimeout(Duration.ofSeconds(5))
.setCookieSpec(StandardCookieSpec.IGNORE) .writeTimeout(Duration.ofSeconds(5))
.setConnectTimeout(Timeout.ofSeconds(2))
.setResponseTimeout(Timeout.ofSeconds(2))
.setConnectionRequestTimeout(Timeout.ofSeconds(1))
.build()
)
.setConnectionManager(
PoolingHttpClientConnectionManagerBuilder.create()
.setMaxConnTotal(500)
.setMaxConnPerRoute(500)
.build()
)
.build() .build()
) )
val client = val client =
ClientFilters.SetBaseUriFrom(remoteSettings.imageServer) ClientFilters.SetBaseUriFrom(remoteSettings.imageServer)
.then(ClientFilters.MicrometerMetrics.RequestTimer(registry)) .then(ClientFilters.MicrometerMetrics.RequestTimer(registry))
.then(apache) .then(okHttpClient)
val imageServer = ImageServer( val imageServer = ImageServer(
storage = storage, storage = storage,
@ -311,7 +297,7 @@ fun getServer(
).withFilter( ).withFilter(
ServerFilters.MicrometerMetrics.RequestTimer(registry, labeler = PostTransactionLabeler()) ServerFilters.MicrometerMetrics.RequestTimer(registry, labeler = PostTransactionLabeler())
).withFilter( ).withFilter(
GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry, apache).build() GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry).build()
) )
) )
.asServer(Netty(remoteSettings.tls!!, serverSettings, statistics)) .asServer(Netty(remoteSettings.tls!!, serverSettings, statistics))

View file

@ -35,6 +35,7 @@ data class ClientSettings(
data class ServerSettings( data class ServerSettings(
@field:Secret val secret: String, @field:Secret val secret: String,
val externalPort: Int = 0, val externalPort: Int = 0,
val externalIp: String? = null,
val gracefulShutdownWaitSeconds: Int = 60, val gracefulShutdownWaitSeconds: Int = 60,
val hostname: String = "0.0.0.0", val hostname: String = "0.0.0.0",
val maxKilobitsPerSecond: Long = 0, val maxKilobitsPerSecond: Long = 0,

View file

@ -0,0 +1,39 @@
/*
Mangadex@Home
Copyright (c) 2020, MangaDex Network
This file is part of MangaDex@Home.
MangaDex@Home is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MangaDex@Home is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet.settings
import com.fasterxml.jackson.databind.PropertyNamingStrategies
import com.fasterxml.jackson.databind.annotation.JsonNaming
import dev.afanasev.sekret.Secret
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
data class SettingsRequest(
@field:Secret val secret: String,
val ipAddress: String?,
val port: Int,
val diskSpace: Long,
val networkSpeed: Long,
val buildVersion: Int,
val tlsCreatedAt: String?,
)
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
data class LogoutRequest(
@field:Secret val secret: String,
)