gitlab-com-mangadex-pub/mangadex_at_home
1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00
Code Releases Activity

Use HTTP/2

Browse source
This commit is contained in:
carbotaniuman 2021-03-02 12:22:24 -06:00
parent 2252a9f02d
commit 1be9ee8506
8 changed files with 71 additions and 83 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
CHANGELOG.md
View file

@ -8,19 +8,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
### Changed
- [2020-02-10] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9]
- [2020-02-10] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9]
- [2020-02-10] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9]
### Deprecated
### Removed
### Fixed
- [2020-02-21] Fix pipeline [@_tde9]
### Security
## [2.0.0-rc14] - 2021-03-02
### Changed
- [2020-02-10] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9].
- [2020-02-10] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9].
- [2020-02-10] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9].
- [2020-02-25] Use HTTP/2 to download when possible [@carbotaniuman].
### Fixed
- [2020-02-21] Fix pipeline [@_tde9].
## [2.0.0-rc13] - 2021-02-19
### Changed
- [2021-02-19] Back to sqlite we go [@carbotaniuman].
@ -369,7 +375,8 @@ This release contains many breaking changes! Of note are the changes to the cach
### Fixed
- [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...HEAD
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...HEAD
[2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14
[2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13
[2.0.0-rc12]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc11...2.0.0-rc12
[2.0.0-rc11]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc10...2.0.0-rc11

20
build.gradle
View file

@ -1,7 +1,7 @@
plugins {
id "jacoco"
id "java"
id "org.jetbrains.kotlin.jvm" version "1.4.20"
id "org.jetbrains.kotlin.jvm" version "1.4.30"
id "org.jetbrains.kotlin.kapt" version "1.4.0"
id "application"
id "com.github.johnrengelman.shadow" version "5.2.0"
@ -32,18 +32,22 @@ dependencies {
implementation group: "io.micrometer", name: "micrometer-registry-prometheus", version: "1.6.2"
implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0"
implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-resilience4j", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-bom", version: "4.3.5.4"
implementation platform(group: "org.http4k", name: "http4k-bom", version: "4.3.5.4")
implementation group: "org.http4k", name: "http4k-core"
implementation group: "org.http4k", name: "http4k-resilience4j"
implementation group: "io.github.resilience4j", name: "resilience4j-micrometer", version: "1.6.1"
implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-format-jackson"
implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1"
implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1"
implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-metrics-micrometer", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-client-okhttp"
implementation group: "org.http4k", name: "http4k-metrics-micrometer"
implementation group: "org.http4k", name: "http4k-server-netty"
implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64"
implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.3.Final", classifier: "linux-x86_64"
testImplementation group: "org.http4k", name: "http4k-testing-kotest", version: "$http_4k_version"
testImplementation group: "org.http4k", name: "http4k-testing-kotest"
runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final"
implementation group: 'com.zaxxer', name: 'HikariCP', version: '4.0.1'

5
gradle.properties
View file

@ -1,4 +1,3 @@
http_4k_version=4.1.0.0
exposed_version=0.26.2
kotest_version=4.4.0.RC1
http_4k_version=4.3.0.0
kotest_version=4.4.1
ktorm_version=3.2.0

4
src/main/kotlin/mdnet/BackendApi.kt
View file

@ -23,7 +23,6 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule
import mdnet.ServerHandlerJackson.auto
import mdnet.logging.info
import mdnet.settings.*
import org.http4k.client.ApacheClient
import org.http4k.core.Body
import org.http4k.core.HttpHandler
import org.http4k.core.Method
@ -42,9 +41,8 @@ object ServerHandlerJackson : ConfigurableJackson(
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
)
class BackendApi(private val settings: ClientSettings) {
class BackendApi(private val settings: ClientSettings, private val client: HttpHandler) {
private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS
private val client = ApacheClient()
fun logoutFromControl(): Boolean {
val serverSettings = settings.serverSettings

2
src/main/kotlin/mdnet/Constants.kt
View file

@ -21,7 +21,7 @@ package mdnet
import java.time.Duration
object Constants {
const val CLIENT_BUILD = 28
const val CLIENT_BUILD = 29
@JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)

4
src/main/kotlin/mdnet/Main.kt
View file

@ -77,6 +77,10 @@ class Main : Runnable {
|if you are not, manually move update your --database args!
|note: the database file itself should be named metadata.{extension}
|where {extension} can be `.db` or `.mv.db`
|
|If this is your first time seeing this message, please check out the support
|channel as things HAVE changed. Failure to do so WILL require
|a cache wipe.
""".trimMargin()
)
println()

64
src/main/kotlin/mdnet/ServerManager.kt
View file

@ -20,6 +20,7 @@ package mdnet
import io.micrometer.prometheus.PrometheusConfig
import io.micrometer.prometheus.PrometheusMeterRegistry
import io.netty.util.internal.SystemPropertyUtil
import mdnet.cache.ImageStorage
import mdnet.data.Statistics
import mdnet.logging.error
@ -29,19 +30,16 @@ import mdnet.metrics.DefaultMicrometerMetrics
import mdnet.server.getServer
import mdnet.settings.ClientSettings
import mdnet.settings.RemoteSettings
import org.apache.hc.client5.http.config.RequestConfig
import org.apache.hc.client5.http.cookie.StandardCookieSpec
import org.apache.hc.client5.http.impl.classic.HttpClients
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
import org.apache.hc.core5.util.TimeValue
import org.apache.hc.core5.util.Timeout
import org.http4k.client.ApacheClient
import okhttp3.OkHttpClient
import okhttp3.Protocol
import org.http4k.client.OkHttp
import org.http4k.core.BodyMode
import org.http4k.core.then
import org.http4k.filter.ClientFilters
import org.http4k.filter.MicrometerMetrics
import org.http4k.server.Http4kServer
import org.slf4j.LoggerFactory
import java.time.Duration
import java.util.concurrent.CountDownLatch
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
@ -73,26 +71,24 @@ class ServerManager(
private val executor = Executors.newSingleThreadScheduledExecutor()
private val registry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
private val statistics = Statistics()
private val connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
.setMaxConnTotal(500)
.setMaxConnPerRoute(500)
.build()
private val apache = ClientFilters.MicrometerMetrics.RequestCounter(registry)
private val okhttp = ClientFilters.MicrometerMetrics.RequestCounter(registry)
.then(ClientFilters.MicrometerMetrics.RequestTimer(registry))
.then(
ApacheClient(
responseBodyMode = BodyMode.Stream,
client = HttpClients.custom()
.disableConnectionState()
.setDefaultRequestConfig(
RequestConfig.custom()
.setCookieSpec(StandardCookieSpec.IGNORE)
.setConnectTimeout(Timeout.ofSeconds(2))
.setResponseTimeout(Timeout.ofSeconds(2))
.setConnectionRequestTimeout(Timeout.ofSeconds(1))
.build()
)
.setConnectionManager(connectionManager)
OkHttp(
bodyMode = BodyMode.Stream,
client = OkHttpClient.Builder()
.callTimeout(Duration.ofSeconds(30))
.connectTimeout(Duration.ofSeconds(1))
.writeTimeout(Duration.ofSeconds(5))
.readTimeout(Duration.ofSeconds(5))
.let {
if (SystemPropertyUtil.get("no-client-http2").toBoolean()) {
it.protocols(listOf(Protocol.HTTP_1_1))
} else {
it
}
}
.build()
)
)
@ -104,7 +100,7 @@ class ServerManager(
init {
state = Uninitialized
backendApi = BackendApi(settings)
backendApi = BackendApi(settings, OkHttp())
}
fun start() {
@ -205,20 +201,6 @@ class ServerManager(
45, 45, TimeUnit.SECONDS
)
executor.scheduleWithFixedDelay(
{
try {
LOGGER.info { "Closing old Apache HTTP connections" }
connectionManager.closeExpired()
connectionManager.closeIdle(TimeValue.ofSeconds(30))
} catch (e: Exception) {
LOGGER.warn(e) { "Old Apache HTTP connection closer failed" }
}
},
45, 45, TimeUnit.SECONDS
)
LOGGER.info { "Image server has started" }
}
@ -271,7 +253,7 @@ class ServerManager(
settings.metricsSettings,
statistics,
registry,
apache,
okhttp,
).start()
this.state = Running(server, remoteSettings)

28
src/main/kotlin/mdnet/metrics/GeoIpMetricsFilter.kt
View file

@ -25,8 +25,7 @@ import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.logging.debug
import mdnet.logging.warn
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.io.IOUtils
import org.http4k.client.ApacheClient
import org.http4k.client.OkHttp
import org.http4k.core.Filter
import org.http4k.core.HttpHandler
import org.http4k.core.Method
@ -37,7 +36,6 @@ import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.net.InetAddress
import java.net.UnknownHostException
import java.nio.file.Files
class GeoIpMetricsFilter(
private val databaseReader: DatabaseReader?,
@ -92,7 +90,8 @@ class GeoIpMetricsFilterBuilder(
private val license: String,
private val registry: PrometheusMeterRegistry,
) {
val client = ApacheClient()
private val client = OkHttp()
fun build(): GeoIpMetricsFilter {
return if (enableGeoIp) {
LOGGER.info("GeoIp initialising")
@ -105,33 +104,28 @@ class GeoIpMetricsFilterBuilder(
}
private fun initDatabase(): DatabaseReader {
val databaseFileDir = Files.createTempDirectory("mangadex-geoip")
val databaseFile = Files.createTempFile(databaseFileDir, "geoip2_country", ".mmdb")
val geoIpDatabaseUri = GEOIP2_COUNTRY_URI_FORMAT.format(license)
val response = client(Request(Method.GET, geoIpDatabaseUri))
if (response.status != Status.OK) {
throw IllegalStateException("Couldn't download GeoIP 2 database (http status: ${response.status})")
}
response.use {
val archiveStream = TarArchiveInputStream(it.body.gunzippedStream().stream)
var entry = archiveStream.nextTarEntry
return response.use { data ->
TarArchiveInputStream(data.body.gunzippedStream().stream).use {
var entry = it.nextTarEntry
while (!entry.name.endsWith(".mmdb")) {
LOGGER.debug { "Skipped non-database file: ${entry.name}" }
entry = archiveStream.nextTarEntry
entry = it.nextTarEntry
}
// reads only the current entry to its end
val dbBytes = IOUtils.toByteArray(archiveStream)
Files.write(databaseFile, dbBytes)
}
return DatabaseReader
.Builder(databaseFile.toFile())
DatabaseReader
.Builder(it)
.withCache(CHMCache())
.build()
}
}
}
companion object {
private val LOGGER = LoggerFactory.getLogger(GeoIpMetricsFilterBuilder::class.java)