gitlab-com-mangadex-pub/mangadex_at_home
1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00
Code Releases Activity

Merge branch 'next' into 'master'

Browse source 
Next

See merge request mangadex-pub/mangadex_at_home!82
This commit is contained in:
carbotaniuman 2021-03-11 20:09:14 +00:00
parent 8178f1b0d6 f9cf5729b4
commit 4fed127e80
16 changed files with 139 additions and 196 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
CHANGELOG.md
View file

@ -14,15 +14,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed
### Fixed
- [2021-03-11] Fixed Access-Control-Expose-Headers typo [@lflare].
### Security
## [2.0.0] - 2021-03-11
### Changed
- [2021-03-11] Switch back to HTTP/1.1 [@carbotaniuman].
- [2021-03-11] Tune connection pool [@carbotaniuman].
### Fixed
- [2021-03-11] Fixed Access-Control-Expose-Headers typo [@lflare].
- [2021-03-11] Throw exceptions less frequently [@carbotaniuman].
- [2021-03-11] Fix various Netty issues [@carbotaniuman].
- [2021-03-11] Don't log IOUring and Epoll errors [@carbotaniuman].
- [2021-03-11] Ignore IPV6 when pinging [@carbotaniuman].
## [2.0.0-rc14] - 2021-03-02
### Changed
- [2021-02-10] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9].
- [2021-02-10] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9].
- [2021-02-10] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9].
- [2021-03-02] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9].
- [2021-03-02] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9].
- [2021-03-02] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9].
- [2021-02-25] Use HTTP/2 to download when possible [@carbotaniuman].
### Fixed
@ -376,7 +387,8 @@ This release contains many breaking changes! Of note are the changes to the cach
### Fixed
- [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...HEAD
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...HEAD
[2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0
[2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14
[2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13
[2.0.0-rc12]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc11...2.0.0-rc12

23
build.gradle
View file

@ -25,39 +25,38 @@ configurations {
dependencies {
compileOnly group: "dev.afanasev", name: "sekret-annotation", version: "0.0.7"
implementation group: "commons-io", name: "commons-io", version: "2.7"
implementation group: "commons-io", name: "commons-io", version: "2.8.0"
implementation group: "org.apache.commons", name: "commons-compress", version: "1.20"
implementation group: "ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha4"
implementation group: "io.micrometer", name: "micrometer-registry-prometheus", version: "1.6.2"
implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0"
implementation group: "org.http4k", name: "http4k-bom", version: "4.3.5.4"
implementation platform(group: "org.http4k", name: "http4k-bom", version: "4.3.5.4")
implementation platform(group: "com.fasterxml.jackson", name: "jackson-bom", version: "2.12.1")
implementation platform(group: "io.netty", name: "netty-bom", version: "4.1.60.Final")
implementation group: "org.http4k", name: "http4k-core"
implementation group: "org.http4k", name: "http4k-resilience4j"
implementation group: "io.github.resilience4j", name: "resilience4j-micrometer", version: "1.6.1"
implementation group: "org.http4k", name: "http4k-format-jackson"
implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1"
implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1"
implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml"
implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310"
implementation group: "org.http4k", name: "http4k-client-okhttp"
implementation group: "org.http4k", name: "http4k-metrics-micrometer"
implementation group: "org.http4k", name: "http4k-server-netty"
implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64"
implementation group: "io.netty", name: "netty-transport-native-epoll", classifier: "linux-x86_64"
implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.3.Final", classifier: "linux-x86_64"
testImplementation group: "org.http4k", name: "http4k-testing-kotest"
runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final"
runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.36.Final"
implementation group: 'com.zaxxer', name: 'HikariCP', version: '4.0.1'
implementation group: "com.h2database", name: "h2", version: "1.4.200"
implementation group: 'org.xerial', name: 'sqlite-jdbc', version: '3.34.0'
implementation group: "com.zaxxer", name: "HikariCP", version: "4.0.2"
implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.34.0"
implementation "org.ktorm:ktorm-core:$ktorm_version"
implementation "org.ktorm:ktorm-jackson:$ktorm_version"
implementation "info.picocli:picocli:4.5.0"
kapt "info.picocli:picocli-codegen:4.5.0"
implementation "info.picocli:picocli:$picocli_version"
kapt "info.picocli:picocli-codegen:$picocli_version"
testImplementation "io.kotest:kotest-runner-junit5:$kotest_version"
testImplementation "io.kotest:kotest-assertions-core:$kotest_version"

3
gradle.properties
View file

@ -1,3 +1,4 @@
http_4k_version=4.3.0.0
kotest_version=4.4.1
ktorm_version=3.2.0
ktorm_version=3.3.0
picocli_version=4.6.1

14
src/main/java/mdnet/cache/CachingInputStream.java vendored
View file

@ -30,6 +30,7 @@ public class CachingInputStream extends ProxyInputStream {
private final OutputStream cache;
private final ExecutorService executor;
private final Runnable onClose;
private boolean eofReached = false;
public CachingInputStream(InputStream response, ExecutorService executor, OutputStream cache, Runnable onClose) {
super(response);
@ -40,7 +41,7 @@ public class CachingInputStream extends ProxyInputStream {
@Override
public void close() throws IOException {
if (read() == EOF) {
if (eofReached) {
try {
in.close();
} catch (IOException ignored) {
@ -50,9 +51,7 @@ public class CachingInputStream extends ProxyInputStream {
} catch (IOException ignored) {
}
onClose.run();
return;
}
} else {
executor.submit(() -> {
try {
IOUtils.copy(in, cache);
@ -70,6 +69,7 @@ public class CachingInputStream extends ProxyInputStream {
}
});
}
}
@Override
public int read() throws IOException {
@ -80,6 +80,8 @@ public class CachingInputStream extends ProxyInputStream {
} catch (IOException ignored) {
// don't let write failures affect the image loading
}
} else {
eofReached = true;
}
return ch;
}
@ -93,6 +95,8 @@ public class CachingInputStream extends ProxyInputStream {
} catch (IOException ignored) {
// don't let write failures affect the image loading
}
} else {
eofReached = true;
}
return n;
}
@ -106,6 +110,8 @@ public class CachingInputStream extends ProxyInputStream {
} catch (IOException ignored) {
// don't let write failures affect the image loading
}
} else {
eofReached = true;
}
return n;
}

21
src/main/kotlin/mdnet/BackendApi.kt
View file

@ -23,15 +23,17 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule
import mdnet.ServerHandlerJackson.auto
import mdnet.logging.info
import mdnet.settings.*
import org.http4k.core.Body
import org.http4k.core.HttpHandler
import org.http4k.core.Method
import org.http4k.core.Request
import okhttp3.Dns
import okhttp3.OkHttpClient
import org.http4k.client.OkHttp
import org.http4k.core.*
import org.http4k.format.ConfigurableJackson
import org.http4k.format.asConfigurable
import org.http4k.format.withStandardMappings
import org.http4k.lens.LensFailure
import org.slf4j.LoggerFactory
import java.net.Inet4Address
import java.net.InetAddress
object ServerHandlerJackson : ConfigurableJackson(
KotlinModule()
@ -41,7 +43,16 @@ object ServerHandlerJackson : ConfigurableJackson(
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
)
class BackendApi(private val settings: ClientSettings, private val client: HttpHandler) {
class BackendApi(private val settings: ClientSettings) {
private val client = OkHttp(
client = OkHttpClient.Builder()
.dns(object : Dns {
override fun lookup(hostname: String): List<InetAddress> {
return Dns.SYSTEM.lookup(hostname).filterIsInstance<Inet4Address>()
}
})
.build()
)
private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS
fun logoutFromControl(): Boolean {

2
src/main/kotlin/mdnet/Constants.kt
View file

@ -21,7 +21,7 @@ package mdnet
import java.time.Duration
object Constants {
const val CLIENT_BUILD = 29
const val CLIENT_BUILD = 30
@JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)

2
src/main/kotlin/mdnet/Main.kt
View file

@ -97,8 +97,6 @@ class Main : Runnable {
throw IllegalArgumentException("Cache folder $cacheFolder must be a directory")
}
migrate(databaseFolder)
val client = MangaDexClient(settingsFile, databaseFolder, cacheFolder)
val hook = Thread {
client.shutdown()

2
src/main/kotlin/mdnet/MangaDexClient.kt
View file

@ -198,7 +198,7 @@ class MangaDexClient(private val settingsFile: File, databaseFolder: Path, cache
}
private fun validateSettings(settings: ClientSettings) {
if (settings.maxCacheSizeInMebibytes < 20480) {
if (settings.maxCacheSizeInMebibytes < 40960) {
throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 20480 MiB (20 GiB)")
}

75
src/main/kotlin/mdnet/Migrator.kt
View file

@ -1,75 +0,0 @@
/*
Mangadex@Home
Copyright (c) 2020, MangaDex Network
This file is part of MangaDex@Home.
MangaDex@Home is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
MangaDex@Home is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
*/
package mdnet
import mdnet.cache.DbImage
import mdnet.cache.INIT_TABLE
import org.ktorm.database.Database
import org.ktorm.dsl.*
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
fun main() {
migrate(Paths.get("./"))
}
fun migrate(path: Path) {
val h2file = path.resolve("metadata.mv.db")
if (!Files.exists(h2file)) {
return
}
println("Migrating database - this may take a long time")
Class.forName("org.sqlite.JDBC")
val sqliteDb = path.resolve("metadata.db")
Files.deleteIfExists(sqliteDb)
val sqlite = Database.connect("jdbc:sqlite:$sqliteDb")
sqlite.useConnection { conn ->
conn.prepareStatement(INIT_TABLE).use {
it.execute()
}
}
val db = path.resolve("metadata")
val h2 = Database.connect("jdbc:h2:$db")
h2.useConnection { conn ->
conn.prepareStatement(INIT_TABLE).use {
it.execute()
}
}
h2.from(DbImage).select().asIterable().chunked(1000).forEach { list ->
sqlite.batchInsert(DbImage) {
for (data in list) {
item {
set(DbImage.id, data[DbImage.id])
set(DbImage.accessed, data[DbImage.accessed])
set(DbImage.size, data[DbImage.size])
}
}
}
}
Files.move(h2file, path.resolve("metadata.mv.db.old"))
}

26
src/main/kotlin/mdnet/ServerManager.kt
View file

@ -20,7 +20,6 @@ package mdnet
import io.micrometer.prometheus.PrometheusConfig
import io.micrometer.prometheus.PrometheusMeterRegistry
import io.netty.util.internal.SystemPropertyUtil
import mdnet.cache.ImageStorage
import mdnet.data.Statistics
import mdnet.logging.error
@ -30,6 +29,7 @@ import mdnet.metrics.DefaultMicrometerMetrics
import mdnet.server.getServer
import mdnet.settings.ClientSettings
import mdnet.settings.RemoteSettings
import okhttp3.ConnectionPool
import okhttp3.OkHttpClient
import okhttp3.Protocol
import org.http4k.client.OkHttp
@ -78,17 +78,17 @@ class ServerManager(
OkHttp(
bodyMode = BodyMode.Stream,
client = OkHttpClient.Builder()
.callTimeout(Duration.ofSeconds(30))
.connectTimeout(Duration.ofSeconds(1))
.writeTimeout(Duration.ofSeconds(5))
.readTimeout(Duration.ofSeconds(5))
.let {
if (SystemPropertyUtil.get("no-client-http2").toBoolean()) {
it.protocols(listOf(Protocol.HTTP_1_1))
} else {
it
}
}
.connectTimeout(Duration.ofSeconds(2))
.connectionPool(
ConnectionPool(
maxIdleConnections = 100,
keepAliveDuration = 1,
timeUnit = TimeUnit.MINUTES
)
)
.writeTimeout(Duration.ofSeconds(10))
.readTimeout(Duration.ofSeconds(10))
.protocols(listOf(Protocol.HTTP_1_1))
.build()
)
)
@ -100,7 +100,7 @@ class ServerManager(
init {
state = Uninitialized
backendApi = BackendApi(settings, OkHttp())
backendApi = BackendApi(settings)
}
fun start() {

25
src/main/kotlin/mdnet/netty/ApplicationNetty.kt
View file

@ -64,6 +64,7 @@ import java.security.PrivateKey
import java.security.cert.CertificateFactory
import java.security.cert.X509Certificate
import java.util.Locale
import java.util.concurrent.TimeUnit
import javax.net.ssl.SSLException
sealed class NettyTransport(threads: Int) {
@ -76,12 +77,6 @@ sealed class NettyTransport(threads: Int) {
}
)
fun shutdownGracefully() {
bossGroup.shutdownGracefully().sync()
workerGroup.shutdownGracefully().sync()
executor.shutdownGracefully().sync()
}
private class NioTransport(threads: Int) : NettyTransport(threads) {
override val bossGroup = NioEventLoopGroup(1)
override val workerGroup = NioEventLoopGroup(8)
@ -108,7 +103,7 @@ sealed class NettyTransport(threads: Int) {
val name = SystemPropertyUtil.get("os.name").toLowerCase(Locale.UK).trim { it <= ' ' }
val threadsToUse = if (threads == 0) defaultNumThreads() else threads
LOGGER.info { "Choosing a transport using $threadsToUse threads" }
LOGGER.info { "Choosing a transport with $threadsToUse threads" }
if (name.startsWith("linux")) {
if (!SystemPropertyUtil.get("no-iouring").toBoolean()) {
@ -116,7 +111,7 @@ sealed class NettyTransport(threads: Int) {
LOGGER.info { "Using IOUring transport" }
return IOUringTransport(threadsToUse)
} else {
LOGGER.info(IOUring.unavailabilityCause()) {
LOGGER.info {
"IOUring transport not available (this may be normal)"
}
}
@ -127,7 +122,7 @@ sealed class NettyTransport(threads: Int) {
LOGGER.info { "Using Epoll transport" }
return EpollTransport(threadsToUse)
} else {
LOGGER.info(Epoll.unavailabilityCause()) {
LOGGER.info {
"Epoll transport not available (this may be normal)"
}
}
@ -145,7 +140,7 @@ class Netty(
private val serverSettings: ServerSettings,
private val statistics: Statistics
) : ServerConfig {
override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer {
override fun toServer(http: HttpHandler): Http4kServer = object : Http4kServer {
private val transport = NettyTransport.bestForPlatform(serverSettings.threads)
private lateinit var channel: Channel
@ -190,7 +185,7 @@ class Netty(
)
ch.pipeline().addLast("streamer", ChunkedWriteHandler())
ch.pipeline().addLast(transport.executor, "handler", Http4kChannelHandler(httpHandler))
ch.pipeline().addLast(transport.executor, "handler", Http4kChannelHandler(http))
ch.pipeline().addLast(
"exceptions",
@ -216,10 +211,14 @@ class Netty(
override fun stop() = apply {
channel.close().sync()
transport.shutdownGracefully()
transport.run {
bossGroup.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).sync()
workerGroup.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).sync()
executor.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).sync()
}
}
override fun port(): Int = serverSettings.port
override fun port(): Int = (channel.localAddress() as InetSocketAddress).port
}
companion object {

3
src/main/kotlin/mdnet/server/ImageServer.kt
View file

@ -105,9 +105,6 @@ fun getServer(
val verifier = TokenVerifier(
tokenKey = remoteSettings.tokenKey,
shouldVerify = { chapter, _ ->
!remoteSettings.disableTokens && !(chapter == "1b682e7b24ae7dbdc5064eeeb8e8e353" || chapter == "8172a46adc798f4f4ace6663322a383e")
}
)
return timeRequest()

5
src/main/kotlin/mdnet/server/TokenVerifier.kt
View file

@ -37,15 +37,13 @@ import org.slf4j.LoggerFactory
import java.time.OffsetDateTime
import java.util.Base64
class TokenVerifier(tokenKey: ByteArray, private val shouldVerify: (String, String) -> Boolean) : Filter {
class TokenVerifier(tokenKey: ByteArray) : Filter {
private val box = TweetNaclFast.SecretBox(tokenKey)
override fun invoke(next: HttpHandler): HttpHandler {
return then@{
val chapterHash = Path.of("chapterHash")(it)
val fileName = Path.of("fileName")(it)
if (shouldVerify(chapterHash, fileName)) {
val cleanedUri = it.uri.path.replaceBefore("/data", "/{token}")
val tokenArr = try {
@ -87,7 +85,6 @@ class TokenVerifier(tokenKey: ByteArray, private val shouldVerify: (String, Stri
LOGGER.info { "Request for $cleanedUri rejected for inapplicable token" }
return@then Response(Status.FORBIDDEN).body("Token is inapplicable for the image")
}
}
return@then next(it)
}

4
src/test/kotlin/mdnet/cache/ImageStorageTest.kt vendored
View file

@ -42,7 +42,7 @@ class ImageStorageTest : FreeSpec() {
val imageStorage = ImageStorage(
maxSize = 5,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:h2:${tempfile()}"),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
)
@ -160,7 +160,7 @@ class ImageStorageSlowTest : FreeSpec() {
val imageStorage = ImageStorage(
maxSize = 4097,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:h2:${tempfile()}"),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
)
"autoPrune" - {

4
src/test/kotlin/mdnet/server/ImageServerTest.kt
View file

@ -112,7 +112,7 @@ class ImageServerTest : FreeSpec() {
val storage = ImageStorage(
maxSize = 100000,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:h2:${tempfile()}"),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
)
@ -157,7 +157,7 @@ class ImageServerTest : FreeSpec() {
val storage = ImageStorage(
maxSize = 100000,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:h2:${tempfile()}"),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
)

4
src/test/kotlin/mdnet/server/TokenVerifierTest.kt
View file

@ -31,9 +31,7 @@ class TokenVerifierTest : FreeSpec() {
val clientKeys = TweetNaclFast.Box.keyPair()
val box = TweetNaclFast.Box(clientKeys.publicKey, remoteKeys.secretKey)
val backend = TokenVerifier(box.before()) { _, _ ->
true
}.then {
val backend = TokenVerifier(box.before()).then {
Response(Status.OK)
}