mirror of
https://gitlab.com/mangadex-pub/mangadex_at_home.git
synced 2024-01-19 02:48:37 +00:00
Merge branch 'next' into 'master'
Next See merge request mangadex-pub/mangadex_at_home!82
This commit is contained in:
commit
4fed127e80
22
CHANGELOG.md
22
CHANGELOG.md
|
@ -14,15 +14,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||||
### Removed
|
### Removed
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
- [2021-03-11] Fixed Access-Control-Expose-Headers typo [@lflare].
|
|
||||||
|
|
||||||
### Security
|
### Security
|
||||||
|
|
||||||
|
## [2.0.0] - 2021-03-11
|
||||||
|
### Changed
|
||||||
|
- [2021-03-11] Switch back to HTTP/1.1 [@carbotaniuman].
|
||||||
|
- [2021-03-11] Tune connection pool [@carbotaniuman].
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- [2021-03-11] Fixed Access-Control-Expose-Headers typo [@lflare].
|
||||||
|
- [2021-03-11] Throw exceptions less frequently [@carbotaniuman].
|
||||||
|
- [2021-03-11] Fix various Netty issues [@carbotaniuman].
|
||||||
|
- [2021-03-11] Don't log IOUring and Epoll errors [@carbotaniuman].
|
||||||
|
- [2021-03-11] Ignore IPV6 when pinging [@carbotaniuman].
|
||||||
|
|
||||||
## [2.0.0-rc14] - 2021-03-02
|
## [2.0.0-rc14] - 2021-03-02
|
||||||
### Changed
|
### Changed
|
||||||
- [2021-02-10] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9].
|
- [2021-03-02] Fix Prometheus to 2.24.1 and Grafana to 7.4.0 [@_tde9].
|
||||||
- [2021-02-10] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9].
|
- [2021-03-02] Update and rearrange the embedded dashboard with the new Timeseries panel from Grafana 7.4 [@_tde9].
|
||||||
- [2021-02-10] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9].
|
- [2021-03-02] Update sample dashboard screenshot thanks to DLMSweet :smile: [@_tde9].
|
||||||
- [2021-02-25] Use HTTP/2 to download when possible [@carbotaniuman].
|
- [2021-02-25] Use HTTP/2 to download when possible [@carbotaniuman].
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
@ -376,7 +387,8 @@ This release contains many breaking changes! Of note are the changes to the cach
|
||||||
### Fixed
|
### Fixed
|
||||||
- [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
|
- [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
|
||||||
|
|
||||||
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...HEAD
|
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...HEAD
|
||||||
|
[2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0
|
||||||
[2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14
|
[2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14
|
||||||
[2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13
|
[2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13
|
||||||
[2.0.0-rc12]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc11...2.0.0-rc12
|
[2.0.0-rc12]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc11...2.0.0-rc12
|
||||||
|
|
23
build.gradle
23
build.gradle
|
@ -25,39 +25,38 @@ configurations {
|
||||||
dependencies {
|
dependencies {
|
||||||
compileOnly group: "dev.afanasev", name: "sekret-annotation", version: "0.0.7"
|
compileOnly group: "dev.afanasev", name: "sekret-annotation", version: "0.0.7"
|
||||||
|
|
||||||
implementation group: "commons-io", name: "commons-io", version: "2.7"
|
implementation group: "commons-io", name: "commons-io", version: "2.8.0"
|
||||||
implementation group: "org.apache.commons", name: "commons-compress", version: "1.20"
|
implementation group: "org.apache.commons", name: "commons-compress", version: "1.20"
|
||||||
implementation group: "ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha4"
|
implementation group: "ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha4"
|
||||||
|
|
||||||
implementation group: "io.micrometer", name: "micrometer-registry-prometheus", version: "1.6.2"
|
implementation group: "io.micrometer", name: "micrometer-registry-prometheus", version: "1.6.2"
|
||||||
implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0"
|
implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0"
|
||||||
|
|
||||||
implementation group: "org.http4k", name: "http4k-bom", version: "4.3.5.4"
|
|
||||||
|
|
||||||
implementation platform(group: "org.http4k", name: "http4k-bom", version: "4.3.5.4")
|
implementation platform(group: "org.http4k", name: "http4k-bom", version: "4.3.5.4")
|
||||||
|
implementation platform(group: "com.fasterxml.jackson", name: "jackson-bom", version: "2.12.1")
|
||||||
|
implementation platform(group: "io.netty", name: "netty-bom", version: "4.1.60.Final")
|
||||||
|
|
||||||
implementation group: "org.http4k", name: "http4k-core"
|
implementation group: "org.http4k", name: "http4k-core"
|
||||||
implementation group: "org.http4k", name: "http4k-resilience4j"
|
implementation group: "org.http4k", name: "http4k-resilience4j"
|
||||||
implementation group: "io.github.resilience4j", name: "resilience4j-micrometer", version: "1.6.1"
|
implementation group: "io.github.resilience4j", name: "resilience4j-micrometer", version: "1.6.1"
|
||||||
implementation group: "org.http4k", name: "http4k-format-jackson"
|
implementation group: "org.http4k", name: "http4k-format-jackson"
|
||||||
implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1"
|
implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml"
|
||||||
implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1"
|
implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310"
|
||||||
implementation group: "org.http4k", name: "http4k-client-okhttp"
|
implementation group: "org.http4k", name: "http4k-client-okhttp"
|
||||||
implementation group: "org.http4k", name: "http4k-metrics-micrometer"
|
implementation group: "org.http4k", name: "http4k-metrics-micrometer"
|
||||||
implementation group: "org.http4k", name: "http4k-server-netty"
|
implementation group: "org.http4k", name: "http4k-server-netty"
|
||||||
implementation group: "io.netty", name: "netty-transport-native-epoll", version: "4.1.58.Final", classifier: "linux-x86_64"
|
implementation group: "io.netty", name: "netty-transport-native-epoll", classifier: "linux-x86_64"
|
||||||
implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.3.Final", classifier: "linux-x86_64"
|
implementation group: "io.netty.incubator", name: "netty-incubator-transport-native-io_uring", version: "0.0.3.Final", classifier: "linux-x86_64"
|
||||||
testImplementation group: "org.http4k", name: "http4k-testing-kotest"
|
testImplementation group: "org.http4k", name: "http4k-testing-kotest"
|
||||||
runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.34.Final"
|
runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.36.Final"
|
||||||
|
|
||||||
implementation group: 'com.zaxxer', name: 'HikariCP', version: '4.0.1'
|
implementation group: "com.zaxxer", name: "HikariCP", version: "4.0.2"
|
||||||
implementation group: "com.h2database", name: "h2", version: "1.4.200"
|
implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.34.0"
|
||||||
implementation group: 'org.xerial', name: 'sqlite-jdbc', version: '3.34.0'
|
|
||||||
implementation "org.ktorm:ktorm-core:$ktorm_version"
|
implementation "org.ktorm:ktorm-core:$ktorm_version"
|
||||||
implementation "org.ktorm:ktorm-jackson:$ktorm_version"
|
implementation "org.ktorm:ktorm-jackson:$ktorm_version"
|
||||||
|
|
||||||
implementation "info.picocli:picocli:4.5.0"
|
implementation "info.picocli:picocli:$picocli_version"
|
||||||
kapt "info.picocli:picocli-codegen:4.5.0"
|
kapt "info.picocli:picocli-codegen:$picocli_version"
|
||||||
|
|
||||||
testImplementation "io.kotest:kotest-runner-junit5:$kotest_version"
|
testImplementation "io.kotest:kotest-runner-junit5:$kotest_version"
|
||||||
testImplementation "io.kotest:kotest-assertions-core:$kotest_version"
|
testImplementation "io.kotest:kotest-assertions-core:$kotest_version"
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
http_4k_version=4.3.0.0
|
http_4k_version=4.3.0.0
|
||||||
kotest_version=4.4.1
|
kotest_version=4.4.1
|
||||||
ktorm_version=3.2.0
|
ktorm_version=3.3.0
|
||||||
|
picocli_version=4.6.1
|
|
@ -30,6 +30,7 @@ public class CachingInputStream extends ProxyInputStream {
|
||||||
private final OutputStream cache;
|
private final OutputStream cache;
|
||||||
private final ExecutorService executor;
|
private final ExecutorService executor;
|
||||||
private final Runnable onClose;
|
private final Runnable onClose;
|
||||||
|
private boolean eofReached = false;
|
||||||
|
|
||||||
public CachingInputStream(InputStream response, ExecutorService executor, OutputStream cache, Runnable onClose) {
|
public CachingInputStream(InputStream response, ExecutorService executor, OutputStream cache, Runnable onClose) {
|
||||||
super(response);
|
super(response);
|
||||||
|
@ -40,7 +41,7 @@ public class CachingInputStream extends ProxyInputStream {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void close() throws IOException {
|
public void close() throws IOException {
|
||||||
if (read() == EOF) {
|
if (eofReached) {
|
||||||
try {
|
try {
|
||||||
in.close();
|
in.close();
|
||||||
} catch (IOException ignored) {
|
} catch (IOException ignored) {
|
||||||
|
@ -50,25 +51,24 @@ public class CachingInputStream extends ProxyInputStream {
|
||||||
} catch (IOException ignored) {
|
} catch (IOException ignored) {
|
||||||
}
|
}
|
||||||
onClose.run();
|
onClose.run();
|
||||||
|
} else {
|
||||||
return;
|
executor.submit(() -> {
|
||||||
|
try {
|
||||||
|
IOUtils.copy(in, cache);
|
||||||
|
} catch (IOException ignored) {
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
in.close();
|
||||||
|
} catch (IOException ignored) {
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
cache.close();
|
||||||
|
} catch (IOException ignored) {
|
||||||
|
}
|
||||||
|
onClose.run();
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
executor.submit(() -> {
|
|
||||||
try {
|
|
||||||
IOUtils.copy(in, cache);
|
|
||||||
} catch (IOException ignored) {
|
|
||||||
} finally {
|
|
||||||
try {
|
|
||||||
in.close();
|
|
||||||
} catch (IOException ignored) {
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
cache.close();
|
|
||||||
} catch (IOException ignored) {
|
|
||||||
}
|
|
||||||
onClose.run();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -80,6 +80,8 @@ public class CachingInputStream extends ProxyInputStream {
|
||||||
} catch (IOException ignored) {
|
} catch (IOException ignored) {
|
||||||
// don't let write failures affect the image loading
|
// don't let write failures affect the image loading
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
eofReached = true;
|
||||||
}
|
}
|
||||||
return ch;
|
return ch;
|
||||||
}
|
}
|
||||||
|
@ -93,6 +95,8 @@ public class CachingInputStream extends ProxyInputStream {
|
||||||
} catch (IOException ignored) {
|
} catch (IOException ignored) {
|
||||||
// don't let write failures affect the image loading
|
// don't let write failures affect the image loading
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
eofReached = true;
|
||||||
}
|
}
|
||||||
return n;
|
return n;
|
||||||
}
|
}
|
||||||
|
@ -106,6 +110,8 @@ public class CachingInputStream extends ProxyInputStream {
|
||||||
} catch (IOException ignored) {
|
} catch (IOException ignored) {
|
||||||
// don't let write failures affect the image loading
|
// don't let write failures affect the image loading
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
eofReached = true;
|
||||||
}
|
}
|
||||||
return n;
|
return n;
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,15 +23,17 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule
|
||||||
import mdnet.ServerHandlerJackson.auto
|
import mdnet.ServerHandlerJackson.auto
|
||||||
import mdnet.logging.info
|
import mdnet.logging.info
|
||||||
import mdnet.settings.*
|
import mdnet.settings.*
|
||||||
import org.http4k.core.Body
|
import okhttp3.Dns
|
||||||
import org.http4k.core.HttpHandler
|
import okhttp3.OkHttpClient
|
||||||
import org.http4k.core.Method
|
import org.http4k.client.OkHttp
|
||||||
import org.http4k.core.Request
|
import org.http4k.core.*
|
||||||
import org.http4k.format.ConfigurableJackson
|
import org.http4k.format.ConfigurableJackson
|
||||||
import org.http4k.format.asConfigurable
|
import org.http4k.format.asConfigurable
|
||||||
import org.http4k.format.withStandardMappings
|
import org.http4k.format.withStandardMappings
|
||||||
import org.http4k.lens.LensFailure
|
import org.http4k.lens.LensFailure
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
|
import java.net.Inet4Address
|
||||||
|
import java.net.InetAddress
|
||||||
|
|
||||||
object ServerHandlerJackson : ConfigurableJackson(
|
object ServerHandlerJackson : ConfigurableJackson(
|
||||||
KotlinModule()
|
KotlinModule()
|
||||||
|
@ -41,7 +43,16 @@ object ServerHandlerJackson : ConfigurableJackson(
|
||||||
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
|
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
|
||||||
)
|
)
|
||||||
|
|
||||||
class BackendApi(private val settings: ClientSettings, private val client: HttpHandler) {
|
class BackendApi(private val settings: ClientSettings) {
|
||||||
|
private val client = OkHttp(
|
||||||
|
client = OkHttpClient.Builder()
|
||||||
|
.dns(object : Dns {
|
||||||
|
override fun lookup(hostname: String): List<InetAddress> {
|
||||||
|
return Dns.SYSTEM.lookup(hostname).filterIsInstance<Inet4Address>()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.build()
|
||||||
|
)
|
||||||
private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS
|
private val serverAddress = settings.devSettings.devUrl ?: SERVER_ADDRESS
|
||||||
|
|
||||||
fun logoutFromControl(): Boolean {
|
fun logoutFromControl(): Boolean {
|
||||||
|
|
|
@ -21,7 +21,7 @@ package mdnet
|
||||||
import java.time.Duration
|
import java.time.Duration
|
||||||
|
|
||||||
object Constants {
|
object Constants {
|
||||||
const val CLIENT_BUILD = 29
|
const val CLIENT_BUILD = 30
|
||||||
|
|
||||||
@JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
|
@JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
|
||||||
|
|
||||||
|
|
|
@ -97,8 +97,6 @@ class Main : Runnable {
|
||||||
throw IllegalArgumentException("Cache folder $cacheFolder must be a directory")
|
throw IllegalArgumentException("Cache folder $cacheFolder must be a directory")
|
||||||
}
|
}
|
||||||
|
|
||||||
migrate(databaseFolder)
|
|
||||||
|
|
||||||
val client = MangaDexClient(settingsFile, databaseFolder, cacheFolder)
|
val client = MangaDexClient(settingsFile, databaseFolder, cacheFolder)
|
||||||
val hook = Thread {
|
val hook = Thread {
|
||||||
client.shutdown()
|
client.shutdown()
|
||||||
|
|
|
@ -198,7 +198,7 @@ class MangaDexClient(private val settingsFile: File, databaseFolder: Path, cache
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun validateSettings(settings: ClientSettings) {
|
private fun validateSettings(settings: ClientSettings) {
|
||||||
if (settings.maxCacheSizeInMebibytes < 20480) {
|
if (settings.maxCacheSizeInMebibytes < 40960) {
|
||||||
throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 20480 MiB (20 GiB)")
|
throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 20480 MiB (20 GiB)")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,75 +0,0 @@
|
||||||
/*
|
|
||||||
Mangadex@Home
|
|
||||||
Copyright (c) 2020, MangaDex Network
|
|
||||||
This file is part of MangaDex@Home.
|
|
||||||
|
|
||||||
MangaDex@Home is free software: you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
MangaDex@Home is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
|
||||||
along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
*/
|
|
||||||
package mdnet
|
|
||||||
|
|
||||||
import mdnet.cache.DbImage
|
|
||||||
import mdnet.cache.INIT_TABLE
|
|
||||||
import org.ktorm.database.Database
|
|
||||||
import org.ktorm.dsl.*
|
|
||||||
import java.nio.file.Files
|
|
||||||
import java.nio.file.Path
|
|
||||||
import java.nio.file.Paths
|
|
||||||
|
|
||||||
fun main() {
|
|
||||||
migrate(Paths.get("./"))
|
|
||||||
}
|
|
||||||
|
|
||||||
fun migrate(path: Path) {
|
|
||||||
val h2file = path.resolve("metadata.mv.db")
|
|
||||||
if (!Files.exists(h2file)) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
println("Migrating database - this may take a long time")
|
|
||||||
|
|
||||||
Class.forName("org.sqlite.JDBC")
|
|
||||||
|
|
||||||
val sqliteDb = path.resolve("metadata.db")
|
|
||||||
Files.deleteIfExists(sqliteDb)
|
|
||||||
|
|
||||||
val sqlite = Database.connect("jdbc:sqlite:$sqliteDb")
|
|
||||||
sqlite.useConnection { conn ->
|
|
||||||
conn.prepareStatement(INIT_TABLE).use {
|
|
||||||
it.execute()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
val db = path.resolve("metadata")
|
|
||||||
|
|
||||||
val h2 = Database.connect("jdbc:h2:$db")
|
|
||||||
h2.useConnection { conn ->
|
|
||||||
conn.prepareStatement(INIT_TABLE).use {
|
|
||||||
it.execute()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
h2.from(DbImage).select().asIterable().chunked(1000).forEach { list ->
|
|
||||||
sqlite.batchInsert(DbImage) {
|
|
||||||
for (data in list) {
|
|
||||||
item {
|
|
||||||
set(DbImage.id, data[DbImage.id])
|
|
||||||
set(DbImage.accessed, data[DbImage.accessed])
|
|
||||||
set(DbImage.size, data[DbImage.size])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Files.move(h2file, path.resolve("metadata.mv.db.old"))
|
|
||||||
}
|
|
|
@ -20,7 +20,6 @@ package mdnet
|
||||||
|
|
||||||
import io.micrometer.prometheus.PrometheusConfig
|
import io.micrometer.prometheus.PrometheusConfig
|
||||||
import io.micrometer.prometheus.PrometheusMeterRegistry
|
import io.micrometer.prometheus.PrometheusMeterRegistry
|
||||||
import io.netty.util.internal.SystemPropertyUtil
|
|
||||||
import mdnet.cache.ImageStorage
|
import mdnet.cache.ImageStorage
|
||||||
import mdnet.data.Statistics
|
import mdnet.data.Statistics
|
||||||
import mdnet.logging.error
|
import mdnet.logging.error
|
||||||
|
@ -30,6 +29,7 @@ import mdnet.metrics.DefaultMicrometerMetrics
|
||||||
import mdnet.server.getServer
|
import mdnet.server.getServer
|
||||||
import mdnet.settings.ClientSettings
|
import mdnet.settings.ClientSettings
|
||||||
import mdnet.settings.RemoteSettings
|
import mdnet.settings.RemoteSettings
|
||||||
|
import okhttp3.ConnectionPool
|
||||||
import okhttp3.OkHttpClient
|
import okhttp3.OkHttpClient
|
||||||
import okhttp3.Protocol
|
import okhttp3.Protocol
|
||||||
import org.http4k.client.OkHttp
|
import org.http4k.client.OkHttp
|
||||||
|
@ -78,17 +78,17 @@ class ServerManager(
|
||||||
OkHttp(
|
OkHttp(
|
||||||
bodyMode = BodyMode.Stream,
|
bodyMode = BodyMode.Stream,
|
||||||
client = OkHttpClient.Builder()
|
client = OkHttpClient.Builder()
|
||||||
.callTimeout(Duration.ofSeconds(30))
|
.connectTimeout(Duration.ofSeconds(2))
|
||||||
.connectTimeout(Duration.ofSeconds(1))
|
.connectionPool(
|
||||||
.writeTimeout(Duration.ofSeconds(5))
|
ConnectionPool(
|
||||||
.readTimeout(Duration.ofSeconds(5))
|
maxIdleConnections = 100,
|
||||||
.let {
|
keepAliveDuration = 1,
|
||||||
if (SystemPropertyUtil.get("no-client-http2").toBoolean()) {
|
timeUnit = TimeUnit.MINUTES
|
||||||
it.protocols(listOf(Protocol.HTTP_1_1))
|
)
|
||||||
} else {
|
)
|
||||||
it
|
.writeTimeout(Duration.ofSeconds(10))
|
||||||
}
|
.readTimeout(Duration.ofSeconds(10))
|
||||||
}
|
.protocols(listOf(Protocol.HTTP_1_1))
|
||||||
.build()
|
.build()
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
@ -100,7 +100,7 @@ class ServerManager(
|
||||||
|
|
||||||
init {
|
init {
|
||||||
state = Uninitialized
|
state = Uninitialized
|
||||||
backendApi = BackendApi(settings, OkHttp())
|
backendApi = BackendApi(settings)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun start() {
|
fun start() {
|
||||||
|
|
|
@ -64,6 +64,7 @@ import java.security.PrivateKey
|
||||||
import java.security.cert.CertificateFactory
|
import java.security.cert.CertificateFactory
|
||||||
import java.security.cert.X509Certificate
|
import java.security.cert.X509Certificate
|
||||||
import java.util.Locale
|
import java.util.Locale
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
import javax.net.ssl.SSLException
|
import javax.net.ssl.SSLException
|
||||||
|
|
||||||
sealed class NettyTransport(threads: Int) {
|
sealed class NettyTransport(threads: Int) {
|
||||||
|
@ -76,12 +77,6 @@ sealed class NettyTransport(threads: Int) {
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
fun shutdownGracefully() {
|
|
||||||
bossGroup.shutdownGracefully().sync()
|
|
||||||
workerGroup.shutdownGracefully().sync()
|
|
||||||
executor.shutdownGracefully().sync()
|
|
||||||
}
|
|
||||||
|
|
||||||
private class NioTransport(threads: Int) : NettyTransport(threads) {
|
private class NioTransport(threads: Int) : NettyTransport(threads) {
|
||||||
override val bossGroup = NioEventLoopGroup(1)
|
override val bossGroup = NioEventLoopGroup(1)
|
||||||
override val workerGroup = NioEventLoopGroup(8)
|
override val workerGroup = NioEventLoopGroup(8)
|
||||||
|
@ -108,7 +103,7 @@ sealed class NettyTransport(threads: Int) {
|
||||||
val name = SystemPropertyUtil.get("os.name").toLowerCase(Locale.UK).trim { it <= ' ' }
|
val name = SystemPropertyUtil.get("os.name").toLowerCase(Locale.UK).trim { it <= ' ' }
|
||||||
|
|
||||||
val threadsToUse = if (threads == 0) defaultNumThreads() else threads
|
val threadsToUse = if (threads == 0) defaultNumThreads() else threads
|
||||||
LOGGER.info { "Choosing a transport using $threadsToUse threads" }
|
LOGGER.info { "Choosing a transport with $threadsToUse threads" }
|
||||||
|
|
||||||
if (name.startsWith("linux")) {
|
if (name.startsWith("linux")) {
|
||||||
if (!SystemPropertyUtil.get("no-iouring").toBoolean()) {
|
if (!SystemPropertyUtil.get("no-iouring").toBoolean()) {
|
||||||
|
@ -116,7 +111,7 @@ sealed class NettyTransport(threads: Int) {
|
||||||
LOGGER.info { "Using IOUring transport" }
|
LOGGER.info { "Using IOUring transport" }
|
||||||
return IOUringTransport(threadsToUse)
|
return IOUringTransport(threadsToUse)
|
||||||
} else {
|
} else {
|
||||||
LOGGER.info(IOUring.unavailabilityCause()) {
|
LOGGER.info {
|
||||||
"IOUring transport not available (this may be normal)"
|
"IOUring transport not available (this may be normal)"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -127,7 +122,7 @@ sealed class NettyTransport(threads: Int) {
|
||||||
LOGGER.info { "Using Epoll transport" }
|
LOGGER.info { "Using Epoll transport" }
|
||||||
return EpollTransport(threadsToUse)
|
return EpollTransport(threadsToUse)
|
||||||
} else {
|
} else {
|
||||||
LOGGER.info(Epoll.unavailabilityCause()) {
|
LOGGER.info {
|
||||||
"Epoll transport not available (this may be normal)"
|
"Epoll transport not available (this may be normal)"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -145,7 +140,7 @@ class Netty(
|
||||||
private val serverSettings: ServerSettings,
|
private val serverSettings: ServerSettings,
|
||||||
private val statistics: Statistics
|
private val statistics: Statistics
|
||||||
) : ServerConfig {
|
) : ServerConfig {
|
||||||
override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer {
|
override fun toServer(http: HttpHandler): Http4kServer = object : Http4kServer {
|
||||||
private val transport = NettyTransport.bestForPlatform(serverSettings.threads)
|
private val transport = NettyTransport.bestForPlatform(serverSettings.threads)
|
||||||
|
|
||||||
private lateinit var channel: Channel
|
private lateinit var channel: Channel
|
||||||
|
@ -190,7 +185,7 @@ class Netty(
|
||||||
)
|
)
|
||||||
|
|
||||||
ch.pipeline().addLast("streamer", ChunkedWriteHandler())
|
ch.pipeline().addLast("streamer", ChunkedWriteHandler())
|
||||||
ch.pipeline().addLast(transport.executor, "handler", Http4kChannelHandler(httpHandler))
|
ch.pipeline().addLast(transport.executor, "handler", Http4kChannelHandler(http))
|
||||||
|
|
||||||
ch.pipeline().addLast(
|
ch.pipeline().addLast(
|
||||||
"exceptions",
|
"exceptions",
|
||||||
|
@ -216,10 +211,14 @@ class Netty(
|
||||||
|
|
||||||
override fun stop() = apply {
|
override fun stop() = apply {
|
||||||
channel.close().sync()
|
channel.close().sync()
|
||||||
transport.shutdownGracefully()
|
transport.run {
|
||||||
|
bossGroup.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).sync()
|
||||||
|
workerGroup.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).sync()
|
||||||
|
executor.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).sync()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun port(): Int = serverSettings.port
|
override fun port(): Int = (channel.localAddress() as InetSocketAddress).port
|
||||||
}
|
}
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
|
|
|
@ -105,9 +105,6 @@ fun getServer(
|
||||||
|
|
||||||
val verifier = TokenVerifier(
|
val verifier = TokenVerifier(
|
||||||
tokenKey = remoteSettings.tokenKey,
|
tokenKey = remoteSettings.tokenKey,
|
||||||
shouldVerify = { chapter, _ ->
|
|
||||||
!remoteSettings.disableTokens && !(chapter == "1b682e7b24ae7dbdc5064eeeb8e8e353" || chapter == "8172a46adc798f4f4ace6663322a383e")
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return timeRequest()
|
return timeRequest()
|
||||||
|
|
|
@ -37,56 +37,53 @@ import org.slf4j.LoggerFactory
|
||||||
import java.time.OffsetDateTime
|
import java.time.OffsetDateTime
|
||||||
import java.util.Base64
|
import java.util.Base64
|
||||||
|
|
||||||
class TokenVerifier(tokenKey: ByteArray, private val shouldVerify: (String, String) -> Boolean) : Filter {
|
class TokenVerifier(tokenKey: ByteArray) : Filter {
|
||||||
private val box = TweetNaclFast.SecretBox(tokenKey)
|
private val box = TweetNaclFast.SecretBox(tokenKey)
|
||||||
|
|
||||||
override fun invoke(next: HttpHandler): HttpHandler {
|
override fun invoke(next: HttpHandler): HttpHandler {
|
||||||
return then@{
|
return then@{
|
||||||
val chapterHash = Path.of("chapterHash")(it)
|
val chapterHash = Path.of("chapterHash")(it)
|
||||||
val fileName = Path.of("fileName")(it)
|
|
||||||
|
|
||||||
if (shouldVerify(chapterHash, fileName)) {
|
val cleanedUri = it.uri.path.replaceBefore("/data", "/{token}")
|
||||||
val cleanedUri = it.uri.path.replaceBefore("/data", "/{token}")
|
|
||||||
|
|
||||||
val tokenArr = try {
|
val tokenArr = try {
|
||||||
val toDecode = try {
|
val toDecode = try {
|
||||||
Path.of("token")(it)
|
Path.of("token")(it)
|
||||||
} catch (e: LensFailure) {
|
} catch (e: LensFailure) {
|
||||||
LOGGER.info(e) { "Request for $cleanedUri rejected for missing token" }
|
LOGGER.info(e) { "Request for $cleanedUri rejected for missing token" }
|
||||||
return@then Response(Status.FORBIDDEN).body("Token is missing")
|
return@then Response(Status.FORBIDDEN).body("Token is missing")
|
||||||
}
|
|
||||||
Base64.getUrlDecoder().decode(toDecode)
|
|
||||||
} catch (e: IllegalArgumentException) {
|
|
||||||
LOGGER.info(e) { "Request for $cleanedUri rejected for non-base64 token" }
|
|
||||||
return@then Response(Status.FORBIDDEN).body("Token is invalid base64")
|
|
||||||
}
|
}
|
||||||
if (tokenArr.size < 24) {
|
Base64.getUrlDecoder().decode(toDecode)
|
||||||
LOGGER.info { "Request for $cleanedUri rejected for invalid token" }
|
} catch (e: IllegalArgumentException) {
|
||||||
return@then Response(Status.FORBIDDEN)
|
LOGGER.info(e) { "Request for $cleanedUri rejected for non-base64 token" }
|
||||||
}
|
return@then Response(Status.FORBIDDEN).body("Token is invalid base64")
|
||||||
val token = try {
|
}
|
||||||
JACKSON.readValue<Token>(
|
if (tokenArr.size < 24) {
|
||||||
box.open(tokenArr.sliceArray(24 until tokenArr.size), tokenArr.sliceArray(0 until 24)).apply {
|
LOGGER.info { "Request for $cleanedUri rejected for invalid token" }
|
||||||
if (this == null) {
|
return@then Response(Status.FORBIDDEN)
|
||||||
LOGGER.info { "Request for $cleanedUri rejected for invalid token" }
|
}
|
||||||
return@then Response(Status.FORBIDDEN)
|
val token = try {
|
||||||
}
|
JACKSON.readValue<Token>(
|
||||||
|
box.open(tokenArr.sliceArray(24 until tokenArr.size), tokenArr.sliceArray(0 until 24)).apply {
|
||||||
|
if (this == null) {
|
||||||
|
LOGGER.info { "Request for $cleanedUri rejected for invalid token" }
|
||||||
|
return@then Response(Status.FORBIDDEN)
|
||||||
}
|
}
|
||||||
)
|
}
|
||||||
} catch (e: JsonProcessingException) {
|
)
|
||||||
LOGGER.info(e) { "Request for $cleanedUri rejected for invalid token" }
|
} catch (e: JsonProcessingException) {
|
||||||
return@then Response(Status.FORBIDDEN).body("Token is invalid")
|
LOGGER.info(e) { "Request for $cleanedUri rejected for invalid token" }
|
||||||
}
|
return@then Response(Status.FORBIDDEN).body("Token is invalid")
|
||||||
|
}
|
||||||
|
|
||||||
if (OffsetDateTime.now().isAfter(token.expires)) {
|
if (OffsetDateTime.now().isAfter(token.expires)) {
|
||||||
LOGGER.info { "Request for $cleanedUri rejected for expired token" }
|
LOGGER.info { "Request for $cleanedUri rejected for expired token" }
|
||||||
return@then Response(Status.GONE).body("Token has expired")
|
return@then Response(Status.GONE).body("Token has expired")
|
||||||
}
|
}
|
||||||
|
|
||||||
if (token.hash != chapterHash) {
|
if (token.hash != chapterHash) {
|
||||||
LOGGER.info { "Request for $cleanedUri rejected for inapplicable token" }
|
LOGGER.info { "Request for $cleanedUri rejected for inapplicable token" }
|
||||||
return@then Response(Status.FORBIDDEN).body("Token is inapplicable for the image")
|
return@then Response(Status.FORBIDDEN).body("Token is inapplicable for the image")
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return@then next(it)
|
return@then next(it)
|
||||||
|
|
|
@ -42,7 +42,7 @@ class ImageStorageTest : FreeSpec() {
|
||||||
val imageStorage = ImageStorage(
|
val imageStorage = ImageStorage(
|
||||||
maxSize = 5,
|
maxSize = 5,
|
||||||
cacheDirectory = tempdir().toPath(),
|
cacheDirectory = tempdir().toPath(),
|
||||||
database = Database.connect("jdbc:h2:${tempfile()}"),
|
database = Database.connect("jdbc:sqlite:${tempfile()}"),
|
||||||
autoPrune = false,
|
autoPrune = false,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -160,7 +160,7 @@ class ImageStorageSlowTest : FreeSpec() {
|
||||||
val imageStorage = ImageStorage(
|
val imageStorage = ImageStorage(
|
||||||
maxSize = 4097,
|
maxSize = 4097,
|
||||||
cacheDirectory = tempdir().toPath(),
|
cacheDirectory = tempdir().toPath(),
|
||||||
database = Database.connect("jdbc:h2:${tempfile()}"),
|
database = Database.connect("jdbc:sqlite:${tempfile()}"),
|
||||||
)
|
)
|
||||||
|
|
||||||
"autoPrune" - {
|
"autoPrune" - {
|
||||||
|
|
|
@ -112,7 +112,7 @@ class ImageServerTest : FreeSpec() {
|
||||||
val storage = ImageStorage(
|
val storage = ImageStorage(
|
||||||
maxSize = 100000,
|
maxSize = 100000,
|
||||||
cacheDirectory = tempdir().toPath(),
|
cacheDirectory = tempdir().toPath(),
|
||||||
database = Database.connect("jdbc:h2:${tempfile()}"),
|
database = Database.connect("jdbc:sqlite:${tempfile()}"),
|
||||||
autoPrune = false,
|
autoPrune = false,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -157,7 +157,7 @@ class ImageServerTest : FreeSpec() {
|
||||||
val storage = ImageStorage(
|
val storage = ImageStorage(
|
||||||
maxSize = 100000,
|
maxSize = 100000,
|
||||||
cacheDirectory = tempdir().toPath(),
|
cacheDirectory = tempdir().toPath(),
|
||||||
database = Database.connect("jdbc:h2:${tempfile()}"),
|
database = Database.connect("jdbc:sqlite:${tempfile()}"),
|
||||||
autoPrune = false,
|
autoPrune = false,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -31,9 +31,7 @@ class TokenVerifierTest : FreeSpec() {
|
||||||
val clientKeys = TweetNaclFast.Box.keyPair()
|
val clientKeys = TweetNaclFast.Box.keyPair()
|
||||||
val box = TweetNaclFast.Box(clientKeys.publicKey, remoteKeys.secretKey)
|
val box = TweetNaclFast.Box(clientKeys.publicKey, remoteKeys.secretKey)
|
||||||
|
|
||||||
val backend = TokenVerifier(box.before()) { _, _ ->
|
val backend = TokenVerifier(box.before()).then {
|
||||||
true
|
|
||||||
}.then {
|
|
||||||
Response(Status.OK)
|
Response(Status.OK)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue