gitlab-com-mangadex-pub/mangadex_at_home
1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00
Code Releases Activity

Minor changes + new CLI

Browse source
This commit is contained in:
carbotaniuman 2020-08-11 14:12:01 -05:00
parent 398ab05788
commit 9d07c18de1
12 changed files with 154 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
CHANGELOG.md
View file

@ -6,15 +6,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- [2020-08-11] New CLI for specifying database location, cache folder, and settings [@carbotaniuman].
### Changed
-- [2020-08-11] Change logging defaults [@carbotaniuman].
- [2020-08-11] Change logging defaults [@carbotaniuman].
### Deprecated
### Removed
### Fixed
- [2020-08-11] Bugs relating to `settings.json` changes [@carbotaniuman].
- [2020-08-11] Logs taking up an absurd amount of space [@carbotaniuman].
- [2020-08-11] Random crashes for no reason [@carbotaniuman].
- [2020-08-11] SQLException is noww properly handled [@carbotaniuman].
### Security

10
build.gradle
View file

@ -1,6 +1,7 @@
plugins {
id "java"
id "org.jetbrains.kotlin.jvm" version "1.3.72"
id "org.jetbrains.kotlin.kapt" version "1.3.72"
id "application"
id "com.github.johnrengelman.shadow" version "5.2.0"
id "com.diffplug.gradle.spotless" version "4.4.0"
@ -45,6 +46,15 @@ dependencies {
implementation "com.goterl.lazycode:lazysodium-java:4.3.0"
implementation "net.java.dev.jna:jna:5.5.0"
implementation "info.picocli:picocli:4.5.0"
kapt "info.picocli:picocli-codegen:4.5.0"
}
kapt {
arguments {
arg("project", "${project.group}/${project.name}")
}
}
java {

55
src/main/kotlin/mdnet/base/Main.kt
View file

@ -19,15 +19,48 @@ along with this MangaDex@Home. If not, see <http://www.gnu.org/licenses/>.
package mdnet.base
import ch.qos.logback.classic.LoggerContext
import java.io.File
import kotlin.system.exitProcess
import mdnet.BuildInfo
import org.slf4j.LoggerFactory
import picocli.CommandLine
object Main {
private val LOGGER = LoggerFactory.getLogger(Main::class.java)
@JvmStatic
fun main(args: Array<String>) {
CommandLine(ClientArgs()).execute(*args)
}
fun dieWithError(e: Throwable): Nothing {
LOGGER.error(e) { "Critical Error" }
(LoggerFactory.getILoggerFactory() as LoggerContext).stop()
exitProcess(1)
}
fun dieWithError(error: String): Nothing {
LOGGER.error { "Critical Error: $error" }
(LoggerFactory.getILoggerFactory() as LoggerContext).stop()
exitProcess(1)
}
}
@CommandLine.Command(name = "java -jar <jar>", usageHelpWidth = 120, version = ["Client Version ${BuildInfo.VERSION} (Build ${Constants.CLIENT_BUILD})"])
data class ClientArgs(
@field:CommandLine.Option(names = ["-s", "--settings"], defaultValue = "settings.json", paramLabel = "<settings>", description = ["the settings file (default: \${DEFAULT-VALUE})"])
var settingsFile: File = File("settings.json"),
@field:CommandLine.Option(names = ["-d", "--database"], defaultValue = "cache\${sys:file.separator}data.db", paramLabel = "<settings>", description = ["the database file (default: \${DEFAULT-VALUE})"])
var databaseFile: File = File("cache${File.separator}data.db"),
@field:CommandLine.Option(names = ["-c", "--cache"], defaultValue = "cache", paramLabel = "<settings>", description = ["the cache folder (default: \${DEFAULT-VALUE})"])
var cacheFolder: File = File("cache"),
@field:CommandLine.Option(names = ["-h", "--help"], usageHelp = true, description = ["show this help message and exit"])
var helpRequested: Boolean = false,
@field:CommandLine.Option(names = ["-v", "--version"], versionHelp = true, description = ["show the version message and exit"])
var versionRequested: Boolean = false
) : Runnable {
override fun run() {
println(
"Mangadex@Home Client Version ${BuildInfo.VERSION} (Build ${Constants.CLIENT_BUILD}) initializing"
)
@ -48,31 +81,11 @@ object Main {
along with Mangadex@Home. If not, see <https://www.gnu.org/licenses/>.
""".trimIndent())
var file = "settings.json"
if (args.size == 1) {
file = args[0]
} else if (args.isNotEmpty()) {
dieWithError("Expected one argument: path to config file, or nothing")
}
val client = MangaDexClient(file)
val client = MangaDexClient(settingsFile, databaseFile, cacheFolder)
Runtime.getRuntime().addShutdownHook(Thread {
client.shutdown()
(LoggerFactory.getILoggerFactory() as LoggerContext).stop()
})
client.runLoop()
}
fun dieWithError(e: Throwable): Nothing {
LOGGER.error(e) { "Critical Error" }
(LoggerFactory.getILoggerFactory() as LoggerContext).stop()
exitProcess(1)
}
fun dieWithError(error: String): Nothing {
LOGGER.error { "Critical Error: $error" }
(LoggerFactory.getILoggerFactory() as LoggerContext).stop()
exitProcess(1)
}
}

42
src/main/kotlin/mdnet/base/MangaDexClient.kt
View file

@ -38,14 +38,16 @@ import mdnet.base.settings.*
import mdnet.cache.DiskLruCache
import mdnet.cache.HeaderMismatchException
import org.http4k.server.Http4kServer
import org.jetbrains.exposed.sql.Database
import org.slf4j.LoggerFactory
// Exception class to handle when Client Settings have invalid values
class ClientSettingsException(message: String) : Exception(message)
class MangaDexClient(private val clientSettingsFile: String) {
class MangaDexClient(private val settingsFile: File, databaseFile: File, cacheFolder: File) {
// just for scheduling one task, so single-threaded
private val executor = Executors.newSingleThreadScheduledExecutor()
private val database: Database
private val cache: DiskLruCache
private var settings: ClientSettings
@ -65,9 +67,13 @@ class MangaDexClient(private val clientSettingsFile: String) {
dieWithError(e)
}
LOGGER.info { "Client settings loaded: $settings" }
database = Database.connect("jdbc:sqlite:$databaseFile", "org.sqlite.JDBC")
try {
cache = DiskLruCache.open(
File("cache"), 1, 1,
cacheFolder, 1, 1,
(settings.maxCacheSizeInMebibytes * 1024 * 1024 * 0.8).toLong() /* MiB to bytes */
)
} catch (e: HeaderMismatchException) {
@ -88,6 +94,9 @@ class MangaDexClient(private val clientSettingsFile: String) {
LOGGER.warn(e) { "Reload of ClientSettings failed" }
}
}, 1, 1, TimeUnit.MINUTES)
startImageServer()
startWebUi()
}
// Precondition: settings must be filled with up-to-date settings and `imageServer` must not be null
@ -96,37 +105,53 @@ class MangaDexClient(private val clientSettingsFile: String) {
val imageServer = requireNotNull(imageServer)
if (webUi != null) throw AssertionError()
LOGGER.info { "WebUI starting" }
webUi = getUiServer(webSettings, imageServer.statistics, imageServer.statsMap).also {
it.start()
}
LOGGER.info { "WebUI was successfully started" }
LOGGER.info { "WebUI started" }
}
}
// Precondition: settings must be filled with up-to-date settings
private fun startImageServer() {
if (imageServer != null) throw AssertionError()
imageServer = ServerManager(settings.serverSettings, settings.devSettings, settings.maxCacheSizeInMebibytes, cache).also {
LOGGER.info { "Server manager starting" }
imageServer = ServerManager(settings.serverSettings, settings.devSettings, settings.maxCacheSizeInMebibytes, cache, database).also {
it.start()
}
LOGGER.info { "Server manager was successfully started" }
LOGGER.info { "Server manager started" }
}
private fun stopImageServer() {
LOGGER.info { "Server manager stopping" }
requireNotNull(imageServer).shutdown()
LOGGER.info { "Server manager was successfully stopped" }
imageServer = null
LOGGER.info { "Server manager stopped" }
}
private fun stopWebUi() {
LOGGER.info { "WebUI stopping" }
requireNotNull(webUi).stop()
LOGGER.info { "Server manager was successfully stopped" }
webUi = null
LOGGER.info { "WebUI stopped" }
}
fun shutdown() {
LOGGER.info { "Mangadex@Home Client shutting down" }
if (webUi != null) {
stopWebUi()
}
if (imageServer != null) {
stopImageServer()
}
LOGGER.info { "Mangadex@Home Client has shut down" }
try {
cache.close()
} catch (e: IOException) {
LOGGER.error(e) { "Cache failed to close" }
}
}
/**
@ -142,6 +167,7 @@ class MangaDexClient(private val clientSettingsFile: String) {
LOGGER.info { "Client settings unchanged" }
return
}
LOGGER.info { "New settings loaded: $newSettings" }
cache.maxSize = (newSettings.maxCacheSizeInMebibytes * 1024 * 1024 * 0.8).toLong()
@ -215,7 +241,7 @@ class MangaDexClient(private val clientSettingsFile: String) {
}
private fun readClientSettings(): ClientSettings {
return JACKSON.readValue<ClientSettings>(FileReader(clientSettingsFile)).apply(::validateSettings)
return JACKSON.readValue<ClientSettings>(FileReader(settingsFile)).apply(::validateSettings)
}
companion object {

15
src/main/kotlin/mdnet/base/ServerManager.kt
View file

@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.SerializationFeature
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import java.io.IOException
import java.time.Instant
import java.util.Collections
import java.util.LinkedHashMap
@ -21,6 +20,7 @@ import mdnet.base.settings.RemoteSettings
import mdnet.base.settings.ServerSettings
import mdnet.cache.DiskLruCache
import org.http4k.server.Http4kServer
import org.jetbrains.exposed.sql.Database
import org.slf4j.LoggerFactory
sealed class State
@ -33,7 +33,7 @@ data class GracefulStop(val lastRunning: Running, val counts: Int = 0, val nextS
// server is currently running
data class Running(val server: Http4kServer, val settings: RemoteSettings, val serverSettings: ServerSettings, val devSettings: DevSettings) : State()
class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, maxCacheSizeInMebibytes: Long, private val cache: DiskLruCache) {
class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, maxCacheSizeInMebibytes: Long, private val cache: DiskLruCache, private val database: Database) {
// this must remain single-threaded because of how the state mechanism works
private val executor = Executors.newSingleThreadScheduledExecutor()
@ -68,6 +68,7 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma
}
fun start() {
LOGGER.info { "Image server starting" }
loginAndStartServer()
statsMap[Instant.now()] = statistics.get()
@ -165,6 +166,8 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma
LOGGER.warn(e) { "Graceful shutdown checker failed" }
}
}, 45, 45, TimeUnit.SECONDS)
LOGGER.info { "Image server has started" }
}
private fun pingControl() {
@ -197,7 +200,7 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma
val remoteSettings = serverHandler.loginToControl()
?: Main.dieWithError("Failed to get a login response from server - check API secret for validity")
val server = getServer(cache, remoteSettings, state.serverSettings, statistics, isHandled).start()
val server = getServer(cache, database, remoteSettings, state.serverSettings, statistics, isHandled).start()
if (remoteSettings.latestBuild > Constants.CLIENT_BUILD) {
LOGGER.warn {
@ -253,12 +256,6 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma
}, 0, TimeUnit.SECONDS)
latch.await()
try {
cache.close()
} catch (e: IOException) {
LOGGER.error(e) { "Cache failed to close" }
}
executor.shutdown()
LOGGER.info { "Image server has shut down" }
}

4
src/main/kotlin/mdnet/base/netty/ApplicationNetty.kt
View file

@ -57,9 +57,9 @@ import org.http4k.server.Http4kServer
import org.http4k.server.ServerConfig
import org.slf4j.LoggerFactory
private val LOGGER = LoggerFactory.getLogger("Application")
private val LOGGER = LoggerFactory.getLogger("AppNetty")
class Netty(private val tls: TlsCert, internal val serverSettings: ServerSettings, private val statistics: AtomicReference<Statistics>) : ServerConfig {
class Netty(private val tls: TlsCert, private val serverSettings: ServerSettings, private val statistics: AtomicReference<Statistics>) : ServerConfig {
override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer {
private val masterGroup = NioEventLoopGroup(serverSettings.threads)
private val workerGroup = NioEventLoopGroup(serverSettings.threads)

2
src/main/kotlin/mdnet/base/netty/Keys.kt
View file

@ -32,7 +32,7 @@ private const val PKCS_1_PEM_FOOTER = "-----END RSA PRIVATE KEY-----"
private const val PKCS_8_PEM_HEADER = "-----BEGIN PRIVATE KEY-----"
private const val PKCS_8_PEM_FOOTER = "-----END PRIVATE KEY-----"
internal fun loadKey(keyDataString: String): PrivateKey? {
fun loadKey(keyDataString: String): PrivateKey? {
if (keyDataString.contains(PKCS_1_PEM_HEADER)) {
val fixedString = keyDataString.replace(PKCS_1_PEM_HEADER, "").replace(
PKCS_1_PEM_FOOTER, "")

7
src/main/kotlin/mdnet/base/netty/WebUiNetty.kt
View file

@ -32,7 +32,6 @@ import io.netty.handler.codec.http.HttpServerCodec
import io.netty.handler.codec.http.HttpServerKeepAliveHandler
import io.netty.handler.stream.ChunkedWriteHandler
import java.net.InetSocketAddress
import java.util.concurrent.TimeUnit
import org.http4k.core.HttpHandler
import org.http4k.server.Http4kChannelHandler
import org.http4k.server.Http4kServer
@ -67,9 +66,9 @@ class WebUiNetty(private val hostname: String, private val port: Int) : ServerCo
}
override fun stop() = apply {
masterGroup.shutdownGracefully(5, 15, TimeUnit.SECONDS).sync()
workerGroup.shutdownGracefully(5, 15, TimeUnit.SECONDS).sync()
closeFuture.sync()
closeFuture.cancel(false)
workerGroup.shutdownGracefully()
masterGroup.shutdownGracefully()
}
override fun port(): Int = address.port

11
src/main/kotlin/mdnet/base/server/ImageServer.kt
View file

@ -66,6 +66,7 @@ import org.http4k.routing.bind
import org.http4k.routing.routes
import org.http4k.server.Http4kServer
import org.http4k.server.asServer
import org.jetbrains.exposed.exceptions.ExposedSQLException
import org.jetbrains.exposed.sql.Database
import org.jetbrains.exposed.sql.SchemaUtils
import org.jetbrains.exposed.sql.transactions.transaction
@ -250,6 +251,7 @@ class ImageServer(
LOGGER.trace { "Request for $sanitizedUri is being cached and served" }
if (imageDatum == null) {
try {
synchronized(database) {
transaction(database) {
ImageDatum.new(imageId) {
@ -258,6 +260,12 @@ class ImageServer(
}
}
}
} catch (_: ExposedSQLException) {
// some other code got to the database first, fall back to just serving
editor.abort()
LOGGER.trace { "Request for $sanitizedUri is being served" }
respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false)
}
}
val tee = CachingInputStream(
@ -331,8 +339,7 @@ class ImageServer(
private fun String.isImageMimetype() = this.toLowerCase().startsWith("image/")
fun getServer(cache: DiskLruCache, remoteSettings: RemoteSettings, serverSettings: ServerSettings, statistics: AtomicReference<Statistics>, isHandled: AtomicBoolean): Http4kServer {
val database = Database.connect("jdbc:sqlite:cache/data.db", "org.sqlite.JDBC")
fun getServer(cache: DiskLruCache, database: Database, remoteSettings: RemoteSettings, serverSettings: ServerSettings, statistics: AtomicReference<Statistics>, isHandled: AtomicBoolean): Http4kServer {
val client = ApacheClient(responseBodyMode = BodyMode.Stream, client = HttpClients.custom()
.disableConnectionState()
.setDefaultRequestConfig(

8
src/main/kotlin/mdnet/base/server/naclbox.kt
View file

@ -27,7 +27,7 @@ import javax.crypto.Cipher
import javax.crypto.spec.SecretKeySpec
@Throws(SodiumException::class)
internal fun LazySodiumJava.cryptoBoxOpenEasyAfterNm(cipherBytes: ByteArray, nonce: ByteArray, sharedKey: ByteArray): String {
fun LazySodiumJava.cryptoBoxOpenEasyAfterNm(cipherBytes: ByteArray, nonce: ByteArray, sharedKey: ByteArray): String {
if (!Box.Checker.checkNonce(nonce.size)) {
throw SodiumException("Incorrect nonce length.")
}
@ -44,18 +44,18 @@ internal fun LazySodiumJava.cryptoBoxOpenEasyAfterNm(cipherBytes: ByteArray, non
return str(message)
}
internal fun getRc4(key: ByteArray): Cipher {
fun getRc4(key: ByteArray): Cipher {
val rc4 = Cipher.getInstance("RC4")
rc4.init(Cipher.ENCRYPT_MODE, SecretKeySpec(key, "RC4"))
return rc4
}
internal fun md5Bytes(stringToHash: String): ByteArray {
fun md5Bytes(stringToHash: String): ByteArray {
val digest = MessageDigest.getInstance("MD5")
return digest.digest(stringToHash.toByteArray())
}
internal fun printHexString(bytes: ByteArray): String {
fun printHexString(bytes: ByteArray): String {
val sb = StringBuilder()
for (b in bytes) {
sb.append(String.format("%02x", b))

38
src/main/kotlin/mdnet/base/settings/ClientSettings.kt
View file

@ -24,13 +24,43 @@ import com.fasterxml.jackson.databind.annotation.JsonNaming
import dev.afanasev.sekret.Secret
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy::class)
data class ClientSettings(
class ClientSettings(
val maxCacheSizeInMebibytes: Long = 20480,
@JsonUnwrapped
val serverSettings: ServerSettings = ServerSettings(),
val webSettings: WebSettings? = null,
val devSettings: DevSettings = DevSettings(isDev = false)
)
) {
// FIXME: jackson doesn't work with data classes and JsonUnwrapped
// fix this in 2.0 when we can break the settings file
// and remove the `@JsonUnwrapped`
@field:JsonUnwrapped
lateinit var serverSettings: ServerSettings
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as ClientSettings
if (maxCacheSizeInMebibytes != other.maxCacheSizeInMebibytes) return false
if (webSettings != other.webSettings) return false
if (devSettings != other.devSettings) return false
if (serverSettings != other.serverSettings) return false
return true
}
override fun hashCode(): Int {
var result = maxCacheSizeInMebibytes.hashCode()
result = 31 * result + (webSettings?.hashCode() ?: 0)
result = 31 * result + devSettings.hashCode()
result = 31 * result + serverSettings.hashCode()
return result
}
override fun toString(): String {
return "ClientSettings(maxCacheSizeInMebibytes=$maxCacheSizeInMebibytes, webSettings=$webSettings, devSettings=$devSettings, serverSettings=$serverSettings)"
}
}
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy::class)
data class ServerSettings(

8
src/main/resources/logback.xml
View file

@ -5,12 +5,12 @@
</filter>
<file>log/latest.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>log/logFile.%d{yyyy-MM-dd_HH}.log</fileNamePattern>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>log/logFile.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>12</maxHistory>
<maxFileSize>100MB</maxFileSize>
<totalSizeCap>1GB</totalSizeCap>
</rollingPolicy>
</rollingPolicy>-->
<encoder>
<pattern>%d{YYYY-MM-dd HH:mm:ss} %-5level %logger{36} - %msg%n</pattern>
@ -37,6 +37,6 @@
<appender-ref ref="ASYNC"/>
</root>
<logger name="Exposed" level="ERROR"/
<logger name="Exposed" level="ERROR"/>
<logger name="io.netty" level="INFO"/>
</configuration>