Minor changes + new CLI

CHANGELOG.md

@@ -6,15 +6,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Added
+- [2020-08-11] New CLI for specifying database location, cache folder, and settings [@carbotaniuman].
 
 ### Changed
--- [2020-08-11] Change logging defaults [@carbotaniuman].
+- [2020-08-11] Change logging defaults [@carbotaniuman].
 
 ### Deprecated
 
 ### Removed
 
 ### Fixed
+- [2020-08-11] Bugs relating to `settings.json` changes [@carbotaniuman].
+- [2020-08-11] Logs taking up an absurd amount of space [@carbotaniuman].
+- [2020-08-11] Random crashes for no reason [@carbotaniuman].
+- [2020-08-11] SQLException is noww properly handled [@carbotaniuman].
 
 ### Security
 

build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id "java"
     id "org.jetbrains.kotlin.jvm" version "1.3.72"
+    id "org.jetbrains.kotlin.kapt" version "1.3.72"
     id "application"
     id "com.github.johnrengelman.shadow" version "5.2.0"
     id "com.diffplug.gradle.spotless" version "4.4.0"
@@ -45,6 +46,15 @@ dependencies {
 
     implementation "com.goterl.lazycode:lazysodium-java:4.3.0"
     implementation "net.java.dev.jna:jna:5.5.0"
+
+    implementation "info.picocli:picocli:4.5.0"
+    kapt "info.picocli:picocli-codegen:4.5.0"
+}
+
+kapt {
+    arguments {
+        arg("project", "${project.group}/${project.name}")
+    }
 }
 
 java {

src/main/kotlin/mdnet/base/Main.kt

@@ -19,15 +19,48 @@ along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
 package mdnet.base
 
 import ch.qos.logback.classic.LoggerContext
+import java.io.File
 import kotlin.system.exitProcess
 import mdnet.BuildInfo
 import org.slf4j.LoggerFactory
+import picocli.CommandLine
 
 object Main {
     private val LOGGER = LoggerFactory.getLogger(Main::class.java)
 
     @JvmStatic
     fun main(args: Array<String>) {
+        CommandLine(ClientArgs()).execute(*args)
+    }
+
+    fun dieWithError(e: Throwable): Nothing {
+            LOGGER.error(e) { "Critical Error" }
+        (LoggerFactory.getILoggerFactory() as LoggerContext).stop()
+        exitProcess(1)
+    }
+
+    fun dieWithError(error: String): Nothing {
+        LOGGER.error { "Critical Error: $error" }
+
+        (LoggerFactory.getILoggerFactory() as LoggerContext).stop()
+        exitProcess(1)
+    }
+}
+
+@CommandLine.Command(name = "java -jar <jar>", usageHelpWidth = 120, version = ["Client Version ${BuildInfo.VERSION} (Build ${Constants.CLIENT_BUILD})"])
+data class ClientArgs(
+    @field:CommandLine.Option(names = ["-s", "--settings"], defaultValue = "settings.json", paramLabel = "<settings>", description = ["the settings file (default: \${DEFAULT-VALUE})"])
+    var settingsFile: File = File("settings.json"),
+    @field:CommandLine.Option(names = ["-d", "--database"], defaultValue = "cache\${sys:file.separator}data.db", paramLabel = "<settings>", description = ["the database file (default: \${DEFAULT-VALUE})"])
+    var databaseFile: File = File("cache${File.separator}data.db"),
+    @field:CommandLine.Option(names = ["-c", "--cache"], defaultValue = "cache", paramLabel = "<settings>", description = ["the cache folder (default: \${DEFAULT-VALUE})"])
+    var cacheFolder: File = File("cache"),
+    @field:CommandLine.Option(names = ["-h", "--help"], usageHelp = true, description = ["show this help message and exit"])
+    var helpRequested: Boolean = false,
+    @field:CommandLine.Option(names = ["-v", "--version"], versionHelp = true, description = ["show the version message and exit"])
+    var versionRequested: Boolean = false
+) : Runnable {
+    override fun run() {
         println(
             "Mangadex@Home Client Version ${BuildInfo.VERSION} (Build ${Constants.CLIENT_BUILD}) initializing"
         )
@@ -48,31 +81,11 @@ object Main {
             along with Mangadex@Home.  If not, see <https://www.gnu.org/licenses/>.
         """.trimIndent())
 
-        var file = "settings.json"
+        val client = MangaDexClient(settingsFile, databaseFile, cacheFolder)
-        if (args.size == 1) {
-            file = args[0]
-        } else if (args.isNotEmpty()) {
-            dieWithError("Expected one argument: path to config file, or nothing")
-        }
-
-        val client = MangaDexClient(file)
         Runtime.getRuntime().addShutdownHook(Thread {
             client.shutdown()
             (LoggerFactory.getILoggerFactory() as LoggerContext).stop()
         })
         client.runLoop()
     }
-
-    fun dieWithError(e: Throwable): Nothing {
-            LOGGER.error(e) { "Critical Error" }
-        (LoggerFactory.getILoggerFactory() as LoggerContext).stop()
-        exitProcess(1)
-    }
-
-    fun dieWithError(error: String): Nothing {
-        LOGGER.error { "Critical Error: $error" }
-
-        (LoggerFactory.getILoggerFactory() as LoggerContext).stop()
-        exitProcess(1)
-    }
 }

src/main/kotlin/mdnet/base/MangaDexClient.kt

@@ -38,14 +38,16 @@ import mdnet.base.settings.*
 import mdnet.cache.DiskLruCache
 import mdnet.cache.HeaderMismatchException
 import org.http4k.server.Http4kServer
+import org.jetbrains.exposed.sql.Database
 import org.slf4j.LoggerFactory
 
 // Exception class to handle when Client Settings have invalid values
 class ClientSettingsException(message: String) : Exception(message)
 
-class MangaDexClient(private val clientSettingsFile: String) {
+class MangaDexClient(private val settingsFile: File, databaseFile: File, cacheFolder: File) {
     // just for scheduling one task, so single-threaded
     private val executor = Executors.newSingleThreadScheduledExecutor()
+    private val database: Database
     private val cache: DiskLruCache
     private var settings: ClientSettings
 
@@ -65,9 +67,13 @@ class MangaDexClient(private val clientSettingsFile: String) {
             dieWithError(e)
         }
 
+        LOGGER.info { "Client settings loaded: $settings" }
+
+        database = Database.connect("jdbc:sqlite:$databaseFile", "org.sqlite.JDBC")
+
         try {
             cache = DiskLruCache.open(
-                File("cache"), 1, 1,
+                cacheFolder, 1, 1,
                 (settings.maxCacheSizeInMebibytes * 1024 * 1024 * 0.8).toLong() /* MiB to bytes */
             )
         } catch (e: HeaderMismatchException) {
@@ -88,6 +94,9 @@ class MangaDexClient(private val clientSettingsFile: String) {
                 LOGGER.warn(e) { "Reload of ClientSettings failed" }
             }
         }, 1, 1, TimeUnit.MINUTES)
+
+        startImageServer()
+        startWebUi()
     }
 
     // Precondition: settings must be filled with up-to-date settings and `imageServer` must not be null
@@ -96,37 +105,53 @@ class MangaDexClient(private val clientSettingsFile: String) {
             val imageServer = requireNotNull(imageServer)
 
             if (webUi != null) throw AssertionError()
+            LOGGER.info { "WebUI starting" }
             webUi = getUiServer(webSettings, imageServer.statistics, imageServer.statsMap).also {
                 it.start()
             }
-            LOGGER.info { "WebUI was successfully started" }
+            LOGGER.info { "WebUI started" }
         }
     }
 
     // Precondition: settings must be filled with up-to-date settings
     private fun startImageServer() {
         if (imageServer != null) throw AssertionError()
-        imageServer = ServerManager(settings.serverSettings, settings.devSettings, settings.maxCacheSizeInMebibytes, cache).also {
+        LOGGER.info { "Server manager starting" }
+        imageServer = ServerManager(settings.serverSettings, settings.devSettings, settings.maxCacheSizeInMebibytes, cache, database).also {
             it.start()
         }
-        LOGGER.info { "Server manager was successfully started" }
+        LOGGER.info { "Server manager started" }
     }
 
     private fun stopImageServer() {
+        LOGGER.info { "Server manager stopping" }
         requireNotNull(imageServer).shutdown()
-        LOGGER.info { "Server manager was successfully stopped" }
+        imageServer = null
+        LOGGER.info { "Server manager stopped" }
     }
 
     private fun stopWebUi() {
+        LOGGER.info { "WebUI stopping" }
         requireNotNull(webUi).stop()
-        LOGGER.info { "Server manager was successfully stopped" }
+        webUi = null
+        LOGGER.info { "WebUI stopped" }
     }
 
     fun shutdown() {
         LOGGER.info { "Mangadex@Home Client shutting down" }
-        stopWebUi()
+        if (webUi != null) {
-        stopImageServer()
+            stopWebUi()
+        }
+        if (imageServer != null) {
+            stopImageServer()
+        }
         LOGGER.info { "Mangadex@Home Client has shut down" }
+
+        try {
+            cache.close()
+        } catch (e: IOException) {
+            LOGGER.error(e) { "Cache failed to close" }
+        }
     }
 
     /**
@@ -142,6 +167,7 @@ class MangaDexClient(private val clientSettingsFile: String) {
                 LOGGER.info { "Client settings unchanged" }
                 return
             }
+            LOGGER.info { "New settings loaded: $newSettings" }
 
             cache.maxSize = (newSettings.maxCacheSizeInMebibytes * 1024 * 1024 * 0.8).toLong()
 
@@ -215,7 +241,7 @@ class MangaDexClient(private val clientSettingsFile: String) {
     }
 
     private fun readClientSettings(): ClientSettings {
-        return JACKSON.readValue<ClientSettings>(FileReader(clientSettingsFile)).apply(::validateSettings)
+        return JACKSON.readValue<ClientSettings>(FileReader(settingsFile)).apply(::validateSettings)
     }
 
     companion object {

src/main/kotlin/mdnet/base/ServerManager.kt

@@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.databind.SerializationFeature
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
-import java.io.IOException
 import java.time.Instant
 import java.util.Collections
 import java.util.LinkedHashMap
@@ -21,6 +20,7 @@ import mdnet.base.settings.RemoteSettings
 import mdnet.base.settings.ServerSettings
 import mdnet.cache.DiskLruCache
 import org.http4k.server.Http4kServer
+import org.jetbrains.exposed.sql.Database
 import org.slf4j.LoggerFactory
 
 sealed class State
@@ -33,7 +33,7 @@ data class GracefulStop(val lastRunning: Running, val counts: Int = 0, val nextS
 // server is currently running
 data class Running(val server: Http4kServer, val settings: RemoteSettings, val serverSettings: ServerSettings, val devSettings: DevSettings) : State()
 
-class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, maxCacheSizeInMebibytes: Long, private val cache: DiskLruCache) {
+class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, maxCacheSizeInMebibytes: Long, private val cache: DiskLruCache, private val database: Database) {
     // this must remain single-threaded because of how the state mechanism works
     private val executor = Executors.newSingleThreadScheduledExecutor()
 
@@ -68,6 +68,7 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma
     }
 
     fun start() {
+        LOGGER.info { "Image server starting" }
         loginAndStartServer()
         statsMap[Instant.now()] = statistics.get()
 
@@ -165,6 +166,8 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma
                 LOGGER.warn(e) { "Graceful shutdown checker failed" }
             }
         }, 45, 45, TimeUnit.SECONDS)
+
+        LOGGER.info { "Image server has started" }
     }
 
     private fun pingControl() {
@@ -197,7 +200,7 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma
 
         val remoteSettings = serverHandler.loginToControl()
             ?: Main.dieWithError("Failed to get a login response from server - check API secret for validity")
-        val server = getServer(cache, remoteSettings, state.serverSettings, statistics, isHandled).start()
+        val server = getServer(cache, database, remoteSettings, state.serverSettings, statistics, isHandled).start()
 
         if (remoteSettings.latestBuild > Constants.CLIENT_BUILD) {
             LOGGER.warn {
@@ -253,12 +256,6 @@ class ServerManager(serverSettings: ServerSettings, devSettings: DevSettings, ma
         }, 0, TimeUnit.SECONDS)
         latch.await()
 
-        try {
-            cache.close()
-        } catch (e: IOException) {
-            LOGGER.error(e) { "Cache failed to close" }
-        }
-
         executor.shutdown()
         LOGGER.info { "Image server has shut down" }
     }

src/main/kotlin/mdnet/base/netty/ApplicationNetty.kt

@@ -57,9 +57,9 @@ import org.http4k.server.Http4kServer
 import org.http4k.server.ServerConfig
 import org.slf4j.LoggerFactory
 
-private val LOGGER = LoggerFactory.getLogger("Application")
+private val LOGGER = LoggerFactory.getLogger("AppNetty")
 
-class Netty(private val tls: TlsCert, internal val serverSettings: ServerSettings, private val statistics: AtomicReference<Statistics>) : ServerConfig {
+class Netty(private val tls: TlsCert, private val serverSettings: ServerSettings, private val statistics: AtomicReference<Statistics>) : ServerConfig {
     override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer {
         private val masterGroup = NioEventLoopGroup(serverSettings.threads)
         private val workerGroup = NioEventLoopGroup(serverSettings.threads)

src/main/kotlin/mdnet/base/netty/Keys.kt

@@ -32,7 +32,7 @@ private const val PKCS_1_PEM_FOOTER = "-----END RSA PRIVATE KEY-----"
 private const val PKCS_8_PEM_HEADER = "-----BEGIN PRIVATE KEY-----"
 private const val PKCS_8_PEM_FOOTER = "-----END PRIVATE KEY-----"
 
-internal fun loadKey(keyDataString: String): PrivateKey? {
+fun loadKey(keyDataString: String): PrivateKey? {
     if (keyDataString.contains(PKCS_1_PEM_HEADER)) {
         val fixedString = keyDataString.replace(PKCS_1_PEM_HEADER, "").replace(
             PKCS_1_PEM_FOOTER, "")

src/main/kotlin/mdnet/base/netty/WebUiNetty.kt

@@ -32,7 +32,6 @@ import io.netty.handler.codec.http.HttpServerCodec
 import io.netty.handler.codec.http.HttpServerKeepAliveHandler
 import io.netty.handler.stream.ChunkedWriteHandler
 import java.net.InetSocketAddress
-import java.util.concurrent.TimeUnit
 import org.http4k.core.HttpHandler
 import org.http4k.server.Http4kChannelHandler
 import org.http4k.server.Http4kServer
@@ -67,9 +66,9 @@ class WebUiNetty(private val hostname: String, private val port: Int) : ServerCo
         }
 
         override fun stop() = apply {
-            masterGroup.shutdownGracefully(5, 15, TimeUnit.SECONDS).sync()
+            closeFuture.cancel(false)
-            workerGroup.shutdownGracefully(5, 15, TimeUnit.SECONDS).sync()
+            workerGroup.shutdownGracefully()
-            closeFuture.sync()
+            masterGroup.shutdownGracefully()
         }
 
         override fun port(): Int = address.port

src/main/kotlin/mdnet/base/server/ImageServer.kt

@@ -66,6 +66,7 @@ import org.http4k.routing.bind
 import org.http4k.routing.routes
 import org.http4k.server.Http4kServer
 import org.http4k.server.asServer
+import org.jetbrains.exposed.exceptions.ExposedSQLException
 import org.jetbrains.exposed.sql.Database
 import org.jetbrains.exposed.sql.SchemaUtils
 import org.jetbrains.exposed.sql.transactions.transaction
@@ -250,13 +251,20 @@ class ImageServer(
             LOGGER.trace { "Request for $sanitizedUri is being cached and served" }
 
             if (imageDatum == null) {
-                synchronized(database) {
+                try {
-                    transaction(database) {
+                    synchronized(database) {
-                        ImageDatum.new(imageId) {
+                        transaction(database) {
-                            this.contentType = contentType
+                            ImageDatum.new(imageId) {
-                            this.lastModified = lastModified
+                                this.contentType = contentType
+                                this.lastModified = lastModified
+                            }
                         }
                     }
+                } catch (_: ExposedSQLException) {
+                    // some other code got to the database first, fall back to just serving
+                    editor.abort()
+                    LOGGER.trace { "Request for $sanitizedUri is being served" }
+                    respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false)
                 }
             }
 
@@ -331,8 +339,7 @@ class ImageServer(
 
 private fun String.isImageMimetype() = this.toLowerCase().startsWith("image/")
 
-fun getServer(cache: DiskLruCache, remoteSettings: RemoteSettings, serverSettings: ServerSettings, statistics: AtomicReference<Statistics>, isHandled: AtomicBoolean): Http4kServer {
+fun getServer(cache: DiskLruCache, database: Database, remoteSettings: RemoteSettings, serverSettings: ServerSettings, statistics: AtomicReference<Statistics>, isHandled: AtomicBoolean): Http4kServer {
-    val database = Database.connect("jdbc:sqlite:cache/data.db", "org.sqlite.JDBC")
     val client = ApacheClient(responseBodyMode = BodyMode.Stream, client = HttpClients.custom()
         .disableConnectionState()
         .setDefaultRequestConfig(

src/main/kotlin/mdnet/base/server/naclbox.kt

@@ -27,7 +27,7 @@ import javax.crypto.Cipher
 import javax.crypto.spec.SecretKeySpec
 
 @Throws(SodiumException::class)
-internal fun LazySodiumJava.cryptoBoxOpenEasyAfterNm(cipherBytes: ByteArray, nonce: ByteArray, sharedKey: ByteArray): String {
+fun LazySodiumJava.cryptoBoxOpenEasyAfterNm(cipherBytes: ByteArray, nonce: ByteArray, sharedKey: ByteArray): String {
     if (!Box.Checker.checkNonce(nonce.size)) {
         throw SodiumException("Incorrect nonce length.")
     }
@@ -44,18 +44,18 @@ internal fun LazySodiumJava.cryptoBoxOpenEasyAfterNm(cipherBytes: ByteArray, non
     return str(message)
 }
 
-internal fun getRc4(key: ByteArray): Cipher {
+fun getRc4(key: ByteArray): Cipher {
     val rc4 = Cipher.getInstance("RC4")
     rc4.init(Cipher.ENCRYPT_MODE, SecretKeySpec(key, "RC4"))
     return rc4
 }
 
-internal fun md5Bytes(stringToHash: String): ByteArray {
+fun md5Bytes(stringToHash: String): ByteArray {
     val digest = MessageDigest.getInstance("MD5")
     return digest.digest(stringToHash.toByteArray())
 }
 
-internal fun printHexString(bytes: ByteArray): String {
+fun printHexString(bytes: ByteArray): String {
     val sb = StringBuilder()
     for (b in bytes) {
         sb.append(String.format("%02x", b))

src/main/kotlin/mdnet/base/settings/ClientSettings.kt

@@ -24,13 +24,43 @@ import com.fasterxml.jackson.databind.annotation.JsonNaming
 import dev.afanasev.sekret.Secret
 
 @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy::class)
-data class ClientSettings(
+class ClientSettings(
     val maxCacheSizeInMebibytes: Long = 20480,
-    @JsonUnwrapped
-    val serverSettings: ServerSettings = ServerSettings(),
     val webSettings: WebSettings? = null,
     val devSettings: DevSettings = DevSettings(isDev = false)
-)
+) {
+    // FIXME: jackson doesn't work with data classes and JsonUnwrapped
+    // fix this in 2.0 when we can break the settings file
+    // and remove the `@JsonUnwrapped`
+    @field:JsonUnwrapped
+    lateinit var serverSettings: ServerSettings
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as ClientSettings
+
+        if (maxCacheSizeInMebibytes != other.maxCacheSizeInMebibytes) return false
+        if (webSettings != other.webSettings) return false
+        if (devSettings != other.devSettings) return false
+        if (serverSettings != other.serverSettings) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = maxCacheSizeInMebibytes.hashCode()
+        result = 31 * result + (webSettings?.hashCode() ?: 0)
+        result = 31 * result + devSettings.hashCode()
+        result = 31 * result + serverSettings.hashCode()
+        return result
+    }
+
+    override fun toString(): String {
+        return "ClientSettings(maxCacheSizeInMebibytes=$maxCacheSizeInMebibytes, webSettings=$webSettings, devSettings=$devSettings, serverSettings=$serverSettings)"
+    }
+}
 
 @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy::class)
 data class ServerSettings(

src/main/resources/logback.xml

@@ -5,12 +5,12 @@
         </filter>
 
         <file>log/latest.log</file>
-        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-            <fileNamePattern>log/logFile.%d{yyyy-MM-dd_HH}.log</fileNamePattern>
+            <fileNamePattern>log/logFile.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
             <maxHistory>12</maxHistory>
             <maxFileSize>100MB</maxFileSize>
             <totalSizeCap>1GB</totalSizeCap>
-        </rollingPolicy>
+        </rollingPolicy>-->
 
         <encoder>
             <pattern>%d{YYYY-MM-dd HH:mm:ss} %-5level %logger{36} - %msg%n</pattern>
@@ -37,6 +37,6 @@
         <appender-ref ref="ASYNC"/>
     </root>
 
-    <logger name="Exposed" level="ERROR"/
+    <logger name="Exposed" level="ERROR"/>
     <logger name="io.netty" level="INFO"/>
 </configuration>