From 1c64f05ac69e4f9cb8c1ca090dcee6df642be714 Mon Sep 17 00:00:00 2001 From: carbotaniuman <41451839+carbotaniuman@users.noreply.github.com> Date: Thu, 11 Feb 2021 15:54:21 -0600 Subject: [PATCH] Fix graceful shutdown --- CHANGELOG.md | 15 +++- build.gradle | 2 +- docker/docker-compose.yml | 4 +- src/main/kotlin/mdnet/Constants.kt | 2 +- src/main/kotlin/mdnet/Main.kt | 82 +++++++++++--------- src/main/kotlin/mdnet/MangaDexClient.kt | 37 ++++----- src/main/kotlin/mdnet/server/ImageHandler.kt | 4 +- 7 files changed, 85 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 376a499..5f8749f 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +## [2.0.0-rc10] - 2021-02-11 +### Added +- [2021-02-11] Add ability to disable io_uring and epoll [@carbotaniuman]. + +### Changed +- [2021-02-11] Changed `databaseFile` to `databaseFolder` in run args [@carbotaniuman]. +- [2021-02-11] Added more error recovery and logging [@carbotaniuman]. +- [2021-02-11] Enforce stricter bounds on disk size [@carbotaniuman]. + +### Fixed +- [2021-02-11] Fixed issues with Netty not shutting down properly [@carbotaniuman]. + ## [2.0.0-rc9] - 2021-02-06 ### Added - [2021-02-06] Add Micrometer and Resilience4J dashboards to defaults [@_tde9]. @@ -341,7 +353,8 @@ This release contains many breaking changes! Of note are the changes to the cach ### Fixed - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman]. -[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc9...HEAD +[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc10...HEAD +[2.0.0-rc10]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc9...2.0.0-rc10 [2.0.0-rc9]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc8...2.0.0-rc9 [2.0.0-rc8]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc7...2.0.0-rc8 [2.0.0-rc7]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc6...2.0.0-rc7 diff --git a/build.gradle b/build.gradle index 602e252..e2b417c 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ plugins { group = "com.mangadex" version = "git describe --tags --dirty".execute().text.trim() -mainClassName = "mdnet.Main" +mainClassName = "mdnet.MainKt" repositories { mavenCentral() diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 3a0cf16..ae8fc3a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,8 +20,8 @@ services: -Dfile-level=off \ -Dstdout-level=info \ -jar mangadex_at_home.jar \ - --cache /mangahome/data/images \ - --database /mangahome/data/metadata" + --cache /mangahome/data/images/ \ + --database /mangahome/data/" ] logging: driver: "json-file" diff --git a/src/main/kotlin/mdnet/Constants.kt b/src/main/kotlin/mdnet/Constants.kt index d932c64..15e7d8b 100644 --- a/src/main/kotlin/mdnet/Constants.kt +++ b/src/main/kotlin/mdnet/Constants.kt @@ -21,7 +21,7 @@ package mdnet import java.time.Duration object Constants { - const val CLIENT_BUILD = 26 + const val CLIENT_BUILD = 27 @JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14) diff --git a/src/main/kotlin/mdnet/Main.kt b/src/main/kotlin/mdnet/Main.kt index aad7511..39902cf 100644 --- a/src/main/kotlin/mdnet/Main.kt +++ b/src/main/kotlin/mdnet/Main.kt @@ -23,49 +23,28 @@ import mdnet.logging.error import org.slf4j.LoggerFactory import picocli.CommandLine import java.io.File +import java.lang.IllegalArgumentException +import java.nio.file.Files import java.nio.file.Path -import java.nio.file.Paths -import kotlin.system.exitProcess +import kotlin.io.path.isRegularFile -object Main { - private val LOGGER = LoggerFactory.getLogger(Main::class.java) - - @JvmStatic - fun main(args: Array) { - try { - CommandLine(ClientArgs()).execute(*args) - } catch (e: Throwable) { - LOGGER.error(e) { "Critical Error " } - } - } - - fun dieWithError(e: Throwable): Nothing { - LOGGER.error(e) { "Critical Error" } - (LoggerFactory.getILoggerFactory() as LoggerContext).stop() - exitProcess(1) - } - - fun dieWithError(error: String): Nothing { - LOGGER.error { "Critical Error: $error" } - - (LoggerFactory.getILoggerFactory() as LoggerContext).stop() - exitProcess(1) - } +fun main(args: Array) { + CommandLine(Main()).execute(*args) } @CommandLine.Command(name = "java -jar ", usageHelpWidth = 120, version = ["Client Version ${BuildInfo.VERSION} (Build ${Constants.CLIENT_BUILD})"]) -class ClientArgs( - @field:CommandLine.Option(names = ["-s", "--settings"], defaultValue = "settings.yaml", paramLabel = "", description = ["the settings file (default: \${DEFAULT-VALUE})"]) - var settingsFile: File = File("settings.yaml"), - @field:CommandLine.Option(names = ["-d", "--database"], defaultValue = ".\${sys:file.separator}metadata", paramLabel = "", description = ["the database file (default: \${DEFAULT-VALUE})"]) - var databaseFile: File = File(".${File.separator}metadata"), - @field:CommandLine.Option(names = ["-c", "--cache"], defaultValue = "images", paramLabel = "", description = ["the cache folder (default: \${DEFAULT-VALUE})"]) - var cacheFolder: Path = Paths.get("images"), +class Main : Runnable { + @field:CommandLine.Option(names = ["-s", "--settings"], defaultValue = ".\\settings.yaml", paramLabel = "", description = ["the settings file (default: \${DEFAULT-VALUE})"]) + lateinit var settingsFile: File + @field:CommandLine.Option(names = ["-d", "--database"], defaultValue = ".\\", paramLabel = "", description = ["the database folder (default: \${DEFAULT-VALUE})"]) + lateinit var databaseFolder: Path + @field:CommandLine.Option(names = ["-c", "--cache"], defaultValue = ".\\images", paramLabel = "", description = ["the cache folder (default: \${DEFAULT-VALUE})"]) + lateinit var cacheFolder: Path @field:CommandLine.Option(names = ["-h", "--help"], usageHelp = true, description = ["show this help message and exit"]) - var helpRequested: Boolean = false, + var helpRequested: Boolean = false @field:CommandLine.Option(names = ["-v", "--version"], versionHelp = true, description = ["show the version message and exit"]) var versionRequested: Boolean = false -) : Runnable { + override fun run() { println( "Mangadex@Home Client Version ${BuildInfo.VERSION} (Build ${Constants.CLIENT_BUILD}) initializing" @@ -89,7 +68,33 @@ class ClientArgs( """.trimIndent() ) - val client = MangaDexClient(settingsFile, databaseFile, cacheFolder) + if (!Files.isDirectory(databaseFolder) || Files.isRegularFile(databaseFolder.resolveSibling(databaseFolder.fileName.toString() + ".mv.db"))) { + println() + println() + println( + """the --database option now takes in the folder with the database file! + |(it previously took in the path to the file without any extensions) + |if you are using docker update your docker mount settings! + |if you are not, manually move update your --database args! + |note: the database file itself should be named metadata.{extension} + |where {extension} can be `.db` or `.mv.db` + """.trimMargin() + ) + println() + println() + + throw IllegalArgumentException() + } + + if (!Files.isDirectory(databaseFolder)) { + throw IllegalArgumentException("Database folder $databaseFolder must be a directory") + } + + if (!Files.isDirectory(cacheFolder)) { + throw IllegalArgumentException("Database folder $cacheFolder must be a directory") + } + + val client = MangaDexClient(settingsFile, databaseFolder, cacheFolder) val hook = Thread { client.shutdown() (LoggerFactory.getILoggerFactory() as LoggerContext).stop() @@ -101,6 +106,7 @@ class ClientArgs( try { client.runLoop() } catch (e: Throwable) { + LOGGER.error(e) { "Failure when starting main loop" } Runtime.getRuntime().removeShutdownHook( hook ) @@ -108,4 +114,8 @@ class ClientArgs( throw e } } + + companion object { + private val LOGGER = LoggerFactory.getLogger(Main::class.java) + } } diff --git a/src/main/kotlin/mdnet/MangaDexClient.kt b/src/main/kotlin/mdnet/MangaDexClient.kt index 17cff98..40b7b5b 100644 --- a/src/main/kotlin/mdnet/MangaDexClient.kt +++ b/src/main/kotlin/mdnet/MangaDexClient.kt @@ -26,7 +26,6 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule import com.fasterxml.jackson.module.kotlin.readValue import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource -import mdnet.Main.dieWithError import mdnet.cache.ImageStorage import mdnet.logging.info import mdnet.logging.warn @@ -46,7 +45,7 @@ import java.util.regex.Pattern // Exception class to handle when Client Settings have invalid values class ClientSettingsException(message: String) : Exception(message) -class MangaDexClient(private val settingsFile: File, databaseFile: File, cacheFolder: Path) { +class MangaDexClient(private val settingsFile: File, databaseFolder: Path, cacheFolder: Path) { // this must remain single-threaded because of how the state mechanism works private val executor = Executors.newSingleThreadScheduledExecutor() private var scheduledFuture: ScheduledFuture<*>? = null @@ -62,22 +61,13 @@ class MangaDexClient(private val settingsFile: File, databaseFile: File, cacheFo // end protected state init { - settings = try { - readClientSettings() - } catch (e: UnrecognizedPropertyException) { - dieWithError("'${e.propertyName}' is not a valid setting") - } catch (e: JsonProcessingException) { - dieWithError(e) - } catch (e: ClientSettingsException) { - dieWithError(e) - } catch (e: IOException) { - dieWithError(e) - } + settings = readClientSettings() LOGGER.info { "Client settings loaded: $settings" } val config = HikariConfig() - config.jdbcUrl = "jdbc:h2:$databaseFile" + val db = databaseFolder.resolve("metadata") + config.jdbcUrl = "jdbc:h2:$db" config.addDataSourceProperty("cachePrepStmts", "true") config.addDataSourceProperty("prepStmtCacheSize", "100") config.addDataSourceProperty("prepStmtCacheSqlLimit", "1000") @@ -99,7 +89,6 @@ class MangaDexClient(private val settingsFile: File, databaseFile: File, cacheFo scheduledFuture = executor.scheduleWithFixedDelay( { try { - // this blocks the executor, so no worries about concurrency reloadClientSettings() } catch (e: Exception) { LOGGER.warn(e) { "Reload of ClientSettings failed" } @@ -107,13 +96,25 @@ class MangaDexClient(private val settingsFile: File, databaseFile: File, cacheFo }, 1, 1, TimeUnit.MINUTES ) + + scheduledFuture = executor.scheduleWithFixedDelay( + { + try { + if (imageServer == null) { + LOGGER.info { "Restarting image server that failed to start" } + startImageServer() + } + } catch (e: Exception) { + LOGGER.warn(e) { "Image server restart failed" } + } + }, + 1, 1, TimeUnit.MINUTES + ) } // Precondition: settings must be filled with up-to-date settings private fun startImageServer() { - if (imageServer != null) { - throw AssertionError() - } + require(imageServer == null) { "imageServer was not null" } LOGGER.info { "Server manager starting" } imageServer = ServerManager( settings, diff --git a/src/main/kotlin/mdnet/server/ImageHandler.kt b/src/main/kotlin/mdnet/server/ImageHandler.kt index b0d5cf7..208e60c 100644 --- a/src/main/kotlin/mdnet/server/ImageHandler.kt +++ b/src/main/kotlin/mdnet/server/ImageHandler.kt @@ -86,7 +86,7 @@ class ImageServer( Response(Status.NOT_MODIFIED) .header("Last-Modified", lastModified) } else { - LOGGER.info { "Request for $sanitizedUri is being served" } + LOGGER.info { "Request for $sanitizedUri is being served from cache" } respondWithImage( BufferedInputStream(image.stream), @@ -147,7 +147,7 @@ class ImageServer( } respondWithImage(tee, contentLength, contentType, lastModified, false) } else { - LOGGER.info { "Request for $sanitizedUri is being served" } + LOGGER.info { "Request for $sanitizedUri is being served due to write errors" } respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false) } }