diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 918e26e..1987d76 100755
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
stages:
- build
- publish
+ - publish_latest
- publish_docker
cache:
@@ -26,8 +27,22 @@ publish:
name: "mangadex_at_home"
paths:
- "*.jar"
+ - "mangadex_at_home-*.zip"
- settings.sample.json
+publish_latest:
+ image: alpine
+ stage: publish
+ before_script:
+ - apk update && apk add git
+ - export VERSION=`git describe --tags --dirty`
+ script:
+ - cp build/libs/mangadex_at_home-${VERSION}-all.jar build/libs/mangadex_at_home-latest-all.jar
+ artifacts:
+ name: "mangadex_at_home-latest"
+ paths:
+ - "build/libs/mangadex_at_home-latest-all.jar"
+
publish_docker:
image: docker:git
stage: publish
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8953ec5..1d31087 100755
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,14 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
+- [2020-07-13] Added reloading client setting without stopping client by [@radonbark].
### Changed
+- [2020-07-29] Disallow unsafe ports [@m3ch_mania].
### Deprecated
### Removed
### Fixed
+- [2020-07-29] Fixed stupid libsodium bugs [@carbotaniuman].
+- [2020-07-29] Fixed issues from the Great Cache Propagation [@carbotaniuman].
### Security
diff --git a/src/main/kotlin/mdnet/base/Constants.kt b/src/main/kotlin/mdnet/base/Constants.kt
index 53293fe..7f4697c 100644
--- a/src/main/kotlin/mdnet/base/Constants.kt
+++ b/src/main/kotlin/mdnet/base/Constants.kt
@@ -22,8 +22,83 @@ import java.time.Duration
object Constants {
const val CLIENT_BUILD = 16
- val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
+
+ @JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
const val MAX_READ_TIME_SECONDS = 300
const val MAX_WRITE_TIME_SECONDS = 60
+
+ // General list of ports to which Firefox and Chromium will not send HTTP requests for security reasons
+ // See:
+ // * https://chromium.googlesource.com/chromium/src.git/+/refs/heads/master/net/base/port_util.cc
+ // * https://developer.mozilla.org/en-US/docs/Mozilla/Mozilla_Port_Blocking#Blocked_Ports
+ @JvmField val RESTRICTED_PORTS = intArrayOf(
+ 1, // tcpmux
+ 7, // echo
+ 9, // discard
+ 11, // systat
+ 13, // daytime
+ 15, // netstat
+ 17, // qotd
+ 19, // chargen
+ 20, // ftp data
+ 21, // ftp access
+ 22, // ssh
+ 23, // telnet
+ 25, // smtp
+ 37, // time
+ 42, // name
+ 43, // nicname
+ 53, // domain
+ 77, // priv-rjs
+ 79, // finger
+ 87, // ttylink
+ 95, // supdup
+ 101, // hostriame
+ 102, // iso-tsap
+ 103, // gppitnp
+ 104, // acr-nema
+ 109, // pop2
+ 110, // pop3
+ 111, // sunrpc
+ 113, // auth
+ 115, // sftp
+ 117, // uucp-path
+ 119, // nntp
+ 123, // NTP
+ 135, // loc-srv /epmap
+ 139, // netbios
+ 143, // imap2
+ 179, // BGP
+ 389, // ldap
+ 427, // SLP (Also used by Apple Filing Protocol)
+ 465, // smtp+ssl
+ 512, // print / exec
+ 513, // login
+ 514, // shell
+ 515, // printer
+ 526, // tempo
+ 530, // courier
+ 531, // chat
+ 532, // netnews
+ 540, // uucp
+ 548, // AFP (Apple Filing Protocol)
+ 556, // remotefs
+ 563, // nntp+ssl
+ 587, // smtp (rfc6409)
+ 601, // syslog-conn (rfc3195)
+ 636, // ldap+ssl
+ 993, // ldap+ssl
+ 995, // pop3+ssl
+ 2049, // nfs
+ 3659, // apple-sasl / PasswordServer
+ 4045, // lockd
+ 6000, // X11
+ 6665, // Alternate IRC [Apple addition]
+ 6666, // Alternate IRC [Apple addition]
+ 6667, // Standard IRC [Apple addition]
+ 6668, // Alternate IRC [Apple addition]
+ 6669, // Alternate IRC [Apple addition]
+ 6697 // IRC + TLS
+ )
}
diff --git a/src/main/kotlin/mdnet/base/Main.kt b/src/main/kotlin/mdnet/base/Main.kt
index 7b1a5b2..5d9584a 100644
--- a/src/main/kotlin/mdnet/base/Main.kt
+++ b/src/main/kotlin/mdnet/base/Main.kt
@@ -19,25 +19,12 @@ along with this MangaDex@Home. If not, see .
package mdnet.base
import ch.qos.logback.classic.LoggerContext
-import com.fasterxml.jackson.core.JsonParser
-import com.fasterxml.jackson.core.JsonProcessingException
-import com.fasterxml.jackson.databind.ObjectMapper
-import com.fasterxml.jackson.databind.SerializationFeature
-import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
-import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
-import com.fasterxml.jackson.module.kotlin.readValue
-import java.io.FileReader
-import java.io.FileWriter
-import java.io.IOException
-import java.util.regex.Pattern
import kotlin.system.exitProcess
import mdnet.BuildInfo
-import mdnet.base.settings.ClientSettings
import org.slf4j.LoggerFactory
object Main {
private val LOGGER = LoggerFactory.getLogger(Main::class.java)
- private val JACKSON: ObjectMapper = jacksonObjectMapper().enable(SerializationFeature.INDENT_OUTPUT).configure(JsonParser.Feature.ALLOW_COMMENTS, true)
@JvmStatic
fun main(args: Array) {
@@ -68,25 +55,7 @@ object Main {
dieWithError("Expected one argument: path to config file, or nothing")
}
- val settings = try {
- JACKSON.readValue(FileReader(file))
- } catch (e: UnrecognizedPropertyException) {
- dieWithError("'${e.propertyName}' is not a valid setting")
- } catch (e: JsonProcessingException) {
- dieWithError(e)
- } catch (ignored: IOException) {
- ClientSettings().also {
- LOGGER.warn("Settings file {} not found, generating file", file)
- try {
- FileWriter(file).use { writer -> JACKSON.writeValue(writer, it) }
- } catch (e: IOException) {
- dieWithError(e)
- }
- }
- }.apply(::validateSettings)
-
- LOGGER.info { "Client settings loaded: $settings" }
- val client = MangaDexClient(settings)
+ val client = MangaDexClient(file)
Runtime.getRuntime().addShutdownHook(Thread { client.shutdown() })
client.runLoop()
}
@@ -103,40 +72,4 @@ object Main {
(LoggerFactory.getILoggerFactory() as LoggerContext).stop()
exitProcess(1)
}
-
- private fun validateSettings(settings: ClientSettings) {
- if (!isSecretValid(settings.clientSecret)) dieWithError("Config Error: API Secret is invalid, must be 52 alphanumeric characters")
- if (settings.clientPort == 0) {
- dieWithError("Config Error: Invalid port number")
- }
- if (settings.maxCacheSizeInMebibytes < 1024) {
- dieWithError("Config Error: Invalid max cache size, must be >= 1024 MiB (1GiB)")
- }
- if (settings.threads < 4) {
- dieWithError("Config Error: Invalid number of threads, must be >= 4")
- }
- if (settings.maxMebibytesPerHour < 0) {
- dieWithError("Config Error: Max bandwidth must be >= 0")
- }
- if (settings.maxKilobitsPerSecond < 0) {
- dieWithError("Config Error: Max burst rate must be >= 0")
- }
- if (settings.gracefulShutdownWaitSeconds < 15) {
- dieWithError("Config Error: Graceful shutdown wait be >= 15")
- }
- if (settings.webSettings != null) {
- if (settings.webSettings.uiPort == 0) {
- dieWithError("Config Error: Invalid UI port number")
- }
- }
- if (settings.experimental != null) {
- if (settings.experimental.maxBufferSizeForCacheHit < 0)
- dieWithError("Config Error: Max cache buffer multiple must be >= 0")
- }
- }
-
- private const val CLIENT_KEY_LENGTH = 52
- private fun isSecretValid(clientSecret: String): Boolean {
- return Pattern.matches("^[a-zA-Z0-9]{$CLIENT_KEY_LENGTH}$", clientSecret)
- }
}
diff --git a/src/main/kotlin/mdnet/base/MangaDexClient.kt b/src/main/kotlin/mdnet/base/MangaDexClient.kt
index 4ff553a..5a33631 100644
--- a/src/main/kotlin/mdnet/base/MangaDexClient.kt
+++ b/src/main/kotlin/mdnet/base/MangaDexClient.kt
@@ -20,10 +20,14 @@ along with this MangaDex@Home. If not, see .
package mdnet.base
import ch.qos.logback.classic.LoggerContext
+import com.fasterxml.jackson.core.JsonProcessingException
import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import java.io.File
+import java.io.FileReader
+import java.io.FileWriter
import java.io.IOException
import java.time.Instant
import java.util.*
@@ -32,6 +36,7 @@ import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicReference
+import java.util.regex.Pattern
import mdnet.base.Main.dieWithError
import mdnet.base.data.Statistics
import mdnet.base.server.getServer
@@ -43,23 +48,28 @@ import mdnet.cache.HeaderMismatchException
import org.http4k.server.Http4kServer
import org.slf4j.LoggerFactory
+private const val CLIENT_KEY_LENGTH = 52
+
+// Exception class to handle when Client Settings have invalid values
+class ClientSettingsException(message: String) : Exception(message)
+
sealed class State
// server is not running
-object Uninitialized : State()
+data class Uninitialized(val clientSettings: ClientSettings) : State()
// server has shut down
object Shutdown : State()
// server is in the process of shutting down
-data class GracefulShutdown(val lastRunning: Running, val counts: Int = 0, val nextState: State = Uninitialized, val action: () -> Unit = {}) : State()
+data class GracefulShutdown(val lastRunning: Running, val counts: Int = 0, val nextState: State = Uninitialized(lastRunning.clientSettings), val action: () -> Unit = {}) : State()
// server is currently running
-data class Running(val server: Http4kServer, val settings: ServerSettings) : State()
-
-class MangaDexClient(private val clientSettings: ClientSettings) {
+data class Running(val server: Http4kServer, val settings: ServerSettings, val clientSettings: ClientSettings) : State()
+// clientSettings must only be accessed from the thread on the executorService
+class MangaDexClient(private val clientSettingsFile: String) {
// this must remain singlethreaded because of how the state mechanism works
private val executorService = Executors.newSingleThreadScheduledExecutor()
// state must only be accessed from the thread on the executorService
- private var state: State = Uninitialized
+ private var state: State
- private val serverHandler: ServerHandler = ServerHandler(clientSettings)
+ private var serverHandler: ServerHandler
private val statsMap: MutableMap = Collections
.synchronizedMap(object : LinkedHashMap(240) {
override fun removeEldestEntry(eldest: Map.Entry): Boolean {
@@ -74,6 +84,32 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
private val cache: DiskLruCache
init {
+ // Read ClientSettings
+ val clientSettings = try {
+ readClientSettings()
+ } catch (e: UnrecognizedPropertyException) {
+ dieWithError("'${e.propertyName}' is not a valid setting")
+ } catch (e: JsonProcessingException) {
+ dieWithError(e)
+ } catch (ignored: IOException) {
+ ClientSettings().also {
+ LOGGER.warn { "Settings file $clientSettingsFile not found, generating file" }
+ try {
+ FileWriter(clientSettingsFile).use { writer -> JACKSON.writeValue(writer, it) }
+ } catch (e: IOException) {
+ dieWithError(e)
+ }
+ }
+ } catch (e: ClientSettingsException) {
+ dieWithError(e)
+ }
+
+ // Initialize things that depend on Client Settings
+ LOGGER.info { "Client settings loaded: $clientSettings" }
+ state = Uninitialized(clientSettings)
+ serverHandler = ServerHandler(clientSettings)
+
+ // Initialize everything else
try {
cache = DiskLruCache.open(
File("cache"), 1, 1,
@@ -94,11 +130,7 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
fun runLoop() {
loginAndStartServer()
statsMap[Instant.now()] = statistics.get()
-
- if (clientSettings.webSettings != null) {
- webUi = getUiServer(clientSettings.webSettings, statistics, statsMap)
- webUi!!.start()
- }
+ startWebUi()
LOGGER.info { "Mangadex@Home Client initialized. Starting normal operation." }
executorService.scheduleAtFixedRate({
@@ -140,11 +172,11 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
}
}, 1, 1, TimeUnit.HOURS)
- val timesToWait = clientSettings.gracefulShutdownWaitSeconds / 15
executorService.scheduleAtFixedRate({
try {
val state = this.state
if (state is GracefulShutdown) {
+ val timesToWait = state.lastRunning.clientSettings.gracefulShutdownWaitSeconds / 15
when {
state.counts == 0 -> {
LOGGER.info { "Starting graceful shutdown" }
@@ -174,7 +206,7 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
}
}
} catch (e: Exception) {
- LOGGER.warn("Main loop failed", e)
+ LOGGER.warn(e) { "Main loop failed" }
}
}, 15, 15, TimeUnit.SECONDS)
@@ -183,7 +215,7 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
val state = this.state
if (state is Running) {
val currentBytesSent = statistics.get().bytesSent - lastBytesSent
- if (clientSettings.maxMebibytesPerHour != 0L && clientSettings.maxMebibytesPerHour * 1024 * 1024 /* MiB to bytes */ < currentBytesSent) {
+ if (state.clientSettings.maxMebibytesPerHour != 0L && state.clientSettings.maxMebibytesPerHour * 1024 * 1024 /* MiB to bytes */ < currentBytesSent) {
LOGGER.info { "Shutting down server as hourly bandwidth limit reached" }
this.state = GracefulShutdown(lastRunning = state)
@@ -195,6 +227,18 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
LOGGER.warn(e) { "Graceful shutdown checker failed" }
}
}, 45, 45, TimeUnit.SECONDS)
+
+ // Check every minute to see if client settings have changed
+ executorService.scheduleWithFixedDelay({
+ try {
+ val state = this.state
+ if (state is Running) {
+ reloadClientSettings()
+ }
+ } catch (e: Exception) {
+ LOGGER.warn(e) { "Reload of ClientSettings failed" }
+ }
+ }, 60, 60, TimeUnit.SECONDS)
}
private fun pingControl() {
@@ -223,11 +267,11 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
}
private fun loginAndStartServer() {
- this.state as Uninitialized
+ val state = this.state as Uninitialized
val serverSettings = serverHandler.loginToControl()
?: dieWithError("Failed to get a login response from server - check API secret for validity")
- val server = getServer(cache, serverSettings, clientSettings, statistics, isHandled).start()
+ val server = getServer(cache, serverSettings, state.clientSettings, statistics, isHandled).start()
if (serverSettings.latestBuild > Constants.CLIENT_BUILD) {
LOGGER.warn {
@@ -235,7 +279,7 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
}
}
- state = Running(server, serverSettings)
+ this.state = Running(server, serverSettings, state.clientSettings)
LOGGER.info { "Internal HTTP server was successfully started" }
}
@@ -243,7 +287,7 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
serverHandler.logoutFromControl()
}
- private fun stopServer(nextState: State = Uninitialized) {
+ private fun stopServer(nextState: State) {
val state = this.state.let {
when (it) {
is Running ->
@@ -262,6 +306,32 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
this.state = nextState
}
+ /**
+ * Starts the WebUI if the ClientSettings demand it.
+ * Because this method checks if the WebUI is needed,
+ * it can be safely called before checking yourself
+ */
+ private fun startWebUi() {
+ val state = this.state
+ // Grab the client settings if available
+ val clientSettings = state.let {
+ when (it) {
+ is Running ->
+ it.clientSettings
+ is Uninitialized ->
+ it.clientSettings
+ else ->
+ null
+ }
+ }
+
+ // Only start the Web UI if the settings demand it
+ if (clientSettings?.webSettings != null) {
+ webUi = getUiServer(clientSettings.webSettings, statistics, statsMap)
+ webUi!!.start()
+ }
+ }
+
fun shutdown() {
LOGGER.info { "Mangadex@Home Client stopping" }
@@ -296,6 +366,131 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
(LoggerFactory.getILoggerFactory() as LoggerContext).stop()
}
+ /**
+ * Reloads the client configuration and restarts the
+ * Web UI and/or the server if needed
+ */
+ private fun reloadClientSettings() {
+ val state = this.state as Running
+ LOGGER.info { "Reloading client settings" }
+ try {
+ val newSettings = readClientSettings()
+
+ if (newSettings == state.clientSettings) {
+ LOGGER.info { "Client Settings have not changed" }
+ return
+ }
+
+ // Setting loaded without issue. Figure out
+ // if there are changes that require a restart
+ val restartServer = newSettings.clientSecret != state.clientSettings.clientSecret ||
+ newSettings.clientHostname != state.clientSettings.clientHostname ||
+ newSettings.clientPort != state.clientSettings.clientPort ||
+ newSettings.clientExternalPort != state.clientSettings.clientExternalPort ||
+ newSettings.threads != state.clientSettings.threads ||
+ newSettings.devSettings?.isDev != state.clientSettings.devSettings?.isDev
+ val stopWebUi = newSettings.webSettings != state.clientSettings.webSettings ||
+ newSettings.webSettings?.uiPort != state.clientSettings.webSettings?.uiPort ||
+ newSettings.webSettings?.uiHostname != state.clientSettings.webSettings?.uiHostname
+ val startWebUi = (stopWebUi && newSettings.webSettings != null)
+
+ // Stop the the WebUI if needed
+ if (stopWebUi) {
+ LOGGER.info { "Stopping WebUI to reload ClientSettings" }
+ webUi?.close()
+ webUi = null
+ }
+
+ if (restartServer) {
+ // If we are restarting the server
+ // We must do it gracefully and set
+ // the new settings later
+ LOGGER.info { "Stopping Server to reload ClientSettings" }
+
+ this.state = GracefulShutdown(state, nextState = Uninitialized(clientSettings = newSettings), action = {
+ serverHandler = ServerHandler(newSettings)
+ LOGGER.info { "Reloaded ClientSettings: $newSettings" }
+
+ LOGGER.info { "Starting Server after reloading ClientSettings" }
+ loginAndStartServer()
+
+ // Start the WebUI if we had to stop it
+ // and still want it
+ if (startWebUi) {
+ LOGGER.info { "Starting WebUI after reloading ClientSettings" }
+ startWebUi()
+ LOGGER.info { "Started WebUI after reloading ClientSettings" }
+ }
+ })
+ } else {
+ // If we aren't restarting the server
+ // We can update the settings now
+ this.state = state.copy(clientSettings = newSettings)
+ serverHandler.setClientSettings(newSettings)
+ LOGGER.info { "Reloaded ClientSettings: $newSettings" }
+
+ // Start the WebUI if we had to stop it
+ // and still want it
+ if (startWebUi) {
+ LOGGER.info { "Starting WebUI after reloading ClientSettings" }
+ startWebUi()
+ LOGGER.info { "Started WebUI after reloading ClientSettings" }
+ }
+ }
+ } catch (e: UnrecognizedPropertyException) {
+ LOGGER.warn { "Settings file is invalid: '$e.propertyName' is not a valid setting" }
+ } catch (e: JsonProcessingException) {
+ LOGGER.warn { "Settings file is invalid: $e.message" }
+ } catch (e: IOException) {
+ LOGGER.warn { "Settings file is could not be found: $e.message" }
+ } catch (e: ClientSettingsException) {
+ LOGGER.warn { "Can't reload client settings: $e.message" }
+ }
+ }
+
+ private fun validateSettings(settings: ClientSettings) {
+ if (!isSecretValid(settings.clientSecret)) throw ClientSettingsException("Config Error: API Secret is invalid, must be 52 alphanumeric characters")
+ if (settings.clientPort == 0) {
+ throw ClientSettingsException("Config Error: Invalid port number")
+ }
+ if (settings.clientPort in Constants.RESTRICTED_PORTS) {
+ throw ClientSettingsException("Config Error: Unsafe port number")
+ }
+ if (settings.maxCacheSizeInMebibytes < 1024) {
+ throw ClientSettingsException("Config Error: Invalid max cache size, must be >= 1024 MiB (1GiB)")
+ }
+ if (settings.threads < 4) {
+ throw ClientSettingsException("Config Error: Invalid number of threads, must be >= 4")
+ }
+ if (settings.maxMebibytesPerHour < 0) {
+ throw ClientSettingsException("Config Error: Max bandwidth must be >= 0")
+ }
+ if (settings.maxKilobitsPerSecond < 0) {
+ throw ClientSettingsException("Config Error: Max burst rate must be >= 0")
+ }
+ if (settings.gracefulShutdownWaitSeconds < 15) {
+ throw ClientSettingsException("Config Error: Graceful shutdown wait must be >= 15")
+ }
+ if (settings.webSettings != null) {
+ if (settings.webSettings.uiPort == 0) {
+ throw ClientSettingsException("Config Error: Invalid UI port number")
+ }
+ }
+ if (settings.experimental != null) {
+ if (settings.experimental.maxBufferSizeForCacheHit < 0)
+ throw ClientSettingsException("Config Error: Max cache buffer multiple must be >= 0")
+ }
+
+ }
+
+ private fun readClientSettings(): ClientSettings {
+ return JACKSON.readValue(FileReader(clientSettingsFile)).apply(::validateSettings)
+ }
+
+ private fun isSecretValid(clientSecret: String): Boolean {
+ return Pattern.matches("^[a-zA-Z0-9]{$CLIENT_KEY_LENGTH}$", clientSecret)
+ }
+
companion object {
private val LOGGER = LoggerFactory.getLogger(MangaDexClient::class.java)
private val JACKSON: ObjectMapper = jacksonObjectMapper()
diff --git a/src/main/kotlin/mdnet/base/ServerHandler.kt b/src/main/kotlin/mdnet/base/ServerHandler.kt
index 9fbff3a..ea540bf 100644
--- a/src/main/kotlin/mdnet/base/ServerHandler.kt
+++ b/src/main/kotlin/mdnet/base/ServerHandler.kt
@@ -40,7 +40,7 @@ object ServerHandlerJackson : ConfigurableJackson(
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
)
-class ServerHandler(private val settings: ClientSettings) {
+class ServerHandler(private var settings: ClientSettings) {
private val client = ApacheClient()
fun logoutFromControl(): Boolean {
@@ -109,6 +109,10 @@ class ServerHandler(private val settings: ClientSettings) {
SERVER_ADDRESS_DEV
}
+ fun setClientSettings(clientSettings: ClientSettings) {
+ this.settings = clientSettings
+ }
+
companion object {
private val LOGGER = LoggerFactory.getLogger(ServerHandler::class.java)
private val STRING_ANY_MAP_LENS = Body.auto