Merge branch 'testing' into 'master'

Cut 2.0.1

See merge request mangadex-pub/mangadex_at_home!87

CHANGELOG.md

@@ -17,6 +17,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security
 
+## [2.0.1] - 2021-05-27
+### Added
+- [2021-05-27] Added SNI check to prevent people from simply scanning nodes [@carbotaniuman].
+
+### Changed
+- [2021-05-21] Update metrics and fix cache directory leak [@carbotaniuman].
+- [2021-05-21] Change headers to be wildcards [@carbotaniuman].
+- [2021-05-27] Make sending the `Server` header configurable but off by default [@carbotaniuman].
+
 ## [2.0.0] - 2021-03-11
 ### Changed
 - [2021-03-11] Switch back to HTTP/1.1 [@carbotaniuman].
@@ -388,6 +397,7 @@ This release contains many breaking changes! Of note are the changes to the cach
 - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
 
 [Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...HEAD
+[2.0.1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...2.0.1
 [2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0
 [2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14
 [2.0.0-rc13]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc12...2.0.0-rc13

src/main/kotlin/mdnet/Constants.kt

@@ -21,7 +21,7 @@ package mdnet
 import java.time.Duration
 
 object Constants {
-    const val CLIENT_BUILD = 30
+    const val CLIENT_BUILD = 31
 
     @JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
 

src/main/kotlin/mdnet/Main.kt

@@ -67,26 +67,8 @@ class Main : Runnable {
             """.trimIndent()
         )
 
-        if (!Files.isDirectory(databaseFolder) || Files.isRegularFile(databaseFolder.resolveSibling(databaseFolder.fileName.toString() + ".mv.db"))) {
+        if (!Files.isDirectory(databaseFolder)) {
-            println()
+            throw IllegalArgumentException("Expected $databaseFolder to be a folder, was a file")
-            println()
-            println(
-                """the --database option now takes in the folder with the database file!
-                |(it previously took in the path to the file without any extensions)
-                |if you are using docker update your docker mount settings!
-                |if you are not, manually move update your --database args!
-                |note: the database file itself should be named metadata.{extension}
-                |where {extension} can be `.db` or `.mv.db`
-                |
-                |If this is your first time seeing this message, please check out the support
-                |channel as things HAVE changed. Failure to do so WILL require
-                |a cache wipe.
-            """.trimMargin()
-            )
-            println()
-            println()
-
-            throw IllegalArgumentException()
         }
 
         if (Files.isRegularFile(databaseFolder)) {

src/main/kotlin/mdnet/ServerManager.kt

@@ -264,6 +264,7 @@ class ServerManager(
             storage,
             remoteSettings,
             settings.serverSettings,
+            settings.devSettings,
             settings.metricsSettings,
             statistics,
             registry,

src/main/kotlin/mdnet/netty/ApplicationNetty.kt

@@ -30,6 +30,8 @@ import io.netty.handler.codec.DecoderException
 import io.netty.handler.codec.http.HttpObjectAggregator
 import io.netty.handler.codec.http.HttpServerCodec
 import io.netty.handler.codec.http.HttpServerKeepAliveHandler
+import io.netty.handler.ssl.SniCompletionEvent
+import io.netty.handler.ssl.SniHandler
 import io.netty.handler.ssl.SslContextBuilder
 import io.netty.handler.stream.ChunkedWriteHandler
 import io.netty.handler.timeout.ReadTimeoutException
@@ -41,6 +43,7 @@ import io.netty.handler.traffic.TrafficCounter
 import io.netty.incubator.channel.uring.IOUring
 import io.netty.incubator.channel.uring.IOUringEventLoopGroup
 import io.netty.incubator.channel.uring.IOUringServerSocketChannel
+import io.netty.util.DomainWildcardMappingBuilder
 import io.netty.util.concurrent.DefaultEventExecutorGroup
 import io.netty.util.internal.SystemPropertyUtil
 import mdnet.Constants
@@ -48,8 +51,9 @@ import mdnet.data.Statistics
 import mdnet.logging.info
 import mdnet.logging.trace
 import mdnet.logging.warn
+import mdnet.settings.DevSettings
+import mdnet.settings.RemoteSettings
 import mdnet.settings.ServerSettings
-import mdnet.settings.TlsCert
 import org.http4k.core.HttpHandler
 import org.http4k.server.Http4kChannelHandler
 import org.http4k.server.Http4kServer
@@ -136,8 +140,9 @@ sealed class NettyTransport(threads: Int) {
 }
 
 class Netty(
-    private val tls: TlsCert,
+    private val remoteSettings: RemoteSettings,
     private val serverSettings: ServerSettings,
+    private val devSettings: DevSettings,
     private val statistics: Statistics
 ) : ServerConfig {
     override fun toServer(http: HttpHandler): Http4kServer = object : Http4kServer {
@@ -155,6 +160,7 @@ class Netty(
 
         override fun start(): Http4kServer = apply {
             LOGGER.info { "Starting Netty!" }
+            val tls = remoteSettings.tls!!
 
             val certs = getX509Certs(tls.certificate)
             val sslContext = SslContextBuilder
@@ -167,7 +173,33 @@ class Netty(
                 .channelFactory(transport.factory)
                 .childHandler(object : ChannelInitializer<SocketChannel>() {
                     public override fun initChannel(ch: SocketChannel) {
-                        ch.pipeline().addLast("ssl", sslContext.newHandler(ch.alloc()))
+                        ch.pipeline().addLast(
+                            "ssl",
+                            SniHandler(DomainWildcardMappingBuilder(sslContext).build())
+                        )
+
+                        ch.pipeline().addLast(
+                            "dropHostname",
+                            object : ChannelInboundHandlerAdapter() {
+                                private val hostToTest = remoteSettings.url.authority.let {
+                                    it.substring(0, it.lastIndexOf(":"))
+                                }
+
+                                override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) {
+                                    if (evt is SniCompletionEvent) {
+                                        if (!devSettings.disableSniCheck) {
+                                            if (!evt.hostname().endsWith(hostToTest) &&
+                                                !evt.hostname().endsWith("localhost")
+                                            ) {
+                                                ctx.close()
+                                            }
+                                        }
+                                    } else {
+                                        ctx.fireUserEventTriggered(evt)
+                                    }
+                                }
+                            }
+                        )
 
                         ch.pipeline().addLast("codec", HttpServerCodec())
                         ch.pipeline().addLast("keepAlive", HttpServerKeepAliveHandler())

src/main/kotlin/mdnet/server/ImageHandler.kt

@@ -147,7 +147,7 @@ class ImageServer(
             }
             respondWithImage(tee, contentLength, contentType, lastModified, false)
         } else {
-            LOGGER.info { "Request for $sanitizedUri is being served due to write errors" }
+            LOGGER.info { "Request for $sanitizedUri is being served as the cache is full" }
             respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false)
         }
     }
@@ -181,10 +181,10 @@ class ImageServer(
                 .then { next: HttpHandler ->
                     { request: Request ->
                         val response = next(request)
-                        response.header("access-control-allow-origin", "https://mangadex.org")
+                        response.header("access-control-allow-origin", "*")
                             .header("access-control-expose-headers", "*")
                             .header("access-control-allow-methods", "GET")
-                            .header("timing-allow-origin", "https://mangadex.org")
+                            .header("timing-allow-origin", "*")
                     }
                 }
     }

src/main/kotlin/mdnet/server/ImageServer.kt

@@ -31,6 +31,7 @@ import mdnet.logging.warn
 import mdnet.metrics.GeoIpMetricsFilterBuilder
 import mdnet.metrics.PostTransactionLabeler
 import mdnet.netty.Netty
+import mdnet.settings.DevSettings
 import mdnet.settings.MetricsSettings
 import mdnet.settings.RemoteSettings
 import mdnet.settings.ServerSettings
@@ -46,6 +47,7 @@ fun getServer(
     storage: ImageStorage,
     remoteSettings: RemoteSettings,
     serverSettings: ServerSettings,
+    devSettings: DevSettings,
     metricsSettings: MetricsSettings,
     statistics: Statistics,
     registry: PrometheusMeterRegistry,
@@ -75,7 +77,7 @@ fun getServer(
     }
 
     circuitBreaker.eventPublisher.onReset {
-        LOGGER.warn { "Circuit breaker has rest" }
+        LOGGER.warn { "Circuit breaker has reset" }
     }
 
     circuitBreaker.eventPublisher.onStateTransition {
@@ -109,7 +111,7 @@ fun getServer(
     )
 
     return timeRequest()
-        .then(addCommonHeaders())
+        .then(addCommonHeaders(devSettings.sendServerHeader))
         .then(catchAllHideDetails())
         .then(
             routes(
@@ -142,7 +144,7 @@ fun getServer(
                 GeoIpMetricsFilterBuilder(metricsSettings.enableGeoip, metricsSettings.geoipLicenseKey, registry).build()
             )
         )
-        .asServer(Netty(remoteSettings.tls!!, serverSettings, statistics))
+        .asServer(Netty(remoteSettings, serverSettings, devSettings, statistics))
 }
 
 private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java)

src/main/kotlin/mdnet/server/common.kt

@@ -35,12 +35,17 @@ import java.util.*
 private val HTTP_TIME_FORMATTER = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss O", Locale.ENGLISH)
 private val LOGGER = LoggerFactory.getLogger("Application")
 
-fun addCommonHeaders(): Filter {
+fun addCommonHeaders(sendServerHeader: Boolean): Filter {
     return Filter { next: HttpHandler ->
         { request: Request ->
             val response = next(request)
-            response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC)))
+            response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC))).let {
-                .header("Server", "MangaDex@Home Node ${BuildInfo.VERSION} (${Constants.CLIENT_BUILD})")
+                if (sendServerHeader) {
+                    it.header("Server", "MangaDex@Home Node ${BuildInfo.VERSION} (${Constants.CLIENT_BUILD})")
+                } else {
+                    it
+                }
+            }
         }
     }
 }

src/main/kotlin/mdnet/server/crypto.kt

@@ -20,7 +20,7 @@ package mdnet.server
 
 import java.security.MessageDigest
 
-fun md5Bytes(stringToHash: String) =
+fun md5Bytes(stringToHash: String): ByteArray =
     MessageDigest.getInstance("MD5")
         .digest(stringToHash.toByteArray())
 

src/main/kotlin/mdnet/settings/ClientSettings.kt

@@ -47,7 +47,9 @@ data class ServerSettings(
 
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)
 data class DevSettings(
-    val devUrl: String? = null
+    val devUrl: String? = null,
+    val disableSniCheck: Boolean = false,
+    val sendServerHeader: Boolean = false,
 )
 
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy::class)

src/test/kotlin/mdnet/server/ImageServerTest.kt

@@ -39,6 +39,7 @@ import org.http4k.core.Response
 import org.http4k.core.Status
 import org.http4k.kotest.shouldHaveHeader
 import org.http4k.kotest.shouldHaveStatus
+import org.http4k.kotest.shouldNotHaveHeader
 import org.http4k.routing.bind
 import org.http4k.routing.routes
 import org.ktorm.database.Database
@@ -106,6 +107,12 @@ class ImageServerTest : FreeSpec() {
                         response.close()
                     }
                 }
+
+                "should not have Server header" {
+                    val response = handler(Request(Method.GET, "/data/02181a8f5fe8cd408720a771dd129fd8/T2.png"))
+                    response.shouldNotHaveHeader("Server")
+                    response.close()
+                }
             }
 
             "with real cache" - {