From 3445f5d569f4d6af40ba68ddb1a2f2b5c86729ca Mon Sep 17 00:00:00 2001 From: carbotaniuman <41451839+carbotaniuman@users.noreply.github.com> Date: Tue, 30 Jun 2020 14:06:12 -0500 Subject: [PATCH] Reorganize some code - add read and write timeouts --- CHANGELOG.md | 1 + build.gradle | 5 ++- src/main/kotlin/mdnet/base/Constants.kt | 3 ++ src/main/kotlin/mdnet/base/MangaDexClient.kt | 5 ++- .../mdnet/base/{ => data}/Statistics.kt | 2 +- .../mdnet/base/{dao => data}/metadata.kt | 2 +- .../mdnet/base/netty/ApplicationNetty.kt | 22 +++++----- .../kotlin/mdnet/base/netty/WebUiNetty.kt | 3 -- .../kotlin/mdnet/base/server/Application.kt | 26 ++++++++++-- .../kotlin/mdnet/base/server/ImageServer.kt | 42 ++++++++++++------- src/main/kotlin/mdnet/base/server/WebUi.kt | 2 +- src/main/kotlin/mdnet/base/server/common.kt | 22 ---------- 12 files changed, 75 insertions(+), 60 deletions(-) rename src/main/kotlin/mdnet/base/{ => data}/Statistics.kt (97%) rename src/main/kotlin/mdnet/base/{dao => data}/metadata.kt (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d8c1ac..34a6db8 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [2020-06-23] Added Gitlab CI integration by [@lflare]. - [2020-06-28] Added `client_external_port setting` [@wedge1001]. - [2020-06-29] Added rudimentary support of Referer checking to mitigate hotlinking by [@lflare]. +- [2020-06-30] Added read and write timeouts to protect against some attacks [@carbotaniuman]. ### Changed diff --git a/build.gradle b/build.gradle index 59a7e72..bb2357b 100644 --- a/build.gradle +++ b/build.gradle @@ -30,12 +30,15 @@ dependencies { implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" runtimeOnly group:"io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.30.Final" - implementation group:"ch.qos.logback", name: "logback-classic", version: "1.2.1" + implementation group:"ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha5" implementation group: "org.jetbrains.exposed", name: "exposed-core", version: "$exposed_version" implementation group: "org.jetbrains.exposed", name: "exposed-dao", version: "$exposed_version" implementation group: "org.jetbrains.exposed", name: "exposed-jdbc", version: "$exposed_version" implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.30.1" + + implementation "com.goterl.lazycode:lazysodium-java:4.2.6" + implementation "net.java.dev.jna:jna:5.5.0" } java { diff --git a/src/main/kotlin/mdnet/base/Constants.kt b/src/main/kotlin/mdnet/base/Constants.kt index d160c45..e86d23c 100644 --- a/src/main/kotlin/mdnet/base/Constants.kt +++ b/src/main/kotlin/mdnet/base/Constants.kt @@ -29,4 +29,7 @@ object Constants { const val WEBUI_VERSION = "0.1.1" val MAX_AGE_CACHE: Duration = Duration.ofDays(14) val JACKSON: ObjectMapper = jacksonObjectMapper().configure(JsonParser.Feature.ALLOW_COMMENTS, true) + + const val MAX_READ_TIME_SECONDS = 300 + const val MAX_WRITE_TIME_SECONDS = 60 } diff --git a/src/main/kotlin/mdnet/base/MangaDexClient.kt b/src/main/kotlin/mdnet/base/MangaDexClient.kt index 8497370..1836cfd 100644 --- a/src/main/kotlin/mdnet/base/MangaDexClient.kt +++ b/src/main/kotlin/mdnet/base/MangaDexClient.kt @@ -23,6 +23,7 @@ import ch.qos.logback.classic.LoggerContext import com.fasterxml.jackson.module.kotlin.readValue import mdnet.base.Constants.JACKSON import mdnet.base.Main.dieWithError +import mdnet.base.data.Statistics import mdnet.base.server.getServer import mdnet.base.server.getUiServer import mdnet.base.settings.ClientSettings @@ -64,7 +65,9 @@ class MangaDexClient(private val clientSettings: ClientSettings) { return this.size > 240 } }) - private val statistics: AtomicReference = AtomicReference(Statistics()) + private val statistics: AtomicReference = AtomicReference( + Statistics() + ) private val isHandled: AtomicBoolean = AtomicBoolean(false) private var webUi: Http4kServer? = null private val cache: DiskLruCache diff --git a/src/main/kotlin/mdnet/base/Statistics.kt b/src/main/kotlin/mdnet/base/data/Statistics.kt similarity index 97% rename from src/main/kotlin/mdnet/base/Statistics.kt rename to src/main/kotlin/mdnet/base/data/Statistics.kt index 192e5d9..00277aa 100644 --- a/src/main/kotlin/mdnet/base/Statistics.kt +++ b/src/main/kotlin/mdnet/base/data/Statistics.kt @@ -16,7 +16,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this MangaDex@Home. If not, see . */ -package mdnet.base +package mdnet.base.data import com.fasterxml.jackson.databind.PropertyNamingStrategy import com.fasterxml.jackson.databind.annotation.JsonNaming diff --git a/src/main/kotlin/mdnet/base/dao/metadata.kt b/src/main/kotlin/mdnet/base/data/metadata.kt similarity index 98% rename from src/main/kotlin/mdnet/base/dao/metadata.kt rename to src/main/kotlin/mdnet/base/data/metadata.kt index 80df223..bed3987 100644 --- a/src/main/kotlin/mdnet/base/dao/metadata.kt +++ b/src/main/kotlin/mdnet/base/data/metadata.kt @@ -16,7 +16,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this MangaDex@Home. If not, see . */ -package mdnet.base.dao +package mdnet.base.data import org.jetbrains.exposed.dao.Entity import org.jetbrains.exposed.dao.EntityClass diff --git a/src/main/kotlin/mdnet/base/netty/ApplicationNetty.kt b/src/main/kotlin/mdnet/base/netty/ApplicationNetty.kt index 02bdd82..9808e62 100644 --- a/src/main/kotlin/mdnet/base/netty/ApplicationNetty.kt +++ b/src/main/kotlin/mdnet/base/netty/ApplicationNetty.kt @@ -16,28 +16,24 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this MangaDex@Home. If not, see . */ +/* ktlint-disable no-wildcard-imports */ package mdnet.base.netty import io.netty.bootstrap.ServerBootstrap -import io.netty.channel.ChannelFactory -import io.netty.channel.ChannelFuture -import io.netty.channel.ChannelHandlerContext -import io.netty.channel.ChannelInboundHandlerAdapter -import io.netty.channel.ChannelInitializer -import io.netty.channel.ChannelOption -import io.netty.channel.ServerChannel +import io.netty.channel.* import io.netty.channel.nio.NioEventLoopGroup import io.netty.channel.socket.SocketChannel import io.netty.channel.socket.nio.NioServerSocketChannel import io.netty.handler.codec.DecoderException -import io.netty.handler.codec.http.HttpObjectAggregator -import io.netty.handler.codec.http.HttpServerCodec -import io.netty.handler.codec.http.HttpServerKeepAliveHandler +import io.netty.handler.codec.http.* import io.netty.handler.ssl.SslContextBuilder import io.netty.handler.stream.ChunkedWriteHandler +import io.netty.handler.timeout.ReadTimeoutHandler +import io.netty.handler.timeout.WriteTimeoutHandler import io.netty.handler.traffic.GlobalTrafficShapingHandler import io.netty.handler.traffic.TrafficCounter -import mdnet.base.Statistics +import mdnet.base.Constants +import mdnet.base.data.Statistics import mdnet.base.settings.ClientSettings import mdnet.base.settings.TlsCert import org.http4k.core.HttpHandler @@ -99,6 +95,10 @@ class Netty(private val tls: TlsCert, private val clientSettings: ClientSettings ch.pipeline().addLast("aggregator", HttpObjectAggregator(65536)) ch.pipeline().addLast("burstLimiter", burstLimiter) + + ch.pipeline().addLast("readTimeoutHandler", ReadTimeoutHandler(Constants.MAX_READ_TIME_SECONDS)) + ch.pipeline().addLast("writeTimeoutHandler", WriteTimeoutHandler(Constants.MAX_WRITE_TIME_SECONDS)) + ch.pipeline().addLast("streamer", ChunkedWriteHandler()) ch.pipeline().addLast("handler", Http4kChannelHandler(httpHandler)) diff --git a/src/main/kotlin/mdnet/base/netty/WebUiNetty.kt b/src/main/kotlin/mdnet/base/netty/WebUiNetty.kt index 682af0b..1bf98f3 100644 --- a/src/main/kotlin/mdnet/base/netty/WebUiNetty.kt +++ b/src/main/kotlin/mdnet/base/netty/WebUiNetty.kt @@ -37,9 +37,6 @@ import org.http4k.core.HttpHandler import org.http4k.server.Http4kChannelHandler import org.http4k.server.Http4kServer import org.http4k.server.ServerConfig -import org.slf4j.LoggerFactory - -private val LOGGER = LoggerFactory.getLogger(WebUiNetty::class.java) class WebUiNetty(private val hostname: String, private val port: Int) : ServerConfig { override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer { diff --git a/src/main/kotlin/mdnet/base/server/Application.kt b/src/main/kotlin/mdnet/base/server/Application.kt index 051c848..7f15e53 100644 --- a/src/main/kotlin/mdnet/base/server/Application.kt +++ b/src/main/kotlin/mdnet/base/server/Application.kt @@ -21,20 +21,22 @@ package mdnet.base.server import mdnet.base.netty.Netty import mdnet.base.settings.ServerSettings -import mdnet.base.Statistics +import mdnet.base.data.Statistics import mdnet.base.settings.ClientSettings import mdnet.cache.DiskLruCache -import org.http4k.core.Method -import org.http4k.core.then +import org.http4k.core.* import org.http4k.filter.ServerFilters import org.http4k.routing.bind import org.http4k.routing.routes import org.http4k.server.Http4kServer import org.http4k.server.asServer import org.jetbrains.exposed.sql.Database +import org.slf4j.LoggerFactory import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicReference +private val LOGGER = LoggerFactory.getLogger("Application") + fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSettings: ClientSettings, statistics: AtomicReference, isHandled: AtomicBoolean): Http4kServer { val database = Database.connect("jdbc:sqlite:cache/data.db", "org.sqlite.JDBC") val imageServer = ImageServer(cache, statistics, serverSettings.imageServer, database, isHandled) @@ -59,3 +61,21 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting ) .asServer(Netty(serverSettings.tls!!, clientSettings, statistics)) } + +fun timeRequest(): Filter { + return Filter { next: HttpHandler -> + { request: Request -> + val start = System.currentTimeMillis() + val response = next(request) + val latency = System.currentTimeMillis() - start + + if (LOGGER.isTraceEnabled && response.header("X-Uri") != null) { + val sanitizedUri = response.header("X-Uri") + if (LOGGER.isInfoEnabled) { + LOGGER.info("Request for $sanitizedUri completed in ${latency}ms") + } + } + response.header("X-Time-Taken", latency.toString()) + } + } +} diff --git a/src/main/kotlin/mdnet/base/server/ImageServer.kt b/src/main/kotlin/mdnet/base/server/ImageServer.kt index dfe3b51..17adbd5 100644 --- a/src/main/kotlin/mdnet/base/server/ImageServer.kt +++ b/src/main/kotlin/mdnet/base/server/ImageServer.kt @@ -20,9 +20,9 @@ along with this MangaDex@Home. If not, see . package mdnet.base.server import mdnet.base.Constants -import mdnet.base.Statistics -import mdnet.base.dao.ImageData -import mdnet.base.dao.ImageDatum +import mdnet.base.data.Statistics +import mdnet.base.data.ImageData +import mdnet.base.data.ImageDatum import mdnet.cache.CachingInputStream import mdnet.cache.DiskLruCache import org.apache.http.client.config.CookieSpecs @@ -30,7 +30,9 @@ import org.apache.http.client.config.RequestConfig import org.apache.http.impl.client.HttpClients import org.http4k.client.ApacheClient import org.http4k.core.* -import org.http4k.filter.MaxAgeTtl +import org.http4k.filter.CachingFilters +import org.http4k.filter.CorsPolicy +import org.http4k.filter.ServerFilters import org.http4k.lens.Path import org.jetbrains.exposed.sql.Database import org.jetbrains.exposed.sql.SchemaUtils @@ -41,6 +43,7 @@ import java.io.BufferedOutputStream import java.io.File import java.io.InputStream import java.security.MessageDigest +import java.time.Clock import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicReference @@ -70,7 +73,7 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi .setMaxConnPerRoute(THREADS_TO_ALLOCATE) .build()) - fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler = { request -> + fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler = baseHandler().then { request -> val chapterHash = Path.of("chapterHash")(request) val fileName = Path.of("fileName")(request) @@ -105,10 +108,7 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi handled.set(true) if (referer != null && !referer.startsWith("https://mangadex.org")) { - if (snapshot != null) { - snapshot.close() - } - + snapshot?.close() Response(Status.FORBIDDEN) } else if (snapshot != null && imageDatum != null) { request.handleCacheHit(sanitizedUri, getRc4(rc4Bytes), snapshot, imageDatum) @@ -255,13 +255,6 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi Response(Status.OK) .header("Content-Type", type) .header("X-Content-Type-Options", "nosniff") - .header( - "Cache-Control", - listOf("public", MaxAgeTtl(Constants.MAX_AGE_CACHE).toHeaderValue()).joinToString(", ") - ) - .header("Access-Control-Allow-Origin", "https://mangadex.org") - .header("Access-Control-Expose-Headers", "*") - .header("Timing-Allow-Origin", "https://mangadex.org") .let { if (length != null) { it.body(input, length.toLong()).header("Content-Length", length) @@ -280,6 +273,23 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi companion object { private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java) + + private fun baseHandler(): Filter = + CachingFilters.Response.MaxAge(Clock.systemUTC(), Constants.MAX_AGE_CACHE) + .then(ServerFilters.Cors( + CorsPolicy( + origins = listOf("https://mangadex.org"), + headers = listOf("*"), + methods = Method.values().toList() + ) + ) + ) + .then(Filter { next: HttpHandler -> + { request: Request -> + val response = next(request) + response.header("timing-allow-origin", "https://mangadex.org") + } + }) } } diff --git a/src/main/kotlin/mdnet/base/server/WebUi.kt b/src/main/kotlin/mdnet/base/server/WebUi.kt index b2f9e71..22ffacc 100644 --- a/src/main/kotlin/mdnet/base/server/WebUi.kt +++ b/src/main/kotlin/mdnet/base/server/WebUi.kt @@ -21,7 +21,7 @@ package mdnet.base.server import java.time.Instant import java.util.concurrent.atomic.AtomicReference -import mdnet.base.Statistics +import mdnet.base.data.Statistics import mdnet.base.netty.WebUiNetty import mdnet.base.settings.WebSettings import org.http4k.core.Body diff --git a/src/main/kotlin/mdnet/base/server/common.kt b/src/main/kotlin/mdnet/base/server/common.kt index 841d027..c224627 100644 --- a/src/main/kotlin/mdnet/base/server/common.kt +++ b/src/main/kotlin/mdnet/base/server/common.kt @@ -58,25 +58,3 @@ fun catchAllHideDetails(): Filter { } } } - -fun timeRequest(): Filter { - return Filter { next: HttpHandler -> - { request: Request -> - val start = System.currentTimeMillis() - val response = next(request) - val latency = System.currentTimeMillis() - start - if (LOGGER.isTraceEnabled && response.header("X-Uri") != null) { - // Dirty hack to get sanitizedUri from ImageServer - val sanitizedUri = response.header("X-Uri") - // Log in TRACE - if (LOGGER.isInfoEnabled) { - LOGGER.info("Request for $sanitizedUri completed in ${latency}ms") - } - // Delete response header entirely - response.header("X-Uri", null) - } - // Set response header with processing times - response.header("X-Time-Taken", latency.toString()) - } - } -}