Reorganize some code - add read and write timeouts

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [2020-06-23] Added Gitlab CI integration by [@lflare].
 - [2020-06-28] Added `client_external_port setting` [@wedge1001].
 - [2020-06-29] Added rudimentary support of Referer checking to mitigate hotlinking by [@lflare].
+- [2020-06-30] Added read and write timeouts to protect against some attacks [@carbotaniuman].
 
 ### Changed
 

build.gradle

@@ -30,12 +30,15 @@ dependencies {
   implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
   runtimeOnly group:"io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.30.Final"
 
-  implementation group:"ch.qos.logback", name: "logback-classic", version: "1.2.1"
+  implementation group:"ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha5"
 
   implementation group: "org.jetbrains.exposed", name: "exposed-core",  version: "$exposed_version"
   implementation group: "org.jetbrains.exposed", name: "exposed-dao",  version: "$exposed_version"
   implementation group: "org.jetbrains.exposed", name: "exposed-jdbc",  version: "$exposed_version"
   implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.30.1"
+
+  implementation "com.goterl.lazycode:lazysodium-java:4.2.6"
+  implementation "net.java.dev.jna:jna:5.5.0"
 }
 
 java {

src/main/kotlin/mdnet/base/Constants.kt

@@ -29,4 +29,7 @@ object Constants {
     const val WEBUI_VERSION = "0.1.1"
     val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
     val JACKSON: ObjectMapper = jacksonObjectMapper().configure(JsonParser.Feature.ALLOW_COMMENTS, true)
+
+    const val MAX_READ_TIME_SECONDS = 300
+    const val MAX_WRITE_TIME_SECONDS = 60
 }

src/main/kotlin/mdnet/base/MangaDexClient.kt

@@ -23,6 +23,7 @@ import ch.qos.logback.classic.LoggerContext
 import com.fasterxml.jackson.module.kotlin.readValue
 import mdnet.base.Constants.JACKSON
 import mdnet.base.Main.dieWithError
+import mdnet.base.data.Statistics
 import mdnet.base.server.getServer
 import mdnet.base.server.getUiServer
 import mdnet.base.settings.ClientSettings
@@ -64,7 +65,9 @@ class MangaDexClient(private val clientSettings: ClientSettings) {
                 return this.size > 240
             }
         })
-    private val statistics: AtomicReference<Statistics> = AtomicReference(Statistics())
+    private val statistics: AtomicReference<Statistics> = AtomicReference(
+        Statistics()
+    )
     private val isHandled: AtomicBoolean = AtomicBoolean(false)
     private var webUi: Http4kServer? = null
     private val cache: DiskLruCache

src/main/kotlin/mdnet/base/data/Statistics.kt

@@ -16,7 +16,7 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
  */
-package mdnet.base
+package mdnet.base.data
 
 import com.fasterxml.jackson.databind.PropertyNamingStrategy
 import com.fasterxml.jackson.databind.annotation.JsonNaming

src/main/kotlin/mdnet/base/data/metadata.kt

@@ -16,7 +16,7 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
  */
-package mdnet.base.dao
+package mdnet.base.data
 
 import org.jetbrains.exposed.dao.Entity
 import org.jetbrains.exposed.dao.EntityClass

src/main/kotlin/mdnet/base/netty/ApplicationNetty.kt

@@ -16,28 +16,24 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
  */
+/* ktlint-disable no-wildcard-imports */
 package mdnet.base.netty
 
 import io.netty.bootstrap.ServerBootstrap
-import io.netty.channel.ChannelFactory
+import io.netty.channel.*
-import io.netty.channel.ChannelFuture
-import io.netty.channel.ChannelHandlerContext
-import io.netty.channel.ChannelInboundHandlerAdapter
-import io.netty.channel.ChannelInitializer
-import io.netty.channel.ChannelOption
-import io.netty.channel.ServerChannel
 import io.netty.channel.nio.NioEventLoopGroup
 import io.netty.channel.socket.SocketChannel
 import io.netty.channel.socket.nio.NioServerSocketChannel
 import io.netty.handler.codec.DecoderException
-import io.netty.handler.codec.http.HttpObjectAggregator
+import io.netty.handler.codec.http.*
-import io.netty.handler.codec.http.HttpServerCodec
-import io.netty.handler.codec.http.HttpServerKeepAliveHandler
 import io.netty.handler.ssl.SslContextBuilder
 import io.netty.handler.stream.ChunkedWriteHandler
+import io.netty.handler.timeout.ReadTimeoutHandler
+import io.netty.handler.timeout.WriteTimeoutHandler
 import io.netty.handler.traffic.GlobalTrafficShapingHandler
 import io.netty.handler.traffic.TrafficCounter
-import mdnet.base.Statistics
+import mdnet.base.Constants
+import mdnet.base.data.Statistics
 import mdnet.base.settings.ClientSettings
 import mdnet.base.settings.TlsCert
 import org.http4k.core.HttpHandler
@@ -99,6 +95,10 @@ class Netty(private val tls: TlsCert, private val clientSettings: ClientSettings
                             ch.pipeline().addLast("aggregator", HttpObjectAggregator(65536))
 
                             ch.pipeline().addLast("burstLimiter", burstLimiter)
+
+                            ch.pipeline().addLast("readTimeoutHandler", ReadTimeoutHandler(Constants.MAX_READ_TIME_SECONDS))
+                            ch.pipeline().addLast("writeTimeoutHandler", WriteTimeoutHandler(Constants.MAX_WRITE_TIME_SECONDS))
+
                             ch.pipeline().addLast("streamer", ChunkedWriteHandler())
                             ch.pipeline().addLast("handler", Http4kChannelHandler(httpHandler))
 

src/main/kotlin/mdnet/base/netty/WebUiNetty.kt

@@ -37,9 +37,6 @@ import org.http4k.core.HttpHandler
 import org.http4k.server.Http4kChannelHandler
 import org.http4k.server.Http4kServer
 import org.http4k.server.ServerConfig
-import org.slf4j.LoggerFactory
-
-private val LOGGER = LoggerFactory.getLogger(WebUiNetty::class.java)
 
 class WebUiNetty(private val hostname: String, private val port: Int) : ServerConfig {
     override fun toServer(httpHandler: HttpHandler): Http4kServer = object : Http4kServer {

src/main/kotlin/mdnet/base/server/Application.kt

@@ -21,20 +21,22 @@ package mdnet.base.server
 
 import mdnet.base.netty.Netty
 import mdnet.base.settings.ServerSettings
-import mdnet.base.Statistics
+import mdnet.base.data.Statistics
 import mdnet.base.settings.ClientSettings
 import mdnet.cache.DiskLruCache
-import org.http4k.core.Method
+import org.http4k.core.*
-import org.http4k.core.then
 import org.http4k.filter.ServerFilters
 import org.http4k.routing.bind
 import org.http4k.routing.routes
 import org.http4k.server.Http4kServer
 import org.http4k.server.asServer
 import org.jetbrains.exposed.sql.Database
+import org.slf4j.LoggerFactory
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
 
+private val LOGGER = LoggerFactory.getLogger("Application")
+
 fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSettings: ClientSettings, statistics: AtomicReference<Statistics>, isHandled: AtomicBoolean): Http4kServer {
     val database = Database.connect("jdbc:sqlite:cache/data.db", "org.sqlite.JDBC")
     val imageServer = ImageServer(cache, statistics, serverSettings.imageServer, database, isHandled)
@@ -59,3 +61,21 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
             )
             .asServer(Netty(serverSettings.tls!!, clientSettings, statistics))
 }
+
+fun timeRequest(): Filter {
+    return Filter { next: HttpHandler ->
+        { request: Request ->
+            val start = System.currentTimeMillis()
+            val response = next(request)
+            val latency = System.currentTimeMillis() - start
+
+            if (LOGGER.isTraceEnabled && response.header("X-Uri") != null) {
+                val sanitizedUri = response.header("X-Uri")
+                if (LOGGER.isInfoEnabled) {
+                    LOGGER.info("Request for $sanitizedUri completed in ${latency}ms")
+                }
+            }
+            response.header("X-Time-Taken", latency.toString())
+        }
+    }
+}

src/main/kotlin/mdnet/base/server/ImageServer.kt

@@ -20,9 +20,9 @@ along with this MangaDex@Home.  If not, see <http://www.gnu.org/licenses/>.
 package mdnet.base.server
 
 import mdnet.base.Constants
-import mdnet.base.Statistics
+import mdnet.base.data.Statistics
-import mdnet.base.dao.ImageData
+import mdnet.base.data.ImageData
-import mdnet.base.dao.ImageDatum
+import mdnet.base.data.ImageDatum
 import mdnet.cache.CachingInputStream
 import mdnet.cache.DiskLruCache
 import org.apache.http.client.config.CookieSpecs
@@ -30,7 +30,9 @@ import org.apache.http.client.config.RequestConfig
 import org.apache.http.impl.client.HttpClients
 import org.http4k.client.ApacheClient
 import org.http4k.core.*
-import org.http4k.filter.MaxAgeTtl
+import org.http4k.filter.CachingFilters
+import org.http4k.filter.CorsPolicy
+import org.http4k.filter.ServerFilters
 import org.http4k.lens.Path
 import org.jetbrains.exposed.sql.Database
 import org.jetbrains.exposed.sql.SchemaUtils
@@ -41,6 +43,7 @@ import java.io.BufferedOutputStream
 import java.io.File
 import java.io.InputStream
 import java.security.MessageDigest
+import java.time.Clock
 import java.util.concurrent.Executors
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
@@ -70,7 +73,7 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
         .setMaxConnPerRoute(THREADS_TO_ALLOCATE)
         .build())
 
-    fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler = { request ->
+    fun handler(dataSaver: Boolean, tokenized: Boolean = false): HttpHandler = baseHandler().then { request ->
         val chapterHash = Path.of("chapterHash")(request)
         val fileName = Path.of("fileName")(request)
 
@@ -105,10 +108,7 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
 
         handled.set(true)
         if (referer != null && !referer.startsWith("https://mangadex.org")) {
-            if (snapshot != null) {
+            snapshot?.close()
-                snapshot.close()
-            }
-
             Response(Status.FORBIDDEN)
         } else if (snapshot != null && imageDatum != null) {
             request.handleCacheHit(sanitizedUri, getRc4(rc4Bytes), snapshot, imageDatum)
@@ -255,13 +255,6 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
         Response(Status.OK)
             .header("Content-Type", type)
             .header("X-Content-Type-Options", "nosniff")
-            .header(
-                "Cache-Control",
-                listOf("public", MaxAgeTtl(Constants.MAX_AGE_CACHE).toHeaderValue()).joinToString(", ")
-            )
-            .header("Access-Control-Allow-Origin", "https://mangadex.org")
-            .header("Access-Control-Expose-Headers", "*")
-            .header("Timing-Allow-Origin", "https://mangadex.org")
             .let {
                 if (length != null) {
                     it.body(input, length.toLong()).header("Content-Length", length)
@@ -280,6 +273,23 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
 
     companion object {
         private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java)
+
+        private fun baseHandler(): Filter =
+            CachingFilters.Response.MaxAge(Clock.systemUTC(), Constants.MAX_AGE_CACHE)
+                .then(ServerFilters.Cors(
+                        CorsPolicy(
+                            origins = listOf("https://mangadex.org"),
+                            headers = listOf("*"),
+                            methods = Method.values().toList()
+                        )
+                    )
+                )
+                .then(Filter { next: HttpHandler ->
+                    { request: Request ->
+                        val response = next(request)
+                        response.header("timing-allow-origin", "https://mangadex.org")
+                    }
+                })
     }
 }
 

src/main/kotlin/mdnet/base/server/WebUi.kt

@@ -21,7 +21,7 @@ package mdnet.base.server
 
 import java.time.Instant
 import java.util.concurrent.atomic.AtomicReference
-import mdnet.base.Statistics
+import mdnet.base.data.Statistics
 import mdnet.base.netty.WebUiNetty
 import mdnet.base.settings.WebSettings
 import org.http4k.core.Body

src/main/kotlin/mdnet/base/server/common.kt

@@ -58,25 +58,3 @@ fun catchAllHideDetails(): Filter {
         }
     }
 }
-
-fun timeRequest(): Filter {
-    return Filter { next: HttpHandler ->
-        { request: Request ->
-            val start = System.currentTimeMillis()
-            val response = next(request)
-            val latency = System.currentTimeMillis() - start
-            if (LOGGER.isTraceEnabled && response.header("X-Uri") != null) {
-                // Dirty hack to get sanitizedUri from ImageServer
-                val sanitizedUri = response.header("X-Uri")
-                // Log in TRACE
-                if (LOGGER.isInfoEnabled) {
-                    LOGGER.info("Request for $sanitizedUri completed in ${latency}ms")
-                }
-                // Delete response header entirely
-                response.header("X-Uri", null)
-            }
-            // Set response header with processing times
-            response.header("X-Time-Taken", latency.toString())
-        }
-    }
-}