From 3e68224cfaf15f6d5ca2fb416d2a185ef1800921 Mon Sep 17 00:00:00 2001 From: carbotaniuman Date: Fri, 5 Feb 2021 18:22:02 +0000 Subject: [PATCH] Add circuit breaker --- .gitlab-ci.yml | 1 + CHANGELOG.md | 15 +- build.gradle | 2 + src/main/kotlin/mdnet/Constants.kt | 2 +- src/main/kotlin/mdnet/server/ImageHandler.kt | 191 +++++++++++ src/main/kotlin/mdnet/server/ImageServer.kt | 299 +++--------------- src/main/kotlin/mdnet/server/TokenVerifier.kt | 102 ++++++ .../kotlin/mdnet/server/ImageServerTest.kt | 41 --- .../kotlin/mdnet/server/TokenVerifierTest.kt | 69 ++++ 9 files changed, 426 insertions(+), 296 deletions(-) create mode 100644 src/main/kotlin/mdnet/server/ImageHandler.kt create mode 100644 src/main/kotlin/mdnet/server/TokenVerifier.kt create mode 100644 src/test/kotlin/mdnet/server/TokenVerifierTest.kt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4782c7..ed94a2f 100755 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ build: stage: build script: - ./gradlew build + - "ls -lah build/libs" publish: image: alpine diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fcdfc7..5019f61 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,16 +17,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +## [2.0.0-rc8] - 2021-02-04 +### Added +- [2021-02-04] Installed a circuit breaker in order to save upstream [@carbotaniuman]. + ## [2.0.0-rc7] - 2021-01-31 ### Added -- [2021-01-28] Add distinct timerange and total egress stats to dashboard [@_tde9] -- [2021-01-28] Add average cache hitrate panel [@_tde9] +- [2021-01-31] Add distinct timerange and total egress stats to dashboard [@_tde9]. +- [2021-01-31] Add average cache hitrate panel [@_tde9]. ### Changed -- [2021-01-29] Add HikariCP connection pool [@carbotaniuman]. +- [2021-01-31] Add HikariCP connection pool [@carbotaniuman]. ### Fixed -- [2021-01-28] Fix client reqs dashboard panel [@_tde9] +- [2021-01-31] Fix client reqs dashboard panel [@_tde9]. ## [2.0.0-rc6] - 2021-01-28 ### Fixed @@ -328,7 +332,8 @@ This release contains many breaking changes! Of note are the changes to the cach ### Fixed - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman]. -[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc7...HEAD +[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc8...HEAD +[2.0.0-rc8]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc7...2.0.0-rc8 [2.0.0-rc7]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc6...2.0.0-rc7 [2.0.0-rc6]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc5...2.0.0-rc6 [2.0.0-rc5]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc4...2.0.0-rc5 diff --git a/build.gradle b/build.gradle index c35a659..602e252 100644 --- a/build.gradle +++ b/build.gradle @@ -33,6 +33,8 @@ dependencies { implementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.15.0" implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version" + implementation group: "org.http4k", name: "http4k-resilience4j", version: "$http_4k_version" + implementation group: "io.github.resilience4j", name: "resilience4j-micrometer", version: "1.6.1" implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version" implementation group: "com.fasterxml.jackson.dataformat", name: "jackson-dataformat-yaml", version: "2.12.1" implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.12.1" diff --git a/src/main/kotlin/mdnet/Constants.kt b/src/main/kotlin/mdnet/Constants.kt index 06b624d..d932c64 100644 --- a/src/main/kotlin/mdnet/Constants.kt +++ b/src/main/kotlin/mdnet/Constants.kt @@ -21,7 +21,7 @@ package mdnet import java.time.Duration object Constants { - const val CLIENT_BUILD = 25 + const val CLIENT_BUILD = 26 @JvmField val MAX_AGE_CACHE: Duration = Duration.ofDays(14) diff --git a/src/main/kotlin/mdnet/server/ImageHandler.kt b/src/main/kotlin/mdnet/server/ImageHandler.kt new file mode 100644 index 0000000..b0d5cf7 --- /dev/null +++ b/src/main/kotlin/mdnet/server/ImageHandler.kt @@ -0,0 +1,191 @@ +/* +Mangadex@Home +Copyright (c) 2020, MangaDex Network +This file is part of MangaDex@Home. + +MangaDex@Home is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +MangaDex@Home is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this MangaDex@Home. If not, see . +*/ +package mdnet.server + +import io.micrometer.core.instrument.Timer +import io.micrometer.prometheus.PrometheusMeterRegistry +import mdnet.Constants +import mdnet.cache.CachingInputStream +import mdnet.cache.Image +import mdnet.cache.ImageMetadata +import mdnet.cache.ImageStorage +import mdnet.logging.info +import mdnet.logging.trace +import mdnet.logging.warn +import org.http4k.core.* +import org.http4k.filter.CachingFilters +import org.http4k.lens.Path +import org.slf4j.LoggerFactory +import java.io.BufferedInputStream +import java.io.BufferedOutputStream +import java.io.InputStream +import java.time.Clock +import java.util.concurrent.Executors + +class ImageServer( + private val storage: ImageStorage, + private val upstream: HttpHandler, + registry: PrometheusMeterRegistry +) { + private val executor = Executors.newCachedThreadPool() + private val cacheLookupTimer = Timer.builder("cache_lookup") + .publishPercentiles(0.5, 0.75, 0.9, 0.99) + .register(registry) + + // This is part of the ImageServer, and it expects `chapterHash` and `fileName` path segments. + fun handler(dataSaver: Boolean): HttpHandler = baseHandler().then { request -> + val chapterHash = Path.of("chapterHash")(request) + val fileName = Path.of("fileName")(request) + + val sanitizedUri = if (dataSaver) { + "/data-saver" + } else { + "/data" + } + "/$chapterHash/$fileName" + + val imageId = if (dataSaver) { + md5Bytes("saver$chapterHash.$fileName") + } else { + md5Bytes("$chapterHash.$fileName") + }.let { + printHexString(it) + } + + val image: Image? = cacheLookupTimer.recordCallable { storage.loadImage(imageId) } + + if (image != null) { + request.handleCacheHit(sanitizedUri, image) + } else { + request.handleCacheMiss(sanitizedUri, imageId) + } + } + + private fun Request.handleCacheHit(sanitizedUri: String, image: Image): Response { + // our files never change, so it's safe to use the browser cache + return if (this.header("If-Modified-Since") != null) { + LOGGER.info { "Request for $sanitizedUri cached by browser" } + + val lastModified = image.data.lastModified + + Response(Status.NOT_MODIFIED) + .header("Last-Modified", lastModified) + } else { + LOGGER.info { "Request for $sanitizedUri is being served" } + + respondWithImage( + BufferedInputStream(image.stream), + image.data.size, image.data.contentType, image.data.lastModified, + true + ) + } + } + + private fun Request.handleCacheMiss(sanitizedUri: String, imageId: String): Response { + val mdResponse = upstream(Request(Method.GET, sanitizedUri)) + + if (mdResponse.status != Status.OK) { + LOGGER.warn { "Upstream query for $sanitizedUri errored with status ${mdResponse.status}" } + + mdResponse.close() + return Response(mdResponse.status) + } + + val contentType = mdResponse.header("Content-Type")!! + val contentLength = mdResponse.header("Content-Length")?.toInt() + val lastModified = mdResponse.header("Last-Modified") + + if (!contentType.isImageMimetype()) { + LOGGER.warn { "Upstream query for $sanitizedUri returned bad mimetype $contentType" } + mdResponse.close() + return Response(Status.INTERNAL_SERVER_ERROR) + } + + // bad upstream responses mean we can't cache, so bail + if (contentLength == null || lastModified == null) { + LOGGER.info { "Request for $sanitizedUri is being served due to upstream issues" } + return respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false) + } + + LOGGER.trace { "Upstream query for $sanitizedUri succeeded" } + + val writer = storage.storeImage(imageId, ImageMetadata(contentType, lastModified, contentLength)) + + // A null writer means that this file is being written to + // concurrently so we skip the cache process + return if (writer != null) { + LOGGER.info { "Request for $sanitizedUri is being cached and served" } + + val tee = CachingInputStream( + mdResponse.body.stream, + executor, BufferedOutputStream(writer.stream), + ) { + try { + if (writer.commit(contentLength)) { + LOGGER.info { "Cache download for $sanitizedUri committed" } + } else { + LOGGER.info { "Cache download for $sanitizedUri aborted" } + } + } catch (e: Exception) { + LOGGER.warn(e) { "Cache go/no go for $sanitizedUri failed" } + } + } + respondWithImage(tee, contentLength, contentType, lastModified, false) + } else { + LOGGER.info { "Request for $sanitizedUri is being served" } + respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false) + } + } + + private fun respondWithImage(input: InputStream, length: Int?, type: String, lastModified: String?, cached: Boolean): Response = + Response(Status.OK) + .header("Content-Type", type) + .header("X-Content-Type-Options", "nosniff") + .let { + if (length != null) { + it.body(input, length.toLong()).header("Content-Length", length.toString()) + } else { + it.body(input).header("Transfer-Encoding", "chunked") + } + } + .let { + if (lastModified != null) { + it.header("Last-Modified", lastModified) + } else { + it + } + } + .header("X-Cache", if (cached) "HIT" else "MISS") + + companion object { + private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java) + private fun String.isImageMimetype() = this.toLowerCase().startsWith("image/") + + private fun baseHandler(): Filter = + CachingFilters.Response.MaxAge(Clock.systemUTC(), Constants.MAX_AGE_CACHE) + .then { next: HttpHandler -> + { request: Request -> + val response = next(request) + response.header("access-control-allow-origin", "https://mangadex.org") + .header("access-control-allow-headers", "*") + .header("access-control-allow-methods", "GET") + .header("timing-allow-origin", "https://mangadex.org") + } + } + } +} diff --git a/src/main/kotlin/mdnet/server/ImageServer.kt b/src/main/kotlin/mdnet/server/ImageServer.kt index 7136b97..329c4fa 100644 --- a/src/main/kotlin/mdnet/server/ImageServer.kt +++ b/src/main/kotlin/mdnet/server/ImageServer.kt @@ -18,208 +18,28 @@ along with this MangaDex@Home. If not, see . */ package mdnet.server -import com.fasterxml.jackson.core.JsonProcessingException -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.fasterxml.jackson.module.kotlin.readValue +import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry +import io.github.resilience4j.micrometer.tagged.TaggedCircuitBreakerMetrics import io.micrometer.core.instrument.FunctionCounter -import io.micrometer.core.instrument.Timer import io.micrometer.prometheus.PrometheusMeterRegistry -import mdnet.Constants -import mdnet.cache.CachingInputStream -import mdnet.cache.Image -import mdnet.cache.ImageMetadata import mdnet.cache.ImageStorage import mdnet.data.Statistics -import mdnet.data.Token import mdnet.logging.info -import mdnet.logging.trace import mdnet.logging.warn import mdnet.metrics.GeoIpMetricsFilterBuilder import mdnet.metrics.PostTransactionLabeler import mdnet.netty.Netty -import mdnet.security.TweetNaclFast import mdnet.settings.MetricsSettings import mdnet.settings.RemoteSettings import mdnet.settings.ServerSettings import org.http4k.core.* -import org.http4k.filter.CachingFilters -import org.http4k.filter.ClientFilters -import org.http4k.filter.MicrometerMetrics -import org.http4k.filter.ServerFilters -import org.http4k.lens.LensFailure -import org.http4k.lens.Path -import org.http4k.routing.bind -import org.http4k.routing.routes +import org.http4k.filter.* +import org.http4k.routing.* import org.http4k.server.Http4kServer import org.http4k.server.asServer import org.slf4j.LoggerFactory -import java.io.BufferedInputStream -import java.io.BufferedOutputStream -import java.io.InputStream -import java.time.Clock -import java.time.OffsetDateTime -import java.util.* -import java.util.concurrent.Executors - -private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java) -private val JACKSON: ObjectMapper = jacksonObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .registerModule(JavaTimeModule()) - -class ImageServer( - private val storage: ImageStorage, - private val upstream: HttpHandler, - registry: PrometheusMeterRegistry -) { - private val executor = Executors.newCachedThreadPool() - private val cacheLookupTimer = Timer - .builder("cache_lookup") - .publishPercentiles(0.5, 0.75, 0.9, 0.99) - .register(registry) - - // This is part of the ImageServer, and it expects `chapterHash` and `fileName` path segments. - fun handler(dataSaver: Boolean): HttpHandler = baseHandler().then { request -> - val chapterHash = Path.of("chapterHash")(request) - val fileName = Path.of("fileName")(request) - - val sanitizedUri = if (dataSaver) { - "/data-saver" - } else { - "/data" - } + "/$chapterHash/$fileName" - - val imageId = if (dataSaver) { - md5Bytes("saver$chapterHash.$fileName") - } else { - md5Bytes("$chapterHash.$fileName") - }.let { - printHexString(it) - } - - val image: Image? = cacheLookupTimer.recordCallable { storage.loadImage(imageId) } - - if (image != null) { - request.handleCacheHit(sanitizedUri, image) - } else { - request.handleCacheMiss(sanitizedUri, imageId) - } - } - - private fun Request.handleCacheHit(sanitizedUri: String, image: Image): Response { - // our files never change, so it's safe to use the browser cache - return if (this.header("If-Modified-Since") != null) { - LOGGER.info { "Request for $sanitizedUri cached by browser" } - - val lastModified = image.data.lastModified - - Response(Status.NOT_MODIFIED) - .header("Last-Modified", lastModified) - } else { - LOGGER.info { "Request for $sanitizedUri is being served" } - - respondWithImage( - BufferedInputStream(image.stream), - image.data.size, image.data.contentType, image.data.lastModified, - true - ) - } - } - - private fun Request.handleCacheMiss(sanitizedUri: String, imageId: String): Response { - val mdResponse = upstream(Request(Method.GET, sanitizedUri)) - - if (mdResponse.status != Status.OK) { - LOGGER.warn { "Upstream query for $sanitizedUri errored with status ${mdResponse.status}" } - - mdResponse.close() - return Response(mdResponse.status) - } - - val contentType = mdResponse.header("Content-Type")!! - val contentLength = mdResponse.header("Content-Length")?.toInt() - val lastModified = mdResponse.header("Last-Modified") - - if (!contentType.isImageMimetype()) { - LOGGER.warn { "Upstream query for $sanitizedUri returned bad mimetype $contentType" } - mdResponse.close() - return Response(Status.INTERNAL_SERVER_ERROR) - } - - // bad upstream responses mean we can't cache, so bail - if (contentLength == null || lastModified == null) { - LOGGER.info { "Request for $sanitizedUri is being served due to upstream issues" } - return respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false) - } - - LOGGER.trace { "Upstream query for $sanitizedUri succeeded" } - - val writer = storage.storeImage(imageId, ImageMetadata(contentType, lastModified, contentLength)) - - // A null writer means that this file is being written to - // concurrently so we skip the cache process - return if (writer != null) { - LOGGER.info { "Request for $sanitizedUri is being cached and served" } - - val tee = CachingInputStream( - mdResponse.body.stream, - executor, BufferedOutputStream(writer.stream), - ) { - try { - if (writer.commit(contentLength)) { - LOGGER.info { "Cache download for $sanitizedUri committed" } - } else { - LOGGER.info { "Cache download for $sanitizedUri aborted" } - } - } catch (e: Exception) { - LOGGER.warn(e) { "Cache go/no go for $sanitizedUri failed" } - } - } - respondWithImage(tee, contentLength, contentType, lastModified, false) - } else { - LOGGER.info { "Request for $sanitizedUri is being served" } - respondWithImage(mdResponse.body.stream, contentLength, contentType, lastModified, false) - } - } - - private fun respondWithImage(input: InputStream, length: Int?, type: String, lastModified: String?, cached: Boolean): Response = - Response(Status.OK) - .header("Content-Type", type) - .header("X-Content-Type-Options", "nosniff") - .let { - if (length != null) { - it.body(input, length.toLong()).header("Content-Length", length.toString()) - } else { - it.body(input).header("Transfer-Encoding", "chunked") - } - } - .let { - if (lastModified != null) { - it.header("Last-Modified", lastModified) - } else { - it - } - } - .header("X-Cache", if (cached) "HIT" else "MISS") - - companion object { - private fun baseHandler(): Filter = - CachingFilters.Response.MaxAge(Clock.systemUTC(), Constants.MAX_AGE_CACHE) - .then { next: HttpHandler -> - { request: Request -> - val response = next(request) - response.header("access-control-allow-origin", "https://mangadex.org") - .header("access-control-allow-headers", "*") - .header("access-control-allow-methods", "GET") - .header("timing-allow-origin", "https://mangadex.org") - } - } - } -} - -private fun String.isImageMimetype() = this.toLowerCase().startsWith("image/") +import java.time.Duration fun getServer( storage: ImageStorage, @@ -230,10 +50,46 @@ fun getServer( registry: PrometheusMeterRegistry, client: HttpHandler ): Http4kServer { - val upstream = - ClientFilters.SetBaseUriFrom(remoteSettings.imageServer) - .then(ClientFilters.MicrometerMetrics.RequestTimer(registry)) - .then(client) + val circuitBreakerRegistry = CircuitBreakerRegistry.ofDefaults() + val circuitBreaker = circuitBreakerRegistry.circuitBreaker( + "upstream", + CircuitBreakerConfig.custom() + .slidingWindow(50, 20, CircuitBreakerConfig.SlidingWindowType.COUNT_BASED) + .permittedNumberOfCallsInHalfOpenState(10) + .slowCallDurationThreshold(Duration.ofSeconds(20)) + .waitDurationInOpenState(Duration.ofMinutes(2)) + .build() + ) + + circuitBreaker.eventPublisher.onFailureRateExceeded { + LOGGER.warn { "Circuit breaker has exceeded failure rate" } + } + + circuitBreaker.eventPublisher.onSlowCallRateExceeded { + LOGGER.warn { "Circuit breaker has exceeded slow call rate" } + } + + circuitBreaker.eventPublisher.onReset { + LOGGER.warn { "Circuit breaker has rest" } + } + + circuitBreaker.eventPublisher.onStateTransition { + LOGGER.warn { "Circuit breaker has moved from ${it.stateTransition.fromState} to ${it.stateTransition.toState}" } + } + + TaggedCircuitBreakerMetrics + .ofCircuitBreakerRegistry(circuitBreakerRegistry) + .bindTo(registry) + + val circuited = ResilienceFilters.CircuitBreak( + circuitBreaker, + isError = { r: Response -> !r.status.successful } + ) + + val upstream = ClientFilters.MicrometerMetrics.RequestTimer(registry) + .then(ClientFilters.SetBaseUriFrom(remoteSettings.imageServer)) + .then(circuited) + .then(client) val imageServer = ImageServer( storage = storage, @@ -247,7 +103,7 @@ fun getServer( { it.bytesSent.get().toDouble() } ).register(registry) - val verifier = tokenVerifier( + val verifier = TokenVerifier( tokenKey = remoteSettings.tokenKey, shouldVerify = { chapter, _ -> !remoteSettings.disableTokens && !(chapter == "1b682e7b24ae7dbdc5064eeeb8e8e353" || chapter == "8172a46adc798f4f4ace6663322a383e") @@ -281,7 +137,7 @@ fun getServer( ), "/prometheus" bind Method.GET to { Response(Status.OK).body(registry.scrape()) - } + }, ).withFilter( ServerFilters.MicrometerMetrics.RequestTimer(registry, labeler = PostTransactionLabeler()) ).withFilter( @@ -291,6 +147,8 @@ fun getServer( .asServer(Netty(remoteSettings.tls!!, serverSettings, statistics)) } +private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java) + fun timeRequest(): Filter { return Filter { next: HttpHandler -> { request: Request -> @@ -308,60 +166,3 @@ fun timeRequest(): Filter { } } } - -fun tokenVerifier(tokenKey: ByteArray, shouldVerify: (String, String) -> Boolean): Filter { - val box = TweetNaclFast.SecretBox(tokenKey) - - return Filter { next -> - then@{ - val chapterHash = Path.of("chapterHash")(it) - val fileName = Path.of("fileName")(it) - - if (shouldVerify(chapterHash, fileName)) { - val cleanedUri = it.uri.path.replaceBefore("/data", "/{token}") - - val tokenArr = try { - val toDecode = try { - Path.of("token")(it) - } catch (e: LensFailure) { - LOGGER.info(e) { "Request for $cleanedUri rejected for missing token" } - return@then Response(Status.FORBIDDEN).body("Token is missing") - } - Base64.getUrlDecoder().decode(toDecode) - } catch (e: IllegalArgumentException) { - LOGGER.info(e) { "Request for $cleanedUri rejected for non-base64 token" } - return@then Response(Status.FORBIDDEN).body("Token is invalid base64") - } - if (tokenArr.size < 24) { - LOGGER.info { "Request for $cleanedUri rejected for invalid token" } - return@then Response(Status.FORBIDDEN) - } - val token = try { - JACKSON.readValue( - box.open(tokenArr.sliceArray(24 until tokenArr.size), tokenArr.sliceArray(0 until 24)).apply { - if (this == null) { - LOGGER.info { "Request for $cleanedUri rejected for invalid token" } - return@then Response(Status.FORBIDDEN) - } - } - ) - } catch (e: JsonProcessingException) { - LOGGER.info(e) { "Request for $cleanedUri rejected for invalid token" } - return@then Response(Status.FORBIDDEN).body("Token is invalid") - } - - if (OffsetDateTime.now().isAfter(token.expires)) { - LOGGER.info { "Request for $cleanedUri rejected for expired token" } - return@then Response(Status.GONE).body("Token has expired") - } - - if (token.hash != chapterHash) { - LOGGER.info { "Request for $cleanedUri rejected for inapplicable token" } - return@then Response(Status.FORBIDDEN).body("Token is inapplicable for the image") - } - } - - return@then next(it) - } - } -} diff --git a/src/main/kotlin/mdnet/server/TokenVerifier.kt b/src/main/kotlin/mdnet/server/TokenVerifier.kt new file mode 100644 index 0000000..c2f5d3f --- /dev/null +++ b/src/main/kotlin/mdnet/server/TokenVerifier.kt @@ -0,0 +1,102 @@ +/* +Mangadex@Home +Copyright (c) 2020, MangaDex Network +This file is part of MangaDex@Home. + +MangaDex@Home is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +MangaDex@Home is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this MangaDex@Home. If not, see . +*/ +package mdnet.server + +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import mdnet.data.Token +import mdnet.logging.info +import mdnet.security.TweetNaclFast +import org.http4k.core.Filter +import org.http4k.core.HttpHandler +import org.http4k.core.Response +import org.http4k.core.Status +import org.http4k.lens.LensFailure +import org.http4k.lens.Path +import org.slf4j.LoggerFactory +import java.time.OffsetDateTime +import java.util.Base64 + +class TokenVerifier(tokenKey: ByteArray, private val shouldVerify: (String, String) -> Boolean) : Filter { + private val box = TweetNaclFast.SecretBox(tokenKey) + + override fun invoke(next: HttpHandler): HttpHandler { + return then@{ + val chapterHash = Path.of("chapterHash")(it) + val fileName = Path.of("fileName")(it) + + if (shouldVerify(chapterHash, fileName)) { + val cleanedUri = it.uri.path.replaceBefore("/data", "/{token}") + + val tokenArr = try { + val toDecode = try { + Path.of("token")(it) + } catch (e: LensFailure) { + LOGGER.info(e) { "Request for $cleanedUri rejected for missing token" } + return@then Response(Status.FORBIDDEN).body("Token is missing") + } + Base64.getUrlDecoder().decode(toDecode) + } catch (e: IllegalArgumentException) { + LOGGER.info(e) { "Request for $cleanedUri rejected for non-base64 token" } + return@then Response(Status.FORBIDDEN).body("Token is invalid base64") + } + if (tokenArr.size < 24) { + LOGGER.info { "Request for $cleanedUri rejected for invalid token" } + return@then Response(Status.FORBIDDEN) + } + val token = try { + JACKSON.readValue( + box.open(tokenArr.sliceArray(24 until tokenArr.size), tokenArr.sliceArray(0 until 24)).apply { + if (this == null) { + LOGGER.info { "Request for $cleanedUri rejected for invalid token" } + return@then Response(Status.FORBIDDEN) + } + } + ) + } catch (e: JsonProcessingException) { + LOGGER.info(e) { "Request for $cleanedUri rejected for invalid token" } + return@then Response(Status.FORBIDDEN).body("Token is invalid") + } + + if (OffsetDateTime.now().isAfter(token.expires)) { + LOGGER.info { "Request for $cleanedUri rejected for expired token" } + return@then Response(Status.GONE).body("Token has expired") + } + + if (token.hash != chapterHash) { + LOGGER.info { "Request for $cleanedUri rejected for inapplicable token" } + return@then Response(Status.FORBIDDEN).body("Token is inapplicable for the image") + } + } + + return@then next(it) + } + } + + companion object { + private val LOGGER = LoggerFactory.getLogger(TokenVerifier::class.java) + private val JACKSON: ObjectMapper = jacksonObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .registerModule(JavaTimeModule()) + } +} diff --git a/src/test/kotlin/mdnet/server/ImageServerTest.kt b/src/test/kotlin/mdnet/server/ImageServerTest.kt index dc902b3..a1a409c 100644 --- a/src/test/kotlin/mdnet/server/ImageServerTest.kt +++ b/src/test/kotlin/mdnet/server/ImageServerTest.kt @@ -31,17 +31,14 @@ import io.mockk.every import io.mockk.mockk import io.mockk.verify import mdnet.cache.ImageStorage -import mdnet.security.TweetNaclFast import org.apache.commons.io.IOUtils import org.http4k.core.HttpHandler import org.http4k.core.Method import org.http4k.core.Request import org.http4k.core.Response import org.http4k.core.Status -import org.http4k.core.then import org.http4k.kotest.shouldHaveHeader import org.http4k.kotest.shouldHaveStatus -import org.http4k.kotest.shouldNotHaveStatus import org.http4k.routing.bind import org.http4k.routing.routes import org.ktorm.database.Database @@ -249,41 +246,3 @@ class ImageServerTest : FreeSpec() { .header("Content-Length", "${data.size}") .header("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") } - -class TokenVerifierTest : FreeSpec() { - init { - val remoteKeys = TweetNaclFast.Box.keyPair() - val clientKeys = TweetNaclFast.Box.keyPair() - val box = TweetNaclFast.Box(clientKeys.publicKey, remoteKeys.secretKey) - - val backend = tokenVerifier(box.before()) { _, _ -> - true - }.then { - Response(Status.OK) - } - - val handler = routes( - "/data/{chapterHash}/{fileName}" bind Method.GET to backend, - "/data-saver/{chapterHash}/{fileName}" bind Method.GET to backend, - "/{token}/data/{chapterHash}/{fileName}" bind Method.GET to backend, - "/{token}/data-saver/{chapterHash}/{fileName}" bind Method.GET to backend, - ) - - "invalid" - { - "missing token should fail" { - val response = handler(Request(Method.GET, "/data/02181a8f5fe8cd408720a771dd129fd8/T2.png")) - response.shouldNotHaveStatus(Status.OK) - } - - "too short token should fail" { - val response = handler(Request(Method.GET, "/a/data/02181a8f5fe8cd408720a771dd129fd8/T2.png")) - response.shouldNotHaveStatus(Status.OK) - } - - "invalid token should fail" { - val response = handler(Request(Method.GET, "/MTIzM2Vhd2Z3YWVmbG1pbzJuM29pNG4yaXAzNG1wMSwyWzMscHdxZWVlZWVlZXBscWFkcVt3ZGwxWzJsM3BbMWwycFsxZSxwMVssZmRbcGF3LGZwW2F3ZnBbLA==/data/02181a8f5fe8cd408720a771dd129fd8/T2.png")) - response.shouldNotHaveStatus(Status.OK) - } - } - } -} diff --git a/src/test/kotlin/mdnet/server/TokenVerifierTest.kt b/src/test/kotlin/mdnet/server/TokenVerifierTest.kt new file mode 100644 index 0000000..96561c0 --- /dev/null +++ b/src/test/kotlin/mdnet/server/TokenVerifierTest.kt @@ -0,0 +1,69 @@ +/* +Mangadex@Home +Copyright (c) 2020, MangaDex Network +This file is part of MangaDex@Home. + +MangaDex@Home is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +MangaDex@Home is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this MangaDex@Home. If not, see . +*/ +package mdnet.server + +import io.kotest.core.spec.style.FreeSpec +import mdnet.security.TweetNaclFast +import org.http4k.core.* +import org.http4k.kotest.shouldNotHaveStatus +import org.http4k.routing.bind +import org.http4k.routing.routes + +class TokenVerifierTest : FreeSpec() { + init { + val remoteKeys = TweetNaclFast.Box.keyPair() + val clientKeys = TweetNaclFast.Box.keyPair() + val box = TweetNaclFast.Box(clientKeys.publicKey, remoteKeys.secretKey) + + val backend = TokenVerifier(box.before()) { _, _ -> + true + }.then { + Response(Status.OK) + } + + val handler = routes( + "/data/{chapterHash}/{fileName}" bind Method.GET to backend, + "/data-saver/{chapterHash}/{fileName}" bind Method.GET to backend, + "/{token}/data/{chapterHash}/{fileName}" bind Method.GET to backend, + "/{token}/data-saver/{chapterHash}/{fileName}" bind Method.GET to backend, + ) + + "invalid" - { + "missing token should fail" { + val response = handler(Request(Method.GET, "/data/02181a8f5fe8cd408720a771dd129fd8/T2.png")) + response.shouldNotHaveStatus(Status.OK) + } + + "too short token should fail" { + val response = handler(Request(Method.GET, "/a/data/02181a8f5fe8cd408720a771dd129fd8/T2.png")) + response.shouldNotHaveStatus(Status.OK) + } + + "invalid token should fail" { + val response = handler( + Request( + Method.GET, + "/MTIzM2Vhd2Z3YWVmbG1pbzJuM29pNG4yaXAzNG1wMSwyWzMscHdxZWVlZWVlZXBscWFkcVt3ZGwxWzJsM3BbMWwycFsxZSxwMVssZmRbcGF3LGZwW2F3ZnBbLA==/data/02181a8f5fe8cd408720a771dd129fd8/T2.png" + ) + ) + response.shouldNotHaveStatus(Status.OK) + } + } + } +}