From bba979c2125c1268f55bdb41c2e182265f38c215 Mon Sep 17 00:00:00 2001 From: carbotaniuman <41451839+carbotaniuman@users.noreply.github.com> Date: Sun, 12 Jul 2020 18:23:16 -0500 Subject: [PATCH 1/4] Bump version --- build.gradle | 158 +++++++++++++++++++++++++-------------------------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/build.gradle b/build.gradle index 55ed557..25ce7a6 100644 --- a/build.gradle +++ b/build.gradle @@ -1,79 +1,79 @@ -plugins { - id "java" - id "org.jetbrains.kotlin.jvm" version "1.3.72" - id "application" - id "com.github.johnrengelman.shadow" version "5.2.0" - id "com.diffplug.gradle.spotless" version "4.4.0" - id "dev.afanasev.sekret" version "0.0.3" -} - -group = "com.mangadex" -version = "git describe --tags --dirty".execute().text.trim() -mainClassName = "mdnet.base.Main" - -repositories { - mavenCentral() - jcenter() -} - -configurations { - runtime.exclude group: "org.jetbrains.kotlinx", module: "kotlinx-coroutines-core" - runtime.exclude group: "com.sun.mail", module: "javax.mail" -} - -dependencies { - compileOnly group: "dev.afanasev", name: "sekret-annotation", version: "0.0.3" - - implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" - implementation "org.jetbrains.kotlin:kotlin-reflect" - - implementation group: "commons-io", name: "commons-io", version: "2.7" - - implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version" - implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version" - implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.11.1" - implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version" - implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" - runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.30.Final" - - implementation group: "ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha4" - implementation group: "org.jetbrains.exposed", name: "exposed-core", version: "$exposed_version" - implementation group: "org.jetbrains.exposed", name: "exposed-dao", version: "$exposed_version" - implementation group: "org.jetbrains.exposed", name: "exposed-jdbc", version: "$exposed_version" - - implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.30.1" - - implementation "com.goterl.lazycode:lazysodium-java:4.2.6" - implementation "net.java.dev.jna:jna:5.5.0" -} - -java { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 -} - -spotless { - lineEndings 'UNIX' - java { - targetExclude("build/generated/**/*") - eclipse() - removeUnusedImports() - trimTrailingWhitespace() - endWithNewline() - } - kotlin { - ktlint() - trimTrailingWhitespace() - endWithNewline() - } -} - -tasks.register("generateVersion", Copy) { - def templateContext = [version: version] - inputs.properties templateContext - from "src/template/java" - into "$buildDir/generated/java" - expand templateContext -} - -sourceSets.main.java.srcDir generateVersion.outputs.files +plugins { + id "java" + id "org.jetbrains.kotlin.jvm" version "1.3.72" + id "application" + id "com.github.johnrengelman.shadow" version "5.2.0" + id "com.diffplug.gradle.spotless" version "4.4.0" + id "dev.afanasev.sekret" version "0.0.3" +} + +group = "com.mangadex" +version = "git describe --tags --dirty".execute().text.trim() +mainClassName = "mdnet.base.Main" + +repositories { + mavenCentral() + jcenter() +} + +configurations { + runtime.exclude group: "org.jetbrains.kotlinx", module: "kotlinx-coroutines-core" + runtime.exclude group: "com.sun.mail", module: "javax.mail" +} + +dependencies { + compileOnly group: "dev.afanasev", name: "sekret-annotation", version: "0.0.3" + + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" + implementation "org.jetbrains.kotlin:kotlin-reflect" + + implementation group: "commons-io", name: "commons-io", version: "2.7" + + implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version" + implementation group: "org.http4k", name: "http4k-format-jackson", version: "$http_4k_version" + implementation group: "com.fasterxml.jackson.datatype", name: "jackson-datatype-jsr310", version: "2.11.1" + implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version" + implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" + runtimeOnly group: "io.netty", name: "netty-tcnative-boringssl-static", version: "2.0.30.Final" + + implementation group: "ch.qos.logback", name: "logback-classic", version: "1.3.0-alpha4" + implementation group: "org.jetbrains.exposed", name: "exposed-core", version: "$exposed_version" + implementation group: "org.jetbrains.exposed", name: "exposed-dao", version: "$exposed_version" + implementation group: "org.jetbrains.exposed", name: "exposed-jdbc", version: "$exposed_version" + + implementation group: "org.xerial", name: "sqlite-jdbc", version: "3.30.1" + + implementation "com.goterl.lazycode:lazysodium-java:4.3.0" + implementation "net.java.dev.jna:jna:5.5.0" +} + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + +spotless { + lineEndings 'UNIX' + java { + targetExclude("build/generated/**/*") + eclipse() + removeUnusedImports() + trimTrailingWhitespace() + endWithNewline() + } + kotlin { + ktlint() + trimTrailingWhitespace() + endWithNewline() + } +} + +tasks.register("generateVersion", Copy) { + def templateContext = [version: version] + inputs.properties templateContext + from "src/template/java" + into "$buildDir/generated/java" + expand templateContext +} + +sourceSets.main.java.srcDir generateVersion.outputs.files From 3558b5ad502b6cd29e00bdc31e406919708a0e2a Mon Sep 17 00:00:00 2001 From: AviKav Date: Tue, 14 Jul 2020 03:50:42 -0400 Subject: [PATCH 2/4] Fixes caching HTML from 404s --- .../kotlin/mdnet/base/server/ImageServer.kt | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/mdnet/base/server/ImageServer.kt b/src/main/kotlin/mdnet/base/server/ImageServer.kt index aebc2aa..7208a1a 100644 --- a/src/main/kotlin/mdnet/base/server/ImageServer.kt +++ b/src/main/kotlin/mdnet/base/server/ImageServer.kt @@ -133,13 +133,25 @@ class ImageServer( } val imageId = printHexString(rc4Bytes) - val snapshot = cache.getUnsafe(imageId.toCacheId()) - val imageDatum = synchronized(database) { + // These shouldn't be var but I don't want to touch the original conditional logic + var snapshot = cache.getUnsafe(imageId.toCacheId()) + var imageDatum = synchronized(database) { transaction(database) { ImageDatum.findById(imageId) } } + // TODO: Cleanup messy conditionals + + // Ensure that cached file isn't a non-image from the Great Cache Propagation + val flaresFault = imageDatum != null && imageDatum.contentType.isImage().not() + + if (flaresFault) { + snapshot?.close(); snapshot = null + cache.removeUnsafe(imageId.toCacheId()); imageDatum = null + LOGGER.warn { "Removing cache file for $sanitizedUri left over the Great Cache Propagation" } + } + if (snapshot != null && imageDatum != null) { request.handleCacheHit(sanitizedUri, getRc4(rc4Bytes), snapshot, imageDatum) } else { @@ -169,6 +181,8 @@ class ImageServer( } } + private fun String.isImage() = this.toLowerCase().startsWith("image/") + private fun Request.handleCacheHit(sanitizedUri: String, cipher: Cipher, snapshot: DiskLruCache.Snapshot, imageDatum: ImageDatum): Response { // our files never change, so it's safe to use the browser cache return if (this.header("If-Modified-Since") != null) { @@ -207,19 +221,26 @@ class ImageServer( val mdResponse = client(Request(Method.GET, "${serverSettings.imageServer}$sanitizedUri")) + val contentType = mdResponse.header("Content-Type") + val contentLength = mdResponse.header("Content-Length") + val lastModified = mdResponse.header("Last-Modified") + if (mdResponse.status != Status.OK) { LOGGER.trace { "Upstream query for $sanitizedUri errored with status ${mdResponse.status}" } + mdResponse.close() + return Response(mdResponse.status) + } + contentType!! + + if (!contentType.isImage()) { + LOGGER.trace { "Upstream query for $sanitizedUri returned bad mime-type" } mdResponse.close() return Response(mdResponse.status) } LOGGER.trace { "Upstream query for $sanitizedUri succeeded" } - val contentType = mdResponse.header("Content-Type")!! - val contentLength = mdResponse.header("Content-Length") - val lastModified = mdResponse.header("Last-Modified") - val editor = cache.editUnsafe(imageId.toCacheId()) // A null editor means that this file is being written to From 0caa1f00a6ad0276b91bb47ce876808ee594b887 Mon Sep 17 00:00:00 2001 From: AviKav Date: Tue, 14 Jul 2020 03:56:16 -0400 Subject: [PATCH 3/4] Add info to bad mimetype log entry and change spelling --- src/main/kotlin/mdnet/base/server/ImageServer.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/mdnet/base/server/ImageServer.kt b/src/main/kotlin/mdnet/base/server/ImageServer.kt index 7208a1a..8466cb7 100644 --- a/src/main/kotlin/mdnet/base/server/ImageServer.kt +++ b/src/main/kotlin/mdnet/base/server/ImageServer.kt @@ -234,7 +234,7 @@ class ImageServer( contentType!! if (!contentType.isImage()) { - LOGGER.trace { "Upstream query for $sanitizedUri returned bad mime-type" } + LOGGER.trace { "Upstream query for $sanitizedUri returned bad mime-type $contentType" } mdResponse.close() return Response(mdResponse.status) } From 80402f3c529dde39120ed0c215d4bc20f218f7c7 Mon Sep 17 00:00:00 2001 From: AviKav Date: Tue, 14 Jul 2020 03:59:07 -0400 Subject: [PATCH 4/4] Rename mimetype check --- src/main/kotlin/mdnet/base/server/ImageServer.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/mdnet/base/server/ImageServer.kt b/src/main/kotlin/mdnet/base/server/ImageServer.kt index 8466cb7..dec52dc 100644 --- a/src/main/kotlin/mdnet/base/server/ImageServer.kt +++ b/src/main/kotlin/mdnet/base/server/ImageServer.kt @@ -144,7 +144,7 @@ class ImageServer( // TODO: Cleanup messy conditionals // Ensure that cached file isn't a non-image from the Great Cache Propagation - val flaresFault = imageDatum != null && imageDatum.contentType.isImage().not() + val flaresFault = imageDatum != null && imageDatum.contentType.isImageMimetype().not() if (flaresFault) { snapshot?.close(); snapshot = null @@ -181,7 +181,7 @@ class ImageServer( } } - private fun String.isImage() = this.toLowerCase().startsWith("image/") + private fun String.isImageMimetype() = this.toLowerCase().startsWith("image/") private fun Request.handleCacheHit(sanitizedUri: String, cipher: Cipher, snapshot: DiskLruCache.Snapshot, imageDatum: ImageDatum): Response { // our files never change, so it's safe to use the browser cache @@ -233,8 +233,8 @@ class ImageServer( contentType!! - if (!contentType.isImage()) { - LOGGER.trace { "Upstream query for $sanitizedUri returned bad mime-type $contentType" } + if (!contentType.isImageMimetype()) { + LOGGER.trace { "Upstream query for $sanitizedUri returned bad mimetype $contentType" } mdResponse.close() return Response(mdResponse.status) }