Merge branch 'fix-tls-and-crashes' into 'master'

Fix tls and crashes

See merge request mangadex-pub/mangadex_at_home!99

CHANGELOG.md

@@ -17,6 +17,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security
 
+## [2.0.2] - 2022-02-16
+### Removed
+- [2022-02-16] Remove TLS 1.0 and 1.1 support [@carbotaniuman].
+
+### Fixed
+- [2022-02-16] Fix uncatched exceptions killing threads and not being logged [@carbotaniuman].
+
 ## [2.0.1] - 2021-05-27
 ### Added
 - [2021-05-27] Added SNI check to prevent people from simply scanning nodes [@carbotaniuman].
@@ -396,7 +403,8 @@ This release contains many breaking changes! Of note are the changes to the cach
 ### Fixed
 - [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
 
-[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...HEAD
+[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.2...HEAD
+[2.0.2]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.1...2.0.2
 [2.0.1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...2.0.1
 [2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0
 [2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14

src/main/kotlin/mdnet/ServerManager.kt

@@ -24,7 +24,6 @@ import io.micrometer.prometheus.PrometheusConfig
 import io.micrometer.prometheus.PrometheusMeterRegistry
 import mdnet.cache.ImageStorage
 import mdnet.data.Statistics
-import mdnet.logging.error
 import mdnet.logging.info
 import mdnet.logging.warn
 import mdnet.metrics.DefaultMicrometerMetrics
@@ -188,7 +187,7 @@ class ServerManager(
                         }
                     }
                 } catch (e: Exception) {
-                    LOGGER.error(e) { "Main loop failed" }
+                    LOGGER.warn(e) { "Main loop failed" }
                 }
             },
             5, 5, TimeUnit.SECONDS

src/main/kotlin/mdnet/cache/ImageStorage.kt

@@ -25,6 +25,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import mdnet.logging.info
 import mdnet.logging.trace
+import mdnet.logging.warn
 import org.apache.commons.io.file.PathUtils
 import org.ktorm.database.Database
 import org.ktorm.dsl.*
@@ -58,7 +59,7 @@ class ImageStorage(
     private val cacheDirectory: Path,
     private val database: Database,
     autoPrune: Boolean = true
-) {
+) : AutoCloseable {
     private val tempCacheDirectory = cacheDirectory.resolve("tmp")
 
     private val evictor: ScheduledExecutorService = Executors.newScheduledThreadPool(2)
@@ -89,36 +90,43 @@ class ImageStorage(
 
         evictor.scheduleWithFixedDelay(
             {
-                val toUpdate = HashSet<String>()
+                try {
-                queue.drainTo(toUpdate)
+                    val toUpdate = HashSet<String>()
-                val now = Instant.now()
+                    queue.drainTo(toUpdate)
+                    val now = Instant.now()
 
-                LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" }
+                    LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" }
-                synchronized(database) {
+                    synchronized(database) {
-                    database.batchUpdate(DbImage) {
+                        database.batchUpdate(DbImage) {
-                        for (id in toUpdate) {
+                            for (id in toUpdate) {
-                            item {
+                                item {
-                                set(DbImage.accessed, now)
+                                    set(DbImage.accessed, now)
-                                where {
+                                    where {
-                                    DbImage.id eq id
+                                        DbImage.id eq id
+                                    }
                                 }
                             }
                         }
                     }
+                    calculateSize()
+                } catch (e: Exception) {
+                    LOGGER.warn(e) { "Error updating LRU $this" }
                 }
-                calculateSize()
             },
-            1, 1, TimeUnit.MINUTES
+            30, 30, TimeUnit.SECONDS
         )
 
-        // evict LRU cache every 3 minutes
         if (autoPrune) {
             evictor.scheduleWithFixedDelay(
                 {
-                    calculateSize()
+                    try {
-                    pruneImages()
+                        calculateSize()
+                        pruneImages()
+                    } catch (e: Exception) {
+                        LOGGER.warn(e) { "Error pruning images" }
+                    }
                 },
-                0, 3, TimeUnit.MINUTES
+                0, 1, TimeUnit.MINUTES
             )
         }
     }
@@ -255,7 +263,7 @@ class ImageStorage(
         }
     }
 
-    fun close() {
+    override fun close() {
         evictor.shutdown()
         evictor.awaitTermination(10, TimeUnit.SECONDS)
     }

src/main/kotlin/mdnet/netty/ApplicationNetty.kt

@@ -175,7 +175,7 @@ class Netty(
             val certs = getX509Certs(tls.certificate)
             val sslContext = SslContextBuilder
                 .forServer(getPrivateKey(tls.privateKey), certs)
-                .protocols("TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1")
+                .protocols("TLSv1.3", "TLSv1.2")
                 .build()
 
             val bootstrap = ServerBootstrap()

src/test/kotlin/mdnet/cache/ImageStorageTest.kt

@@ -39,11 +39,13 @@ class ImageStorageTest : FreeSpec() {
     override fun isolationMode() = IsolationMode.InstancePerTest
 
     init {
-        val imageStorage = ImageStorage(
+        val imageStorage = autoClose(
-            maxSize = 5,
+            ImageStorage(
-            cacheDirectory = tempdir().toPath(),
+                maxSize = 5,
-            database = Database.connect("jdbc:sqlite:${tempfile()}"),
+                cacheDirectory = tempdir().toPath(),
-            autoPrune = false,
+                database = Database.connect("jdbc:sqlite:${tempfile()}"),
+                autoPrune = false,
+            )
         )
 
         val testMeta = ImageMetadata("a", "a", 123)
@@ -81,7 +83,8 @@ class ImageStorageTest : FreeSpec() {
                 writer.stream.write(ByteArray(12))
                 writer.abort()
 
-                "should not update size" {
+                "should not update size even if calculated" {
+                    imageStorage.calculateSize()
                     imageStorage.size.shouldBeZero()
                 }
             }
@@ -157,14 +160,17 @@ class ImageStorageSlowTest : FreeSpec() {
     override fun isolationMode() = IsolationMode.InstancePerTest
 
     init {
-        val imageStorage = ImageStorage(
+        val imageStorage = autoClose(
-            maxSize = 4097,
+            ImageStorage(
-            cacheDirectory = tempdir().toPath(),
+                maxSize = 4097,
-            database = Database.connect("jdbc:sqlite:${tempfile()}"),
+                cacheDirectory = tempdir().toPath(),
+                database = Database.connect("jdbc:sqlite:${tempfile()}"),
+            )
         )
 
         "autoPrune" - {
             "should update size eventually" {
+                println("1 - $imageStorage")
                 val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 4096))
                 writer.shouldNotBeNull()
 
@@ -177,6 +183,7 @@ class ImageStorageSlowTest : FreeSpec() {
             }
 
             "should prune if insufficient size eventually" {
+                println("2 - $imageStorage")
                 imageStorage.maxSize = 10000
 
                 val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 123))
@@ -185,6 +192,7 @@ class ImageStorageSlowTest : FreeSpec() {
                 writer.stream.write(ByteArray(8192))
                 writer.commit(8192).shouldBeTrue()
 
+                imageStorage.calculateSize()
                 eventually(Duration.minutes(5)) {
                     imageStorage.size.shouldBeZero()
                 }

src/test/kotlin/mdnet/server/ImageServerTest.kt

@@ -116,11 +116,13 @@ class ImageServerTest : FreeSpec() {
             }
 
             "with real cache" - {
-                val storage = ImageStorage(
+                val storage = autoClose(
-                    maxSize = 100000,
+                    ImageStorage(
-                    cacheDirectory = tempdir().toPath(),
+                        maxSize = 100000,
-                    database = Database.connect("jdbc:sqlite:${tempfile()}"),
+                        cacheDirectory = tempdir().toPath(),
-                    autoPrune = false,
+                        database = Database.connect("jdbc:sqlite:${tempfile()}"),
+                        autoPrune = false,
+                    )
                 )
 
                 val server = ImageServer(
@@ -161,11 +163,13 @@ class ImageServerTest : FreeSpec() {
         "failed upstream responses" - {
             val client = mockk<HttpHandler>()
 
-            val storage = ImageStorage(
+            val storage = autoClose(
-                maxSize = 100000,
+                ImageStorage(
-                cacheDirectory = tempdir().toPath(),
+                    maxSize = 100000,
-                database = Database.connect("jdbc:sqlite:${tempfile()}"),
+                    cacheDirectory = tempdir().toPath(),
-                autoPrune = false,
+                    database = Database.connect("jdbc:sqlite:${tempfile()}"),
+                    autoPrune = false,
+                )
             )
 
             val server = ImageServer(