gitlab-com-mangadex-pub/mangadex_at_home
1
0
Fork 1
mirror of https://gitlab.com/mangadex-pub/mangadex_at_home.git synced 2024-01-19 02:48:37 +00:00
Code Releases Activity

Merge branch 'fix-tls-and-crashes' into 'master'

Browse source 
Fix tls and crashes

See merge request mangadex-pub/mangadex_at_home!99
This commit is contained in:
carbotaniuman 2022-02-17 06:27:44 +00:00
parent c7830161c1 d198c27de3
commit 85dce402df
6 changed files with 70 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
CHANGELOG.md
View file

@ -17,6 +17,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Security
## [2.0.2] - 2022-02-16
### Removed
- [2022-02-16] Remove TLS 1.0 and 1.1 support [@carbotaniuman].
### Fixed
- [2022-02-16] Fix uncatched exceptions killing threads and not being logged [@carbotaniuman].
## [2.0.1] - 2021-05-27
### Added
- [2021-05-27] Added SNI check to prevent people from simply scanning nodes [@carbotaniuman].
@ -396,7 +403,8 @@ This release contains many breaking changes! Of note are the changes to the cach
### Fixed
- [2020-06-11] Tweaked logging configuration to reduce log file sizes by [@carbotaniuman].
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...HEAD
[Unreleased]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.2...HEAD
[2.0.2]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.1...2.0.2
[2.0.1]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0...2.0.1
[2.0.0]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc14...2.0.0
[2.0.0-rc14]: https://gitlab.com/mangadex/mangadex_at_home/-/compare/2.0.0-rc13...2.0.0-rc14

3
src/main/kotlin/mdnet/ServerManager.kt
View file

@ -24,7 +24,6 @@ import io.micrometer.prometheus.PrometheusConfig
import io.micrometer.prometheus.PrometheusMeterRegistry
import mdnet.cache.ImageStorage
import mdnet.data.Statistics
import mdnet.logging.error
import mdnet.logging.info
import mdnet.logging.warn
import mdnet.metrics.DefaultMicrometerMetrics
@ -188,7 +187,7 @@ class ServerManager(
}
}
} catch (e: Exception) {
LOGGER.error(e) { "Main loop failed" }
LOGGER.warn(e) { "Main loop failed" }
}
},
5, 5, TimeUnit.SECONDS

46
src/main/kotlin/mdnet/cache/ImageStorage.kt vendored
View file

@ -25,6 +25,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import mdnet.logging.info
import mdnet.logging.trace
import mdnet.logging.warn
import org.apache.commons.io.file.PathUtils
import org.ktorm.database.Database
import org.ktorm.dsl.*
@ -58,7 +59,7 @@ class ImageStorage(
private val cacheDirectory: Path,
private val database: Database,
autoPrune: Boolean = true
) {
) : AutoCloseable {
private val tempCacheDirectory = cacheDirectory.resolve("tmp")
private val evictor: ScheduledExecutorService = Executors.newScheduledThreadPool(2)
@ -89,36 +90,43 @@ class ImageStorage(
evictor.scheduleWithFixedDelay(
{
val toUpdate = HashSet<String>()
queue.drainTo(toUpdate)
val now = Instant.now()
try {
val toUpdate = HashSet<String>()
queue.drainTo(toUpdate)
val now = Instant.now()
LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" }
synchronized(database) {
database.batchUpdate(DbImage) {
for (id in toUpdate) {
item {
set(DbImage.accessed, now)
where {
DbImage.id eq id
LOGGER.info { "Updating LRU times for ${toUpdate.size} entries" }
synchronized(database) {
database.batchUpdate(DbImage) {
for (id in toUpdate) {
item {
set(DbImage.accessed, now)
where {
DbImage.id eq id
}
}
}
}
}
calculateSize()
} catch (e: Exception) {
LOGGER.warn(e) { "Error updating LRU $this" }
}
calculateSize()
},
1, 1, TimeUnit.MINUTES
30, 30, TimeUnit.SECONDS
)
// evict LRU cache every 3 minutes
if (autoPrune) {
evictor.scheduleWithFixedDelay(
{
calculateSize()
pruneImages()
try {
calculateSize()
pruneImages()
} catch (e: Exception) {
LOGGER.warn(e) { "Error pruning images" }
}
},
0, 3, TimeUnit.MINUTES
0, 1, TimeUnit.MINUTES
)
}
}
@ -255,7 +263,7 @@ class ImageStorage(
}
}
fun close() {
override fun close() {
evictor.shutdown()
evictor.awaitTermination(10, TimeUnit.SECONDS)
}

2
src/main/kotlin/mdnet/netty/ApplicationNetty.kt
View file

@ -175,7 +175,7 @@ class Netty(
val certs = getX509Certs(tls.certificate)
val sslContext = SslContextBuilder
.forServer(getPrivateKey(tls.privateKey), certs)
.protocols("TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1")
.protocols("TLSv1.3", "TLSv1.2")
.build()
val bootstrap = ServerBootstrap()

28
src/test/kotlin/mdnet/cache/ImageStorageTest.kt vendored
View file

@ -39,11 +39,13 @@ class ImageStorageTest : FreeSpec() {
override fun isolationMode() = IsolationMode.InstancePerTest
init {
val imageStorage = ImageStorage(
maxSize = 5,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
val imageStorage = autoClose(
ImageStorage(
maxSize = 5,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
)
)
val testMeta = ImageMetadata("a", "a", 123)
@ -81,7 +83,8 @@ class ImageStorageTest : FreeSpec() {
writer.stream.write(ByteArray(12))
writer.abort()
"should not update size" {
"should not update size even if calculated" {
imageStorage.calculateSize()
imageStorage.size.shouldBeZero()
}
}
@ -157,14 +160,17 @@ class ImageStorageSlowTest : FreeSpec() {
override fun isolationMode() = IsolationMode.InstancePerTest
init {
val imageStorage = ImageStorage(
maxSize = 4097,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
val imageStorage = autoClose(
ImageStorage(
maxSize = 4097,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
)
)
"autoPrune" - {
"should update size eventually" {
println("1 - $imageStorage")
val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 4096))
writer.shouldNotBeNull()
@ -177,6 +183,7 @@ class ImageStorageSlowTest : FreeSpec() {
}
"should prune if insufficient size eventually" {
println("2 - $imageStorage")
imageStorage.maxSize = 10000
val writer = imageStorage.storeImage("test", ImageMetadata("a", "a", 123))
@ -185,6 +192,7 @@ class ImageStorageSlowTest : FreeSpec() {
writer.stream.write(ByteArray(8192))
writer.commit(8192).shouldBeTrue()
imageStorage.calculateSize()
eventually(Duration.minutes(5)) {
imageStorage.size.shouldBeZero()
}

24
src/test/kotlin/mdnet/server/ImageServerTest.kt
View file

@ -116,11 +116,13 @@ class ImageServerTest : FreeSpec() {
}
"with real cache" - {
val storage = ImageStorage(
maxSize = 100000,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
val storage = autoClose(
ImageStorage(
maxSize = 100000,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
)
)
val server = ImageServer(
@ -161,11 +163,13 @@ class ImageServerTest : FreeSpec() {
"failed upstream responses" - {
val client = mockk<HttpHandler>()
val storage = ImageStorage(
maxSize = 100000,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
val storage = autoClose(
ImageStorage(
maxSize = 100000,
cacheDirectory = tempdir().toPath(),
database = Database.connect("jdbc:sqlite:${tempfile()}"),
autoPrune = false,
)
)
val server = ImageServer(