terribleplan/mangadex_at_home
1
0
Fork 0
forked from gitlab-com-mangadex-pub/mangadex_at_home
Code Issues Pull requests Releases Wiki Activity

merge

Browse source
This commit is contained in:
M 2020-06-12 12:58:10 -05:00
parent d7bf45af0b
commit 541890190a
5 changed files with 61 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/mdnet/base/Constants.java
View file

@ -3,7 +3,7 @@ package mdnet.base;
import java.time.Duration; import java.time.Duration;
public class Constants { public class Constants {
public static final int CLIENT_BUILD = 3; public static final int CLIENT_BUILD = 4;
public static final String CLIENT_VERSION = "1.0"; public static final String CLIENT_VERSION = "1.0";
public static final Duration MAX_AGE_CACHE = Duration.ofDays(14); public static final Duration MAX_AGE_CACHE = Duration.ofDays(14);
} }

2
src/main/java/mdnet/base/settings/ClientSettings.java
View file

@ -32,7 +32,7 @@ public final class ClientSettings {
} }
public ClientSettings(long maxCacheSizeMib, long maxBandwidthMibPerHour, long maxBurstRateKibPerSecond, public ClientSettings(long maxCacheSizeMib, long maxBandwidthMibPerHour, long maxBurstRateKibPerSecond,
int clientPort, String clientSecret, int threads, WebSettings webSettings) { int clientPort, String clientSecret, int threads, WebSettings webSettings) {
this.maxCacheSizeMib = maxCacheSizeMib; this.maxCacheSizeMib = maxCacheSizeMib;
this.maxBandwidthMibPerHour = maxBandwidthMibPerHour; this.maxBandwidthMibPerHour = maxBandwidthMibPerHour;
this.maxBurstRateKibPerSecond = maxBurstRateKibPerSecond; this.maxBurstRateKibPerSecond = maxBurstRateKibPerSecond;

8
src/main/java/mdnet/cache/DiskLruCache.java vendored
View file

@ -966,6 +966,10 @@ public final class DiskLruCache implements Closeable {
// Move files to new caching tree if exists // Move files to new caching tree if exists
Path oldCache = Paths.get(directory + File.separator + key + "." + i); Path oldCache = Paths.get(directory + File.separator + key + "." + i);
Path newCache = Paths.get(directory + subKeyPath + File.separator + key + "." + i); Path newCache = Paths.get(directory + subKeyPath + File.separator + key + "." + i);
File newCacheDirectory = new File(directory + subKeyPath, key + "." + i + ".tmp");
newCacheDirectory.getParentFile().mkdirs();
if (Files.exists(oldCache)) { if (Files.exists(oldCache)) {
try { try {
Files.move(oldCache, newCache, StandardCopyOption.ATOMIC_MOVE); Files.move(oldCache, newCache, StandardCopyOption.ATOMIC_MOVE);
@ -985,6 +989,10 @@ public final class DiskLruCache implements Closeable {
// Move files to new caching tree if exists // Move files to new caching tree if exists
Path oldCache = Paths.get(directory + File.separator + key + "." + i + ".tmp"); Path oldCache = Paths.get(directory + File.separator + key + "." + i + ".tmp");
Path newCache = Paths.get(directory + subKeyPath + File.separator + key + "." + i + ".tmp"); Path newCache = Paths.get(directory + subKeyPath + File.separator + key + "." + i + ".tmp");
File newCacheDirectory = new File(directory + subKeyPath, key + "." + i + ".tmp");
newCacheDirectory.getParentFile().mkdirs();
if (Files.exists(oldCache)) { if (Files.exists(oldCache)) {
try { try {
Files.move(oldCache, newCache, StandardCopyOption.ATOMIC_MOVE); Files.move(oldCache, newCache, StandardCopyOption.ATOMIC_MOVE);

98
src/main/kotlin/mdnet/base/Application.kt
View file

@ -19,10 +19,8 @@ import org.http4k.filter.CachingFilters
import org.http4k.filter.MaxAgeTtl import org.http4k.filter.MaxAgeTtl
import org.http4k.filter.ServerFilters import org.http4k.filter.ServerFilters
import org.http4k.lens.Path import org.http4k.lens.Path
import org.http4k.routing.ResourceLoader
import org.http4k.routing.bind import org.http4k.routing.bind
import org.http4k.routing.routes import org.http4k.routing.routes
import org.http4k.routing.singlePageApp
import org.http4k.server.Http4kServer import org.http4k.server.Http4kServer
import org.http4k.server.asServer import org.http4k.server.asServer
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
@ -52,16 +50,16 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
} }
val client = ApacheClient(responseBodyMode = BodyMode.Stream, client = HttpClients.custom() val client = ApacheClient(responseBodyMode = BodyMode.Stream, client = HttpClients.custom()
.setDefaultRequestConfig(RequestConfig.custom() .setDefaultRequestConfig(RequestConfig.custom()
.setCookieSpec(CookieSpecs.IGNORE_COOKIES) .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.setConnectTimeout(3000) .setConnectTimeout(3000)
.setSocketTimeout(3000) .setSocketTimeout(3000)
.setConnectionRequestTimeout(3000) .setConnectionRequestTimeout(3000)
.build())
.setMaxConnTotal(THREADS_TO_ALLOCATE)
.setMaxConnPerRoute(THREADS_TO_ALLOCATE)
// Have it at the maximum open sockets a user can have in most modern OSes. No reason to limit this, just limit it at the Netty side.
.build()) .build())
.setMaxConnTotal(THREADS_TO_ALLOCATE)
.setMaxConnPerRoute(THREADS_TO_ALLOCATE)
// Have it at the maximum open sockets a user can have in most modern OSes. No reason to limit this, just limit it at the Netty side.
.build())
val app = { dataSaver: Boolean -> val app = { dataSaver: Boolean ->
{ request: Request -> { request: Request ->
@ -89,28 +87,28 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
// Netty doesn't do Content-Length or Content-Type, so we have the pleasure of doing that ourselves // Netty doesn't do Content-Length or Content-Type, so we have the pleasure of doing that ourselves
fun respondWithImage(input: InputStream, length: String?, type: String, lastModified: String?): Response = fun respondWithImage(input: InputStream, length: String?, type: String, lastModified: String?): Response =
Response(Status.OK) Response(Status.OK)
.header("Content-Type", type) .header("Content-Type", type)
.header("X-Content-Type-Options", "nosniff") .header("X-Content-Type-Options", "nosniff")
.header( .header(
"Cache-Control", "Cache-Control",
listOf("public", MaxAgeTtl(Constants.MAX_AGE_CACHE).toHeaderValue()).joinToString(", ") listOf("public", MaxAgeTtl(Constants.MAX_AGE_CACHE).toHeaderValue()).joinToString(", ")
) )
.header("Timing-Allow-Origin", "https://mangadex.org") .header("Timing-Allow-Origin", "https://mangadex.org")
.let { .let {
if (length != null) { if (length != null) {
it.body(input, length.toLong()).header("Content-Length", length) it.body(input, length.toLong()).header("Content-Length", length)
} else { } else {
it.body(input).header("Transfer-Encoding", "chunked") it.body(input).header("Transfer-Encoding", "chunked")
} }
} }
.let { .let {
if (lastModified != null) { if (lastModified != null) {
it.header("Last-Modified", lastModified) it.header("Last-Modified", lastModified)
} else { } else {
it it
} }
} }
val snapshot = cache.get(cacheId) val snapshot = cache.get(cacheId)
if (snapshot != null) { if (snapshot != null) {
@ -126,15 +124,15 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
snapshot.close() snapshot.close()
Response(Status.NOT_MODIFIED) Response(Status.NOT_MODIFIED)
.header("Last-Modified", lastModified) .header("Last-Modified", lastModified)
} else { } else {
if (LOGGER.isInfoEnabled) { if (LOGGER.isInfoEnabled) {
LOGGER.info("Request for $sanitizedUri hit cache") LOGGER.info("Request for $sanitizedUri hit cache")
} }
respondWithImage( respondWithImage(
CipherInputStream(BufferedInputStream(snapshot.getInputStream(0)), getRc4(rc4Bytes)), CipherInputStream(BufferedInputStream(snapshot.getInputStream(0)), getRc4(rc4Bytes)),
snapshot.getLength(0).toString(), snapshot.getString(1), snapshot.getString(2) snapshot.getLength(0).toString(), snapshot.getString(1), snapshot.getString(2)
) )
} }
} else { } else {
@ -171,8 +169,8 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
editor.setString(2, lastModified) editor.setString(2, lastModified)
val tee = CachingInputStream( val tee = CachingInputStream(
mdResponse.body.stream, mdResponse.body.stream,
executor, CipherOutputStream(BufferedOutputStream(editor.newOutputStream(0)), getRc4(rc4Bytes)) executor, CipherOutputStream(BufferedOutputStream(editor.newOutputStream(0)), getRc4(rc4Bytes))
) { ) {
// Note: if neither of the options get called/are in the log // Note: if neither of the options get called/are in the log
// check that tee gets closed and for exceptions in this lambda // check that tee gets closed and for exceptions in this lambda
@ -207,19 +205,17 @@ fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSetting
CachingFilters CachingFilters
return catchAllHideDetails() return catchAllHideDetails()
.then(ServerFilters.CatchLensFailure) .then(ServerFilters.CatchLensFailure)
.then(addCommonHeaders()) .then(addCommonHeaders())
.then( .then(
routes( routes(
"/data/{chapterHash}/{fileName}" bind Method.GET to app(false), "/data/{chapterHash}/{fileName}" bind Method.GET to app(false),
"/data-saver/{chapterHash}/{fileName}" bind Method.GET to app(true), "/data-saver/{chapterHash}/{fileName}" bind Method.GET to app(true),
"/{token}/data/{chapterHash}/{fileName}" bind Method.GET to app(false), "/{token}/data/{chapterHash}/{fileName}" bind Method.GET to app(false),
"/{token}/data-saver/{chapterHash}/{fileName}" bind Method.GET to app(true), "/{token}/data-saver/{chapterHash}/{fileName}" bind Method.GET to app(true)
)
singlePageApp(ResourceLoader.Classpath("/webui"))
) )
) .asServer(Netty(serverSettings.tls, clientSettings, statistics))
.asServer(Netty(serverSettings.tls, clientSettings, statistics))
} }
private fun getRc4(key: ByteArray): Cipher { private fun getRc4(key: ByteArray): Cipher {
@ -235,7 +231,7 @@ private fun addCommonHeaders(): Filter {
{ request: Request -> { request: Request ->
val response = next(request) val response = next(request)
response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC))) response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC)))
.header("Server", "Mangadex@Home Node ${Constants.CLIENT_VERSION} (${Constants.CLIENT_BUILD})") .header("Server", "Mangadex@Home Node ${Constants.CLIENT_VERSION} (${Constants.CLIENT_BUILD})")
} }
} }
} }

8
src/main/kotlin/mdnet/base/Netty.kt
View file

@ -46,7 +46,7 @@ class Netty(private val tls: ServerSettings.TlsCert, private val clientSettings:
private lateinit var address: InetSocketAddress private lateinit var address: InetSocketAddress
private val burstLimiter = object : GlobalTrafficShapingHandler( private val burstLimiter = object : GlobalTrafficShapingHandler(
workerGroup, 1024 * clientSettings.maxBurstRateKibPerSecond, 0, 50) { workerGroup, 1024 * clientSettings.maxBurstRateKibPerSecond, 0, 50) {
override fun doAccounting(counter: TrafficCounter) { override fun doAccounting(counter: TrafficCounter) {
stats.get().bytesSent.getAndAdd(counter.cumulativeWrittenBytes()) stats.get().bytesSent.getAndAdd(counter.cumulativeWrittenBytes())
counter.resetCumulativeTime() counter.resetCumulativeTime()
@ -60,9 +60,9 @@ class Netty(private val tls: ServerSettings.TlsCert, private val clientSettings:
val (mainCert, chainCert) = getX509Certs(tls.certificate) val (mainCert, chainCert) = getX509Certs(tls.certificate)
val sslContext = SslContextBuilder val sslContext = SslContextBuilder
.forServer(getPrivateKey(tls.privateKey), mainCert, chainCert) .forServer(getPrivateKey(tls.privateKey), mainCert, chainCert)
.protocols("TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1") .protocols("TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1")
.build() .build()
val bootstrap = ServerBootstrap() val bootstrap = ServerBootstrap()
bootstrap.group(masterGroup, workerGroup) bootstrap.group(masterGroup, workerGroup)