terribleplan/mangadex_at_home
1
0
Fork 0
forked from gitlab-com-mangadex-pub/mangadex_at_home
Code Issues Pull requests Releases Wiki Activity

Stuff

Browse source
This commit is contained in:
carbotaniuman 2020-06-12 14:56:51 -05:00
parent 9086e09c60
commit f153a11717
11 changed files with 170 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
build.gradle
View file

@ -22,8 +22,12 @@ dependencies {
implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-format-gson", version: "3.249.0"
implementation group: "commons-io", name: "commons-io", version: "2.7"
compile "org.java-websocket:Java-WebSocket:1.5.1"
implementation group: 'org.java-websocket', name: 'Java-WebSocket', version: '1.5.1'
implementation "ch.qos.logback:logback-classic:$logback_version"
runtimeOnly 'io.netty:netty-tcnative-boringssl-static:2.0.30.Final'
}
@ -35,12 +39,14 @@ java {
spotless {
java {
indentWithSpaces(4)
eclipse()
removeUnusedImports()
trimTrailingWhitespace()
endWithNewline()
}
kotlin {
indentWithSpaces(4)
ktlint()
trimTrailingWhitespace()
endWithNewline()

12
src/main/java/mdnet/base/MangaDexClient.java
View file

@ -3,6 +3,8 @@ package mdnet.base;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import mdnet.base.settings.ClientSettings;
import mdnet.base.web.ApplicationKt;
import mdnet.base.web.WebUiKt;
import mdnet.cache.DiskLruCache;
import mdnet.webui.WebConsole;
import org.http4k.server.Http4kServer;
@ -29,6 +31,7 @@ public class MangaDexClient {
// if this is null, then the server has shutdown
private Http4kServer engine;
private Http4kServer webUi;
private DiskLruCache cache;
public MangaDexClient(ClientSettings clientSettings) {
@ -46,7 +49,7 @@ public class MangaDexClient {
// This function also does most of the program initialization.
public void runLoop() {
statistics.set(new Statistics());
statistics.set(new Statistics(0));
loginAndStartServer();
if (serverSettings.getLatestBuild() > Constants.CLIENT_BUILD) {
if (LOGGER.isWarnEnabled()) {
@ -59,6 +62,9 @@ public class MangaDexClient {
LOGGER.info("MDNet initialization completed successfully. Starting normal operation.");
}
webUi = WebUiKt.getUiServer(clientSettings.getWebSettings(), statistics);
webUi.start();
// we don't really care about the Atomic part here
AtomicInteger counter = new AtomicInteger();
// ping keep-alive every 45 seconds
@ -71,7 +77,7 @@ public class MangaDexClient {
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Hourly update: refreshing statistics");
}
statistics.set(new Statistics());
statistics.set(new Statistics(statistics.get().getSequenceNumber() + 1));
if (engine == null) {
if (LOGGER.isInfoEnabled()) {
@ -218,7 +224,7 @@ public class MangaDexClient {
// TODO: system.out redirect
ClientSettings finalSettings = settings;
new Thread(() -> {
WebConsole webConsole = new WebConsole(finalSettings.getWebSettings().getClientWebsocketPort()) {
WebConsole webConsole = new WebConsole(finalSettings.getWebSettings().getUiWebsocketPort()) {
@Override
protected void parseMessage(String message) {
System.out.println(message);

24
src/main/java/mdnet/base/Statistics.java
View file

@ -1,19 +1,28 @@
package mdnet.base;
import com.google.gson.annotations.SerializedName;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
public class Statistics {
@SerializedName("requests_served")
private final AtomicInteger requestsServed;
@SerializedName("cache_hits")
private final AtomicInteger cacheHits;
@SerializedName("cache_misses")
private final AtomicInteger cacheMisses;
@SerializedName("bytes_sent")
private final AtomicLong bytesSent;
@SerializedName("sequence_number")
private final int sequenceNumber;
public Statistics() {
public Statistics(int sequenceNumber) {
requestsServed = new AtomicInteger();
cacheHits = new AtomicInteger();
cacheMisses = new AtomicInteger();
bytesSent = new AtomicLong();
this.sequenceNumber = sequenceNumber;
}
public AtomicInteger getRequestsServed() {
@ -32,9 +41,18 @@ public class Statistics {
return bytesSent;
}
public int getSequenceNumber() {
return sequenceNumber;
}
@Override
public String toString() {
return "Statistics{" + "requestsServed=" + requestsServed + ", cacheHits=" + cacheHits + ", cacheMisses="
+ cacheMisses + ", bytesSent=" + bytesSent + '}';
return "Statistics{" +
"requestsServed=" + requestsServed +
", cacheHits=" + cacheHits +
", cacheMisses=" + cacheMisses +
", bytesSent=" + bytesSent +
", sequenceNumber=" + sequenceNumber +
'}';
}
}

2
src/main/java/mdnet/base/settings/ClientSettings.java
View file

@ -32,7 +32,7 @@ public final class ClientSettings {
}
public ClientSettings(long maxCacheSizeMib, long maxBandwidthMibPerHour, long maxBurstRateKibPerSecond,
int clientPort, String clientSecret, int threads, WebSettings webSettings) {
int clientPort, String clientSecret, int threads, WebSettings webSettings) {
this.maxCacheSizeMib = maxCacheSizeMib;
this.maxBandwidthMibPerHour = maxBandwidthMibPerHour;
this.maxBurstRateKibPerSecond = maxBurstRateKibPerSecond;

24
src/main/java/mdnet/base/settings/WebSettings.java
View file

@ -3,23 +3,31 @@ package mdnet.base.settings;
import com.google.gson.annotations.SerializedName;
public final class WebSettings {
@SerializedName("client_websocket_port")
private final int clientWebsocketPort;
@SerializedName("ui_websocket_port")
private final int uiWebsocketPort;
@SerializedName("ui_port")
private final int uiPort;
public WebSettings() {
this.clientWebsocketPort = 33333;
this.uiWebsocketPort = 33333;
this.uiPort = 8080;
}
public WebSettings(int clientWebsocketPort) {
this.clientWebsocketPort = clientWebsocketPort;
public WebSettings(int uiWebsocketPort, int uiPort) {
this.uiWebsocketPort = uiWebsocketPort;
this.uiPort = uiPort;
}
public int getClientWebsocketPort() {
return clientWebsocketPort;
public int getUiWebsocketPort() {
return uiWebsocketPort;
}
public int getUiPort() {
return uiPort;
}
@Override
public String toString() {
return "WebSettings{" + "clientWebsocketPort=" + clientWebsocketPort + '}';
return "WebSettings{" + "uiWebsocketPort=" + uiWebsocketPort + ", uiPort=" + uiPort + '}';
}
}

2
src/main/java/mdnet/base/CachingInputStream.java → src/main/java/mdnet/cache/CachingInputStream.java vendored
View file

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package mdnet.base;
package mdnet.cache;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.ProxyInputStream;

7
src/main/java/mdnet/cache/DiskLruCache.java vendored
View file

@ -82,10 +82,9 @@ import java.util.regex.Pattern;
* <li>When an entry is being <strong>edited</strong>, it is not necessary to
* supply data for every value; values default to their previous value.
* </ul>
* Every {@link #editImpl} call must be matched by a call to
* {@link Editor#commit} or {@link Editor#abort}. Committing is atomic: a read
* observes the full set of values as they were before or after the commit, but
* never a mix of values.
* Every {@link #edit} call must be matched by a call to {@link Editor#commit}
* or {@link Editor#abort}. Committing is atomic: a read observes the full set
* of values as they were before or after the commit, but never a mix of values.
*
* <p>
* Clients call {@link #get} to read a snapshot of an entry. The read will

52
src/main/java/mdnet/webui/WebConsole.java
View file

@ -2,8 +2,6 @@ package mdnet.webui;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicReference;
import mdnet.base.Statistics;
import org.java_websocket.WebSocket;
import org.java_websocket.handshake.ClientHandshake;
import org.java_websocket.server.WebSocketServer;
@ -78,30 +76,30 @@ public abstract class WebConsole extends WebSocketServer {
// }
public void sendMessage(String type, Object message) {
// JSONObject out = new JSONObject();
// switch (type) {
// case "command" :
// out.put("type", "command");
// out.put("data", message.toString());
// break;
// case "stats" :
// out.put("type", "stats");
// AtomicReference<Statistics> temp = (AtomicReference<Statistics>) message;
// out.put("hits", temp.get().getCacheHits());
// out.put("misses", temp.get().getCacheMisses());
// out.put("bytes_sent", temp.get().getBytesSent());
// out.put("req_served", temp.get().getRequestsServed());
// out.put("dataval", "empty");
// out.put("dataval", "empty");
// out.put("dataval", "empty");
// break;
// case "auth" :
// break;
// default :
// out.put("type", "command");
// out.put("data", message.toString());
// break;
// }
// broadcast(out.toString());
// JSONObject out = new JSONObject();
// switch (type) {
// case "command" :
// out.put("type", "command");
// out.put("data", message.toString());
// break;
// case "stats" :
// out.put("type", "stats");
// AtomicReference<Statistics> temp = (AtomicReference<Statistics>) message;
// out.put("hits", temp.get().getCacheHits());
// out.put("misses", temp.get().getCacheMisses());
// out.put("bytes_sent", temp.get().getBytesSent());
// out.put("req_served", temp.get().getRequestsServed());
// out.put("dataval", "empty");
// out.put("dataval", "empty");
// out.put("dataval", "empty");
// break;
// case "auth" :
// break;
// default :
// out.put("type", "command");
// out.put("data", message.toString());
// break;
// }
// broadcast(out.toString());
}
}

37
src/main/kotlin/mdnet/base/Application.kt → src/main/kotlin/mdnet/base/web/Application.kt
View file

@ -1,15 +1,18 @@
/* ktlint-disable no-wildcard-imports */
package mdnet.base
package mdnet.base.web
import mdnet.base.Constants
import mdnet.base.Netty
import mdnet.base.ServerSettings
import mdnet.base.Statistics
import mdnet.base.settings.ClientSettings
import mdnet.cache.CachingInputStream
import mdnet.cache.DiskLruCache
import org.apache.http.client.config.CookieSpecs
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.HttpClients
import org.http4k.client.ApacheClient
import org.http4k.core.BodyMode
import org.http4k.core.Filter
import org.http4k.core.HttpHandler
import org.http4k.core.Method
import org.http4k.core.Request
import org.http4k.core.Response
@ -28,8 +31,6 @@ import java.io.BufferedInputStream
import java.io.BufferedOutputStream
import java.io.InputStream
import java.security.MessageDigest
import java.time.ZoneOffset
import java.time.ZonedDateTime
import java.time.format.DateTimeFormatter
import java.util.*
import java.util.concurrent.Executors
@ -40,7 +41,7 @@ import javax.crypto.CipherOutputStream
import javax.crypto.spec.SecretKeySpec
private val LOGGER = LoggerFactory.getLogger("Application")
private val THREADS_TO_ALLOCATE = 262144 // 2**18 // Honestly, no reason to not just let 'er rip. Inactive connections will expire on their own :D
private const val THREADS_TO_ALLOCATE = 262144 // 2**18 // Honestly, no reason to not just let 'er rip. Inactive connections will expire on their own :D
fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSettings: ClientSettings, statistics: AtomicReference<Statistics>): Http4kServer {
val executor = Executors.newCachedThreadPool()
@ -226,30 +227,6 @@ private fun getRc4(key: ByteArray): Cipher {
private val HTTP_TIME_FORMATTER = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss O", Locale.ENGLISH)
private fun addCommonHeaders(): Filter {
return Filter { next: HttpHandler ->
{ request: Request ->
val response = next(request)
response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC)))
.header("Server", "Mangadex@Home Node ${Constants.CLIENT_VERSION} (${Constants.CLIENT_BUILD})")
}
}
}
private fun catchAllHideDetails(): Filter {
return Filter { next: HttpHandler ->
{ request: Request ->
try {
next(request)
} catch (e: Exception) {
if (LOGGER.isWarnEnabled) {
LOGGER.warn("Request error detected", e)
}
Response(Status.INTERNAL_SERVER_ERROR)
}
}
}
}
private fun md5Bytes(stringToHash: String): ByteArray {
val digest = MessageDigest.getInstance("MD5")

37
src/main/kotlin/mdnet/base/web/WebUi.kt Normal file
View file

@ -0,0 +1,37 @@
/* ktlint-disable no-wildcard-imports */
package mdnet.base.web
import mdnet.base.Statistics
import mdnet.base.settings.WebSettings
import org.http4k.core.Body
import org.http4k.core.Method
import org.http4k.core.Response
import org.http4k.core.Status
import org.http4k.core.then
import org.http4k.filter.ServerFilters
import org.http4k.routing.ResourceLoader
import org.http4k.routing.bind
import org.http4k.routing.routes
import org.http4k.routing.singlePageApp
import org.http4k.server.Http4kServer
import org.http4k.server.Netty
import org.http4k.server.asServer
import java.util.concurrent.atomic.AtomicReference
import org.http4k.format.Gson.auto
fun getUiServer(webSettings: WebSettings, statistics: AtomicReference<Statistics>): Http4kServer {
val statisticsLens = Body.auto<Statistics>().toLens()
return catchAllHideDetails()
.then(ServerFilters.CatchLensFailure)
.then(addCommonHeaders())
.then(
routes(
"/api/stats" bind Method.GET to {
statisticsLens(statistics.get(), Response(Status.OK))
},
singlePageApp(ResourceLoader.Classpath("/webui"))
)
)
.asServer(Netty(webSettings.uiPort))
}

43
src/main/kotlin/mdnet/base/web/common.kt Normal file
View file

@ -0,0 +1,43 @@
/* ktlint-disable no-wildcard-imports */
package mdnet.base.web
import mdnet.base.Constants
import org.http4k.core.Filter
import org.http4k.core.HttpHandler
import org.http4k.core.Request
import org.http4k.core.Response
import org.http4k.core.Status
import org.slf4j.LoggerFactory
import java.time.ZoneOffset
import java.time.ZonedDateTime
import java.time.format.DateTimeFormatter
import java.util.*
private val HTTP_TIME_FORMATTER = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss O", Locale.ENGLISH)
private val LOGGER = LoggerFactory.getLogger("Application")
fun addCommonHeaders(): Filter {
return Filter { next: HttpHandler ->
{ request: Request ->
val response = next(request)
response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC)))
.header("Server", "Mangadex@Home Node ${Constants.CLIENT_VERSION} (${Constants.CLIENT_BUILD})")
}
}
}
fun catchAllHideDetails(): Filter {
return Filter { next: HttpHandler ->
{ request: Request ->
try {
next(request)
} catch (e: Exception) {
if (LOGGER.isWarnEnabled) {
LOGGER.warn("Request error detected", e)
}
Response(Status.INTERNAL_SERVER_ERROR)
}
}
}
}