From f153a11717022298f3df6ea6af4d27c1c976eb55 Mon Sep 17 00:00:00 2001 From: carbotaniuman <41451839+carbotaniuman@users.noreply.github.com> Date: Fri, 12 Jun 2020 14:56:51 -0500 Subject: [PATCH] Stuff --- build.gradle | 8 ++- src/main/java/mdnet/base/MangaDexClient.java | 12 +++-- src/main/java/mdnet/base/Statistics.java | 24 +++++++-- .../mdnet/base/settings/ClientSettings.java | 2 +- .../java/mdnet/base/settings/WebSettings.java | 24 ++++++--- .../{base => cache}/CachingInputStream.java | 2 +- src/main/java/mdnet/cache/DiskLruCache.java | 7 ++- src/main/java/mdnet/webui/WebConsole.java | 52 +++++++++---------- .../mdnet/base/{ => web}/Application.kt | 37 +++---------- src/main/kotlin/mdnet/base/web/WebUi.kt | 37 +++++++++++++ src/main/kotlin/mdnet/base/web/common.kt | 43 +++++++++++++++ 11 files changed, 170 insertions(+), 78 deletions(-) rename src/main/java/mdnet/{base => cache}/CachingInputStream.java (99%) rename src/main/kotlin/mdnet/base/{ => web}/Application.kt (90%) create mode 100644 src/main/kotlin/mdnet/base/web/WebUi.kt create mode 100644 src/main/kotlin/mdnet/base/web/common.kt diff --git a/build.gradle b/build.gradle index 5ce2bc6..09482d7 100644 --- a/build.gradle +++ b/build.gradle @@ -22,8 +22,12 @@ dependencies { implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version" + implementation group: "org.http4k", name: "http4k-format-gson", version: "3.249.0" + implementation group: "commons-io", name: "commons-io", version: "2.7" - compile "org.java-websocket:Java-WebSocket:1.5.1" + + implementation group: 'org.java-websocket', name: 'Java-WebSocket', version: '1.5.1' + implementation "ch.qos.logback:logback-classic:$logback_version" runtimeOnly 'io.netty:netty-tcnative-boringssl-static:2.0.30.Final' } @@ -35,12 +39,14 @@ java { spotless { java { + indentWithSpaces(4) eclipse() removeUnusedImports() trimTrailingWhitespace() endWithNewline() } kotlin { + indentWithSpaces(4) ktlint() trimTrailingWhitespace() endWithNewline() diff --git a/src/main/java/mdnet/base/MangaDexClient.java b/src/main/java/mdnet/base/MangaDexClient.java index e818258..3a09e17 100644 --- a/src/main/java/mdnet/base/MangaDexClient.java +++ b/src/main/java/mdnet/base/MangaDexClient.java @@ -3,6 +3,8 @@ package mdnet.base; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import mdnet.base.settings.ClientSettings; +import mdnet.base.web.ApplicationKt; +import mdnet.base.web.WebUiKt; import mdnet.cache.DiskLruCache; import mdnet.webui.WebConsole; import org.http4k.server.Http4kServer; @@ -29,6 +31,7 @@ public class MangaDexClient { // if this is null, then the server has shutdown private Http4kServer engine; + private Http4kServer webUi; private DiskLruCache cache; public MangaDexClient(ClientSettings clientSettings) { @@ -46,7 +49,7 @@ public class MangaDexClient { // This function also does most of the program initialization. public void runLoop() { - statistics.set(new Statistics()); + statistics.set(new Statistics(0)); loginAndStartServer(); if (serverSettings.getLatestBuild() > Constants.CLIENT_BUILD) { if (LOGGER.isWarnEnabled()) { @@ -59,6 +62,9 @@ public class MangaDexClient { LOGGER.info("MDNet initialization completed successfully. Starting normal operation."); } + webUi = WebUiKt.getUiServer(clientSettings.getWebSettings(), statistics); + webUi.start(); + // we don't really care about the Atomic part here AtomicInteger counter = new AtomicInteger(); // ping keep-alive every 45 seconds @@ -71,7 +77,7 @@ public class MangaDexClient { if (LOGGER.isInfoEnabled()) { LOGGER.info("Hourly update: refreshing statistics"); } - statistics.set(new Statistics()); + statistics.set(new Statistics(statistics.get().getSequenceNumber() + 1)); if (engine == null) { if (LOGGER.isInfoEnabled()) { @@ -218,7 +224,7 @@ public class MangaDexClient { // TODO: system.out redirect ClientSettings finalSettings = settings; new Thread(() -> { - WebConsole webConsole = new WebConsole(finalSettings.getWebSettings().getClientWebsocketPort()) { + WebConsole webConsole = new WebConsole(finalSettings.getWebSettings().getUiWebsocketPort()) { @Override protected void parseMessage(String message) { System.out.println(message); diff --git a/src/main/java/mdnet/base/Statistics.java b/src/main/java/mdnet/base/Statistics.java index 29651d3..1da1fee 100644 --- a/src/main/java/mdnet/base/Statistics.java +++ b/src/main/java/mdnet/base/Statistics.java @@ -1,19 +1,28 @@ package mdnet.base; +import com.google.gson.annotations.SerializedName; + import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; public class Statistics { + @SerializedName("requests_served") private final AtomicInteger requestsServed; + @SerializedName("cache_hits") private final AtomicInteger cacheHits; + @SerializedName("cache_misses") private final AtomicInteger cacheMisses; + @SerializedName("bytes_sent") private final AtomicLong bytesSent; + @SerializedName("sequence_number") + private final int sequenceNumber; - public Statistics() { + public Statistics(int sequenceNumber) { requestsServed = new AtomicInteger(); cacheHits = new AtomicInteger(); cacheMisses = new AtomicInteger(); bytesSent = new AtomicLong(); + this.sequenceNumber = sequenceNumber; } public AtomicInteger getRequestsServed() { @@ -32,9 +41,18 @@ public class Statistics { return bytesSent; } + public int getSequenceNumber() { + return sequenceNumber; + } + @Override public String toString() { - return "Statistics{" + "requestsServed=" + requestsServed + ", cacheHits=" + cacheHits + ", cacheMisses=" - + cacheMisses + ", bytesSent=" + bytesSent + '}'; + return "Statistics{" + + "requestsServed=" + requestsServed + + ", cacheHits=" + cacheHits + + ", cacheMisses=" + cacheMisses + + ", bytesSent=" + bytesSent + + ", sequenceNumber=" + sequenceNumber + + '}'; } } diff --git a/src/main/java/mdnet/base/settings/ClientSettings.java b/src/main/java/mdnet/base/settings/ClientSettings.java index 3e424e2..8e854c5 100644 --- a/src/main/java/mdnet/base/settings/ClientSettings.java +++ b/src/main/java/mdnet/base/settings/ClientSettings.java @@ -32,7 +32,7 @@ public final class ClientSettings { } public ClientSettings(long maxCacheSizeMib, long maxBandwidthMibPerHour, long maxBurstRateKibPerSecond, - int clientPort, String clientSecret, int threads, WebSettings webSettings) { + int clientPort, String clientSecret, int threads, WebSettings webSettings) { this.maxCacheSizeMib = maxCacheSizeMib; this.maxBandwidthMibPerHour = maxBandwidthMibPerHour; this.maxBurstRateKibPerSecond = maxBurstRateKibPerSecond; diff --git a/src/main/java/mdnet/base/settings/WebSettings.java b/src/main/java/mdnet/base/settings/WebSettings.java index 7708789..03ea12d 100644 --- a/src/main/java/mdnet/base/settings/WebSettings.java +++ b/src/main/java/mdnet/base/settings/WebSettings.java @@ -3,23 +3,31 @@ package mdnet.base.settings; import com.google.gson.annotations.SerializedName; public final class WebSettings { - @SerializedName("client_websocket_port") - private final int clientWebsocketPort; + @SerializedName("ui_websocket_port") + private final int uiWebsocketPort; + @SerializedName("ui_port") + private final int uiPort; public WebSettings() { - this.clientWebsocketPort = 33333; + this.uiWebsocketPort = 33333; + this.uiPort = 8080; } - public WebSettings(int clientWebsocketPort) { - this.clientWebsocketPort = clientWebsocketPort; + public WebSettings(int uiWebsocketPort, int uiPort) { + this.uiWebsocketPort = uiWebsocketPort; + this.uiPort = uiPort; } - public int getClientWebsocketPort() { - return clientWebsocketPort; + public int getUiWebsocketPort() { + return uiWebsocketPort; + } + + public int getUiPort() { + return uiPort; } @Override public String toString() { - return "WebSettings{" + "clientWebsocketPort=" + clientWebsocketPort + '}'; + return "WebSettings{" + "uiWebsocketPort=" + uiWebsocketPort + ", uiPort=" + uiPort + '}'; } } diff --git a/src/main/java/mdnet/base/CachingInputStream.java b/src/main/java/mdnet/cache/CachingInputStream.java similarity index 99% rename from src/main/java/mdnet/base/CachingInputStream.java rename to src/main/java/mdnet/cache/CachingInputStream.java index c544e2b..9ea5fcd 100644 --- a/src/main/java/mdnet/base/CachingInputStream.java +++ b/src/main/java/mdnet/cache/CachingInputStream.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package mdnet.base; +package mdnet.cache; import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.ProxyInputStream; diff --git a/src/main/java/mdnet/cache/DiskLruCache.java b/src/main/java/mdnet/cache/DiskLruCache.java index 91ad816..e8c0556 100644 --- a/src/main/java/mdnet/cache/DiskLruCache.java +++ b/src/main/java/mdnet/cache/DiskLruCache.java @@ -82,10 +82,9 @@ import java.util.regex.Pattern; *
  • When an entry is being edited, it is not necessary to * supply data for every value; values default to their previous value. * - * Every {@link #editImpl} call must be matched by a call to - * {@link Editor#commit} or {@link Editor#abort}. Committing is atomic: a read - * observes the full set of values as they were before or after the commit, but - * never a mix of values. + * Every {@link #edit} call must be matched by a call to {@link Editor#commit} + * or {@link Editor#abort}. Committing is atomic: a read observes the full set + * of values as they were before or after the commit, but never a mix of values. * *

    * Clients call {@link #get} to read a snapshot of an entry. The read will diff --git a/src/main/java/mdnet/webui/WebConsole.java b/src/main/java/mdnet/webui/WebConsole.java index 9515793..7644b82 100644 --- a/src/main/java/mdnet/webui/WebConsole.java +++ b/src/main/java/mdnet/webui/WebConsole.java @@ -2,8 +2,6 @@ package mdnet.webui; import java.net.InetSocketAddress; import java.nio.ByteBuffer; -import java.util.concurrent.atomic.AtomicReference; -import mdnet.base.Statistics; import org.java_websocket.WebSocket; import org.java_websocket.handshake.ClientHandshake; import org.java_websocket.server.WebSocketServer; @@ -78,30 +76,30 @@ public abstract class WebConsole extends WebSocketServer { // } public void sendMessage(String type, Object message) { -// JSONObject out = new JSONObject(); -// switch (type) { -// case "command" : -// out.put("type", "command"); -// out.put("data", message.toString()); -// break; -// case "stats" : -// out.put("type", "stats"); -// AtomicReference temp = (AtomicReference) message; -// out.put("hits", temp.get().getCacheHits()); -// out.put("misses", temp.get().getCacheMisses()); -// out.put("bytes_sent", temp.get().getBytesSent()); -// out.put("req_served", temp.get().getRequestsServed()); -// out.put("dataval", "empty"); -// out.put("dataval", "empty"); -// out.put("dataval", "empty"); -// break; -// case "auth" : -// break; -// default : -// out.put("type", "command"); -// out.put("data", message.toString()); -// break; -// } -// broadcast(out.toString()); + // JSONObject out = new JSONObject(); + // switch (type) { + // case "command" : + // out.put("type", "command"); + // out.put("data", message.toString()); + // break; + // case "stats" : + // out.put("type", "stats"); + // AtomicReference temp = (AtomicReference) message; + // out.put("hits", temp.get().getCacheHits()); + // out.put("misses", temp.get().getCacheMisses()); + // out.put("bytes_sent", temp.get().getBytesSent()); + // out.put("req_served", temp.get().getRequestsServed()); + // out.put("dataval", "empty"); + // out.put("dataval", "empty"); + // out.put("dataval", "empty"); + // break; + // case "auth" : + // break; + // default : + // out.put("type", "command"); + // out.put("data", message.toString()); + // break; + // } + // broadcast(out.toString()); } } diff --git a/src/main/kotlin/mdnet/base/Application.kt b/src/main/kotlin/mdnet/base/web/Application.kt similarity index 90% rename from src/main/kotlin/mdnet/base/Application.kt rename to src/main/kotlin/mdnet/base/web/Application.kt index 88b7e18..3daead4 100644 --- a/src/main/kotlin/mdnet/base/Application.kt +++ b/src/main/kotlin/mdnet/base/web/Application.kt @@ -1,15 +1,18 @@ /* ktlint-disable no-wildcard-imports */ -package mdnet.base +package mdnet.base.web +import mdnet.base.Constants +import mdnet.base.Netty +import mdnet.base.ServerSettings +import mdnet.base.Statistics import mdnet.base.settings.ClientSettings +import mdnet.cache.CachingInputStream import mdnet.cache.DiskLruCache import org.apache.http.client.config.CookieSpecs import org.apache.http.client.config.RequestConfig import org.apache.http.impl.client.HttpClients import org.http4k.client.ApacheClient import org.http4k.core.BodyMode -import org.http4k.core.Filter -import org.http4k.core.HttpHandler import org.http4k.core.Method import org.http4k.core.Request import org.http4k.core.Response @@ -28,8 +31,6 @@ import java.io.BufferedInputStream import java.io.BufferedOutputStream import java.io.InputStream import java.security.MessageDigest -import java.time.ZoneOffset -import java.time.ZonedDateTime import java.time.format.DateTimeFormatter import java.util.* import java.util.concurrent.Executors @@ -40,7 +41,7 @@ import javax.crypto.CipherOutputStream import javax.crypto.spec.SecretKeySpec private val LOGGER = LoggerFactory.getLogger("Application") -private val THREADS_TO_ALLOCATE = 262144 // 2**18 // Honestly, no reason to not just let 'er rip. Inactive connections will expire on their own :D +private const val THREADS_TO_ALLOCATE = 262144 // 2**18 // Honestly, no reason to not just let 'er rip. Inactive connections will expire on their own :D fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSettings: ClientSettings, statistics: AtomicReference): Http4kServer { val executor = Executors.newCachedThreadPool() @@ -226,30 +227,6 @@ private fun getRc4(key: ByteArray): Cipher { private val HTTP_TIME_FORMATTER = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss O", Locale.ENGLISH) -private fun addCommonHeaders(): Filter { - return Filter { next: HttpHandler -> - { request: Request -> - val response = next(request) - response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC))) - .header("Server", "Mangadex@Home Node ${Constants.CLIENT_VERSION} (${Constants.CLIENT_BUILD})") - } - } -} - -private fun catchAllHideDetails(): Filter { - return Filter { next: HttpHandler -> - { request: Request -> - try { - next(request) - } catch (e: Exception) { - if (LOGGER.isWarnEnabled) { - LOGGER.warn("Request error detected", e) - } - Response(Status.INTERNAL_SERVER_ERROR) - } - } - } -} private fun md5Bytes(stringToHash: String): ByteArray { val digest = MessageDigest.getInstance("MD5") diff --git a/src/main/kotlin/mdnet/base/web/WebUi.kt b/src/main/kotlin/mdnet/base/web/WebUi.kt new file mode 100644 index 0000000..baa75c1 --- /dev/null +++ b/src/main/kotlin/mdnet/base/web/WebUi.kt @@ -0,0 +1,37 @@ +/* ktlint-disable no-wildcard-imports */ +package mdnet.base.web + +import mdnet.base.Statistics +import mdnet.base.settings.WebSettings +import org.http4k.core.Body +import org.http4k.core.Method +import org.http4k.core.Response +import org.http4k.core.Status +import org.http4k.core.then +import org.http4k.filter.ServerFilters +import org.http4k.routing.ResourceLoader +import org.http4k.routing.bind +import org.http4k.routing.routes +import org.http4k.routing.singlePageApp +import org.http4k.server.Http4kServer +import org.http4k.server.Netty +import org.http4k.server.asServer +import java.util.concurrent.atomic.AtomicReference +import org.http4k.format.Gson.auto + +fun getUiServer(webSettings: WebSettings, statistics: AtomicReference): Http4kServer { + val statisticsLens = Body.auto().toLens() + + return catchAllHideDetails() + .then(ServerFilters.CatchLensFailure) + .then(addCommonHeaders()) + .then( + routes( + "/api/stats" bind Method.GET to { + statisticsLens(statistics.get(), Response(Status.OK)) + }, + singlePageApp(ResourceLoader.Classpath("/webui")) + ) + ) + .asServer(Netty(webSettings.uiPort)) +} diff --git a/src/main/kotlin/mdnet/base/web/common.kt b/src/main/kotlin/mdnet/base/web/common.kt new file mode 100644 index 0000000..47b2c78 --- /dev/null +++ b/src/main/kotlin/mdnet/base/web/common.kt @@ -0,0 +1,43 @@ +/* ktlint-disable no-wildcard-imports */ +package mdnet.base.web + +import mdnet.base.Constants +import org.http4k.core.Filter +import org.http4k.core.HttpHandler +import org.http4k.core.Request +import org.http4k.core.Response +import org.http4k.core.Status +import org.slf4j.LoggerFactory +import java.time.ZoneOffset +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.util.* + +private val HTTP_TIME_FORMATTER = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss O", Locale.ENGLISH) + +private val LOGGER = LoggerFactory.getLogger("Application") + +fun addCommonHeaders(): Filter { + return Filter { next: HttpHandler -> + { request: Request -> + val response = next(request) + response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC))) + .header("Server", "Mangadex@Home Node ${Constants.CLIENT_VERSION} (${Constants.CLIENT_BUILD})") + } + } +} + +fun catchAllHideDetails(): Filter { + return Filter { next: HttpHandler -> + { request: Request -> + try { + next(request) + } catch (e: Exception) { + if (LOGGER.isWarnEnabled) { + LOGGER.warn("Request error detected", e) + } + Response(Status.INTERNAL_SERVER_ERROR) + } + } + } +}