This commit is contained in:
carbotaniuman 2020-06-12 14:56:51 -05:00
parent 9086e09c60
commit f153a11717
11 changed files with 170 additions and 78 deletions

View file

@ -22,8 +22,12 @@ dependencies {
implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-core", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-server-netty", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version" implementation group: "org.http4k", name: "http4k-client-apache", version: "$http_4k_version"
implementation group: "org.http4k", name: "http4k-format-gson", version: "3.249.0"
implementation group: "commons-io", name: "commons-io", version: "2.7" implementation group: "commons-io", name: "commons-io", version: "2.7"
compile "org.java-websocket:Java-WebSocket:1.5.1"
implementation group: 'org.java-websocket', name: 'Java-WebSocket', version: '1.5.1'
implementation "ch.qos.logback:logback-classic:$logback_version" implementation "ch.qos.logback:logback-classic:$logback_version"
runtimeOnly 'io.netty:netty-tcnative-boringssl-static:2.0.30.Final' runtimeOnly 'io.netty:netty-tcnative-boringssl-static:2.0.30.Final'
} }
@ -35,12 +39,14 @@ java {
spotless { spotless {
java { java {
indentWithSpaces(4)
eclipse() eclipse()
removeUnusedImports() removeUnusedImports()
trimTrailingWhitespace() trimTrailingWhitespace()
endWithNewline() endWithNewline()
} }
kotlin { kotlin {
indentWithSpaces(4)
ktlint() ktlint()
trimTrailingWhitespace() trimTrailingWhitespace()
endWithNewline() endWithNewline()

View file

@ -3,6 +3,8 @@ package mdnet.base;
import com.google.gson.Gson; import com.google.gson.Gson;
import com.google.gson.GsonBuilder; import com.google.gson.GsonBuilder;
import mdnet.base.settings.ClientSettings; import mdnet.base.settings.ClientSettings;
import mdnet.base.web.ApplicationKt;
import mdnet.base.web.WebUiKt;
import mdnet.cache.DiskLruCache; import mdnet.cache.DiskLruCache;
import mdnet.webui.WebConsole; import mdnet.webui.WebConsole;
import org.http4k.server.Http4kServer; import org.http4k.server.Http4kServer;
@ -29,6 +31,7 @@ public class MangaDexClient {
// if this is null, then the server has shutdown // if this is null, then the server has shutdown
private Http4kServer engine; private Http4kServer engine;
private Http4kServer webUi;
private DiskLruCache cache; private DiskLruCache cache;
public MangaDexClient(ClientSettings clientSettings) { public MangaDexClient(ClientSettings clientSettings) {
@ -46,7 +49,7 @@ public class MangaDexClient {
// This function also does most of the program initialization. // This function also does most of the program initialization.
public void runLoop() { public void runLoop() {
statistics.set(new Statistics()); statistics.set(new Statistics(0));
loginAndStartServer(); loginAndStartServer();
if (serverSettings.getLatestBuild() > Constants.CLIENT_BUILD) { if (serverSettings.getLatestBuild() > Constants.CLIENT_BUILD) {
if (LOGGER.isWarnEnabled()) { if (LOGGER.isWarnEnabled()) {
@ -59,6 +62,9 @@ public class MangaDexClient {
LOGGER.info("MDNet initialization completed successfully. Starting normal operation."); LOGGER.info("MDNet initialization completed successfully. Starting normal operation.");
} }
webUi = WebUiKt.getUiServer(clientSettings.getWebSettings(), statistics);
webUi.start();
// we don't really care about the Atomic part here // we don't really care about the Atomic part here
AtomicInteger counter = new AtomicInteger(); AtomicInteger counter = new AtomicInteger();
// ping keep-alive every 45 seconds // ping keep-alive every 45 seconds
@ -71,7 +77,7 @@ public class MangaDexClient {
if (LOGGER.isInfoEnabled()) { if (LOGGER.isInfoEnabled()) {
LOGGER.info("Hourly update: refreshing statistics"); LOGGER.info("Hourly update: refreshing statistics");
} }
statistics.set(new Statistics()); statistics.set(new Statistics(statistics.get().getSequenceNumber() + 1));
if (engine == null) { if (engine == null) {
if (LOGGER.isInfoEnabled()) { if (LOGGER.isInfoEnabled()) {
@ -218,7 +224,7 @@ public class MangaDexClient {
// TODO: system.out redirect // TODO: system.out redirect
ClientSettings finalSettings = settings; ClientSettings finalSettings = settings;
new Thread(() -> { new Thread(() -> {
WebConsole webConsole = new WebConsole(finalSettings.getWebSettings().getClientWebsocketPort()) { WebConsole webConsole = new WebConsole(finalSettings.getWebSettings().getUiWebsocketPort()) {
@Override @Override
protected void parseMessage(String message) { protected void parseMessage(String message) {
System.out.println(message); System.out.println(message);

View file

@ -1,19 +1,28 @@
package mdnet.base; package mdnet.base;
import com.google.gson.annotations.SerializedName;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
public class Statistics { public class Statistics {
@SerializedName("requests_served")
private final AtomicInteger requestsServed; private final AtomicInteger requestsServed;
@SerializedName("cache_hits")
private final AtomicInteger cacheHits; private final AtomicInteger cacheHits;
@SerializedName("cache_misses")
private final AtomicInteger cacheMisses; private final AtomicInteger cacheMisses;
@SerializedName("bytes_sent")
private final AtomicLong bytesSent; private final AtomicLong bytesSent;
@SerializedName("sequence_number")
private final int sequenceNumber;
public Statistics() { public Statistics(int sequenceNumber) {
requestsServed = new AtomicInteger(); requestsServed = new AtomicInteger();
cacheHits = new AtomicInteger(); cacheHits = new AtomicInteger();
cacheMisses = new AtomicInteger(); cacheMisses = new AtomicInteger();
bytesSent = new AtomicLong(); bytesSent = new AtomicLong();
this.sequenceNumber = sequenceNumber;
} }
public AtomicInteger getRequestsServed() { public AtomicInteger getRequestsServed() {
@ -32,9 +41,18 @@ public class Statistics {
return bytesSent; return bytesSent;
} }
public int getSequenceNumber() {
return sequenceNumber;
}
@Override @Override
public String toString() { public String toString() {
return "Statistics{" + "requestsServed=" + requestsServed + ", cacheHits=" + cacheHits + ", cacheMisses=" return "Statistics{" +
+ cacheMisses + ", bytesSent=" + bytesSent + '}'; "requestsServed=" + requestsServed +
", cacheHits=" + cacheHits +
", cacheMisses=" + cacheMisses +
", bytesSent=" + bytesSent +
", sequenceNumber=" + sequenceNumber +
'}';
} }
} }

View file

@ -3,23 +3,31 @@ package mdnet.base.settings;
import com.google.gson.annotations.SerializedName; import com.google.gson.annotations.SerializedName;
public final class WebSettings { public final class WebSettings {
@SerializedName("client_websocket_port") @SerializedName("ui_websocket_port")
private final int clientWebsocketPort; private final int uiWebsocketPort;
@SerializedName("ui_port")
private final int uiPort;
public WebSettings() { public WebSettings() {
this.clientWebsocketPort = 33333; this.uiWebsocketPort = 33333;
this.uiPort = 8080;
} }
public WebSettings(int clientWebsocketPort) { public WebSettings(int uiWebsocketPort, int uiPort) {
this.clientWebsocketPort = clientWebsocketPort; this.uiWebsocketPort = uiWebsocketPort;
this.uiPort = uiPort;
} }
public int getClientWebsocketPort() { public int getUiWebsocketPort() {
return clientWebsocketPort; return uiWebsocketPort;
}
public int getUiPort() {
return uiPort;
} }
@Override @Override
public String toString() { public String toString() {
return "WebSettings{" + "clientWebsocketPort=" + clientWebsocketPort + '}'; return "WebSettings{" + "uiWebsocketPort=" + uiWebsocketPort + ", uiPort=" + uiPort + '}';
} }
} }

View file

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package mdnet.base; package mdnet.cache;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.ProxyInputStream; import org.apache.commons.io.input.ProxyInputStream;

View file

@ -82,10 +82,9 @@ import java.util.regex.Pattern;
* <li>When an entry is being <strong>edited</strong>, it is not necessary to * <li>When an entry is being <strong>edited</strong>, it is not necessary to
* supply data for every value; values default to their previous value. * supply data for every value; values default to their previous value.
* </ul> * </ul>
* Every {@link #editImpl} call must be matched by a call to * Every {@link #edit} call must be matched by a call to {@link Editor#commit}
* {@link Editor#commit} or {@link Editor#abort}. Committing is atomic: a read * or {@link Editor#abort}. Committing is atomic: a read observes the full set
* observes the full set of values as they were before or after the commit, but * of values as they were before or after the commit, but never a mix of values.
* never a mix of values.
* *
* <p> * <p>
* Clients call {@link #get} to read a snapshot of an entry. The read will * Clients call {@link #get} to read a snapshot of an entry. The read will

View file

@ -2,8 +2,6 @@ package mdnet.webui;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicReference;
import mdnet.base.Statistics;
import org.java_websocket.WebSocket; import org.java_websocket.WebSocket;
import org.java_websocket.handshake.ClientHandshake; import org.java_websocket.handshake.ClientHandshake;
import org.java_websocket.server.WebSocketServer; import org.java_websocket.server.WebSocketServer;
@ -78,30 +76,30 @@ public abstract class WebConsole extends WebSocketServer {
// } // }
public void sendMessage(String type, Object message) { public void sendMessage(String type, Object message) {
// JSONObject out = new JSONObject(); // JSONObject out = new JSONObject();
// switch (type) { // switch (type) {
// case "command" : // case "command" :
// out.put("type", "command"); // out.put("type", "command");
// out.put("data", message.toString()); // out.put("data", message.toString());
// break; // break;
// case "stats" : // case "stats" :
// out.put("type", "stats"); // out.put("type", "stats");
// AtomicReference<Statistics> temp = (AtomicReference<Statistics>) message; // AtomicReference<Statistics> temp = (AtomicReference<Statistics>) message;
// out.put("hits", temp.get().getCacheHits()); // out.put("hits", temp.get().getCacheHits());
// out.put("misses", temp.get().getCacheMisses()); // out.put("misses", temp.get().getCacheMisses());
// out.put("bytes_sent", temp.get().getBytesSent()); // out.put("bytes_sent", temp.get().getBytesSent());
// out.put("req_served", temp.get().getRequestsServed()); // out.put("req_served", temp.get().getRequestsServed());
// out.put("dataval", "empty"); // out.put("dataval", "empty");
// out.put("dataval", "empty"); // out.put("dataval", "empty");
// out.put("dataval", "empty"); // out.put("dataval", "empty");
// break; // break;
// case "auth" : // case "auth" :
// break; // break;
// default : // default :
// out.put("type", "command"); // out.put("type", "command");
// out.put("data", message.toString()); // out.put("data", message.toString());
// break; // break;
// } // }
// broadcast(out.toString()); // broadcast(out.toString());
} }
} }

View file

@ -1,15 +1,18 @@
/* ktlint-disable no-wildcard-imports */ /* ktlint-disable no-wildcard-imports */
package mdnet.base package mdnet.base.web
import mdnet.base.Constants
import mdnet.base.Netty
import mdnet.base.ServerSettings
import mdnet.base.Statistics
import mdnet.base.settings.ClientSettings import mdnet.base.settings.ClientSettings
import mdnet.cache.CachingInputStream
import mdnet.cache.DiskLruCache import mdnet.cache.DiskLruCache
import org.apache.http.client.config.CookieSpecs import org.apache.http.client.config.CookieSpecs
import org.apache.http.client.config.RequestConfig import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.HttpClients import org.apache.http.impl.client.HttpClients
import org.http4k.client.ApacheClient import org.http4k.client.ApacheClient
import org.http4k.core.BodyMode import org.http4k.core.BodyMode
import org.http4k.core.Filter
import org.http4k.core.HttpHandler
import org.http4k.core.Method import org.http4k.core.Method
import org.http4k.core.Request import org.http4k.core.Request
import org.http4k.core.Response import org.http4k.core.Response
@ -28,8 +31,6 @@ import java.io.BufferedInputStream
import java.io.BufferedOutputStream import java.io.BufferedOutputStream
import java.io.InputStream import java.io.InputStream
import java.security.MessageDigest import java.security.MessageDigest
import java.time.ZoneOffset
import java.time.ZonedDateTime
import java.time.format.DateTimeFormatter import java.time.format.DateTimeFormatter
import java.util.* import java.util.*
import java.util.concurrent.Executors import java.util.concurrent.Executors
@ -40,7 +41,7 @@ import javax.crypto.CipherOutputStream
import javax.crypto.spec.SecretKeySpec import javax.crypto.spec.SecretKeySpec
private val LOGGER = LoggerFactory.getLogger("Application") private val LOGGER = LoggerFactory.getLogger("Application")
private val THREADS_TO_ALLOCATE = 262144 // 2**18 // Honestly, no reason to not just let 'er rip. Inactive connections will expire on their own :D private const val THREADS_TO_ALLOCATE = 262144 // 2**18 // Honestly, no reason to not just let 'er rip. Inactive connections will expire on their own :D
fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSettings: ClientSettings, statistics: AtomicReference<Statistics>): Http4kServer { fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSettings: ClientSettings, statistics: AtomicReference<Statistics>): Http4kServer {
val executor = Executors.newCachedThreadPool() val executor = Executors.newCachedThreadPool()
@ -226,30 +227,6 @@ private fun getRc4(key: ByteArray): Cipher {
private val HTTP_TIME_FORMATTER = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss O", Locale.ENGLISH) private val HTTP_TIME_FORMATTER = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss O", Locale.ENGLISH)
private fun addCommonHeaders(): Filter {
return Filter { next: HttpHandler ->
{ request: Request ->
val response = next(request)
response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC)))
.header("Server", "Mangadex@Home Node ${Constants.CLIENT_VERSION} (${Constants.CLIENT_BUILD})")
}
}
}
private fun catchAllHideDetails(): Filter {
return Filter { next: HttpHandler ->
{ request: Request ->
try {
next(request)
} catch (e: Exception) {
if (LOGGER.isWarnEnabled) {
LOGGER.warn("Request error detected", e)
}
Response(Status.INTERNAL_SERVER_ERROR)
}
}
}
}
private fun md5Bytes(stringToHash: String): ByteArray { private fun md5Bytes(stringToHash: String): ByteArray {
val digest = MessageDigest.getInstance("MD5") val digest = MessageDigest.getInstance("MD5")

View file

@ -0,0 +1,37 @@
/* ktlint-disable no-wildcard-imports */
package mdnet.base.web
import mdnet.base.Statistics
import mdnet.base.settings.WebSettings
import org.http4k.core.Body
import org.http4k.core.Method
import org.http4k.core.Response
import org.http4k.core.Status
import org.http4k.core.then
import org.http4k.filter.ServerFilters
import org.http4k.routing.ResourceLoader
import org.http4k.routing.bind
import org.http4k.routing.routes
import org.http4k.routing.singlePageApp
import org.http4k.server.Http4kServer
import org.http4k.server.Netty
import org.http4k.server.asServer
import java.util.concurrent.atomic.AtomicReference
import org.http4k.format.Gson.auto
fun getUiServer(webSettings: WebSettings, statistics: AtomicReference<Statistics>): Http4kServer {
val statisticsLens = Body.auto<Statistics>().toLens()
return catchAllHideDetails()
.then(ServerFilters.CatchLensFailure)
.then(addCommonHeaders())
.then(
routes(
"/api/stats" bind Method.GET to {
statisticsLens(statistics.get(), Response(Status.OK))
},
singlePageApp(ResourceLoader.Classpath("/webui"))
)
)
.asServer(Netty(webSettings.uiPort))
}

View file

@ -0,0 +1,43 @@
/* ktlint-disable no-wildcard-imports */
package mdnet.base.web
import mdnet.base.Constants
import org.http4k.core.Filter
import org.http4k.core.HttpHandler
import org.http4k.core.Request
import org.http4k.core.Response
import org.http4k.core.Status
import org.slf4j.LoggerFactory
import java.time.ZoneOffset
import java.time.ZonedDateTime
import java.time.format.DateTimeFormatter
import java.util.*
private val HTTP_TIME_FORMATTER = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss O", Locale.ENGLISH)
private val LOGGER = LoggerFactory.getLogger("Application")
fun addCommonHeaders(): Filter {
return Filter { next: HttpHandler ->
{ request: Request ->
val response = next(request)
response.header("Date", HTTP_TIME_FORMATTER.format(ZonedDateTime.now(ZoneOffset.UTC)))
.header("Server", "Mangadex@Home Node ${Constants.CLIENT_VERSION} (${Constants.CLIENT_BUILD})")
}
}
}
fun catchAllHideDetails(): Filter {
return Filter { next: HttpHandler ->
{ request: Request ->
try {
next(request)
} catch (e: Exception) {
if (LOGGER.isWarnEnabled) {
LOGGER.warn("Request error detected", e)
}
Response(Status.INTERNAL_SERVER_ERROR)
}
}
}
}