Merge branch 'graceful' into 'master'

Add graceful shutdown

See merge request mangadex/mangadex_at_home!33

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 ### Changed
+- [2020-06-16] Reworked graceful shutdown [@carbotaniuman].
 
 ### Deprecated
 

build.gradle

@@ -9,7 +9,7 @@ plugins {
 
 group = "com.mangadex"
 version = "git describe --tags --dirty".execute().text.trim()
-mainClassName = "mdnet.base.MangaDexClient"
+mainClassName = "mdnet.base.Main"
 
 repositories {
   mavenCentral()

src/main/java/mdnet/base/Main.java

@@ -0,0 +1,104 @@
+package mdnet.base;
+
+import mdnet.base.settings.ClientSettings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.regex.Pattern;
+
+import static mdnet.base.Constants.GSON;
+
+public class Main {
+	private final static Logger LOGGER = LoggerFactory.getLogger(Main.class);
+
+	public static void main(String[] args) {
+		System.out.println("Mangadex@Home Client " + Constants.CLIENT_VERSION + " (Build " + Constants.CLIENT_BUILD
+				+ ") initializing\n");
+		System.out.println("Copyright (c) 2020, MangaDex Network");
+
+		String file = "settings.json";
+		if (args.length == 1) {
+			file = args[0];
+		} else if (args.length != 0) {
+			dieWithError("Expected one argument: path to config file, or nothing");
+		}
+
+		ClientSettings settings;
+
+		try {
+			settings = GSON.fromJson(new FileReader(file), ClientSettings.class);
+		} catch (FileNotFoundException ignored) {
+			settings = new ClientSettings();
+			LOGGER.warn("Settings file {} not found, generating file", file);
+			try (FileWriter writer = new FileWriter(file)) {
+				writer.write(GSON.toJson(settings));
+			} catch (IOException e) {
+				dieWithError(e);
+			}
+		}
+
+		validateSettings(settings);
+
+		if (LOGGER.isInfoEnabled()) {
+			LOGGER.info("Client settings loaded: {}", settings);
+		}
+
+		MangaDexClient client = new MangaDexClient(settings);
+		Runtime.getRuntime().addShutdownHook(new Thread(client::shutdown));
+		client.runLoop();
+	}
+
+	public static void dieWithError(Throwable e) {
+		if (LOGGER.isErrorEnabled()) {
+			LOGGER.error("Critical Error", e);
+		}
+		System.exit(1);
+	}
+
+	public static void dieWithError(String error) {
+		if (LOGGER.isErrorEnabled()) {
+			LOGGER.error("Critical Error: {}", error);
+		}
+		System.exit(1);
+	}
+
+	public static void validateSettings(ClientSettings settings) {
+		if (!isSecretValid(settings.getClientSecret()))
+			dieWithError("Config Error: API Secret is invalid, must be 52 alphanumeric characters");
+
+		if (settings.getClientPort() == 0) {
+			dieWithError("Config Error: Invalid port number");
+		}
+
+		if (settings.getMaxCacheSizeMib() < 1024) {
+			dieWithError("Config Error: Invalid max cache size, must be >= 1024 MiB (1GiB)");
+		}
+
+		if (settings.getThreads() < 4) {
+			dieWithError("Config Error: Invalid number of threads, must be >= 4");
+		}
+
+		if (settings.getMaxBandwidthMibPerHour() < 0) {
+			dieWithError("Config Error: Max bandwidth must be >= 0");
+		}
+
+		if (settings.getMaxBurstRateKibPerSecond() < 0) {
+			dieWithError("Config Error: Max burst rate must be >= 0");
+		}
+
+		if (settings.getWebSettings() != null) {
+			if (settings.getWebSettings().getUiPort() == 0) {
+				dieWithError("Config Error: Invalid UI port number");
+			}
+		}
+	}
+
+	public static boolean isSecretValid(String clientSecret) {
+		final int CLIENT_KEY_LENGTH = 52;
+		return Pattern.matches("^[a-zA-Z0-9]{" + CLIENT_KEY_LENGTH + "}$", clientSecret);
+	}
+}

src/main/java/mdnet/base/MangaDexClient.java

@@ -1,7 +1,5 @@
 package mdnet.base;
 
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
 import mdnet.base.settings.ClientSettings;
 import mdnet.base.server.ApplicationKt;
 import mdnet.base.server.WebUiKt;
@@ -18,15 +16,14 @@ import java.util.Map;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.regex.Pattern;
+
+import static mdnet.base.Constants.GSON;
 
 public class MangaDexClient {
-	private final static Gson GSON = new GsonBuilder().setPrettyPrinting().create();
 	private final static Logger LOGGER = LoggerFactory.getLogger(MangaDexClient.class);
 
-	// This lock protects the Http4kServer from concurrent restart attempts
-	private final Object shutdownLock = new Object();
 	private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
 	private final ServerHandler serverHandler;
 	private final ClientSettings clientSettings;
@@ -39,6 +36,7 @@ public class MangaDexClient {
 				}
 			});
 	private final AtomicReference<Statistics> statistics;
+	private final AtomicBoolean isHandled;
 
 	private ServerSettings serverSettings;
 	private Http4kServer engine; // if this is null, then the server has shutdown
@@ -48,11 +46,15 @@ public class MangaDexClient {
 	// these variables are for runLoop();
 	private int counter = 0;
 	private long lastBytesSent = 0;
+	// a non-negative number here means we are shutting down
+	private int gracefulCounter = -1;
+	private Runnable gracefulAction;
 
 	public MangaDexClient(ClientSettings clientSettings) {
 		this.clientSettings = clientSettings;
 		this.serverHandler = new ServerHandler(clientSettings);
 		this.statistics = new AtomicReference<>();
+		this.isHandled = new AtomicBoolean();
 
 		try {
 			cache = DiskLruCache.open(new File("cache"), 1, 1,
@@ -66,7 +68,7 @@ public class MangaDexClient {
 				statistics.set(new Statistics());
 			}
 		} catch (IOException e) {
-			MangaDexClient.dieWithError(e);
+			Main.dieWithError(e);
 		}
 	}
 
@@ -93,26 +95,8 @@ public class MangaDexClient {
 
 		executorService.scheduleWithFixedDelay(() -> {
 			try {
-				statistics.updateAndGet(n -> n.copy(n.getRequestsServed(), n.getCacheHits(), n.getCacheMisses(),
+				// Converting from 15 seconds loop to 45 second loop
-						n.getBrowserCached(), n.getBytesSent(), cache.size()));
+				if (counter / 3 == 80) {
-
-				statsMap.put(Instant.now(), statistics.get());
-
-				DiskLruCache.Editor editor = cache.edit("statistics");
-				if (editor != null) {
-					String json = GSON.toJson(statistics.get(), Statistics.class);
-					editor.setString(0, json);
-					editor.commit();
-				}
-			} catch (Exception e) {
-				LOGGER.warn("statistics update failed", e);
-			}
-
-		}, 15, 15, TimeUnit.SECONDS);
-
-		executorService.scheduleAtFixedRate(() -> {
-			try {
-				if (counter == 80) {
 					counter = 0;
 					lastBytesSent = statistics.get().getBytesSent();
 
@@ -127,20 +111,70 @@ public class MangaDexClient {
 					counter++;
 				}
 
+				if (gracefulCounter == 0) {
+					logout();
+					if (LOGGER.isInfoEnabled()) {
+						LOGGER.info("Waiting another 15 seconds for graceful shutdown ({} out of {} tries)",
+								gracefulCounter + 1, 4);
+					}
+					gracefulCounter++;
+				} else if (gracefulCounter > 0) {
+					if (!isHandled.get() || gracefulCounter == 4) {
+						if (LOGGER.isInfoEnabled()) {
+							if (!isHandled.get()) {
+								LOGGER.info("No requests received, shutting down");
+							} else {
+								LOGGER.info("Max tries attempted, shutting down");
+							}
+						}
+
+						stopServer();
+						if (gracefulAction != null) {
+							gracefulAction.run();
+						}
+
+						// reset variables
+						gracefulCounter = -1;
+						gracefulAction = null;
+					} else {
+						if (LOGGER.isInfoEnabled()) {
+							LOGGER.info("Waiting another 15 seconds for graceful shutdown ({} out of {} tries)",
+									gracefulCounter + 1, 4);
+						}
+						gracefulCounter++;
+					}
+					isHandled.set(false);
+				} else {
+					if (counter % 3 == 0) {
+						pingControl();
+					}
+					updateStats();
+				}
+
+			} catch (Exception e) {
+				LOGGER.warn("statistics update failed", e);
+			}
+
+		}, 15, 15, TimeUnit.SECONDS);
+	}
+
+	private void pingControl() {
 		// if the server is offline then don't try and refresh certs
 		if (engine == null) {
 			return;
 		}
 
 		long currentBytesSent = statistics.get().getBytesSent() - lastBytesSent;
-				if (clientSettings.getMaxBandwidthMibPerHour() != 0 && clientSettings.getMaxBandwidthMibPerHour() * 1024
+		if (clientSettings.getMaxBandwidthMibPerHour() != 0
-						* 1024 /* MiB to bytes */ < currentBytesSent) {
+				&& clientSettings.getMaxBandwidthMibPerHour() * 1024 * 1024 /* MiB to bytes */ < currentBytesSent) {
 			if (LOGGER.isInfoEnabled()) {
 				LOGGER.info("Shutting down server as hourly bandwidth limit reached");
 			}
 
-					synchronized (shutdownLock) {
+			// Give enough time for graceful shutdown
-						logoutAndStopServer();
+			if (240 - counter > 3) {
+				LOGGER.info("Graceful shutdown started");
+				gracefulCounter = 0;
 			}
 		}
 
@@ -164,25 +198,33 @@ public class MangaDexClient {
 					LOGGER.info("Doing internal restart of HTTP server to refresh certs/upstream URL");
 				}
 
-						synchronized (shutdownLock) {
+				LOGGER.info("Graceful shutdown started");
-							logoutAndStopServer();
+				gracefulCounter = 0;
-							loginAndStartServer();
+				gracefulAction = this::loginAndStartServer;
 			}
 		}
 	}
-			} catch (Exception e) {
-				LOGGER.warn("Server ping failed", e);
-			}
-		}, 45, 45, TimeUnit.SECONDS);
 
+	private void updateStats() throws IOException {
+		statistics.updateAndGet(n -> n.copy(n.getRequestsServed(), n.getCacheHits(), n.getCacheMisses(),
+				n.getBrowserCached(), n.getBytesSent(), cache.size()));
+
+		statsMap.put(Instant.now(), statistics.get());
+
+		DiskLruCache.Editor editor = cache.edit("statistics");
+		if (editor != null) {
+			String json = GSON.toJson(statistics.get(), Statistics.class);
+			editor.setString(0, json);
+			editor.commit();
+		}
 	}
 
 	private void loginAndStartServer() {
 		serverSettings = serverHandler.loginToControl();
 		if (serverSettings == null) {
-			MangaDexClient.dieWithError("Failed to get a login response from server - check API secret for validity");
+			Main.dieWithError("Failed to get a login response from server - check API secret for validity");
 		}
-		engine = ApplicationKt.getServer(cache, serverSettings, clientSettings, statistics);
+		engine = ApplicationKt.getServer(cache, serverSettings, clientSettings, statistics, isHandled);
 		engine.start();
 
 		if (LOGGER.isInfoEnabled()) {
@@ -190,11 +232,14 @@ public class MangaDexClient {
 		}
 	}
 
-	private void logoutAndStopServer() {
+	private void logout() {
-		if (LOGGER.isInfoEnabled()) {
-			LOGGER.info("Gracefully shutting down HTTP server");
-		}
 		serverHandler.logoutFromControl();
+	}
+
+	private void stopServer() {
+		if (LOGGER.isInfoEnabled()) {
+			LOGGER.info("Shutting down HTTP server");
+		}
 		engine.stop();
 		if (LOGGER.isInfoEnabled()) {
 			LOGGER.info("Internal HTTP server has gracefully shut down");
@@ -203,14 +248,10 @@ public class MangaDexClient {
 	}
 
 	public void shutdown() {
-		executorService.shutdown();
+		LOGGER.info("Graceful shutdown started");
-		synchronized (shutdownLock) {
+		gracefulCounter = 0;
-			if (engine == null) {
+		AtomicBoolean readyToExit = new AtomicBoolean(false);
-				return;
+		gracefulAction = () -> {
-			}
-
-			logoutAndStopServer();
-		}
 			if (webUi != null) {
 				webUi.close();
 			}
@@ -219,92 +260,10 @@ public class MangaDexClient {
 			} catch (IOException e) {
 				LOGGER.error("Cache failed to close", e);
 			}
-	}
+			executorService.shutdown();
-
+			readyToExit.set(true);
-	public static void main(String[] args) {
+		};
-		System.out.println("Mangadex@Home Client " + Constants.CLIENT_VERSION + " (Build " + Constants.CLIENT_BUILD
+		while (!readyToExit.get()) {
-				+ ") initializing\n");
-		System.out.println("Copyright (c) 2020, MangaDex Network");
-
-		String file = "settings.json";
-		if (args.length == 1) {
-			file = args[0];
-		} else if (args.length != 0) {
-			MangaDexClient.dieWithError("Expected one argument: path to config file, or nothing");
-		}
-
-		ClientSettings settings;
-
-		try {
-			settings = GSON.fromJson(new FileReader(file), ClientSettings.class);
-		} catch (FileNotFoundException ignored) {
-			settings = new ClientSettings();
-			LOGGER.warn("Settings file {} not found, generating file", file);
-			try (FileWriter writer = new FileWriter(file)) {
-				writer.write(GSON.toJson(settings));
-			} catch (IOException e) {
-				MangaDexClient.dieWithError(e);
-			}
-		}
-
-		validateSettings(settings);
-
-		if (LOGGER.isInfoEnabled()) {
-			LOGGER.info("Client settings loaded: {}", settings);
-		}
-
-		MangaDexClient client = new MangaDexClient(settings);
-		Runtime.getRuntime().addShutdownHook(new Thread(client::shutdown));
-		client.runLoop();
-	}
-
-	public static void dieWithError(Throwable e) {
-		if (LOGGER.isErrorEnabled()) {
-			LOGGER.error("Critical Error", e);
-		}
-		System.exit(1);
-	}
-
-	public static void dieWithError(String error) {
-		if (LOGGER.isErrorEnabled()) {
-			LOGGER.error("Critical Error: {}", error);
-		}
-		System.exit(1);
-	}
-
-	public static void validateSettings(ClientSettings settings) {
-		if (!isSecretValid(settings.getClientSecret()))
-			MangaDexClient.dieWithError("Config Error: API Secret is invalid, must be 52 alphanumeric characters");
-
-		if (settings.getClientPort() == 0) {
-			MangaDexClient.dieWithError("Config Error: Invalid port number");
-		}
-
-		if (settings.getMaxCacheSizeMib() < 1024) {
-			MangaDexClient.dieWithError("Config Error: Invalid max cache size, must be >= 1024 MiB (1GiB)");
-		}
-
-		if (settings.getThreads() < 4) {
-			MangaDexClient.dieWithError("Config Error: Invalid number of threads, must be >= 4");
-		}
-
-		if (settings.getMaxBandwidthMibPerHour() < 0) {
-			MangaDexClient.dieWithError("Config Error: Max bandwidth must be >= 0");
-		}
-
-		if (settings.getMaxBurstRateKibPerSecond() < 0) {
-			MangaDexClient.dieWithError("Config Error: Max burst rate must be >= 0");
-		}
-
-		if (settings.getWebSettings() != null) {
-			if (settings.getWebSettings().getUiPort() == 0) {
-				MangaDexClient.dieWithError("Config Error: Invalid UI port number");
 		}
 	}
 }
-
-	public static boolean isSecretValid(String clientSecret) {
-		final int CLIENT_KEY_LENGTH = 52;
-		return Pattern.matches("^[a-zA-Z0-9]{" + CLIENT_KEY_LENGTH + "}$", clientSecret);
-	}
-}

src/main/kotlin/mdnet/base/Constants.kt

@@ -1,9 +1,13 @@
 package mdnet.base
 
+import com.google.gson.Gson
+import com.google.gson.GsonBuilder
 import java.time.Duration
 
 object Constants {
     const val CLIENT_BUILD = 8
     const val CLIENT_VERSION = "1.0"
     val MAX_AGE_CACHE: Duration = Duration.ofDays(14)
+    @JvmField
+    val GSON: Gson = GsonBuilder().setPrettyPrinting().create()
 }

src/main/kotlin/mdnet/base/server/Application.kt

@@ -14,11 +14,12 @@ import org.http4k.routing.routes
 import org.http4k.server.Http4kServer
 import org.http4k.server.asServer
 import org.jetbrains.exposed.sql.Database
+import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
 
-fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSettings: ClientSettings, statistics: AtomicReference<Statistics>): Http4kServer {
+fun getServer(cache: DiskLruCache, serverSettings: ServerSettings, clientSettings: ClientSettings, statistics: AtomicReference<Statistics>, isHandled: AtomicBoolean): Http4kServer {
     val database = Database.connect("jdbc:sqlite:cache/data.db", "org.sqlite.JDBC")
-    val imageServer = ImageServer(cache, statistics, serverSettings.imageServer, database)
+    val imageServer = ImageServer(cache, statistics, serverSettings.imageServer, database, isHandled)
 
     return Timer
             .then(catchAllHideDetails())

src/main/kotlin/mdnet/base/server/ImageServer.kt

@@ -24,6 +24,7 @@ import java.io.File
 import java.io.InputStream
 import java.security.MessageDigest
 import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
 import javax.crypto.Cipher
 import javax.crypto.CipherInputStream
@@ -33,7 +34,7 @@ import javax.crypto.spec.SecretKeySpec
 private const val THREADS_TO_ALLOCATE = 262144 // 2**18 // Honestly, no reason to not just let 'er rip. Inactive connections will expire on their own :D
 private val LOGGER = LoggerFactory.getLogger(ImageServer::class.java)
 
-class ImageServer(private val cache: DiskLruCache, private val statistics: AtomicReference<Statistics>, private val upstreamUrl: String, private val database: Database) {
+class ImageServer(private val cache: DiskLruCache, private val statistics: AtomicReference<Statistics>, private val upstreamUrl: String, private val database: Database, private val handled: AtomicBoolean) {
     init {
         transaction(database) {
             SchemaUtils.create(ImageData)
@@ -83,6 +84,7 @@ class ImageServer(private val cache: DiskLruCache, private val statistics: Atomi
             }
         }
 
+        handled.set(true)
         if (snapshot != null && imageDatum != null) {
             request.handleCacheHit(sanitizedUri, getRc4(rc4Bytes), snapshot, imageDatum)
                 .header("X-Uri", sanitizedUri)