Centralized online connectivity check

See also #2011

docs/configuration/config_yaml.rst

@@ -909,6 +909,18 @@ Use the following settings to configure the server:
        # Command to shut down the system OctoPrint is running on, defaults to being unset
        systemShutdownCommand: sudo shutdown -h now
 
+     # Configuration of the regular online connectivity check
+     onlineCheck:
+
+       # interval in which to check for online connectivity (in seconds)
+       interval: 300
+
+       # DNS host against which to check (default: 8.8.8.8 aka Google's DNS)
+       host: 8.8.8.8
+
+       # DNS port against which to check (default: 53 - the default DNS port)
+       port: 53
+
      # Settings of when to display what disk space warning
      diskspace:
 

docs/events/index.rst

@@ -110,6 +110,14 @@ ClientClosed
 
      * ``remoteAddress``: the remote address (IP) of the client that disconnected
 
+ConnectivityChanged
+   The server's internet connectivity changed
+
+   Payload:
+
+     * ``old``: Old connectivity value (true for online, false for offline)
+     * ``new``: New connectivity value (true for online, false for offline)
+
 Printer communication
 ---------------------
 

docs/plugins/injectedproperties.rst

@@ -45,6 +45,8 @@ An overview of these properties follows.
   OctoPrint's application session manager, an instance of :class:`octoprint.server.util.flask.AppSessionManager`.
 ``self._user_manager``
   OctoPrint's user manager, an instance of :class:`octoprint.users.UserManager`.
+``self._connectivity_checker``
+  OctoPrint's connectivity checker, an instance of :class:`octoprint.util.ConnectivityChecker`.
 
 .. seealso::
 

src/octoprint/events.py

@@ -27,9 +27,10 @@ def all_events():
 
 
 class Events(object):
-	# application startup
+	# server
 	STARTUP = "Startup"
 	SHUTDOWN = "Shutdown"
+	CONNECTIVITY_CHANGED = "ConnectivityChanged"
 
 	# connect/disconnect to printer
 	CONNECTING = "Connecting"

src/octoprint/server/__init__.py

@@ -49,6 +49,7 @@ pluginManager = None
 appSessionManager = None
 pluginLifecycleManager = None
 preemptiveCache = None
+connectivityChecker = None
 
 principals = Principal(app)
 admin_permission = Permission(RoleNeed("admin"))
@@ -183,6 +184,7 @@ class Server(object):
 		global appSessionManager
 		global pluginLifecycleManager
 		global preemptiveCache
+		global connectivityChecker
 		global debug
 		global safe_mode
 
@@ -228,6 +230,35 @@ class Server(object):
 		pluginLifecycleManager = LifecycleManager(pluginManager)
 		preemptiveCache = PreemptiveCache(os.path.join(self._settings.getBaseFolder("data"), "preemptive_cache_config.yaml"))
 
+		# start regular check if we are connected to the internet
+		connectivityInterval = self._settings.getInt(["server", "onlineCheck", "interval"])
+		connectivityHost = self._settings.get(["server", "onlineCheck", "host"])
+		connectivityPort = self._settings.getInt(["server", "onlineCheck", "port"])
+
+		def on_connectivity_change(old_value, new_value):
+			eventManager.fire(events.Events.CONNECTIVITY_CHANGED, payload=dict(old=old_value, new=new_value))
+
+		connectivityChecker = octoprint.util.ConnectivityChecker(connectivityInterval,
+		                                                         connectivityHost,
+		                                                         port=connectivityPort,
+		                                                         on_change=on_connectivity_change)
+
+		def on_settings_update(*args, **kwargs):
+			# make sure our connectivity checker runs with the latest settings
+			connectivityInterval = self._settings.getInt(["server", "onlineCheck", "interval"])
+			connectivityHost = self._settings.get(["server", "onlineCheck", "host"])
+			connectivityPort = self._settings.getInt(["server", "onlineCheck", "port"])
+
+			if connectivityChecker.interval != connectivityInterval \
+					or connectivityChecker.host != connectivityHost \
+					or connectivityChecker.port != connectivityPort:
+				connectivityChecker.interval = connectivityInterval
+				connectivityChecker.host = connectivityHost
+				connectivityChecker.port = connectivityPort
+				connectivityChecker.check_immediately()
+
+		eventManager.subscribe(events.Events.SETTINGS_UPDATED, on_settings_update)
+
 		# setup access control
 		userManagerName = self._settings.get(["accessControl", "userManager"])
 		try:
@@ -249,7 +280,8 @@ class Server(object):
 			app_session_manager=appSessionManager,
 			plugin_lifecycle_manager=pluginLifecycleManager,
 			user_manager=userManager,
-			preemptive_cache=preemptiveCache
+			preemptive_cache=preemptiveCache,
+			connectivity_checker=connectivityChecker
 		)
 
 		# create printer instance

src/octoprint/server/api/settings.py

@@ -202,6 +202,11 @@ def getSettings():
 			"diskspace": {
 				"warning": s.getInt(["server", "diskspace", "warning"]),
 				"critical": s.getInt(["server", "diskspace", "critical"])
+			},
+			"onlineCheck": {
+				"interval": int(s.getInt(["server", "onlineCheck", "interval"]) / 60),
+				"host": s.get(["server", "onlineCheck", "host"]),
+				"port": s.getInt(["server", "onlineCheck", "port"])
 			}
 		}
 	}
@@ -419,6 +424,15 @@ def _saveSettings(data):
 		if "diskspace" in data["server"]:
 			if "warning" in data["server"]["diskspace"]: s.setInt(["server", "diskspace", "warning"], data["server"]["diskspace"]["warning"])
 			if "critical" in data["server"]["diskspace"]: s.setInt(["server", "diskspace", "critical"], data["server"]["diskspace"]["critical"])
+		if "onlineCheck" in data["server"]:
+			if "interval" in data["server"]["onlineCheck"]:
+				try:
+					interval = int(data["server"]["onlineCheck"]["interval"])
+					s.setInt(["server", "onlineCheck", "interval"], interval*60)
+				except ValueError:
+					pass
+			if "host" in data["server"]["onlineCheck"]: s.set(["server", "onlineCheck", "host"], data["server"]["onlineCheck"]["host"])
+			if "port" in data["server"]["onlineCheck"]: s.setInt(["server", "onlineCheck", "port"], data["server"]["onlineCheck"]["port"])
 
 	if "plugins" in data:
 		for plugin in octoprint.plugin.plugin_manager().get_implementations(octoprint.plugin.SettingsPlugin):

src/octoprint/settings.py

@@ -148,6 +148,11 @@ default_settings = {
 			"systemRestartCommand": None,
 			"serverRestartCommand": None
 		},
+		"onlineCheck": {
+			"interval": 15 * 60, # 15 min
+			"host": "8.8.8.8",
+			"port": 53
+		},
 		"diskspace": {
 			"warning": 500 * 1024 * 1024, # 500 MB
 			"critical": 200 * 1024 * 1024, # 200 MB

src/octoprint/static/js/app/viewmodels/settings.js

@@ -213,6 +213,10 @@ $(function() {
         self.server_diskspace_warning_str = sizeObservable(self.server_diskspace_warning);
         self.server_diskspace_critical_str = sizeObservable(self.server_diskspace_critical);
 
+        self.server_onlineCheck_interval = ko.observable();
+        self.server_onlineCheck_host = ko.observable();
+        self.server_onlineCheck_port = ko.observable();
+
         self.settings = undefined;
         self.lastReceivedSettings = undefined;
 

src/octoprint/templates/dialogs/settings/server.jinja2

@@ -4,4 +4,32 @@
     {% include "snippets/settings/server/serverCommandServerRestart.jinja2" %}
     {% include "snippets/settings/server/serverCommandSystemRestart.jinja2" %}
     {% include "snippets/settings/server/serverCommandSystemShutdown.jinja2" %}
+
+    <h3>{{ _('Connectivity check') }}</h3>
+
+    <p>{{ _('You normally should not have to change any of the following settings.') }}</p>
+
+    <div class="control-group" title="{{ _('Interval in which to check for internet connectivity') }}">
+        <label class="control-label" for="settings-serverOnlineCheckInterval">{{ _('Check interval') }}</label>
+        <div class="controls">
+            <span class="input-append">
+                <input type="number" min="1" step="1" class="input-mini" data-bind="value: server_onlineCheck_interval" id="settings-serverOnlineCheckInterval">
+                <span class="add-on">min</span>
+            </span>
+        </div>
+    </div>
+
+    <div class="control-group" title="{{ _('DNS against which to check for internet connectivity') }}">
+        <label class="control-label" for="settings-serverOnlineCheckHost">{{ _('DNS host') }}</label>
+        <div class="controls">
+            <input type="text" class="input-small text-right" data-bind="value: server_onlineCheck_host" id="settings-serverOnlineCheckHost">
+        </div>
+    </div>
+
+    <div class="control-group" title="{{ _('Port against which to check for internet connectivity. Default is 53.') }}">
+        <label class="control-label" for="settings-serverOnlineCheckHost">{{ _('DNS port') }}</label>
+        <div class="controls">
+            <input type="number" min="1" max="65535" step="1" class="input-mini" data-bind="value: server_onlineCheck_port" id="settings-serverOnlineCheckPort">
+        </div>
+    </div>
 </form>

src/octoprint/util/__init__.py

@@ -735,22 +735,47 @@ def interface_addresses(family=None):
 				if not ifaddress["addr"].startswith("169.254."):
 					yield ifaddress["addr"]
 
-def address_for_client(host, port):
+def address_for_client(host, port, timeout=3.05):
 	"""
 	Determines the address of the network interface on this host needed to connect to the indicated client host and port.
 	"""
 
-	import socket
-
 	for address in interface_addresses():
 		try:
-			sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-			sock.bind((address, 0))
-			sock.connect((host, port))
-			return address
+			if server_reachable(host, port, timeout=timeout, proto="udp", source=address):
+				return address
 		except:
 			continue
 
+def server_reachable(host, port, timeout=3.05, proto="tcp", source=None):
+	"""
+	Checks if a server is reachable
+
+	Args:
+		host (str): host to check against
+		port (int): port to check against
+		timeout (float): timeout for check
+		proto (str): ``tcp`` or ``udp``
+		source (str): optional, socket used for check will be bound against this address if provided
+
+	Returns:
+		boolean: True if a connection to the server could be opened, False otherwise
+	"""
+
+	import socket
+
+	if proto not in ("tcp", "udp"):
+		raise ValueError("proto must be either 'tcp' or 'udp'")
+
+	try:
+		sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM if proto == "udp" else socket.SOCK_STREAM)
+		sock.settimeout(timeout)
+		if source is not None:
+			sock.bind((source, 0))
+		sock.connect((host, port))
+		return True
+	except:
+		return False
 
 @contextlib.contextmanager
 def atomic_write(filename, mode="w+b", prefix="tmp", suffix="", permissions=0o644, max_permissions=0o777):
@@ -1132,3 +1157,102 @@ class TypeAlreadyInQueue(Exception):
 		Exception.__init__(self, *args, **kwargs)
 		self.type = t
 
+
+class ConnectivityChecker(object):
+	"""
+	Regularly checks for online connectivity.
+
+	Tries to open a connection to the provided ``host`` and ``port`` every ``interval``
+	seconds and sets the ``online`` status accordingly.
+	"""
+
+	def __init__(self, interval, host, port, on_change=None):
+		self._interval = interval
+		self._host = host
+		self._port = port
+		self._on_change = on_change
+
+		self._logger = logging.getLogger(__name__ + ".connectivity_checker")
+
+		self._last_check = None
+		self._online = False
+
+		self._check_worker = None
+		self._check_mutex = threading.RLock()
+
+		self._run()
+
+	@property
+	def online(self):
+		"""Current online status, True if online, False if offline."""
+		with self._check_mutex:
+			return self._online
+
+	@property
+	def last_check(self):
+		"""Timestamp of last check."""
+		with self._check_mutex:
+			return self._last_check
+
+	@property
+	def host(self):
+		"""DNS host to query."""
+		with self._check_mutex:
+			return self._host
+
+	@host.setter
+	def host(self, value):
+		with self._check_mutex:
+			self._host = value
+
+	@property
+	def port(self):
+		"""DNS port to query."""
+		with self._check_mutex:
+			return self._port
+
+	@port.setter
+	def port(self, value):
+		with self._check_mutex:
+			self._port = value
+
+	@property
+	def interval(self):
+		"""Interval between consecutive automatic checks."""
+		return self._interval
+
+	@interval.setter
+	def interval(self, value):
+		self._interval = value
+
+	def check_immediately(self):
+		"""Check immediately and return result."""
+		with self._check_mutex:
+			self._perform_check()
+			return self.online
+
+	def _run(self):
+		from octoprint.util import RepeatedTimer
+
+		if self._check_worker is not None:
+			raise RuntimeError("Connectivity manager check thread already active")
+
+		self._check_worker = RepeatedTimer(self._interval, self._perform_check, run_first=True)
+		self._check_worker.start()
+
+	def _perform_check(self):
+		import time
+
+		with self._check_mutex:
+			self._logger.debug("Checking against {}:{} if we are online...".format(self._host, self._port))
+
+			old_value = self._online
+			self._online = server_reachable(self._host, port=self._port)
+			self._last_check = time.time()
+
+			if old_value != self._online:
+				self._logger.info("Connectivity changed from {} to {}".format("online" if old_value else "offline",
+				                                                              "online" if self._online else "offline"))
+				if callable(self._on_change):
+					self._on_change(old_value, self._online)
+