From fb8c56be5715da5b303fc903980cedab8e72d9f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 19 Jul 2017 17:19:36 +0200 Subject: [PATCH] SWU: Track network connectivity & handle offline scenarios See also #2011 --- setup.py | 3 +- .../plugins/softwareupdate/__init__.py | 168 ++++++++++++------ .../plugins/softwareupdate/exceptions.py | 19 ++ .../static/css/softwareupdate.css | 2 +- .../static/js/softwareupdate.js | 36 ++++ .../static/less/softwareupdate.less | 6 + .../templates/softwareupdate_settings.jinja2 | 9 +- .../plugins/softwareupdate/updaters/pip.py | 9 +- .../softwareupdate/updaters/python_updater.py | 23 ++- .../softwareupdate/updaters/update_script.py | 11 +- .../version_checks/bitbucket_commit.py | 22 ++- .../version_checks/commandline.py | 7 +- .../version_checks/git_commit.py | 32 ++-- .../version_checks/github_commit.py | 23 ++- .../version_checks/github_release.py | 29 ++- .../version_checks/python_checker.py | 19 +- 16 files changed, 307 insertions(+), 111 deletions(-) diff --git a/setup.py b/setup.py index 79ee74d1..8ab62c20 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,8 @@ INSTALL_REQUIRES = [ "scandir>=1.3,<1.4", "websocket-client>=0.40,<0.41", "python-dateutil>=2.6,<2.7", - "wrapt>=1.10.10,<1.11" + "wrapt>=1.10.10,<1.11", + "futures>=3.1.1,<3.2" ] if sys.platform == "darwin": diff --git a/src/octoprint/plugins/softwareupdate/__init__.py b/src/octoprint/plugins/softwareupdate/__init__.py index f1dafe89..cb2376e5 100644 --- a/src/octoprint/plugins/softwareupdate/__init__.py +++ b/src/octoprint/plugins/softwareupdate/__init__.py @@ -15,6 +15,9 @@ import time import logging import logging.handlers import hashlib +import traceback + +from concurrent import futures from . import version_checks, updaters, exceptions, util, cli @@ -45,6 +48,7 @@ class SoftwareUpdatePlugin(octoprint.plugin.BlueprintPlugin, self._refresh_configured_checks = False self._get_versions_mutex = threading.Lock() + self._get_versions_last = None self._version_cache = dict() self._version_cache_ttl = 0 @@ -467,15 +471,6 @@ class SoftwareUpdatePlugin(octoprint.plugin.BlueprintPlugin, def view(): try: information, update_available, update_possible = self.get_current_versions(check_targets=check_targets, force=force) - - # we don't want to transfer python_checker or python_updater values through json - replace with True - for key, data in information.items(): - if "check" in data: - if "python_checker" in data["check"]: - data["check"]["python_checker"] = True - if "python_updater" in data["check"]: - data["check"]["python_updater"] = True - return flask.jsonify(dict(status="updatePossible" if update_available and update_possible else "updateAvailable" if update_available else "current", information=information, timestamp=self._version_cache_timestamp)) @@ -502,9 +497,11 @@ class SoftwareUpdatePlugin(octoprint.plugin.BlueprintPlugin, hash.update(repr(data["information"])) hash.update(str(data["available"])) hash.update(str(data["possible"])) + hash.update(str(data.get("online", None))) hash.update(",".join(targets)) hash.update(str(self._version_cache_timestamp)) + hash.update(str(self._connectivity_checker.online)) return hash.hexdigest() def condition(): @@ -585,53 +582,89 @@ class SoftwareUpdatePlugin(octoprint.plugin.BlueprintPlugin, information = dict() # we don't want to do the same work twice, so let's use a lock - with self._get_versions_mutex: - for target, check in checks.items(): - if not target in check_targets: - continue + if self._get_versions_mutex.acquire(False): + try: + futures_to_result = dict() + online = self._connectivity_checker.check_immediately() + self._logger.debug("Looks like we are {}".format("online" if online else "offline")) - if not check: - continue + with futures.ThreadPoolExecutor(max_workers=5) as executor: + for target, check in checks.items(): + if not target in check_targets: + continue - try: - populated_check = self._populated_check(target, check) - target_information, target_update_available, target_update_possible = self._get_current_version(target, populated_check, force=force) - if target_information is None: - target_information = dict() - except exceptions.UnknownCheckType: - self._logger.warn("Unknown update check type for target {}: {}".format(target, check.get("type", ""))) - continue + if not check: + continue - target_information = dict_merge(dict(local=dict(name="unknown", value="unknown"), remote=dict(name="unknown", value="unknown", release_notes=None)), target_information) + try: + populated_check = self._populated_check(target, check) + future = executor.submit(self._get_current_version, target, populated_check, force=force) + futures_to_result[future] = (target, populated_check) + except exceptions.UnknownCheckType: + self._logger.warn("Unknown update check type for target {}: {}".format(target, + check.get("type", + ""))) + continue + except: + self._logger.exception("Could not check {} for updates".format(target)) + continue - update_available = update_available or target_update_available - update_possible = update_possible or (target_update_possible and target_update_available) + for future in futures.as_completed(futures_to_result): - local_name = target_information["local"]["name"] - local_value = target_information["local"]["value"] + target, populated_check = futures_to_result[future] + if future.exception() is not None: + self._logger.error("Could not check {} for updates, error: {!r}".format(target, + future.exception())) + continue - release_notes = None - if target_information and target_information["remote"] and target_information["remote"]["value"]: - if "release_notes" in populated_check and populated_check["release_notes"]: - release_notes = populated_check["release_notes"] - elif "release_notes" in target_information["remote"]: - release_notes = target_information["remote"]["release_notes"] + target_information, target_update_available, target_update_possible, target_online, target_error = future.result() - if release_notes: - release_notes = release_notes.format(octoprint_version=VERSION, - target_name=target_information["remote"]["name"], - target_version=target_information["remote"]["value"]) + target_information = dict_merge(dict(local=dict(name="?", value="?"), + remote=dict(name="?", value="?", + release_notes=None), + needs_online=True), target_information) - information[target] = dict(updateAvailable=target_update_available, - updatePossible=target_update_possible, - information=target_information, - displayName=populated_check["displayName"], - displayVersion=populated_check["displayVersion"].format(octoprint_version=VERSION, local_name=local_name, local_value=local_value), - check=populated_check, - releaseNotes=release_notes) + update_available = update_available or target_update_available + update_possible = update_possible or (target_update_possible and target_update_available) + + local_name = target_information["local"]["name"] + local_value = target_information["local"]["value"] + + release_notes = None + if target_information and target_information["remote"] and target_information["remote"][ + "value"]: + if "release_notes" in populated_check and populated_check["release_notes"]: + release_notes = populated_check["release_notes"] + elif "release_notes" in target_information["remote"]: + release_notes = target_information["remote"]["release_notes"] + + if release_notes: + release_notes = release_notes.format(octoprint_version=VERSION, + target_name=target_information["remote"]["name"], + target_version=target_information["remote"]["value"]) + + information[target] = dict(updateAvailable=target_update_available, + updatePossible=target_update_possible, + information=target_information, + displayName=populated_check["displayName"], + displayVersion=populated_check["displayVersion"].format(octoprint_version=VERSION, + local_name=local_name, + local_value=local_value), + releaseNotes=release_notes, + online=target_online, + error=target_error) + + if self._version_cache_dirty: + self._save_version_cache() + + self._get_versions_last = information, update_available, update_possible + finally: + self._get_versions_mutex.release() + + else: # something's already in progress, let's wait for it to complete and use its result + self._get_versions_mutex.wait() + information, update_available, update_possible = self._get_versions_last - if self._version_cache_dirty: - self._save_version_cache() return information, update_available, update_possible def _get_check_hash(self, check): @@ -650,36 +683,51 @@ class SoftwareUpdatePlugin(octoprint.plugin.BlueprintPlugin, hash.update(dict_to_sorted_repr(check)) return hash.hexdigest() - def _get_current_version(self, target, check, force=False): + def _get_current_version(self, target, check, force=False, online=None): """ Determines the current version information for one target based on its check configuration. """ current_hash = self._get_check_hash(check) + if online is None: + online = self._connectivity_checker.online if target in self._version_cache and not force: data = self._version_cache[target] - if data["hash"] == current_hash and data["timestamp"] + self._version_cache_ttl >= time.time() > data["timestamp"]: + if data["hash"] == current_hash \ + and data["timestamp"] + self._version_cache_ttl >= time.time() > data["timestamp"] \ + and data.get("online", None) == online: # we also check that timestamp < now to not get confused too much by clock changes - return data["information"], data["available"], data["possible"] + return data["information"], data["available"], data["possible"], data["online"], data.get("error", None) information = dict() update_available = False + error = None try: version_checker = self._get_version_checker(target, check) - information, is_current = version_checker.get_latest(target, check) + information, is_current = version_checker.get_latest(target, check, online=online) if information is not None and not is_current: update_available = True except exceptions.UnknownCheckType: self._logger.warn("Unknown check type %s for %s" % (check["type"], target)) update_possible = False + error = "unknown_check" + except exceptions.CannotCheckOffline: + self._logger.warn("Cannot check %s while we are offline" % (target,)) + update_possible = False + information["needs_online"] = True + except exceptions.NetworkError: + self._logger.warn("Could not check %s for updates due to a network error" % target) + update_possible = False + error = "network" except: self._logger.exception("Could not check %s for updates" % target) update_possible = False + error = "unknown" else: try: updater = self._get_updater(target, check) - update_possible = updater.can_perform_update(target, check) + update_possible = updater.can_perform_update(target, check, online=online) except: update_possible = False @@ -687,9 +735,11 @@ class SoftwareUpdatePlugin(octoprint.plugin.BlueprintPlugin, hash=current_hash, information=information, available=update_available, - possible=update_possible) + possible=update_possible, + online=online, + error=error) self._version_cache_dirty = True - return information, update_available, update_possible + return information, update_available, update_possible, online, error def perform_updates(self, check_targets=None, force=False): """ @@ -801,7 +851,9 @@ class SoftwareUpdatePlugin(octoprint.plugin.BlueprintPlugin, self._send_client_message("success", dict(results=target_results)) def _perform_update(self, target, check, force): - information, update_available, update_possible = self._get_current_version(target, check) + online = self._connectivity_checker.online + + information, update_available, update_possible, _, _ = self._get_current_version(target, check, online=online) if not update_available and not force: return False, None @@ -824,7 +876,7 @@ class SoftwareUpdatePlugin(octoprint.plugin.BlueprintPlugin, if updater is None: raise exceptions.UnknownUpdateType() - update_result = updater.perform_update(target, populated_check, target_version, log_cb=self._log) + update_result = updater.perform_update(target, populated_check, target_version, log_cb=self._log, online=online) target_result = ("success", update_result) self._logger.info("Update of %s to %s successful!" % (target, target_version)) @@ -833,6 +885,10 @@ class SoftwareUpdatePlugin(octoprint.plugin.BlueprintPlugin, self._send_client_message("update_failed", dict(target=target, version=target_version, name=populated_check["displayName"], reason="Unknown update type")) return False, None + except exceptions.CannotUpdateOffline: + self._logger.warn("Update of %s can not be performed, it's not marked as 'offline' capable but we are apparently offline right now" % target) + self._send_client_message("update_failed", dict(target=target, version=target_version, name=populated_check["displayName"], reason="No internet connection")) + except Exception as e: self._logger.exception("Update of %s can not be performed" % target) if not "ignorable" in populated_check or not populated_check["ignorable"]: diff --git a/src/octoprint/plugins/softwareupdate/exceptions.py b/src/octoprint/plugins/softwareupdate/exceptions.py index 452eabea..75f66d3b 100644 --- a/src/octoprint/plugins/softwareupdate/exceptions.py +++ b/src/octoprint/plugins/softwareupdate/exceptions.py @@ -18,6 +18,20 @@ class UnknownUpdateType(Exception): class UnknownCheckType(Exception): pass +class NetworkError(Exception): + def __init__(self, message=None, cause=None): + Exception.__init__(self) + self.message = message + self.cause = cause + + def __str__(self): + if self.message is not None: + return self.message + elif self.cause is not None: + return "NetworkError caused by {}".format(self.cause) + else: + return "NetworkError" + class UpdateError(Exception): def __init__(self, message, data): self.message = message @@ -35,3 +49,8 @@ class RestartFailed(Exception): class ConfigurationInvalid(Exception): pass +class CannotCheckOffline(Exception): + pass + +class CannotUpdateOffline(Exception): + pass diff --git a/src/octoprint/plugins/softwareupdate/static/css/softwareupdate.css b/src/octoprint/plugins/softwareupdate/static/css/softwareupdate.css index 42c433cb..38ccc7f6 100644 --- a/src/octoprint/plugins/softwareupdate/static/css/softwareupdate.css +++ b/src/octoprint/plugins/softwareupdate/static/css/softwareupdate.css @@ -1 +1 @@ -td.settings_plugin_softwareupdate_column_update{width:16px}#settings_plugin_softwareupdate_workingdialog_output{font-size:.8em}#settings_plugin_softwareupdate_workingdialog_output .message{font-weight:700}#settings_plugin_softwareupdate_workingdialog_output .separator{font-weight:700;color:#666}#settings_plugin_softwareupdate_workingdialog_output .stdout{color:#333}#settings_plugin_softwareupdate_workingdialog_output .stderr{color:#900}#settings_plugin_softwareupdate_workingdialog_output .call{color:#009}#settings_plugin_softwareupdate_workingdialog_output .message_error{font-weight:700;color:#900}.softwareupdate_notification ul{margin:10px 0 10px 25px}.softwareupdate_notification ul .name{text-overflow:ellipsis;white-space:nowrap;overflow:hidden;display:block} \ No newline at end of file +td.settings_plugin_softwareupdate_column_update{width:16px}td.settings_plugin_softwareupdate_column_information .line{display:block}#settings_plugin_softwareupdate_workingdialog_output{font-size:.8em}#settings_plugin_softwareupdate_workingdialog_output .message{font-weight:700}#settings_plugin_softwareupdate_workingdialog_output .separator{font-weight:700;color:#666}#settings_plugin_softwareupdate_workingdialog_output .stdout{color:#333}#settings_plugin_softwareupdate_workingdialog_output .stderr{color:#900}#settings_plugin_softwareupdate_workingdialog_output .call{color:#009}#settings_plugin_softwareupdate_workingdialog_output .message_error{font-weight:700;color:#900}.softwareupdate_notification ul{margin:10px 0 10px 25px}.softwareupdate_notification ul .name{text-overflow:ellipsis;white-space:nowrap;overflow:hidden;display:block} \ No newline at end of file diff --git a/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js b/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js index 6d857aac..9a68d144 100644 --- a/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js +++ b/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js @@ -417,6 +417,42 @@ $(function() { }); }; + self.iconTitleForEntry = function(data) { + if (data.updatePossible) { + return ""; + } else if (!data.online && data.information && data.information.needs_online) { + return gettext("No internet connection"); + } else if (data.error) { + return self.errorTextForEntry(data); + } else { + return gettext("Update not possible"); + } + }; + + self.errorTextForEntry = function(data) { + if (!data.error) { + return ""; + } + + switch (data.error) { + case "unknown_check": { + return gettext("Unknown update check, configuration ok?"); + } + case "needs_online": { + return gettext("Cannot check for update, need online connection"); + } + case "network": { + return gettext("Network error while checking for update"); + } + case "unknown": { + return gettext("Unknown error while checking for update, please check the logs"); + } + default: { + return ""; + } + } + }; + self._markNotificationAsSeen = function(data) { if (!Modernizr.localstorage) return false; diff --git a/src/octoprint/plugins/softwareupdate/static/less/softwareupdate.less b/src/octoprint/plugins/softwareupdate/static/less/softwareupdate.less index 0065ed95..a7a3d683 100644 --- a/src/octoprint/plugins/softwareupdate/static/less/softwareupdate.less +++ b/src/octoprint/plugins/softwareupdate/static/less/softwareupdate.less @@ -2,6 +2,12 @@ td.settings_plugin_softwareupdate_column_update { width: 16px; } +td.settings_plugin_softwareupdate_column_information { + .line { + display: block; + } +} + #settings_plugin_softwareupdate_workingdialog_output { font-size: 0.8em; diff --git a/src/octoprint/plugins/softwareupdate/templates/softwareupdate_settings.jinja2 b/src/octoprint/plugins/softwareupdate/templates/softwareupdate_settings.jinja2 index 954e0285..aa6c9e50 100644 --- a/src/octoprint/plugins/softwareupdate/templates/softwareupdate_settings.jinja2 +++ b/src/octoprint/plugins/softwareupdate/templates/softwareupdate_settings.jinja2 @@ -29,11 +29,12 @@ - :
+ :
- {{ _('Installed:') }}
- {{ _('Available:') }}
- {{ _('Release Notes:') }} + {{ _('Installed:') }} + {{ _('Available:') }} + {{ _('Release Notes:') }} + diff --git a/src/octoprint/plugins/softwareupdate/updaters/pip.py b/src/octoprint/plugins/softwareupdate/updaters/pip.py index a60ff211..3c0e9022 100644 --- a/src/octoprint/plugins/softwareupdate/updaters/pip.py +++ b/src/octoprint/plugins/softwareupdate/updaters/pip.py @@ -18,9 +18,9 @@ console_logger = logging.getLogger("octoprint.plugins.softwareupdate.updaters.pi _pip_callers = dict() _pip_version_dependency_links = pkg_resources.parse_version("1.5") -def can_perform_update(target, check): +def can_perform_update(target, check, online=True): pip_caller = _get_pip_caller(command=check["pip_command"] if "pip_command" in check else None) - return "pip" in check and pip_caller is not None and pip_caller.available + return "pip" in check and pip_caller is not None and pip_caller.available and (online or check.get("offline", False)) def _get_pip_caller(command=None): key = command @@ -35,11 +35,14 @@ def _get_pip_caller(command=None): return _pip_callers[key] -def perform_update(target, check, target_version, log_cb=None): +def perform_update(target, check, target_version, log_cb=None, online=True): pip_command = None if "pip_command" in check: pip_command = check["pip_command"] + if not online and not check.get("offline", False): + raise exceptions.CannotUpdateOffline() + pip_caller = _get_pip_caller(command=pip_command) if pip_caller is None: raise exceptions.UpdateError("Can't run pip", None) diff --git a/src/octoprint/plugins/softwareupdate/updaters/python_updater.py b/src/octoprint/plugins/softwareupdate/updaters/python_updater.py index e3e6ddfd..35dc2696 100644 --- a/src/octoprint/plugins/softwareupdate/updaters/python_updater.py +++ b/src/octoprint/plugins/softwareupdate/updaters/python_updater.py @@ -6,9 +6,24 @@ __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agp __copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" -def can_perform_update(target, check): - return "python_updater" in check and check["python_updater"] is not None and hasattr(check["python_updater"], "perform_update") +def can_perform_update(target, check, online=True): + return "python_updater" in check and check["python_updater"] is not None and hasattr(check["python_updater"], "perform_update") and (online or check.get("offline", False)) -def perform_update(target, check, target_version, log_cb=None): - return check["python_updater"].perform_update(target, check, target_version, log_cb=log_cb) +def perform_update(target, check, target_version, log_cb=None, online=True): + from ..exceptions import CannotUpdateOffline + + if not online and not check("offline", False): + raise CannotUpdateOffline() + + try: + return check["python_updater"].perform_update(target, check, target_version, log_cb=log_cb, online=online) + except: + import inspect + args, _, _, _ = inspect.getargspec(check["python_updater"].perform_update) + if "online" not in args: + # old python_updater footprint, simply leave out the online parameter + return check["python_updater"].perform_update(target, check, target_version, log_cb=log_cb) + + # some other error, raise again + raise diff --git a/src/octoprint/plugins/softwareupdate/updaters/update_script.py b/src/octoprint/plugins/softwareupdate/updaters/update_script.py index b47a7703..8ac125d7 100644 --- a/src/octoprint/plugins/softwareupdate/updaters/update_script.py +++ b/src/octoprint/plugins/softwareupdate/updaters/update_script.py @@ -8,7 +8,7 @@ __copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms import sys import logging -from ..exceptions import ConfigurationInvalid, UpdateError +from ..exceptions import ConfigurationInvalid, UpdateError, CannotUpdateOffline from octoprint.util.commandline import CommandlineCaller, CommandlineError @@ -36,7 +36,7 @@ def _get_caller(log_cb=None): return caller -def can_perform_update(target, check): +def can_perform_update(target, check, online=True): import os script_configured = bool("update_script" in check and check["update_script"]) @@ -47,12 +47,15 @@ def can_perform_update(target, check): folder = check["checkout_folder"] folder_configured = bool(folder and os.path.isdir(folder)) - return script_configured and folder_configured + return script_configured and folder_configured and (online or check.get("offline", False)) -def perform_update(target, check, target_version, log_cb=None): +def perform_update(target, check, target_version, log_cb=None, online=True): logger = logging.getLogger("octoprint.plugins.softwareupdate.updaters.update_script") + if not online and not check("offline", False): + raise CannotUpdateOffline() + if not can_perform_update(target, check): raise ConfigurationInvalid("checkout_folder and update_folder are missing for update target %s, one is needed" % target) diff --git a/src/octoprint/plugins/softwareupdate/version_checks/bitbucket_commit.py b/src/octoprint/plugins/softwareupdate/version_checks/bitbucket_commit.py index ddce43c1..728b0238 100644 --- a/src/octoprint/plugins/softwareupdate/version_checks/bitbucket_commit.py +++ b/src/octoprint/plugins/softwareupdate/version_checks/bitbucket_commit.py @@ -14,12 +14,18 @@ logger = logging.getLogger("octoprint.plugins.softwareupdate.version_checks.bitb def _get_latest_commit(user, repo, branch, api_user=None, api_password=None): + from ..exceptions import NetworkError + url = BRANCH_HEAD_URL.format(user=user, repo=repo, branch=branch) headers = {} if api_user is not None and api_password is not None: auth_value = base64.b64encode(b"{user}:{pw}".format(user=api_user, pw=api_password)) headers["authorization"] = "Basic {}".format(auth_value) - r = requests.get(url, headers=headers) + + try: + r = requests.get(url, headers=headers, timeout=(3.05, 30)) + except requests.ConnectionError as exc: + raise NetworkError(cause=exc) if not r.status_code == requests.codes.ok: return None @@ -31,7 +37,7 @@ def _get_latest_commit(user, repo, branch, api_user=None, api_password=None): return reference["hash"] -def get_latest(target, check): +def get_latest(target, check, online=True): from ..exceptions import ConfigurationInvalid if "user" not in check or "repo" not in check: @@ -46,12 +52,18 @@ def get_latest(target, check): current = check.get("current") - remote_commit = _get_latest_commit(check["user"], check["repo"], branch, api_user, api_password) - information = dict( local=dict(name="Commit {commit}".format(commit=current if current is not None else "unknown"), value=current), - remote=dict(name="Commit {commit}".format(commit=remote_commit if remote_commit is not None else "unknown"), value=remote_commit) + remote=dict(name="?", value="?"), + needs_online=not check.get("offline", False) ) + if not online and information["needs_online"]: + return information, True + + remote_commit = _get_latest_commit(check["user"], check["repo"], branch, api_user, api_password) + remote_name = "Commit {commit}".format(commit=remote_commit) if remote_commit is not None else "-" + + information["remote"] = dict(name=remote_name, value=remote_commit) is_current = (current is not None and current == remote_commit) or remote_commit is None logger.debug("Target: %s, local: %s, remote: %s" % (target, current, remote_commit)) diff --git a/src/octoprint/plugins/softwareupdate/version_checks/commandline.py b/src/octoprint/plugins/softwareupdate/version_checks/commandline.py index bba990ad..fb7e75c3 100644 --- a/src/octoprint/plugins/softwareupdate/version_checks/commandline.py +++ b/src/octoprint/plugins/softwareupdate/version_checks/commandline.py @@ -8,14 +8,17 @@ __copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms import logging -from ..exceptions import ConfigurationInvalid +from ..exceptions import ConfigurationInvalid, CannotCheckOffline from ..util import execute -def get_latest(target, check): +def get_latest(target, check, online=True): command = check.get("command") if command is None: raise ConfigurationInvalid("Update configuration for {} of type commandline needs command set and not None".format(target)) + if not online and not check.get("offline", False): + raise CannotCheckOffline("{} isn't marked as 'offline' capable, but we are apparently offline right now".format(target)) + returncode, stdout, stderr = execute(command, evaluate_returncode=False) # We expect command line check commands to diff --git a/src/octoprint/plugins/softwareupdate/version_checks/git_commit.py b/src/octoprint/plugins/softwareupdate/version_checks/git_commit.py index 59dad51e..f703cd57 100644 --- a/src/octoprint/plugins/softwareupdate/version_checks/git_commit.py +++ b/src/octoprint/plugins/softwareupdate/version_checks/git_commit.py @@ -47,38 +47,40 @@ def _git(args, cwd, hide_stderr=False): return p.returncode, stdout -def get_latest(target, check): +def get_latest(target, check, online=True): checkout_folder = check.get("checkout_folder") if checkout_folder is None: raise ConfigurationInvalid("Update configuration for {} of type git_commit needs checkout_folder set and not None".format(target)) - returncode, _ = _git(["fetch"], checkout_folder) - if returncode != 0: - return None, True - returncode, local_commit = _git(["rev-parse", "@{0}"], checkout_folder) if returncode != 0: return None, True + information = dict( + local=dict(name="Commit %s" % local_commit, value=local_commit), + remote=dict(name="?", value="?"), + needs_online=not check.get("offline", False) + ) + if not online and information["needs_online"]: + return information, True + + returncode, _ = _git(["fetch"], checkout_folder) + if returncode != 0: + return information, True + returncode, remote_commit = _git(["rev-parse", "@{u}"], checkout_folder) if returncode != 0: - return None, True + return information, True returncode, base = _git(["merge-base", "@{0}", "@{u}"], checkout_folder) if returncode != 0: - return None, True + return information, True if local_commit == remote_commit or remote_commit == base: - information = dict( - local=dict(name="Commit %s" % local_commit, value=local_commit), - remote=dict(name="Commit %s" % local_commit, value=local_commit) - ) + information["remote"] = dict(name="Commit %s" % local_commit, value=local_commit) is_current = True else: - information = dict( - local=dict(name="Commit %s" % local_commit, value=local_commit), - remote=dict(name="Commit %s" % remote_commit, value=remote_commit) - ) + information["remote"] = dict(name="Commit %s" % remote_commit, value=remote_commit) is_current = local_commit == remote_commit logger = logging.getLogger("octoprint.plugins.softwareupdate.version_checks.git_commit") diff --git a/src/octoprint/plugins/softwareupdate/version_checks/github_commit.py b/src/octoprint/plugins/softwareupdate/version_checks/github_commit.py index 9df59833..fe3de9fc 100644 --- a/src/octoprint/plugins/softwareupdate/version_checks/github_commit.py +++ b/src/octoprint/plugins/softwareupdate/version_checks/github_commit.py @@ -13,7 +13,12 @@ BRANCH_HEAD_URL = "https://api.github.com/repos/{user}/{repo}/git/refs/heads/{br logger = logging.getLogger("octoprint.plugins.softwareupdate.version_checks.github_commit") def _get_latest_commit(user, repo, branch): - r = requests.get(BRANCH_HEAD_URL.format(user=user, repo=repo, branch=branch), timeout=30) + from ..exceptions import NetworkError + + try: + r = requests.get(BRANCH_HEAD_URL.format(user=user, repo=repo, branch=branch), timeout=(3.05, 30)) + except requests.ConnectionError as exc: + raise NetworkError(cause=exc) from . import log_github_ratelimit log_github_ratelimit(logger, r) @@ -28,7 +33,7 @@ def _get_latest_commit(user, repo, branch): return reference["object"]["sha"] -def get_latest(target, check): +def get_latest(target, check, online=True): from ..exceptions import ConfigurationInvalid user = check.get("user") @@ -43,12 +48,16 @@ def get_latest(target, check): current = check.get("current") - remote_commit = _get_latest_commit(check["user"], check["repo"], branch) + information = dict(local=dict(name="Commit {commit}".format(commit=current if current is not None else "?"), value=current), + remote=dict(name="?", value="?"), + needs_online=not check.get("offline", False)) + if not online and information["needs_online"]: + return information, True - information = dict( - local=dict(name="Commit {commit}".format(commit=current if current is not None else "unknown"), value=current), - remote=dict(name="Commit {commit}".format(commit=remote_commit if remote_commit is not None else "unknown"), value=remote_commit) - ) + remote_commit = _get_latest_commit(check["user"], check["repo"], branch) + remote_name = "Commit {commit}".format(commit=remote_commit) if remote_commit is not None else "-" + + information["remote"] = dict(name=remote_name, value=remote_commit) is_current = (current is not None and current == remote_commit) or remote_commit is None logger.debug("Target: %s, local: %s, remote: %s" % (target, current, remote_commit)) diff --git a/src/octoprint/plugins/softwareupdate/version_checks/github_release.py b/src/octoprint/plugins/softwareupdate/version_checks/github_release.py index 07493e48..53dac73e 100644 --- a/src/octoprint/plugins/softwareupdate/version_checks/github_release.py +++ b/src/octoprint/plugins/softwareupdate/version_checks/github_release.py @@ -96,8 +96,14 @@ def _get_latest_release(user, repo, compare_type, include_prerelease=False, commitish=None, force_base=True): + from ..exceptions import NetworkError + nothing = None, None, None - r = requests.get(RELEASE_URL.format(user=user, repo=repo), timeout=30) + + try: + r = requests.get(RELEASE_URL.format(user=user, repo=repo), timeout=(3.05, 30)) + except requests.ConnectionError as exc: + raise NetworkError(cause=exc) from . import log_github_ratelimit log_github_ratelimit(logger, r) @@ -258,7 +264,7 @@ def _is_current(release_information, compare_type, custom=None, force_base=True) return True -def get_latest(target, check, custom_compare=None): +def get_latest(target, check, custom_compare=None, online=True): from ..exceptions import ConfigurationInvalid user = check.get("user", None) @@ -267,6 +273,14 @@ def get_latest(target, check, custom_compare=None): if user is None or repo is None or current is None: raise ConfigurationInvalid("Update configuration for {} of type github_release needs all of user, repo and current set and not None".format(target)) + information =dict( + local=dict(name=current, value=current), + remote=dict(name="?", value="?", release_notes=None), + needs_online=not check.get("offline", False) + ) + if not online and information["needs_online"]: + return information, True + include_prerelease = check.get("prerelease", False) prerelease_channel = check.get("prerelease_channel", None) @@ -290,10 +304,13 @@ def get_latest(target, check, custom_compare=None): commitish=commitish, force_base=force_base) - information =dict( - local=dict(name=current, value=current), - remote=dict(name=remote_name, value=remote_tag, release_notes=release_notes) - ) + if remote_name is None: + if remote_tag is not None: + remote_name = remote_tag + else: + remote_name = "-" + + information["remote"] = dict(name=remote_name, value=remote_tag, release_notes=release_notes) logger.debug("Target: %s, local: %s, remote: %s" % (target, current, remote_tag)) diff --git a/src/octoprint/plugins/softwareupdate/version_checks/python_checker.py b/src/octoprint/plugins/softwareupdate/version_checks/python_checker.py index cb395851..e0e8199c 100644 --- a/src/octoprint/plugins/softwareupdate/version_checks/python_checker.py +++ b/src/octoprint/plugins/softwareupdate/version_checks/python_checker.py @@ -5,11 +5,24 @@ __author__ = "Gina Häußge " __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' __copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" -from ..exceptions import ConfigurationInvalid +from ..exceptions import ConfigurationInvalid, CannotCheckOffline -def get_latest(target, check, full_data=False): +def get_latest(target, check, full_data=False, online=True): python_checker = check.get("python_checker") if python_checker is None or not hasattr(python_checker, "get_latest"): raise ConfigurationInvalid("Update configuration for {} of type python_checker needs python_checker defined and have an attribute \"get_latest\"".format(target)) - return check["python_checker"].get_latest(target, check, full_data=full_data) + if not online and not check.get("offline", False): + raise CannotCheckOffline("{} isn't marked as 'offline' capable, but we are apparently offline right now".format(target)) + + try: + return check["python_checker"].get_latest(target, check, full_data=full_data, online=online) + except: + import inspect + args, _, _, _ = inspect.getargspec(check["python_checker"].get_latest) + if "online" not in args: + # old python_checker footprint, simply leave out the online parameter + return check["python_checker"].get_latest(target, check, full_data=full_data) + + # some other error, raise again + raise