From 74a94469b6137aff09d901d48ce11df4bc40afb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 17 Aug 2016 14:50:53 +0200 Subject: [PATCH 1/4] Fix an issue with global_get from 3rd party plugins --- src/octoprint/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/octoprint/settings.py b/src/octoprint/settings.py index 830112f1..c99fba5f 100644 --- a/src/octoprint/settings.py +++ b/src/octoprint/settings.py @@ -46,7 +46,7 @@ def settings(init=False, basedir=None, configfile=None): (False, default). If this is set to True and the plugin manager has already been initialized, a :class:`ValueError` will be raised. The same will happen if the plugin manager has not yet been initialized and this is set to False. - basedir (str): Path of the base directoy for all of OctoPrint's settings, log files, uploads etc. If not set + basedir (str): Path of the base directory for all of OctoPrint's settings, log files, uploads etc. If not set the default will be used: ``~/.octoprint`` on Linux, ``%APPDATA%/OctoPrint`` on Windows and ``~/Library/Application Support/OctoPrint`` on MacOS. configfile (str): Path of the configuration file (``config.yaml``) to work on. If not set the default will @@ -874,9 +874,9 @@ class Settings(object): while len(path) > 1: key = path.pop(0) - if key in config and key in defaults: + if key in config: config = config[key] - defaults = defaults[key] + defaults = defaults.get(key, dict()) elif incl_defaults and key in defaults: config = {} defaults = defaults[key] From 6499bd160f613b79e65defa59b65cfeb8ebe7d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 18 Aug 2016 13:56:46 +0200 Subject: [PATCH 2/4] Only wait for preemptive cache if it's enabled for path & view --- src/octoprint/server/util/flask.py | 60 ++++++++++++++++++-------- src/octoprint/server/views.py | 52 +++++++++++++++++++--- src/octoprint/static/intermediary.html | 6 +++ 3 files changed, 93 insertions(+), 25 deletions(-) diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index 11ef9bc3..e12a1188 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -371,7 +371,9 @@ def cached(timeout=5 * 60, key=lambda: "view:%s" % flask.request.path, unless=No return decorator def is_in_cache(key=lambda: "view:%s" % flask.request.path): - return key() in _cache + if callable(key): + key = key() + return key in _cache def cache_check_headers(): return "no-cache" in flask.request.cache_control or "no-cache" in flask.request.pragma @@ -400,10 +402,11 @@ class PreemptiveCache(object): self.cachefile = cachefile self._lock = threading.RLock() + self._log_lock = threading.RLock() self._logger = logging.getLogger(__name__ + "." + self.__class__.__name__) self._log_access = True - def record(self, data, unless=None): + def record(self, data, unless=None, root=None): if callable(unless) and unless(): return @@ -412,12 +415,32 @@ class PreemptiveCache(object): entry_data = entry_data() if entry_data is not None: + if root is None: + from flask import request + root = request.path + self.add_data(root, entry_data) + + def has_record(self, data, root=None): + if callable(data): + data = data() + + if data is None: + return False + + if root is None: from flask import request - self.add_data(request.path, entry_data) + root = request.path + + all_data = self.get_data(root) + for existing in all_data: + if self._compare_data(data, existing): + return True + + return False @contextlib.contextmanager def disable_access_logging(self): - with self._lock: + with self._log_lock: self._log_access = False yield self._log_access = True @@ -482,20 +505,19 @@ class PreemptiveCache(object): self.set_all_data(all_data) def add_data(self, root, data): - from octoprint.util import dict_filter - - def strip_ignored(d): - return dict_filter(d, lambda k, v: not k.startswith("_")) - - def compare(a, b): - return set(strip_ignored(a).items()) == set(strip_ignored(b).items()) + with self._log_lock: + if not self._log_access: + self._logger.debug( + "Not updating timestamp and counter for {} and {!r}, currently flagged as disabled".format(root, + data)) + return def split_matched_and_unmatched(entry, entries): matched = [] unmatched = [] for e in entries: - if compare(e, entry): + if self._compare_data(e, entry): matched.append(e) else: unmatched.append(e) @@ -503,12 +525,6 @@ class PreemptiveCache(object): return matched, unmatched with self._lock: - if not self._log_access: - self._logger.debug( - "Not updating timestamp and counter for {} and {!r}, currently flagged as disabled".format(root, - data)) - return - cache_data = self.get_all_data() if not root in cache_data: @@ -537,6 +553,14 @@ class PreemptiveCache(object): self.set_data(root, [to_persist] + other) + def _compare_data(self, a, b): + from octoprint.util import dict_filter + + def strip_ignored(d): + return dict_filter(d, lambda k, v: not k.startswith("_")) + + return set(strip_ignored(a).items()) == set(strip_ignored(b).items()) + def preemptively_cached(cache, data, unless=None): def decorator(f): diff --git a/src/octoprint/server/views.py b/src/octoprint/server/views.py index bcc29ce1..b26a7b5c 100644 --- a/src/octoprint/server/views.py +++ b/src/octoprint/server/views.py @@ -30,25 +30,63 @@ _logger = logging.getLogger(__name__) _valid_id_re = re.compile("[a-z_]+") _valid_div_re = re.compile("[a-zA-Z_-]+") +def _preemptive_unless(root=None, path=None): + if root is None: + root = request.url_root + if path is None: + path = request.path + + return not settings().getBoolean(["devel", "cache", "preemptive"]) \ + or path in settings().get(["server", "preemptiveCache", "exceptions"]) \ + or not (root.startswith("http://") or root.startswith("https://")) + +def _preemptive_data(path=None, base_url=None): + if path is None: + path = request.path + if base_url is None: + base_url = request.url_root + + return dict(path=path, + base_url=base_url, + query_string="l10n={}".format(g.locale.language) if g.locale else "en") + +def _cache_key(url=None, locale=None): + if url is None: + url = request.base_url + if locale is None: + locale = g.locale.language if g.locale else "en" + + return "view:{}:{}".format(url, locale) + @app.route("/cached.gif") def in_cache(): url = request.base_url.replace("/cached.gif", "/") - key = lambda: "view:{}:{}".format(url, g.locale.language if g.locale else "en") - if not util.flask.is_in_cache(key): - return abort(404) + path = request.path.replace("/cached.gif", "/") + + key = _cache_key(url) + data = _preemptive_data(path=path) response = make_response(bytes(base64.b64decode("R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"))) response.headers["Content-Type"] = "image/gif" - return response + + if _preemptive_unless(root=url, path=path) or not preemptiveCache.has_record(data, root=path): + _logger.info("Preemptive cache not active for path {} and data {!r}, signaling as cached".format(path, data)) + return response + elif util.flask.is_in_cache(key): + _logger.info("Found path {} in cache (key: {}), signaling as cached".format(path, key)) + return response + else: + _logger.info("Path {} not yet cached (key: {}), signaling as missing".format(path, key)) + return abort(404) @app.route("/") @util.flask.preemptively_cached(cache=preemptiveCache, - data=lambda: dict(path=request.path, base_url=request.url_root, query_string="l10n={}".format(g.locale.language) if g.locale else "en"), - unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"]) or not (request.url_root.startswith("http://") or request.url_root.startswith("https://"))) + data=_preemptive_data, + unless=_preemptive_unless) @util.flask.conditional(lambda: _check_etag_and_lastmodified_for_index(), NOT_MODIFIED) @util.flask.cached(timeout=-1, refreshif=lambda cached: _validate_cache_for_index(cached), - key=lambda: "view:{}:{}".format(request.base_url, g.locale.language if g.locale else "en"), + key=_cache_key, unless_response=lambda response: util.flask.cache_check_response_headers(response)) @util.flask.etagged(lambda _: _compute_etag_for_index()) @util.flask.lastmodified(lambda _: _compute_date_for_index()) diff --git a/src/octoprint/static/intermediary.html b/src/octoprint/static/intermediary.html index 56c621e5..173c7128 100644 --- a/src/octoprint/static/intermediary.html +++ b/src/octoprint/static/intermediary.html @@ -162,9 +162,15 @@ var message = window.document.getElementById("message"); + var indexCached = false; var indexCachedCallback = function(result) { if (result == "load") { + if (indexCached) { + return; + } + // our cached.gif loaded, so the index is cached now, let's reload + indexCached = true; message.className = "pulsate1 green"; message.innerText = "OctoPrint server ready, reloading page..."; From 8782f77577ba63e9be293eccab66d96651aff0f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 18 Aug 2016 14:01:25 +0200 Subject: [PATCH 3/4] Slightly modified logging for cached.gif --- src/octoprint/server/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/octoprint/server/views.py b/src/octoprint/server/views.py index b26a7b5c..0dc26965 100644 --- a/src/octoprint/server/views.py +++ b/src/octoprint/server/views.py @@ -76,7 +76,7 @@ def in_cache(): _logger.info("Found path {} in cache (key: {}), signaling as cached".format(path, key)) return response else: - _logger.info("Path {} not yet cached (key: {}), signaling as missing".format(path, key)) + _logger.debug("Path {} not yet cached (key: {}), signaling as missing".format(path, key)) return abort(404) @app.route("/") From 60c46a3fc45a054f7410b300bbff5ec9ffb10169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 18 Aug 2016 14:05:12 +0200 Subject: [PATCH 4/4] Fixed use of preemptive cache exceptions in "unless" implementation --- src/octoprint/server/views.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/octoprint/server/views.py b/src/octoprint/server/views.py index 0dc26965..9c22fa4b 100644 --- a/src/octoprint/server/views.py +++ b/src/octoprint/server/views.py @@ -30,15 +30,13 @@ _logger = logging.getLogger(__name__) _valid_id_re = re.compile("[a-z_]+") _valid_div_re = re.compile("[a-zA-Z_-]+") -def _preemptive_unless(root=None, path=None): - if root is None: - root = request.url_root - if path is None: - path = request.path +def _preemptive_unless(base_url=None): + if base_url is None: + base_url = request.url_root return not settings().getBoolean(["devel", "cache", "preemptive"]) \ - or path in settings().get(["server", "preemptiveCache", "exceptions"]) \ - or not (root.startswith("http://") or root.startswith("https://")) + or base_url in settings().get(["server", "preemptiveCache", "exceptions"]) \ + or not (base_url.startswith("http://") or base_url.startswith("https://")) def _preemptive_data(path=None, base_url=None): if path is None: @@ -69,7 +67,7 @@ def in_cache(): response = make_response(bytes(base64.b64decode("R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"))) response.headers["Content-Type"] = "image/gif" - if _preemptive_unless(root=url, path=path) or not preemptiveCache.has_record(data, root=path): + if _preemptive_unless(base_url=url) or not preemptiveCache.has_record(data, root=path): _logger.info("Preemptive cache not active for path {} and data {!r}, signaling as cached".format(path, data)) return response elif util.flask.is_in_cache(key):