From fab5fc489915cee8cc50be43dcaa78f090f5b879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 23 Nov 2015 17:35:25 +0100 Subject: [PATCH 1/6] Added preemptive caching of / and /i18n//messages.js Introduced a @preemptively_cached decorator that for decorated views persists the provided data in ~/.octoprint/data/preemptive_flask_cache.yaml in a list indexed by the view's path if the data is not yet part of the list. During initialization the server will iterate over the persisted paths and data and for each persisted path and entry in the list initialize a temporary WSGI environment based on the data (which is interpretated as keyword arguments to werkzeug's EnvironmentBuilder) which will then be used to call the view function in the correct context. The current implementation for / and /i18n//messages.js utilizes that decorator to allow preemptive caching of those views (/ being probably the most expensive one in the whole core application) utilizing request base URLs (internal access, external access, reverse proxy with prefix url etc) that had been encountered in the past. Through the new config setting server.preemptiveCaching.exceptions it is possible to define a set of base URLs to never cache. Preemptive caching can be globally disabled by setting devel.cache.preemptive to false. --- src/octoprint/server/__init__.py | 30 +++++++++++++- src/octoprint/server/util/flask.py | 63 ++++++++++++++++++++++++++++++ src/octoprint/server/views.py | 9 +++-- src/octoprint/settings.py | 6 ++- 4 files changed, 103 insertions(+), 5 deletions(-) diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index 643e32cc..83014e53 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -467,6 +467,10 @@ class Server(): implementation.on_after_startup() pluginLifecycleManager.add_callback("enabled", call_on_after_startup) + # when we are through with that we also run our preemptive cache + if settings().getBoolean(["devel", "cache", "preemptive"]): + self._execute_preemptive_flask_caching() + import threading threading.Thread(target=work).start() ioloop.add_callback(on_after_startup) @@ -526,7 +530,7 @@ class Server(): if default_language is not None and not default_language == "_default" and default_language in LANGUAGES: return Locale.negotiate([default_language], LANGUAGES) - return request.accept_languages.best_match(LANGUAGES) + return Locale.parse(request.accept_languages.best_match(LANGUAGES)) def _setup_logging(self, debug, logConf=None): defaultConfig = { @@ -663,6 +667,30 @@ class Server(): self._register_template_plugins() + def _execute_preemptive_flask_caching(self): + from octoprint.server.util.flask import get_preemptive_cache_data + from werkzeug.test import EnvironBuilder + + cache_data = get_preemptive_cache_data() + if not cache_data: + return + + def execute_caching(): + for route in sorted(cache_data.keys(), key=lambda x: (x.count("/"), x)): + entries = cache_data[route] + for kwargs in entries: + try: + self._logger.info("Preemptively caching {} for {!r}".format(route, kwargs)) + builder = EnvironBuilder(**kwargs) + app(builder.get_environ(), lambda *a, **kw: None) + except: + self._logger.exception("Error while trying to preemptively cache {} for {!r}".format(route, kwargs)) + + import threading + cache_thread = threading.Thread(target=execute_caching, name="Preemptive Cache Worker") + cache_thread.daemon = True + cache_thread.start() + def _register_template_plugins(self): template_plugins = pluginManager.get_implementations(octoprint.plugin.TemplatePlugin) for plugin in template_plugins: diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index ce59f6d6..0a83f690 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -376,6 +376,69 @@ def cache_check_response_headers(response): return False +_preemptive_flask_cache = "preemptive_flask_cache.yaml" + +def preemptively_cached(data, unless=None): + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + if not (callable(unless) and unless()): + entry_data = data + if callable(entry_data): + entry_data = entry_data() + + if entry_data is not None: + from flask import request + from octoprint.util import atomic_write + import yaml + + data_folder = settings().getBaseFolder("data") + cache_data_file = os.path.join(data_folder, _preemptive_flask_cache) + cache_data = get_preemptive_cache_data() + + if not request.path in cache_data: + cache_data[request.path] = [] + + cache_data_for_path = cache_data.get(request.path, []) + if all(map(lambda entry: entry_data != entry, cache_data_for_path)): + logging.getLogger(__name__).info("Adding {} for {!r} to views to preemptively cache".format(request.path, entry_data)) + cache_data[request.path] = cache_data_for_path + [entry_data] + try: + with atomic_write(cache_data_file, "wb", prefix="octoprint-{}-".format(_preemptive_flask_cache[:-len(".yaml")]), suffix=".yaml") as handle: + yaml.safe_dump(cache_data, handle,default_flow_style=False, indent=" ", allow_unicode=True) + except: + logging.getLogger(__name__).exception("Error while writing {}".format(_preemptive_flask_cache)) + + return f(*args, **kwargs) + + return decorated_function + + return decorator + + +def get_preemptive_cache_data(root=None): + import yaml + + data_folder = settings().getBaseFolder("data") + cache_data_file = os.path.join(data_folder, _preemptive_flask_cache) + if not os.path.isfile(cache_data_file): + return dict() + + cache_data = None + try: + with open(cache_data_file, "r") as f: + cache_data = yaml.safe_load(f) + except: + logging.getLogger(__name__).exception("Error while reading {}".format(_preemptive_flask_cache)) + + if cache_data is None: + cache_data = dict() + + if root: + return cache_data.get(root, dict()) + else: + return cache_data + def add_non_caching_response_headers(response): response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0" diff --git a/src/octoprint/server/views.py b/src/octoprint/server/views.py index 3399a740..34d701a8 100644 --- a/src/octoprint/server/views.py +++ b/src/octoprint/server/views.py @@ -29,10 +29,11 @@ _valid_div_re = re.compile("[a-zA-Z_-]+") @app.route("/") @util.flask.cached(timeout=-1, refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, - key=lambda: "view:{}:{}".format(request.base_url, g.locale), + key=lambda: "view:{}:{}".format(request.base_url, g.locale.language if g.locale else "default"), unless_response=util.flask.cache_check_response_headers) +@util.flask.preemptively_cached(data=lambda: dict(path=request.path, base_url=request.url_root, query_string="l10n={}".format(g.locale.language)) if g.locale else None, + unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) def index(): - #~~ a bunch of settings enable_gcodeviewer = settings().getBoolean(["gcodeViewer", "enabled"]) @@ -404,7 +405,9 @@ def robotsTxt(): @app.route("/i18n//.js") @util.flask.cached(timeout=-1, refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, - key=lambda: "view:{}:{}".format(request.base_url, g.locale)) + key=lambda: "view:{}".format(request.base_url)) +@util.flask.preemptively_cached(data=lambda: dict(path=request.path, base_url=request.url_root) if g.locale else None, + unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) def localeJs(locale, domain): messages = dict() plural_expr = None diff --git a/src/octoprint/settings.py b/src/octoprint/settings.py index a7137e14..ffd972db 100644 --- a/src/octoprint/settings.py +++ b/src/octoprint/settings.py @@ -114,6 +114,9 @@ default_settings = { "diskspace": { "warning": 500 * 1024 * 1024, # 500 MB "critical": 200 * 1024 * 1024, # 200 MB + }, + "preemptiveCache": { + "exceptions": [] } }, "webcam": { @@ -262,7 +265,8 @@ default_settings = { "devel": { "stylesheet": "css", "cache": { - "enabled": True + "enabled": True, + "preemptive": True }, "webassets": { "minify": False, From 6473937b7541332d748dec2e93de852677290464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 24 Nov 2015 14:37:30 +0100 Subject: [PATCH 2/6] Refactore preemptive flask cache into a proper class Also now tracks timestamps of last access to a preemptively cached resource and cleans up stuff that hasn't been accessed in a while (7 days by default) --- src/octoprint/server/__init__.py | 22 +++- src/octoprint/server/util/flask.py | 175 +++++++++++++++++++++-------- src/octoprint/server/views.py | 8 +- src/octoprint/settings.py | 3 +- src/octoprint/util/__init__.py | 42 +++++++ 5 files changed, 194 insertions(+), 56 deletions(-) diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index 83014e53..8ac8a503 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -622,6 +622,9 @@ class Server(): response.headers.add("X-Clacks-Overhead", "GNU Terry Pratchett") return response + preemptive_cache = octoprint.server.util.flask.PreemptiveCache(os.path.join(settings().getBaseFolder("data"), "preemptive_flask_cache.yaml")) + preemptive_cache.attach_to_app(app) + def _setup_i18n(self, app): global babel global LOCALES @@ -668,10 +671,18 @@ class Server(): self._register_template_plugins() def _execute_preemptive_flask_caching(self): - from octoprint.server.util.flask import get_preemptive_cache_data from werkzeug.test import EnvironBuilder + import time - cache_data = get_preemptive_cache_data() + if not hasattr(app, "preemptive_cache"): + return + + # we clean up entries from our preemptive cache settings that haven't been + # accessed longer than server.preemptiveCache.until days + preemptive_cache_timeout = settings().getInt(["server", "preemptiveCache", "until"]) + cutoff_timestamp = time.time() + preemptive_cache_timeout * 24 * 60 * 60 + + cache_data = app.preemptive_cache.clean_all_data(lambda root, entries: filter(lambda entry: "_timestamp" in entry and entry["_timestamp"] <= cutoff_timestamp, entries)) if not cache_data: return @@ -679,10 +690,15 @@ class Server(): for route in sorted(cache_data.keys(), key=lambda x: (x.count("/"), x)): entries = cache_data[route] for kwargs in entries: + additional_request_data = kwargs.get("_additional_request_data", dict()) + kwargs = dict((k, v) for k, v in kwargs.items() if not k.startswith("_")) + kwargs.update(additional_request_data) try: + self._logger.info("Preemptively caching {} for {!r}".format(route, kwargs)) builder = EnvironBuilder(**kwargs) - app(builder.get_environ(), lambda *a, **kw: None) + with app.preemptive_cache.disable_timestamp_update(): + app(builder.get_environ(), lambda *a, **kw: None) except: self._logger.exception("Error while trying to preemptively cache {} for {!r}".format(route, kwargs)) diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index 0a83f690..9d83eb39 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -14,6 +14,7 @@ import flask.ext.assets import webassets.updater import webassets.utils import functools +import contextlib import time import uuid import threading @@ -376,69 +377,147 @@ def cache_check_response_headers(response): return False -_preemptive_flask_cache = "preemptive_flask_cache.yaml" -def preemptively_cached(data, unless=None): - def decorator(f): - @functools.wraps(f) - def decorated_function(*args, **kwargs): - if not (callable(unless) and unless()): - entry_data = data - if callable(entry_data): - entry_data = entry_data() +class PreemptiveCache(object): - if entry_data is not None: - from flask import request - from octoprint.util import atomic_write - import yaml + def __init__(self, cachefile): + self.cachefile = cachefile - data_folder = settings().getBaseFolder("data") - cache_data_file = os.path.join(data_folder, _preemptive_flask_cache) - cache_data = get_preemptive_cache_data() + self._lock = threading.RLock() + self._logger = logging.getLogger(__name__ + "." + self.__class__.__name__) + self._update_timestamp = True - if not request.path in cache_data: - cache_data[request.path] = [] + def recorded(self, data, unless=None): + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + if not (callable(unless) and unless()): + entry_data = data + if callable(entry_data): + entry_data = entry_data() - cache_data_for_path = cache_data.get(request.path, []) - if all(map(lambda entry: entry_data != entry, cache_data_for_path)): - logging.getLogger(__name__).info("Adding {} for {!r} to views to preemptively cache".format(request.path, entry_data)) - cache_data[request.path] = cache_data_for_path + [entry_data] - try: - with atomic_write(cache_data_file, "wb", prefix="octoprint-{}-".format(_preemptive_flask_cache[:-len(".yaml")]), suffix=".yaml") as handle: - yaml.safe_dump(cache_data, handle,default_flow_style=False, indent=" ", allow_unicode=True) - except: - logging.getLogger(__name__).exception("Error while writing {}".format(_preemptive_flask_cache)) + if entry_data is not None: + from flask import request + self.add_data(request.path, entry_data) + return f(*args, **kwargs) + return decorated_function + return decorator - return f(*args, **kwargs) + @contextlib.contextmanager + def disable_timestamp_update(self): + with self._lock: + self._update_timestamp = False + yield + self._update_timestamp = True - return decorated_function + def clean_all_data(self, cleanup_function): + assert callable(cleanup_function) - return decorator + with self._lock: + all_data = self.get_all_data() + for root, entries in all_data.items(): + old_count = len(entries) + entries = cleanup_function(root, entries) + if not entries: + del all_data[root] + self._logger.debug("Removed root {} from preemptive cache".format(root)) + elif len(entries) < old_count: + all_data[root] = entries + self._logger.debug("Removed {} from preemptive cache for root {}".format(old_count - len(entries), root)) + self.set_all_data(all_data) + return all_data -def get_preemptive_cache_data(root=None): - import yaml + def get_all_data(self): + import yaml - data_folder = settings().getBaseFolder("data") - cache_data_file = os.path.join(data_folder, _preemptive_flask_cache) - if not os.path.isfile(cache_data_file): - return dict() + cache_data = None + with self._lock: + try: + with open(self.cachefile, "r") as f: + cache_data = yaml.safe_load(f) + except: + self._logger.exception("Error while reading {}".format(self.cachefile)) - cache_data = None - try: - with open(cache_data_file, "r") as f: - cache_data = yaml.safe_load(f) - except: - logging.getLogger(__name__).exception("Error while reading {}".format(_preemptive_flask_cache)) + if cache_data is None: + cache_data = dict() - if cache_data is None: - cache_data = dict() - - if root: - return cache_data.get(root, dict()) - else: return cache_data + def get_data(self, root): + cache_data = self.get_all_data() + return cache_data.get(root, dict()) + + def set_all_data(self, data): + from octoprint.util import atomic_write + import yaml + + with self._lock: + try: + with atomic_write(self.cachefile, "wb") as handle: + yaml.safe_dump(data, handle,default_flow_style=False, indent=" ", allow_unicode=True) + except: + self._logger.exception("Error while writing {}".format(self.cachefile)) + + def set_data(self, root, data): + with self._lock: + all_data = self.get_all_data() + all_data[root] = data + self.set_all_data(all_data) + + def add_data(self, root, data): + from octoprint.util import dict_filter + + def strip_ignored(d): + return dict_filter(d, lambda k, v: not k.startswith("_")) + + def compare(a, b): + return set(strip_ignored(a).items()) == set(strip_ignored(b).items()) + + def split_matched_and_unmatched(entry, entries): + matched = [] + unmatched = [] + + for e in entries: + if compare(e, entry): + matched.append(e) + else: + unmatched.append(e) + + return matched, unmatched + + with self._lock: + cache_data = self.get_all_data() + + if not root in cache_data: + cache_data[root] = [] + + existing, other = split_matched_and_unmatched(data, cache_data[root]) + + def get_newest(entries): + result = None + for entry in entries: + if "_timestamp" in entry and (result is None or ("_timestamp" in entry and result["_timestamp"] < entry["_timestamp"])): + result = entry + return result + + to_persist = get_newest(existing) + if not to_persist: + import copy + to_persist = copy.deepcopy(data) + to_persist["_timestamp"] = time.time() + self._logger.info("Adding entry for {} and {!r}".format(root, to_persist)) + elif self._update_timestamp: + to_persist["_timestamp"] = time.time() + self._logger.debug("Updating timestamp for {} and {!r}".format(root, data)) + else: + self._logger.debug("Not updating timestamp for {} and {!r}, currently flagged as disabled".format(root, data)) + + self.set_data(root, [to_persist] + other) + + def attach_to_app(self, app): + app.preemptive_cache = self + def add_non_caching_response_headers(response): response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0" diff --git a/src/octoprint/server/views.py b/src/octoprint/server/views.py index 34d701a8..9f99c958 100644 --- a/src/octoprint/server/views.py +++ b/src/octoprint/server/views.py @@ -27,12 +27,12 @@ _valid_id_re = re.compile("[a-z_]+") _valid_div_re = re.compile("[a-zA-Z_-]+") @app.route("/") +@app.preemptive_cache.recorded(data=lambda: dict(path=request.path, base_url=request.url_root, query_string="l10n={}".format(g.locale.language)) if g.locale else None, + unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) @util.flask.cached(timeout=-1, refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, key=lambda: "view:{}:{}".format(request.base_url, g.locale.language if g.locale else "default"), unless_response=util.flask.cache_check_response_headers) -@util.flask.preemptively_cached(data=lambda: dict(path=request.path, base_url=request.url_root, query_string="l10n={}".format(g.locale.language)) if g.locale else None, - unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) def index(): #~~ a bunch of settings @@ -403,11 +403,11 @@ def robotsTxt(): @app.route("/i18n//.js") +@app.preemptive_cache.recorded(data=lambda: dict(path=request.path, base_url=request.url_root), + unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) @util.flask.cached(timeout=-1, refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, key=lambda: "view:{}".format(request.base_url)) -@util.flask.preemptively_cached(data=lambda: dict(path=request.path, base_url=request.url_root) if g.locale else None, - unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) def localeJs(locale, domain): messages = dict() plural_expr = None diff --git a/src/octoprint/settings.py b/src/octoprint/settings.py index ffd972db..b669ff55 100644 --- a/src/octoprint/settings.py +++ b/src/octoprint/settings.py @@ -116,7 +116,8 @@ default_settings = { "critical": 200 * 1024 * 1024, # 200 MB }, "preemptiveCache": { - "exceptions": [] + "exceptions": [], + "until": 7 } }, "webcam": { diff --git a/src/octoprint/util/__init__.py b/src/octoprint/util/__init__.py index 2174f4cf..3426ed67 100644 --- a/src/octoprint/util/__init__.py +++ b/src/octoprint/util/__init__.py @@ -542,6 +542,48 @@ def dict_contains_keys(keys, dictionary): return True + +def dict_filter(dictionary, filter_function): + """ + Filters a dictionary with the provided filter_function + + Example:: + + >>> data = dict(key1="value1", key2="value2", other_key="other_value", foo="bar", bar="foo") + >>> dict_filter(data, lambda k, v: k.startswith("key")) == dict(key1="value1", key2="value2") + True + >>> dict_filter(data, lambda k, v: v.startswith("value")) == dict(key1="value1", key2="value2") + True + >>> dict_filter(data, lambda k, v: k == "foo" or v == "foo") == dict(foo="bar", bar="foo") + True + >>> dict_filter(data, lambda k, v: False) == dict() + True + >>> dict_filter(data, lambda k, v: True) == data + True + >>> dict_filter(None, lambda k, v: True) + Traceback (most recent call last): + ... + AssertionError + >>> dict_filter(data, None) + Traceback (most recent call last): + ... + AssertionError + + Arguments: + dictionary (dict): The dictionary to filter + filter_function (callable): The filter function to apply, called with key and + value of an entry in the dictionary, must return ``True`` for values to + keep and ``False`` for values to strip + + Returns: + dict: A shallow copy of the provided dictionary, stripped of the key-value-pairs + for which the ``filter_function`` returned ``False`` + """ + assert isinstance(dictionary, dict) + assert callable(filter_function) + return dict((k, v) for k, v in dictionary.items() if filter_function(k, v)) + + class Object(object): pass From 3de4f91f506de2734454ba104bd97d1ef66b8656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 24 Nov 2015 18:33:49 +0100 Subject: [PATCH 3/6] Decoupled decorator from PreemptiveCache class Coupling it led to problems (naturally) when there was no PreemptiveCache instance available yet. --- src/octoprint/server/__init__.py | 21 ++++++++-------- src/octoprint/server/util/flask.py | 39 ++++++++++++++++++------------ src/octoprint/server/views.py | 12 +++++---- 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index 8ac8a503..08f1b3bc 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -42,6 +42,7 @@ loginManager = None pluginManager = None appSessionManager = None pluginLifecycleManager = None +preemptiveCache = None principals = Principal(app) admin_permission = Permission(RoleNeed("admin")) @@ -61,6 +62,7 @@ import octoprint.util import octoprint.filemanager.storage import octoprint.filemanager.analysis import octoprint.slicing +from octoprint.server.util.flask import PreemptiveCache from . import util @@ -142,6 +144,7 @@ class Server(): global pluginManager global appSessionManager global pluginLifecycleManager + global preemptiveCache global debug from tornado.ioloop import IOLoop @@ -185,6 +188,7 @@ class Server(): printer = Printer(fileManager, analysisQueue, printerProfileManager) appSessionManager = util.flask.AppSessionManager() pluginLifecycleManager = LifecycleManager(pluginManager) + preemptiveCache = PreemptiveCache(os.path.join(s.getBaseFolder("data"), "preemptive_cache_config.yaml")) def octoprint_plugin_inject_factory(name, implementation): if not isinstance(implementation, octoprint.plugin.OctoPrintPlugin): @@ -199,7 +203,8 @@ class Server(): printer=printer, app_session_manager=appSessionManager, plugin_lifecycle_manager=pluginLifecycleManager, - data_folder=os.path.join(settings().getBaseFolder("data"), name) + data_folder=os.path.join(settings().getBaseFolder("data"), name), + preemptive_cache=preemptiveCache ) def settings_plugin_inject_factory(name, implementation): @@ -469,7 +474,7 @@ class Server(): # when we are through with that we also run our preemptive cache if settings().getBoolean(["devel", "cache", "preemptive"]): - self._execute_preemptive_flask_caching() + self._execute_preemptive_flask_caching(preemptiveCache) import threading threading.Thread(target=work).start() @@ -622,9 +627,6 @@ class Server(): response.headers.add("X-Clacks-Overhead", "GNU Terry Pratchett") return response - preemptive_cache = octoprint.server.util.flask.PreemptiveCache(os.path.join(settings().getBaseFolder("data"), "preemptive_flask_cache.yaml")) - preemptive_cache.attach_to_app(app) - def _setup_i18n(self, app): global babel global LOCALES @@ -670,19 +672,16 @@ class Server(): self._register_template_plugins() - def _execute_preemptive_flask_caching(self): + def _execute_preemptive_flask_caching(self, preemptive_cache): from werkzeug.test import EnvironBuilder import time - if not hasattr(app, "preemptive_cache"): - return - # we clean up entries from our preemptive cache settings that haven't been # accessed longer than server.preemptiveCache.until days preemptive_cache_timeout = settings().getInt(["server", "preemptiveCache", "until"]) cutoff_timestamp = time.time() + preemptive_cache_timeout * 24 * 60 * 60 - cache_data = app.preemptive_cache.clean_all_data(lambda root, entries: filter(lambda entry: "_timestamp" in entry and entry["_timestamp"] <= cutoff_timestamp, entries)) + cache_data = preemptive_cache.clean_all_data(lambda root, entries: filter(lambda entry: "_timestamp" in entry and entry["_timestamp"] <= cutoff_timestamp, entries)) if not cache_data: return @@ -697,7 +696,7 @@ class Server(): self._logger.info("Preemptively caching {} for {!r}".format(route, kwargs)) builder = EnvironBuilder(**kwargs) - with app.preemptive_cache.disable_timestamp_update(): + with preemptive_cache.disable_timestamp_update(): app(builder.get_environ(), lambda *a, **kw: None) except: self._logger.exception("Error while trying to preemptively cache {} for {!r}".format(route, kwargs)) diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index 9d83eb39..3b376ebc 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -387,21 +387,17 @@ class PreemptiveCache(object): self._logger = logging.getLogger(__name__ + "." + self.__class__.__name__) self._update_timestamp = True - def recorded(self, data, unless=None): - def decorator(f): - @functools.wraps(f) - def decorated_function(*args, **kwargs): - if not (callable(unless) and unless()): - entry_data = data - if callable(entry_data): - entry_data = entry_data() + def record(self, data, unless=None): + if callable(unless) and unless(): + return - if entry_data is not None: - from flask import request - self.add_data(request.path, entry_data) - return f(*args, **kwargs) - return decorated_function - return decorator + entry_data = data + if callable(entry_data): + entry_data = entry_data() + + if entry_data is not None: + from flask import request + self.add_data(request.path, entry_data) @contextlib.contextmanager def disable_timestamp_update(self): @@ -436,6 +432,10 @@ class PreemptiveCache(object): try: with open(self.cachefile, "r") as f: cache_data = yaml.safe_load(f) + except IOError as e: + import errno + if e.errno != errno.ENOENT: + raise except: self._logger.exception("Error while reading {}".format(self.cachefile)) @@ -515,8 +515,15 @@ class PreemptiveCache(object): self.set_data(root, [to_persist] + other) - def attach_to_app(self, app): - app.preemptive_cache = self + +def preemptively_cached(cache, data, unless=None): + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + cache.record(data, unless=unless) + return f(*args, **kwargs) + return decorated_function + return decorator def add_non_caching_response_headers(response): diff --git a/src/octoprint/server/views.py b/src/octoprint/server/views.py index 9f99c958..c25671e6 100644 --- a/src/octoprint/server/views.py +++ b/src/octoprint/server/views.py @@ -13,7 +13,7 @@ from flask import request, g, url_for, make_response, render_template, send_from import octoprint.plugin from octoprint.server import app, userManager, pluginManager, gettext, \ - debug, LOCALES, VERSION, DISPLAY_VERSION, UI_API_KEY, BRANCH + debug, LOCALES, VERSION, DISPLAY_VERSION, UI_API_KEY, BRANCH, preemptiveCache from octoprint.settings import settings import re @@ -27,8 +27,9 @@ _valid_id_re = re.compile("[a-z_]+") _valid_div_re = re.compile("[a-zA-Z_-]+") @app.route("/") -@app.preemptive_cache.recorded(data=lambda: dict(path=request.path, base_url=request.url_root, query_string="l10n={}".format(g.locale.language)) if g.locale else None, - unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) +@util.flask.preemptively_cached(cache=preemptiveCache, + data=lambda: dict(path=request.path, base_url=request.url_root, query_string="l10n={}".format(g.locale.language)) if g.locale else None, + unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) @util.flask.cached(timeout=-1, refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, key=lambda: "view:{}:{}".format(request.base_url, g.locale.language if g.locale else "default"), @@ -403,8 +404,9 @@ def robotsTxt(): @app.route("/i18n//.js") -@app.preemptive_cache.recorded(data=lambda: dict(path=request.path, base_url=request.url_root), - unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) +@util.flask.preemptively_cached(cache=preemptiveCache, + data=lambda: dict(path=request.path, base_url=request.url_root), + unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) @util.flask.cached(timeout=-1, refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, key=lambda: "view:{}".format(request.base_url)) From 2e23cd39a42d5397a418c211e00db249805dcfc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 26 Nov 2015 13:32:32 +0100 Subject: [PATCH 4/6] Added ETag and LastModified headers + processing to UI index That should improve performance tremendously. Both ETag and LastModified depend on all files the template rendering depends on. If any of the depended on files changes, both values will change as well. That allows us to track whether our cached copy is still current (and force a refresh if not) and also process IfMatch request headers and reply with a 304 directly so that we do not even have to transfer the data if nothing changed and the browser still has it. --- src/octoprint/server/__init__.py | 1 + src/octoprint/server/util/__init__.py | 14 ++ src/octoprint/server/util/flask.py | 77 ++++++- src/octoprint/server/views.py | 277 +++++++++++++++++++------- src/octoprint/util/jinja.py | 54 ++++- 5 files changed, 345 insertions(+), 78 deletions(-) diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index 08f1b3bc..5011066b 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -25,6 +25,7 @@ import signal SUCCESS = {} NO_CONTENT = ("", 204) +NOT_MODIFIED = ("Not Modified", 304) app = Flask("octoprint") assets = None diff --git a/src/octoprint/server/util/__init__.py b/src/octoprint/server/util/__init__.py index d128e8ef..20640e64 100644 --- a/src/octoprint/server/util/__init__.py +++ b/src/octoprint/server/util/__init__.py @@ -143,6 +143,20 @@ def get_api_key(request): return None +def get_plugin_hash(): + from octoprint.plugin import plugin_manager + + plugin_signature = lambda impl: "{}:{}".format(impl._identifier, impl._plugin_version) + template_plugins = map(plugin_signature, plugin_manager().get_implementations(octoprint.plugin.TemplatePlugin)) + asset_plugins = map(plugin_signature, plugin_manager().get_implementations(octoprint.plugin.AssetPlugin)) + ui_plugins = sorted(set(template_plugins + asset_plugins)) + + import hashlib + plugin_hash = hashlib.sha1() + plugin_hash.update(",".join(ui_plugins)) + return plugin_hash.hexdigest() + + #~~ reverse proxy compatible WSGI middleware diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index 3b376ebc..2a9bcadf 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -331,13 +331,14 @@ def cached(timeout=5 * 60, key=lambda: "view:%s" % flask.request.path, unless=No return f(*args, **kwargs) cache_key = key() + rv = _cache.get(cache_key) # only take the value from the cache if we are not required to refresh it from the wrapped function - if not callable(refreshif) or not refreshif(): - rv = _cache.get(cache_key) - if rv is not None: - logger.debug("Serving entry for {path} from cache".format(path=flask.request.path)) - return rv + if rv is not None and (not callable(refreshif) or not refreshif(rv)): + logger.debug("Serving entry for {path} from cache".format(path=flask.request.path)) + if not "X-From-Cache" in rv.headers: + rv.headers["X-From-Cache"] = "true" + return rv # get value from wrapped function logger.debug("No cache entry or refreshing cache for {path} (key: {key}), calling wrapped function".format(path=flask.request.path, key=cache_key)) @@ -526,6 +527,72 @@ def preemptively_cached(cache, data, unless=None): return decorator +def etagged(etag): + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + rv = f(*args, **kwargs) + if isinstance(rv, flask.Response): + result = etag + if callable(result): + result = result(rv) + if result: + rv.set_etag(result) + return rv + return decorated_function + return decorator + + +def lastmodified(date): + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + rv = f(*args, **kwargs) + if not "last-modified" in rv.headers: + result = date + if callable(result): + result = result(rv) + + if not isinstance(result, basestring): + from werkzeug.http import http_date + result = http_date(result) + + if result: + rv.headers["last-modified"] = result + return rv + return decorated_function + return decorator + + +def conditional(condition, met): + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + if callable(condition) and condition(): + # condition has been met, return met-response + rv = met + if callable(met): + rv = met() + return rv + + # condition hasn't been met, call decorated function + return f(*args, **kwargs) + return decorated_function + return decorator + + +def check_etag(etag): + return flask.request.method in ("GET", "HEAD") and \ + flask.request.if_none_match and \ + etag in flask.request.if_none_match + + +def check_lastmodified(lastmodified): + return flask.request.method in ("GET", "HEAD") and \ + flask.request.if_modified_since and \ + lastmodified >= flask.request.if_modified_since + + def add_non_caching_response_headers(response): response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0" response.headers["Pragma"] = "no-cache" diff --git a/src/octoprint/server/views.py b/src/octoprint/server/views.py index c25671e6..2632323b 100644 --- a/src/octoprint/server/views.py +++ b/src/octoprint/server/views.py @@ -13,7 +13,8 @@ from flask import request, g, url_for, make_response, render_template, send_from import octoprint.plugin from octoprint.server import app, userManager, pluginManager, gettext, \ - debug, LOCALES, VERSION, DISPLAY_VERSION, UI_API_KEY, BRANCH, preemptiveCache + debug, LOCALES, VERSION, DISPLAY_VERSION, UI_API_KEY, BRANCH, preemptiveCache, \ + NOT_MODIFIED from octoprint.settings import settings import re @@ -28,12 +29,15 @@ _valid_div_re = re.compile("[a-zA-Z_-]+") @app.route("/") @util.flask.preemptively_cached(cache=preemptiveCache, - data=lambda: dict(path=request.path, base_url=request.url_root, query_string="l10n={}".format(g.locale.language)) if g.locale else None, + data=lambda: dict(path=request.path, base_url=request.url_root, query_string="l10n={}".format(g.locale.language)) if g.locale else "en", unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) +@util.flask.conditional(lambda: _check_etag_and_lastmodified_for_index(), NOT_MODIFIED) @util.flask.cached(timeout=-1, - refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, - key=lambda: "view:{}:{}".format(request.base_url, g.locale.language if g.locale else "default"), - unless_response=util.flask.cache_check_response_headers) + refreshif=lambda cached: _validate_cache_for_index(cached), + key=lambda: "view:{}:{}".format(request.base_url, g.locale.language if g.locale else "en"), + unless_response=lambda response: util.flask.cache_check_response_headers(response)) +@util.flask.etagged(lambda _: _compute_etag_for_index()) +@util.flask.lastmodified(lambda _: _compute_date_for_index()) def index(): #~~ a bunch of settings @@ -299,13 +303,10 @@ def index(): #~~ render! - import datetime - response = make_response(render_template( "index.jinja2", **render_kwargs )) - response.headers["Last-Modified"] = datetime.datetime.now() if first_run: response = util.flask.add_non_caching_response_headers(response) @@ -356,6 +357,7 @@ def _process_template_configs(name, implementation, configs, rules): return includes + def _process_template_config(name, implementation, rule, config=None, counter=1): if "mandatory" in rule: for mandatory in rule["mandatory"]: @@ -398,81 +400,23 @@ def _process_template_config(name, implementation, rule, config=None, counter=1) return data + @app.route("/robots.txt") +@util.flask.cached(timeout=-1) def robotsTxt(): return send_from_directory(app.static_folder, "robots.txt") @app.route("/i18n//.js") -@util.flask.preemptively_cached(cache=preemptiveCache, - data=lambda: dict(path=request.path, base_url=request.url_root), - unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) -@util.flask.cached(timeout=-1, - refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, - key=lambda: "view:{}".format(request.base_url)) +@util.flask.conditional(lambda: _check_etag_and_lastmodified_for_i18n(), NOT_MODIFIED) +@util.flask.etagged(lambda _: _compute_etag_for_i18n(request.view_args["locale"], request.view_args["domain"])) +@util.flask.lastmodified(lambda _: _compute_date_for_i18n(request.view_args["locale"], request.view_args["domain"])) def localeJs(locale, domain): messages = dict() plural_expr = None if locale != "en": - from flask import _request_ctx_stack - from babel.messages.pofile import read_po - - def messages_from_po(base_path, locale, domain): - path = os.path.join(base_path, locale) - if not os.path.isdir(path): - return None, None - - path = os.path.join(path, "LC_MESSAGES", "{domain}.po".format(**locals())) - if not os.path.isfile(path): - return None, None - - messages = dict() - with file(path) as f: - catalog = read_po(f, locale=locale, domain=domain) - - for message in catalog: - message_id = message.id - if isinstance(message_id, (list, tuple)): - message_id = message_id[0] - messages[message_id] = message.string - - return messages, catalog.plural_expr - - user_base_path = os.path.join(settings().getBaseFolder("translations")) - user_plugin_path = os.path.join(user_base_path, "_plugins") - - # plugin translations - plugins = octoprint.plugin.plugin_manager().enabled_plugins - for name, plugin in plugins.items(): - dirs = [os.path.join(user_plugin_path, name), os.path.join(plugin.location, 'translations')] - for dirname in dirs: - if not os.path.isdir(dirname): - continue - - plugin_messages, _ = messages_from_po(dirname, locale, domain) - - if plugin_messages is not None: - messages = octoprint.util.dict_merge(messages, plugin_messages) - _logger.debug("Using translation folder {dirname} for locale {locale} of plugin {name}".format(**locals())) - break - else: - _logger.debug("No translations for locale {locale} for plugin {name}".format(**locals())) - - # core translations - ctx = _request_ctx_stack.top - base_path = os.path.join(ctx.app.root_path, "translations") - - dirs = [user_base_path, base_path] - for dirname in dirs: - core_messages, plural_expr = messages_from_po(dirname, locale, domain) - - if core_messages is not None: - messages = octoprint.util.dict_merge(messages, core_messages) - _logger.debug("Using translation folder {dirname} for locale {locale} of core translations".format(**locals())) - break - else: - _logger.debug("No core translations for locale {locale}".format(**locals())) + messages, plural_expr = _get_translations(locale, domain) catalog = dict( messages=messages, @@ -490,3 +434,192 @@ def plugin_assets(name, filename): return redirect(url_for("plugin." + name + ".static", filename=filename)) +def _compute_etag_for_index(files=None, lastmodified=None): + if files is None: + files = _files_for_index() + if lastmodified is None: + lastmodified = _compute_date(files) + if lastmodified and not isinstance(lastmodified, basestring): + from werkzeug.http import http_date + lastmodified = http_date(lastmodified) + + from octoprint import __version__ + from octoprint.server import UI_API_KEY + + import hashlib + hash = hashlib.sha1() + hash.update(__version__) + hash.update(UI_API_KEY) + hash.update(",".join(sorted(files))) + if lastmodified: + hash.update(lastmodified) + return hash.hexdigest() + + +def _compute_etag_for_i18n(locale, domain, files=None, lastmodified=None): + if files is None: + files = _get_all_translationfiles(locale, domain) + if lastmodified is None: + lastmodified = _compute_date(files) + if lastmodified and not isinstance(lastmodified, basestring): + from werkzeug.http import http_date + lastmodified = http_date(lastmodified) + + import hashlib + hash = hashlib.sha1() + hash.update(",".join(sorted(files))) + if lastmodified: + hash.update(lastmodified) + return hash.hexdigest() + + +def _compute_date_for_i18n(locale, domain): + return _compute_date(_get_all_translationfiles(locale, domain)) + + +def _compute_date_for_index(): + return _compute_date(_files_for_index()) + + +def _validate_cache_for_index(cached): + no_cache_headers = util.flask.cache_check_headers() + refresh_flag = "_refresh" in request.values + etag_different = _compute_etag_for_index() != cached.get_etag()[0] + + return no_cache_headers or refresh_flag or etag_different + + +def _files_for_index(): + """ + Collects all paths of files that the index page depends on. + + The relevant files are: + + * all jinja2 templates: they might be used within the index page, so + any changes here change the rendering outcome + * all defined assets: if one of them changes, the webassets bundle will + be regenerated and hence the URL included in the cached page won't be + valid anymore + * all translation files used for our current locale: if any of those change + we also need to re-render + """ + + templates = _get_all_templates() + assets = _get_all_assets() + translations = _get_all_translationfiles(g.locale.language if g.locale else "en", "messages") + return sorted(set(templates + assets + translations)) + + +def _compute_date(files): + from datetime import datetime + timestamps = map(lambda path: os.stat(path).st_mtime, files) + max_timestamp = max(*timestamps) if timestamps else None + if max_timestamp: + # we set the micros to 0 since microseconds are not speced for HTTP + max_timestamp = datetime.fromtimestamp(max_timestamp).replace(microsecond=0) + return max_timestamp + + +def _check_etag_and_lastmodified_for_index(): + files = _files_for_index() + lastmodified = _compute_date(files) + lastmodified_ok = util.flask.check_lastmodified(lastmodified) + etag_ok = util.flask.check_etag(_compute_etag_for_index(files, lastmodified)) + return etag_ok and lastmodified_ok + + +def _check_etag_and_lastmodified_for_i18n(): + locale = request.view_args["locale"] + domain = request.view_args["domain"] + + etag_ok = util.flask.check_etag(_compute_etag_for_i18n(request.view_args["locale"], request.view_args["domain"])) + + lastmodified = _compute_date_for_i18n(locale, domain) + lastmodified_ok = lastmodified is None or util.flask.check_lastmodified(lastmodified) + + return etag_ok and lastmodified_ok + + +def _get_all_templates(): + from octoprint.util.jinja import get_all_template_paths + return get_all_template_paths(app.jinja_loader, lambda path: not octoprint.util.is_hidden_path(path)) + + +def _get_all_assets(): + from octoprint.util.jinja import get_all_asset_paths + return get_all_asset_paths(app.jinja_env.assets_environment) + + +def _get_all_translationfiles(locale, domain): + from flask import _request_ctx_stack + + def get_po_path(basedir, locale, domain): + path = os.path.join(basedir, locale) + if not os.path.isdir(path): + return None + + path = os.path.join(path, "LC_MESSAGES", "{domain}.po".format(**locals())) + if not os.path.isfile(path): + return None + + return path + + po_files = [] + + user_base_path = os.path.join(settings().getBaseFolder("translations")) + user_plugin_path = os.path.join(user_base_path, "_plugins") + + # plugin translations + plugins = octoprint.plugin.plugin_manager().enabled_plugins + for name, plugin in plugins.items(): + dirs = [os.path.join(user_plugin_path, name), os.path.join(plugin.location, 'translations')] + for dirname in dirs: + if not os.path.isdir(dirname): + continue + + po_file = get_po_path(dirname, locale, domain) + if po_file: + po_files.append(po_file) + break + + # core translations + ctx = _request_ctx_stack.top + base_path = os.path.join(ctx.app.root_path, "translations") + + dirs = [user_base_path, base_path] + for dirname in dirs: + po_file = get_po_path(dirname, locale, domain) + if po_file: + po_files.append(po_file) + break + + return po_files + + +def _get_translations(locale, domain): + from babel.messages.pofile import read_po + from octoprint.util import dict_merge + + messages = dict() + plural_expr = None + + def messages_from_po(path, locale, domain): + messages = dict() + with file(path) as f: + catalog = read_po(f, locale=locale, domain=domain) + + for message in catalog: + message_id = message.id + if isinstance(message_id, (list, tuple)): + message_id = message_id[0] + messages[message_id] = message.string + + return messages, catalog.plural_expr + + po_files = _get_all_translationfiles(locale, domain) + for po_file in po_files: + po_messages, plural_expr = messages_from_po(po_file, locale, domain) + if po_messages is not None: + messages = dict_merge(messages, po_messages) + + return messages, plural_expr diff --git a/src/octoprint/util/jinja.py b/src/octoprint/util/jinja.py index bf3f8b30..28bfcea4 100644 --- a/src/octoprint/util/jinja.py +++ b/src/octoprint/util/jinja.py @@ -6,7 +6,8 @@ __copyright__ = "Copyright (C) 2015 The OctoPrint Project - Released under terms import os -from jinja2.loaders import FileSystemLoader, TemplateNotFound, split_template_path +from jinja2.loaders import FileSystemLoader, PrefixLoader, ChoiceLoader, \ + ModuleLoader, TemplateNotFound, split_template_path class FilteredFileSystemLoader(FileSystemLoader): """ @@ -48,3 +49,54 @@ class FilteredFileSystemLoader(FileSystemLoader): filter_results = map(lambda x: not os.path.exists(os.path.join(x, path)) or self.path_filter(os.path.join(x, path)), self.searchpath) return all(filter_results) + + +def collect_template_folders(loader): + import copy + + if isinstance(loader, FileSystemLoader): + return copy.copy(loader.searchpath) + elif isinstance(loader, PrefixLoader): + result = [] + for subloader in loader.mapping.values(): + result += collect_template_folders(subloader) + return result + elif isinstance(loader, ChoiceLoader): + result = [] + for subloader in loader.loaders: + result += collect_template_folders(subloader) + return result + elif isinstance(loader, ModuleLoader): + return [loader.module.__path__] + + return [] + + +def get_all_template_paths(loader, filter_function=None): + result = [] + template_folders = collect_template_folders(loader) + for template_folder in template_folders: + walk_dir = os.walk(template_folder, followlinks=True) + for dirpath, dirnames, filenames in walk_dir: + for filename in filenames: + path = os.path.join(dirpath, filename) + if not callable(filter_function) or filter_function(path): + result.append(path) + return result + + +def get_all_asset_paths(env): + result = [] + for bundle in env: + for content in bundle.resolve_contents(): + try: + if not content: + continue + path = content[1] + if not os.path.isfile(path): + continue + result.append(path) + except IndexError: + # intentionally ignored + pass + return result From c595d02f22b9ed21e75c1b1a327a6aa44c6585c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 26 Nov 2015 14:15:06 +0100 Subject: [PATCH 5/6] reload(true) shouldn't be necessary any more UI + assets should actually have ETag and LastModified headers and proper IfMatch handling on the server side and hence the server should do the right thing on its own. --- src/octoprint/server/util/flask.py | 4 ++-- src/octoprint/static/js/app/dataupdater.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index 2a9bcadf..d621671a 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -548,7 +548,7 @@ def lastmodified(date): @functools.wraps(f) def decorated_function(*args, **kwargs): rv = f(*args, **kwargs) - if not "last-modified" in rv.headers: + if not "Last-Modified" in rv.headers: result = date if callable(result): result = result(rv) @@ -558,7 +558,7 @@ def lastmodified(date): result = http_date(result) if result: - rv.headers["last-modified"] = result + rv.headers["Last-Modified"] = result return rv return decorated_function return decorator diff --git a/src/octoprint/static/js/app/dataupdater.js b/src/octoprint/static/js/app/dataupdater.js index 26bd5f7e..d46ce715 100644 --- a/src/octoprint/static/js/app/dataupdater.js +++ b/src/octoprint/static/js/app/dataupdater.js @@ -12,7 +12,7 @@ function DataUpdater(allViewModels) { self._pluginHash = undefined; self.reloadOverlay = $("#reloadui_overlay"); - $("#reloadui_overlay_reload").click(function() { location.reload(true); }); + $("#reloadui_overlay_reload").click(function() { location.reload(); }); self.connect = function() { var options = {}; From 63d095ab00b287cf200651c57a4043dec283a7ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 26 Nov 2015 19:48:44 +0100 Subject: [PATCH 6/6] Refactored implementation of get_all_template_paths --- src/octoprint/util/jinja.py | 66 ++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/src/octoprint/util/jinja.py b/src/octoprint/util/jinja.py index 28bfcea4..ac2d01ba 100644 --- a/src/octoprint/util/jinja.py +++ b/src/octoprint/util/jinja.py @@ -7,7 +7,7 @@ __copyright__ = "Copyright (C) 2015 The OctoPrint Project - Released under terms import os from jinja2.loaders import FileSystemLoader, PrefixLoader, ChoiceLoader, \ - ModuleLoader, TemplateNotFound, split_template_path + TemplateNotFound, split_template_path class FilteredFileSystemLoader(FileSystemLoader): """ @@ -51,38 +51,44 @@ class FilteredFileSystemLoader(FileSystemLoader): return all(filter_results) -def collect_template_folders(loader): - import copy - - if isinstance(loader, FileSystemLoader): - return copy.copy(loader.searchpath) - elif isinstance(loader, PrefixLoader): - result = [] - for subloader in loader.mapping.values(): - result += collect_template_folders(subloader) - return result - elif isinstance(loader, ChoiceLoader): - result = [] - for subloader in loader.loaders: - result += collect_template_folders(subloader) - return result - elif isinstance(loader, ModuleLoader): - return [loader.module.__path__] - - return [] - - -def get_all_template_paths(loader, filter_function=None): - result = [] - template_folders = collect_template_folders(loader) - for template_folder in template_folders: - walk_dir = os.walk(template_folder, followlinks=True) +def get_all_template_paths(loader): + def walk_folder(folder): + files = [] + walk_dir = os.walk(folder, followlinks=True) for dirpath, dirnames, filenames in walk_dir: for filename in filenames: path = os.path.join(dirpath, filename) - if not callable(filter_function) or filter_function(path): - result.append(path) - return result + files.append(path) + return files + + def collect_templates_for_loader(loader): + if isinstance(loader, FilteredFileSystemLoader): + result = [] + for folder in loader.searchpath: + result += walk_folder(folder) + return filter(loader.path_filter, result) + + elif isinstance(loader, FileSystemLoader): + result = [] + for folder in loader.searchpath: + result += walk_folder(folder) + return result + + elif isinstance(loader, PrefixLoader): + result = [] + for subloader in loader.mapping.values(): + result += collect_templates_for_loader(subloader) + return result + + elif isinstance(loader, ChoiceLoader): + result = [] + for subloader in loader.loaders: + result += collect_templates_for_loader(subloader) + return result + + return [] + + return collect_templates_for_loader(loader) def get_all_asset_paths(env):