diff --git a/src/octoprint/plugin/types.py b/src/octoprint/plugin/types.py index 0112d3a1..0223772c 100644 --- a/src/octoprint/plugin/types.py +++ b/src/octoprint/plugin/types.py @@ -661,13 +661,25 @@ class UiPlugin(OctoPrintPlugin, SortablePlugin): return None - def get_ui_additional_key_data_for_cache(self, request): + def get_ui_additional_key_data_for_cache(self): return None - def get_ui_data_for_preemptive_caching(self, request): + def get_ui_additional_tracked_files(self): return None - def get_ui_additional_request_data_for_preemptive_caching(self, request): + def get_ui_custom_tracked_files(self): + return None + + def get_ui_custom_etag(self): + return None + + def get_ui_custom_lastmodified(self): + return None + + def get_ui_data_for_preemptive_caching(self): + return None + + def get_ui_additional_request_data_for_preemptive_caching(self): return None class WizardPlugin(OctoPrintPlugin, ReloadNeedingPlugin): diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index c6a0aa13..c5ef4b60 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -26,6 +26,7 @@ import signal SUCCESS = {} NO_CONTENT = ("", 204) +NOT_MODIFIED = ("Not Modified", 304) app = Flask("octoprint") assets = None diff --git a/src/octoprint/server/util/__init__.py b/src/octoprint/server/util/__init__.py index ab9249d3..78425af8 100644 --- a/src/octoprint/server/util/__init__.py +++ b/src/octoprint/server/util/__init__.py @@ -143,6 +143,20 @@ def get_api_key(request): return None +def get_plugin_hash(): + from octoprint.plugin import plugin_manager + + plugin_signature = lambda impl: "{}:{}".format(impl._identifier, impl._plugin_version) + template_plugins = map(plugin_signature, plugin_manager().get_implementations(octoprint.plugin.TemplatePlugin)) + asset_plugins = map(plugin_signature, plugin_manager().get_implementations(octoprint.plugin.AssetPlugin)) + ui_plugins = sorted(set(template_plugins + asset_plugins)) + + import hashlib + plugin_hash = hashlib.sha1() + plugin_hash.update(",".join(ui_plugins)) + return plugin_hash.hexdigest() + + #~~ reverse proxy compatible WSGI middleware diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index 28f86124..0ba64d4b 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -331,13 +331,14 @@ def cached(timeout=5 * 60, key=lambda: "view:%s" % flask.request.path, unless=No return f(*args, **kwargs) cache_key = key() + rv = _cache.get(cache_key) # only take the value from the cache if we are not required to refresh it from the wrapped function - if not callable(refreshif) or not refreshif(): - rv = _cache.get(cache_key) - if rv is not None: - logger.debug("Serving entry for {path} from cache (key: {key})".format(path=flask.request.path, key=cache_key)) - return rv + if rv is not None and (not callable(refreshif) or not refreshif(rv)): + logger.debug("Serving entry for {path} from cache (key: {key})".format(path=flask.request.path, key=cache_key)) + if not "X-From-Cache" in rv.headers: + rv.headers["X-From-Cache"] = "true" + return rv # get value from wrapped function logger.debug("No cache entry or refreshing cache for {path} (key: {key}), calling wrapped function".format(path=flask.request.path, key=cache_key)) @@ -526,6 +527,72 @@ def preemptively_cached(cache, data, unless=None): return decorator +def etagged(etag): + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + rv = f(*args, **kwargs) + if isinstance(rv, flask.Response): + result = etag + if callable(result): + result = result(rv) + if result: + rv.set_etag(result) + return rv + return decorated_function + return decorator + + +def lastmodified(date): + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + rv = f(*args, **kwargs) + if not "Last-Modified" in rv.headers: + result = date + if callable(result): + result = result(rv) + + if not isinstance(result, basestring): + from werkzeug.http import http_date + result = http_date(result) + + if result: + rv.headers["Last-Modified"] = result + return rv + return decorated_function + return decorator + + +def conditional(condition, met): + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + if callable(condition) and condition(): + # condition has been met, return met-response + rv = met + if callable(met): + rv = met() + return rv + + # condition hasn't been met, call decorated function + return f(*args, **kwargs) + return decorated_function + return decorator + + +def check_etag(etag): + return flask.request.method in ("GET", "HEAD") and \ + flask.request.if_none_match and \ + etag in flask.request.if_none_match + + +def check_lastmodified(lastmodified): + return flask.request.method in ("GET", "HEAD") and \ + flask.request.if_modified_since and \ + lastmodified >= flask.request.if_modified_since + + def add_non_caching_response_headers(response): response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0" response.headers["Pragma"] = "no-cache" diff --git a/src/octoprint/server/views.py b/src/octoprint/server/views.py index 0a3a9e97..d2d5f47c 100644 --- a/src/octoprint/server/views.py +++ b/src/octoprint/server/views.py @@ -14,7 +14,8 @@ from flask import request, g, url_for, make_response, render_template, send_from import octoprint.plugin from octoprint.server import app, userManager, pluginManager, gettext, \ - debug, LOCALES, VERSION, DISPLAY_VERSION, UI_API_KEY, BRANCH, preemptiveCache + debug, LOCALES, VERSION, DISPLAY_VERSION, UI_API_KEY, BRANCH, preemptiveCache, \ + NOT_MODIFIED from octoprint.settings import settings import re @@ -62,36 +63,131 @@ def index(): # add data if we have any if data is not None: - if "query_string" in data: - data["query_string"] = "l10n={}&{}".format(g.locale.language, data["query_string"]) - d.update(data) + try: + if callable(data): + data = data() + if "query_string" in data: + data["query_string"] = "l10n={}&{}".format(g.locale.language, data["query_string"]) + d.update(data) + except: + _logger.exception("Error collecting data for preemptive cache from plugin {}".format(key)) # add additional request data if we have any - if additional_request_data: - d.update(dict( - _additional_request_data = additional_request_data - )) + if callable(additional_request_data): + try: + ard = additional_request_data() + if ard: + d.update(dict( + _additional_request_data = ard + )) + except: + _logger.exception("Error retrieving additional data for preemptive cache from plugin {}".format(key)) # finally decorate our view return util.flask.preemptively_cached(cache=preemptiveCache, data=d, unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"]))(view) - def get_cached_view(key, view, additional_key_data=None): + def get_cached_view(key, view, additional_key_data=None, additional_files=None, custom_files=None, custom_etag=None, custom_lastmodified=None): def cache_key(): k = "ui:{}:{}:{}".format(key, request.base_url, g.locale.language if g.locale else "default") - ak = additional_key_data - if ak: - # we have some additional key components, let's attach them - if not isinstance(ak, (list, tuple)): - ak = [ak] - k = "{}:{}".format(k, ":".join(ak)) + if callable(additional_key_data): + try: + ak = additional_key_data() + if ak: + # we have some additional key components, let's attach them + if not isinstance(ak, (list, tuple)): + ak = [ak] + k = "{}:{}".format(k, ":".join(ak)) + except: + _logger.exception("Error while trying to retrieve additional cache key parts for plugin {}".format(key)) return k - return util.flask.cached(timeout=-1, - refreshif=lambda: force_refresh, - key=cache_key, - unless_response=util.flask.cache_check_response_headers)(view) + def check_etag_and_lastmodified(): + files = collect_files() + lastmodified = compute_lastmodified(files) + lastmodified_ok = util.flask.check_lastmodified(lastmodified) + etag_ok = util.flask.check_etag(compute_etag(files, lastmodified)) + return lastmodified_ok and etag_ok + + def validate_cache(cached): + etag_different = compute_etag() != cached.get_etag()[0] + return force_refresh or etag_different + + def collect_files(): + if callable(custom_files): + try: + files = custom_files() + if files: + return files + except: + _logger.exception("Error while trying to retrieve tracked files for plugin {}".format(key)) + + templates = _get_all_templates() + assets = _get_all_assets() + translations = _get_all_translationfiles(g.locale.language if g.locale else "en", + "messages") + + files = templates + assets + translations + + if callable(additional_files): + try: + af = additional_files() + if af: + files += af + except: + _logger.exception("Error while trying to retrieve additional tracked files for plugin {}".format(key)) + + return sorted(set(files)) + + def compute_lastmodified(files=None): + if callable(custom_lastmodified): + try: + lastmodified = custom_lastmodified() + if lastmodified: + return lastmodified + except: + _logger.exception("Error while trying to retrieve custom LastModified value for plugin {}".format(key)) + + if files is None: + files = collect_files() + return _compute_date(files) + + def compute_etag(files=None, lastmodified=None): + if callable(custom_etag): + try: + etag = custom_etag() + if etag: + return etag + except: + _logger.exception("Error while trying to retrieve custom ETag value for plugin {}".format(key)) + + if files is None: + files = collect_files() + if lastmodified is None: + lastmodified = compute_lastmodified(files) + if lastmodified and not isinstance(lastmodified, basestring): + from werkzeug.http import http_date + lastmodified = http_date(lastmodified) + + import hashlib + hash = hashlib.sha1() + hash.update(octoprint.__version__) + hash.update(octoprint.server.UI_API_KEY) + hash.update(",".join(sorted(files))) + if lastmodified: + hash.update(lastmodified) + return hash.hexdigest() + + decorated_view = view + decorated_view = util.flask.lastmodified(lambda _: compute_lastmodified())(decorated_view) + decorated_view = util.flask.etagged(lambda _: compute_etag())(decorated_view) + decorated_view = util.flask.cached(timeout=-1, + refreshif=validate_cache, + key=cache_key, + unless_response=util.flask.cache_check_response_headers)(decorated_view) + decorated_view = util.flask.conditional(check_etag_and_lastmodified, NOT_MODIFIED)(decorated_view) + return decorated_view ui_plugins = pluginManager.get_implementations(octoprint.plugin.UiPlugin, sorting_context="UiPlugin.on_ui_render") for plugin in ui_plugins: @@ -99,12 +195,16 @@ def index(): # plugin claims responsibility, let it render the UI cached = get_cached_view(plugin._identifier, plugin.on_ui_render, - plugin.get_ui_additional_key_data_for_cache(request)) + additional_key_data=plugin.get_ui_additional_key_data_for_cache, + additional_files=plugin.get_ui_additional_tracked_files, + custom_files=plugin.get_ui_custom_tracked_files, + custom_etag=plugin.get_ui_custom_etag, + custom_lastmodified=plugin.get_ui_custom_lastmodified) preemptively_cached = get_preemptively_cached_view(plugin._identifier, cached, - plugin.get_ui_data_for_preemptive_caching(request), - plugin.get_ui_additional_request_data_for_preemptive_caching(request)) + plugin.get_ui_data_for_preemptive_caching, + plugin.get_ui_additional_request_data_for_preemptive_caching) response = preemptively_cached(now, request, render_kwargs) if response is not None: @@ -132,12 +232,14 @@ def index(): r = util.flask.add_non_caching_response_headers(r) return r - cached = get_cached_view("_default", make_default_ui) - preemptively_cached = get_preemptively_cached_view("_default", cached, dict(), dict()) + cached = get_cached_view("_default", + make_default_ui) + preemptively_cached = get_preemptively_cached_view("_default", + cached, + dict(), + dict()) response = preemptively_cached() - response.headers["Last-Modified"] = now - return response @@ -506,6 +608,7 @@ def _process_template_configs(name, implementation, configs, rules): return includes + def _process_template_config(name, implementation, rule, config=None, counter=1): if "mandatory" in rule: for mandatory in rule["mandatory"]: @@ -550,80 +653,21 @@ def _process_template_config(name, implementation, rule, config=None, counter=1) @app.route("/robots.txt") +@util.flask.cached(timeout=-1) def robotsTxt(): return send_from_directory(app.static_folder, "robots.txt") @app.route("/i18n//.js") -@util.flask.preemptively_cached(cache=preemptiveCache, - data=lambda: dict(path=request.path, base_url=request.url_root), - unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) -@util.flask.cached(timeout=-1, - refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, - key=lambda: "{}".format(request.base_url)) +@util.flask.conditional(lambda: _check_etag_and_lastmodified_for_i18n(), NOT_MODIFIED) +@util.flask.etagged(lambda _: _compute_etag_for_i18n(request.view_args["locale"], request.view_args["domain"])) +@util.flask.lastmodified(lambda _: _compute_date_for_i18n(request.view_args["locale"], request.view_args["domain"])) def localeJs(locale, domain): messages = dict() plural_expr = None if locale != "en": - from flask import _request_ctx_stack - from babel.messages.pofile import read_po - - def messages_from_po(base_path, locale, domain): - path = os.path.join(base_path, locale) - if not os.path.isdir(path): - return None, None - - path = os.path.join(path, "LC_MESSAGES", "{domain}.po".format(**locals())) - if not os.path.isfile(path): - return None, None - - messages = dict() - with file(path) as f: - catalog = read_po(f, locale=locale, domain=domain) - - for message in catalog: - message_id = message.id - if isinstance(message_id, (list, tuple)): - message_id = message_id[0] - messages[message_id] = message.string - - return messages, catalog.plural_expr - - user_base_path = os.path.join(settings().getBaseFolder("translations")) - user_plugin_path = os.path.join(user_base_path, "_plugins") - - # plugin translations - plugins = octoprint.plugin.plugin_manager().enabled_plugins - for name, plugin in plugins.items(): - dirs = [os.path.join(user_plugin_path, name), os.path.join(plugin.location, 'translations')] - for dirname in dirs: - if not os.path.isdir(dirname): - continue - - plugin_messages, _ = messages_from_po(dirname, locale, domain) - - if plugin_messages is not None: - messages = octoprint.util.dict_merge(messages, plugin_messages) - _logger.debug("Using translation folder {dirname} for locale {locale} of plugin {name}".format(**locals())) - break - else: - _logger.debug("No translations for locale {locale} for plugin {name}".format(**locals())) - - # core translations - ctx = _request_ctx_stack.top - base_path = os.path.join(ctx.app.root_path, "translations") - - dirs = [user_base_path, base_path] - for dirname in dirs: - core_messages, plural_expr = messages_from_po(dirname, locale, domain) - - if core_messages is not None: - messages = octoprint.util.dict_merge(messages, core_messages) - _logger.debug("Using translation folder {dirname} for locale {locale} of core translations".format(**locals())) - break - else: - _logger.debug("No core translations for locale {locale}".format(**locals())) + messages, plural_expr = _get_translations(locale, domain) catalog = dict( messages=messages, @@ -640,3 +684,130 @@ def localeJs(locale, domain): def plugin_assets(name, filename): return redirect(url_for("plugin." + name + ".static", filename=filename)) + +def _compute_etag_for_i18n(locale, domain, files=None, lastmodified=None): + if files is None: + files = _get_all_translationfiles(locale, domain) + if lastmodified is None: + lastmodified = _compute_date(files) + if lastmodified and not isinstance(lastmodified, basestring): + from werkzeug.http import http_date + lastmodified = http_date(lastmodified) + + import hashlib + hash = hashlib.sha1() + hash.update(",".join(sorted(files))) + if lastmodified: + hash.update(lastmodified) + return hash.hexdigest() + + +def _compute_date_for_i18n(locale, domain): + return _compute_date(_get_all_translationfiles(locale, domain)) + + +def _compute_date(files): + from datetime import datetime + timestamps = map(lambda path: os.stat(path).st_mtime, files) + max_timestamp = max(*timestamps) if timestamps else None + if max_timestamp: + # we set the micros to 0 since microseconds are not speced for HTTP + max_timestamp = datetime.fromtimestamp(max_timestamp).replace(microsecond=0) + return max_timestamp + + +def _check_etag_and_lastmodified_for_i18n(): + locale = request.view_args["locale"] + domain = request.view_args["domain"] + + etag_ok = util.flask.check_etag(_compute_etag_for_i18n(request.view_args["locale"], request.view_args["domain"])) + + lastmodified = _compute_date_for_i18n(locale, domain) + lastmodified_ok = lastmodified is None or util.flask.check_lastmodified(lastmodified) + + return etag_ok and lastmodified_ok + + +def _get_all_templates(): + from octoprint.util.jinja import get_all_template_paths + return get_all_template_paths(app.jinja_loader) + + +def _get_all_assets(): + from octoprint.util.jinja import get_all_asset_paths + return get_all_asset_paths(app.jinja_env.assets_environment) + + +def _get_all_translationfiles(locale, domain): + from flask import _request_ctx_stack + + def get_po_path(basedir, locale, domain): + path = os.path.join(basedir, locale) + if not os.path.isdir(path): + return None + + path = os.path.join(path, "LC_MESSAGES", "{domain}.po".format(**locals())) + if not os.path.isfile(path): + return None + + return path + + po_files = [] + + user_base_path = os.path.join(settings().getBaseFolder("translations")) + user_plugin_path = os.path.join(user_base_path, "_plugins") + + # plugin translations + plugins = octoprint.plugin.plugin_manager().enabled_plugins + for name, plugin in plugins.items(): + dirs = [os.path.join(user_plugin_path, name), os.path.join(plugin.location, 'translations')] + for dirname in dirs: + if not os.path.isdir(dirname): + continue + + po_file = get_po_path(dirname, locale, domain) + if po_file: + po_files.append(po_file) + break + + # core translations + ctx = _request_ctx_stack.top + base_path = os.path.join(ctx.app.root_path, "translations") + + dirs = [user_base_path, base_path] + for dirname in dirs: + po_file = get_po_path(dirname, locale, domain) + if po_file: + po_files.append(po_file) + break + + return po_files + + +def _get_translations(locale, domain): + from babel.messages.pofile import read_po + from octoprint.util import dict_merge + + messages = dict() + plural_expr = None + + def messages_from_po(path, locale, domain): + messages = dict() + with file(path) as f: + catalog = read_po(f, locale=locale, domain=domain) + + for message in catalog: + message_id = message.id + if isinstance(message_id, (list, tuple)): + message_id = message_id[0] + messages[message_id] = message.string + + return messages, catalog.plural_expr + + po_files = _get_all_translationfiles(locale, domain) + for po_file in po_files: + po_messages, plural_expr = messages_from_po(po_file, locale, domain) + if po_messages is not None: + messages = dict_merge(messages, po_messages) + + return messages, plural_expr diff --git a/src/octoprint/static/js/app/dataupdater.js b/src/octoprint/static/js/app/dataupdater.js index c607016d..ab2acc88 100644 --- a/src/octoprint/static/js/app/dataupdater.js +++ b/src/octoprint/static/js/app/dataupdater.js @@ -7,6 +7,7 @@ function DataUpdater(allViewModels) { self._configHash = undefined; self.reloadOverlay = $("#reloadui_overlay"); + $("#reloadui_overlay_reload").click(function() { location.reload(); }); self.connect = function() { OctoPrint.socket.connect({debug: !!SOCKJS_DEBUG}); diff --git a/src/octoprint/util/jinja.py b/src/octoprint/util/jinja.py index e4bc7dc6..747de519 100644 --- a/src/octoprint/util/jinja.py +++ b/src/octoprint/util/jinja.py @@ -9,7 +9,8 @@ import os from jinja2 import nodes from jinja2.ext import Extension -from jinja2.loaders import FileSystemLoader, TemplateNotFound, split_template_path +from jinja2.loaders import FileSystemLoader, PrefixLoader, ChoiceLoader, \ + ModuleLoader, TemplateNotFound, split_template_path class FilteredFileSystemLoader(FileSystemLoader): """ @@ -101,3 +102,54 @@ class ExceptionHandlerExtension(Extension): return "Unknown error" trycatch = ExceptionHandlerExtension + + +def collect_template_folders(loader): + import copy + + if isinstance(loader, FileSystemLoader): + return copy.copy(loader.searchpath) + elif isinstance(loader, PrefixLoader): + result = [] + for subloader in loader.mapping.values(): + result += collect_template_folders(subloader) + return result + elif isinstance(loader, ChoiceLoader): + result = [] + for subloader in loader.loaders: + result += collect_template_folders(subloader) + return result + elif isinstance(loader, ModuleLoader): + return [loader.module.__path__] + + return [] + + +def get_all_template_paths(loader, filter_function=None): + result = [] + template_folders = collect_template_folders(loader) + for template_folder in template_folders: + walk_dir = os.walk(template_folder, followlinks=True) + for dirpath, dirnames, filenames in walk_dir: + for filename in filenames: + path = os.path.join(dirpath, filename) + if not callable(filter_function) or filter_function(path): + result.append(path) + return result + + +def get_all_asset_paths(env): + result = [] + for bundle in env: + for content in bundle.resolve_contents(): + try: + if not content: + continue + path = content[1] + if not os.path.isfile(path): + continue + result.append(path) + except IndexError: + # intentionally ignored + pass + return result