diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index 08f1b3bc..5011066b 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -25,6 +25,7 @@ import signal SUCCESS = {} NO_CONTENT = ("", 204) +NOT_MODIFIED = ("Not Modified", 304) app = Flask("octoprint") assets = None diff --git a/src/octoprint/server/util/__init__.py b/src/octoprint/server/util/__init__.py index d128e8ef..20640e64 100644 --- a/src/octoprint/server/util/__init__.py +++ b/src/octoprint/server/util/__init__.py @@ -143,6 +143,20 @@ def get_api_key(request): return None +def get_plugin_hash(): + from octoprint.plugin import plugin_manager + + plugin_signature = lambda impl: "{}:{}".format(impl._identifier, impl._plugin_version) + template_plugins = map(plugin_signature, plugin_manager().get_implementations(octoprint.plugin.TemplatePlugin)) + asset_plugins = map(plugin_signature, plugin_manager().get_implementations(octoprint.plugin.AssetPlugin)) + ui_plugins = sorted(set(template_plugins + asset_plugins)) + + import hashlib + plugin_hash = hashlib.sha1() + plugin_hash.update(",".join(ui_plugins)) + return plugin_hash.hexdigest() + + #~~ reverse proxy compatible WSGI middleware diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index 3b376ebc..2a9bcadf 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -331,13 +331,14 @@ def cached(timeout=5 * 60, key=lambda: "view:%s" % flask.request.path, unless=No return f(*args, **kwargs) cache_key = key() + rv = _cache.get(cache_key) # only take the value from the cache if we are not required to refresh it from the wrapped function - if not callable(refreshif) or not refreshif(): - rv = _cache.get(cache_key) - if rv is not None: - logger.debug("Serving entry for {path} from cache".format(path=flask.request.path)) - return rv + if rv is not None and (not callable(refreshif) or not refreshif(rv)): + logger.debug("Serving entry for {path} from cache".format(path=flask.request.path)) + if not "X-From-Cache" in rv.headers: + rv.headers["X-From-Cache"] = "true" + return rv # get value from wrapped function logger.debug("No cache entry or refreshing cache for {path} (key: {key}), calling wrapped function".format(path=flask.request.path, key=cache_key)) @@ -526,6 +527,72 @@ def preemptively_cached(cache, data, unless=None): return decorator +def etagged(etag): + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + rv = f(*args, **kwargs) + if isinstance(rv, flask.Response): + result = etag + if callable(result): + result = result(rv) + if result: + rv.set_etag(result) + return rv + return decorated_function + return decorator + + +def lastmodified(date): + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + rv = f(*args, **kwargs) + if not "last-modified" in rv.headers: + result = date + if callable(result): + result = result(rv) + + if not isinstance(result, basestring): + from werkzeug.http import http_date + result = http_date(result) + + if result: + rv.headers["last-modified"] = result + return rv + return decorated_function + return decorator + + +def conditional(condition, met): + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + if callable(condition) and condition(): + # condition has been met, return met-response + rv = met + if callable(met): + rv = met() + return rv + + # condition hasn't been met, call decorated function + return f(*args, **kwargs) + return decorated_function + return decorator + + +def check_etag(etag): + return flask.request.method in ("GET", "HEAD") and \ + flask.request.if_none_match and \ + etag in flask.request.if_none_match + + +def check_lastmodified(lastmodified): + return flask.request.method in ("GET", "HEAD") and \ + flask.request.if_modified_since and \ + lastmodified >= flask.request.if_modified_since + + def add_non_caching_response_headers(response): response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0" response.headers["Pragma"] = "no-cache" diff --git a/src/octoprint/server/views.py b/src/octoprint/server/views.py index c25671e6..2632323b 100644 --- a/src/octoprint/server/views.py +++ b/src/octoprint/server/views.py @@ -13,7 +13,8 @@ from flask import request, g, url_for, make_response, render_template, send_from import octoprint.plugin from octoprint.server import app, userManager, pluginManager, gettext, \ - debug, LOCALES, VERSION, DISPLAY_VERSION, UI_API_KEY, BRANCH, preemptiveCache + debug, LOCALES, VERSION, DISPLAY_VERSION, UI_API_KEY, BRANCH, preemptiveCache, \ + NOT_MODIFIED from octoprint.settings import settings import re @@ -28,12 +29,15 @@ _valid_div_re = re.compile("[a-zA-Z_-]+") @app.route("/") @util.flask.preemptively_cached(cache=preemptiveCache, - data=lambda: dict(path=request.path, base_url=request.url_root, query_string="l10n={}".format(g.locale.language)) if g.locale else None, + data=lambda: dict(path=request.path, base_url=request.url_root, query_string="l10n={}".format(g.locale.language)) if g.locale else "en", unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) +@util.flask.conditional(lambda: _check_etag_and_lastmodified_for_index(), NOT_MODIFIED) @util.flask.cached(timeout=-1, - refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, - key=lambda: "view:{}:{}".format(request.base_url, g.locale.language if g.locale else "default"), - unless_response=util.flask.cache_check_response_headers) + refreshif=lambda cached: _validate_cache_for_index(cached), + key=lambda: "view:{}:{}".format(request.base_url, g.locale.language if g.locale else "en"), + unless_response=lambda response: util.flask.cache_check_response_headers(response)) +@util.flask.etagged(lambda _: _compute_etag_for_index()) +@util.flask.lastmodified(lambda _: _compute_date_for_index()) def index(): #~~ a bunch of settings @@ -299,13 +303,10 @@ def index(): #~~ render! - import datetime - response = make_response(render_template( "index.jinja2", **render_kwargs )) - response.headers["Last-Modified"] = datetime.datetime.now() if first_run: response = util.flask.add_non_caching_response_headers(response) @@ -356,6 +357,7 @@ def _process_template_configs(name, implementation, configs, rules): return includes + def _process_template_config(name, implementation, rule, config=None, counter=1): if "mandatory" in rule: for mandatory in rule["mandatory"]: @@ -398,81 +400,23 @@ def _process_template_config(name, implementation, rule, config=None, counter=1) return data + @app.route("/robots.txt") +@util.flask.cached(timeout=-1) def robotsTxt(): return send_from_directory(app.static_folder, "robots.txt") @app.route("/i18n//.js") -@util.flask.preemptively_cached(cache=preemptiveCache, - data=lambda: dict(path=request.path, base_url=request.url_root), - unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"])) -@util.flask.cached(timeout=-1, - refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, - key=lambda: "view:{}".format(request.base_url)) +@util.flask.conditional(lambda: _check_etag_and_lastmodified_for_i18n(), NOT_MODIFIED) +@util.flask.etagged(lambda _: _compute_etag_for_i18n(request.view_args["locale"], request.view_args["domain"])) +@util.flask.lastmodified(lambda _: _compute_date_for_i18n(request.view_args["locale"], request.view_args["domain"])) def localeJs(locale, domain): messages = dict() plural_expr = None if locale != "en": - from flask import _request_ctx_stack - from babel.messages.pofile import read_po - - def messages_from_po(base_path, locale, domain): - path = os.path.join(base_path, locale) - if not os.path.isdir(path): - return None, None - - path = os.path.join(path, "LC_MESSAGES", "{domain}.po".format(**locals())) - if not os.path.isfile(path): - return None, None - - messages = dict() - with file(path) as f: - catalog = read_po(f, locale=locale, domain=domain) - - for message in catalog: - message_id = message.id - if isinstance(message_id, (list, tuple)): - message_id = message_id[0] - messages[message_id] = message.string - - return messages, catalog.plural_expr - - user_base_path = os.path.join(settings().getBaseFolder("translations")) - user_plugin_path = os.path.join(user_base_path, "_plugins") - - # plugin translations - plugins = octoprint.plugin.plugin_manager().enabled_plugins - for name, plugin in plugins.items(): - dirs = [os.path.join(user_plugin_path, name), os.path.join(plugin.location, 'translations')] - for dirname in dirs: - if not os.path.isdir(dirname): - continue - - plugin_messages, _ = messages_from_po(dirname, locale, domain) - - if plugin_messages is not None: - messages = octoprint.util.dict_merge(messages, plugin_messages) - _logger.debug("Using translation folder {dirname} for locale {locale} of plugin {name}".format(**locals())) - break - else: - _logger.debug("No translations for locale {locale} for plugin {name}".format(**locals())) - - # core translations - ctx = _request_ctx_stack.top - base_path = os.path.join(ctx.app.root_path, "translations") - - dirs = [user_base_path, base_path] - for dirname in dirs: - core_messages, plural_expr = messages_from_po(dirname, locale, domain) - - if core_messages is not None: - messages = octoprint.util.dict_merge(messages, core_messages) - _logger.debug("Using translation folder {dirname} for locale {locale} of core translations".format(**locals())) - break - else: - _logger.debug("No core translations for locale {locale}".format(**locals())) + messages, plural_expr = _get_translations(locale, domain) catalog = dict( messages=messages, @@ -490,3 +434,192 @@ def plugin_assets(name, filename): return redirect(url_for("plugin." + name + ".static", filename=filename)) +def _compute_etag_for_index(files=None, lastmodified=None): + if files is None: + files = _files_for_index() + if lastmodified is None: + lastmodified = _compute_date(files) + if lastmodified and not isinstance(lastmodified, basestring): + from werkzeug.http import http_date + lastmodified = http_date(lastmodified) + + from octoprint import __version__ + from octoprint.server import UI_API_KEY + + import hashlib + hash = hashlib.sha1() + hash.update(__version__) + hash.update(UI_API_KEY) + hash.update(",".join(sorted(files))) + if lastmodified: + hash.update(lastmodified) + return hash.hexdigest() + + +def _compute_etag_for_i18n(locale, domain, files=None, lastmodified=None): + if files is None: + files = _get_all_translationfiles(locale, domain) + if lastmodified is None: + lastmodified = _compute_date(files) + if lastmodified and not isinstance(lastmodified, basestring): + from werkzeug.http import http_date + lastmodified = http_date(lastmodified) + + import hashlib + hash = hashlib.sha1() + hash.update(",".join(sorted(files))) + if lastmodified: + hash.update(lastmodified) + return hash.hexdigest() + + +def _compute_date_for_i18n(locale, domain): + return _compute_date(_get_all_translationfiles(locale, domain)) + + +def _compute_date_for_index(): + return _compute_date(_files_for_index()) + + +def _validate_cache_for_index(cached): + no_cache_headers = util.flask.cache_check_headers() + refresh_flag = "_refresh" in request.values + etag_different = _compute_etag_for_index() != cached.get_etag()[0] + + return no_cache_headers or refresh_flag or etag_different + + +def _files_for_index(): + """ + Collects all paths of files that the index page depends on. + + The relevant files are: + + * all jinja2 templates: they might be used within the index page, so + any changes here change the rendering outcome + * all defined assets: if one of them changes, the webassets bundle will + be regenerated and hence the URL included in the cached page won't be + valid anymore + * all translation files used for our current locale: if any of those change + we also need to re-render + """ + + templates = _get_all_templates() + assets = _get_all_assets() + translations = _get_all_translationfiles(g.locale.language if g.locale else "en", "messages") + return sorted(set(templates + assets + translations)) + + +def _compute_date(files): + from datetime import datetime + timestamps = map(lambda path: os.stat(path).st_mtime, files) + max_timestamp = max(*timestamps) if timestamps else None + if max_timestamp: + # we set the micros to 0 since microseconds are not speced for HTTP + max_timestamp = datetime.fromtimestamp(max_timestamp).replace(microsecond=0) + return max_timestamp + + +def _check_etag_and_lastmodified_for_index(): + files = _files_for_index() + lastmodified = _compute_date(files) + lastmodified_ok = util.flask.check_lastmodified(lastmodified) + etag_ok = util.flask.check_etag(_compute_etag_for_index(files, lastmodified)) + return etag_ok and lastmodified_ok + + +def _check_etag_and_lastmodified_for_i18n(): + locale = request.view_args["locale"] + domain = request.view_args["domain"] + + etag_ok = util.flask.check_etag(_compute_etag_for_i18n(request.view_args["locale"], request.view_args["domain"])) + + lastmodified = _compute_date_for_i18n(locale, domain) + lastmodified_ok = lastmodified is None or util.flask.check_lastmodified(lastmodified) + + return etag_ok and lastmodified_ok + + +def _get_all_templates(): + from octoprint.util.jinja import get_all_template_paths + return get_all_template_paths(app.jinja_loader, lambda path: not octoprint.util.is_hidden_path(path)) + + +def _get_all_assets(): + from octoprint.util.jinja import get_all_asset_paths + return get_all_asset_paths(app.jinja_env.assets_environment) + + +def _get_all_translationfiles(locale, domain): + from flask import _request_ctx_stack + + def get_po_path(basedir, locale, domain): + path = os.path.join(basedir, locale) + if not os.path.isdir(path): + return None + + path = os.path.join(path, "LC_MESSAGES", "{domain}.po".format(**locals())) + if not os.path.isfile(path): + return None + + return path + + po_files = [] + + user_base_path = os.path.join(settings().getBaseFolder("translations")) + user_plugin_path = os.path.join(user_base_path, "_plugins") + + # plugin translations + plugins = octoprint.plugin.plugin_manager().enabled_plugins + for name, plugin in plugins.items(): + dirs = [os.path.join(user_plugin_path, name), os.path.join(plugin.location, 'translations')] + for dirname in dirs: + if not os.path.isdir(dirname): + continue + + po_file = get_po_path(dirname, locale, domain) + if po_file: + po_files.append(po_file) + break + + # core translations + ctx = _request_ctx_stack.top + base_path = os.path.join(ctx.app.root_path, "translations") + + dirs = [user_base_path, base_path] + for dirname in dirs: + po_file = get_po_path(dirname, locale, domain) + if po_file: + po_files.append(po_file) + break + + return po_files + + +def _get_translations(locale, domain): + from babel.messages.pofile import read_po + from octoprint.util import dict_merge + + messages = dict() + plural_expr = None + + def messages_from_po(path, locale, domain): + messages = dict() + with file(path) as f: + catalog = read_po(f, locale=locale, domain=domain) + + for message in catalog: + message_id = message.id + if isinstance(message_id, (list, tuple)): + message_id = message_id[0] + messages[message_id] = message.string + + return messages, catalog.plural_expr + + po_files = _get_all_translationfiles(locale, domain) + for po_file in po_files: + po_messages, plural_expr = messages_from_po(po_file, locale, domain) + if po_messages is not None: + messages = dict_merge(messages, po_messages) + + return messages, plural_expr diff --git a/src/octoprint/util/jinja.py b/src/octoprint/util/jinja.py index bf3f8b30..28bfcea4 100644 --- a/src/octoprint/util/jinja.py +++ b/src/octoprint/util/jinja.py @@ -6,7 +6,8 @@ __copyright__ = "Copyright (C) 2015 The OctoPrint Project - Released under terms import os -from jinja2.loaders import FileSystemLoader, TemplateNotFound, split_template_path +from jinja2.loaders import FileSystemLoader, PrefixLoader, ChoiceLoader, \ + ModuleLoader, TemplateNotFound, split_template_path class FilteredFileSystemLoader(FileSystemLoader): """ @@ -48,3 +49,54 @@ class FilteredFileSystemLoader(FileSystemLoader): filter_results = map(lambda x: not os.path.exists(os.path.join(x, path)) or self.path_filter(os.path.join(x, path)), self.searchpath) return all(filter_results) + + +def collect_template_folders(loader): + import copy + + if isinstance(loader, FileSystemLoader): + return copy.copy(loader.searchpath) + elif isinstance(loader, PrefixLoader): + result = [] + for subloader in loader.mapping.values(): + result += collect_template_folders(subloader) + return result + elif isinstance(loader, ChoiceLoader): + result = [] + for subloader in loader.loaders: + result += collect_template_folders(subloader) + return result + elif isinstance(loader, ModuleLoader): + return [loader.module.__path__] + + return [] + + +def get_all_template_paths(loader, filter_function=None): + result = [] + template_folders = collect_template_folders(loader) + for template_folder in template_folders: + walk_dir = os.walk(template_folder, followlinks=True) + for dirpath, dirnames, filenames in walk_dir: + for filename in filenames: + path = os.path.join(dirpath, filename) + if not callable(filter_function) or filter_function(path): + result.append(path) + return result + + +def get_all_asset_paths(env): + result = [] + for bundle in env: + for content in bundle.resolve_contents(): + try: + if not content: + continue + path = content[1] + if not os.path.isfile(path): + continue + result.append(path) + except IndexError: + # intentionally ignored + pass + return result