Merge branch 'fix/preemptiveUiCaching' into devel

Conflicts:
	src/octoprint/server/views.py

src/octoprint/plugin/types.py

@@ -661,6 +661,14 @@ class UiPlugin(OctoPrintPlugin, SortablePlugin):
 
 		return None
 
+	def get_ui_additional_key_data_for_cache(self, request):
+		return None
+
+	def get_ui_data_for_preemptive_caching(self, request):
+		return None
+
+	def get_ui_additional_request_data_for_preemptive_caching(self, request):
+		return None
 
 class WizardPlugin(OctoPrintPlugin, ReloadNeedingPlugin):
 	"""

src/octoprint/server/__init__.py

@@ -474,6 +474,10 @@ class Server():
 					implementation.on_after_startup()
 				pluginLifecycleManager.add_callback("enabled", call_on_after_startup)
 
+				# when we are through with that we also run our preemptive cache
+				if settings().getBoolean(["devel", "cache", "preemptive"]):
+					self._execute_preemptive_flask_caching()
+
 			import threading
 			threading.Thread(target=work).start()
 		ioloop.add_callback(on_after_startup)
@@ -534,7 +538,7 @@ class Server():
 		if default_language is not None and not default_language == "_default" and default_language in LANGUAGES:
 			return Locale.negotiate([default_language], LANGUAGES)
 
-		return request.accept_languages.best_match(LANGUAGES)
+		return Locale.parse(request.accept_languages.best_match(LANGUAGES))
 
 	def _setup_app(self):
 		@app.before_request
@@ -655,6 +659,37 @@ class Server():
 
 		self._register_template_plugins()
 
+	def _execute_preemptive_flask_caching(self):
+		from octoprint.server.util.flask import get_preemptive_cache_data
+		from werkzeug.test import EnvironBuilder
+
+		cache_data = get_preemptive_cache_data()
+		if not cache_data:
+			return
+
+		def execute_caching():
+			for route in sorted(cache_data.keys(), key=lambda x: (x.count("/"), x)):
+				entries = cache_data[route]
+				for kwargs in entries:
+					plugin = kwargs.get("plugin", None)
+					additional_request_data = kwargs.get("_additional_request_data", dict())
+					kwargs = dict((k, v) for k, v in kwargs.items() if not k.startswith("_") and not k == "plugin")
+					kwargs.update(additional_request_data)
+					try:
+						if plugin:
+							self._logger.info("Preemptively caching {} (plugin {}) for {!r}".format(route, plugin, kwargs))
+						else:
+							self._logger.info("Preemptively caching {} for {!r}".format(route, kwargs))
+						builder = EnvironBuilder(**kwargs)
+						app(builder.get_environ(), lambda *a, **kw: None)
+					except:
+						self._logger.exception("Error while trying to preemptively cache {} for {!r}".format(route, kwargs))
+
+		import threading
+		cache_thread = threading.Thread(target=execute_caching, name="Preemptive Cache Worker")
+		cache_thread.daemon = True
+		cache_thread.start()
+
 	def _register_template_plugins(self):
 		template_plugins = pluginManager.get_implementations(octoprint.plugin.TemplatePlugin)
 		for plugin in template_plugins:

src/octoprint/server/util/flask.py

@@ -376,6 +376,75 @@ def cache_check_response_headers(response):
 
 	return False
 
+_preemptive_flask_cache = "preemptive_flask_cache.yaml"
+
+def preemptively_cached(data, unless=None):
+	def decorator(f):
+		@functools.wraps(f)
+		def decorated_function(*args, **kwargs):
+			if not (callable(unless) and unless()):
+				entry_data = data
+				if callable(entry_data):
+					entry_data = entry_data()
+
+				if entry_data is not None:
+					from flask import request
+					from octoprint.util import atomic_write
+					import yaml
+
+					data_folder = settings().getBaseFolder("data")
+					cache_data_file = os.path.join(data_folder, _preemptive_flask_cache)
+					cache_data = get_preemptive_cache_data()
+
+					if not request.path in cache_data:
+						cache_data[request.path] = []
+
+					def strip_ignored(d):
+						return dict((k, v) for k, v in d.items() if not k.startswith("_"))
+
+					def compare(a, b):
+						return set(strip_ignored(a).items()) == set(strip_ignored(b).items())
+
+					cache_data_for_path = cache_data.get(request.path, [])
+					if not any(map(lambda entry: compare(entry_data, entry), cache_data_for_path)):
+						logging.getLogger(__name__).info("Adding {} for {!r} to views to preemptively cache".format(request.path, entry_data))
+						cache_data[request.path] = cache_data_for_path + [entry_data]
+						try:
+							with atomic_write(cache_data_file, "wb", prefix="octoprint-{}-".format(_preemptive_flask_cache[:-len(".yaml")]), suffix=".yaml") as handle:
+								yaml.safe_dump(cache_data, handle,default_flow_style=False, indent="    ", allow_unicode=True)
+						except:
+							logging.getLogger(__name__).exception("Error while writing {}".format(_preemptive_flask_cache))
+
+			return f(*args, **kwargs)
+
+		return decorated_function
+
+	return decorator
+
+
+def get_preemptive_cache_data(root=None):
+	import yaml
+
+	data_folder = settings().getBaseFolder("data")
+	cache_data_file = os.path.join(data_folder, _preemptive_flask_cache)
+	if not os.path.isfile(cache_data_file):
+		return dict()
+
+	cache_data = None
+	try:
+		with open(cache_data_file, "r") as f:
+			cache_data = yaml.safe_load(f)
+	except:
+		logging.getLogger(__name__).exception("Error while reading {}".format(_preemptive_flask_cache))
+
+	if cache_data is None:
+		cache_data = dict()
+
+	if root:
+		return cache_data.get(root, dict())
+	else:
+		return cache_data
+
 
 def add_non_caching_response_headers(response):
 	response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0"

src/octoprint/server/views.py

@@ -49,17 +49,62 @@ def index():
 	now = datetime.datetime.utcnow()
 	render_kwargs = _get_render_kwargs(_templates, _plugin_names, _plugin_vars, now)
 
-	def get_cached_view(key, view):
+	def get_preemptively_cached_view(key, view, data=None, additional_request_data=None):
+		if (data is None and additional_request_data is None) or g.locale is None:
+			return view
+
+		d = dict(path=request.path,
+		         base_url=request.base_url,
+		         query_string="l10n={}".format(g.locale.language))
+
+		if key != "_default":
+			d["plugin"] = key
+
+		# add data if we have any
+		if data is not None:
+			if "query_string" in data:
+				data["query_string"] = "l10n={}&{}".format(g.locale.language, data["query_string"])
+			d.update(data)
+
+		# add additional request data if we have any
+		if additional_request_data:
+			d.update(dict(
+				_additional_request_data = additional_request_data
+			))
+
+		# finally decorate our view
+		return util.flask.preemptively_cached(data=d,
+		                                      unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"]))(view)
+
+	def get_cached_view(key, view, additional_key_data=None):
+		def cache_key():
+			k = "ui:{}:{}:{}".format(key, request.base_url, g.locale.language if g.locale else "default")
+			ak = additional_key_data
+			if ak:
+				# we have some additional key components, let's attach them
+				if not isinstance(ak, (list, tuple)):
+					ak = [ak]
+				k = "{}:{}".format(k, ":".join(ak))
+			return k
+
 		return util.flask.cached(timeout=-1,
 		                         refreshif=lambda: force_refresh,
-		                         key=lambda: "ui:{}:{}:{}".format(key, request.base_url, g.locale),
+		                         key=cache_key,
 		                         unless_response=util.flask.cache_check_response_headers)(view)
 
 	ui_plugins = pluginManager.get_implementations(octoprint.plugin.UiPlugin, sorting_context="UiPlugin.on_ui_render")
 	for plugin in ui_plugins:
 		if plugin.will_handle_ui(request):
 			# plugin claims responsibility, let it render the UI
-			cached = get_cached_view(plugin._identifier, plugin.on_ui_render)
+			preemptively_cached = get_preemptively_cached_view(plugin._identifier,
+			                                                   plugin.on_ui_render,
+			                                                   plugin.get_ui_data_for_preemptive_caching(request),
+			                                                   plugin.get_ui_additional_request_data_for_preemptive_caching(request))
+
+			cached = get_cached_view(plugin._identifier,
+			                         preemptively_cached,
+			                         plugin.get_ui_additional_key_data_for_cache(request))
+
 			response = cached(now, request, render_kwargs)
 			if response is not None:
 				break
@@ -86,7 +131,8 @@ def index():
 				r = util.flask.add_non_caching_response_headers(r)
 			return r
 
-		cached = get_cached_view("_default", make_default_ui)
+		preemptively_cached = get_preemptively_cached_view("_default", make_default_ui, dict(), dict())
+		cached = get_cached_view("_default", preemptively_cached)
 		response = cached()
 
 	response.headers["Last-Modified"] = now
@@ -510,7 +556,9 @@ def robotsTxt():
 @app.route("/i18n/<string:locale>/<string:domain>.js")
 @util.flask.cached(timeout=-1,
                    refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values,
-                   key=lambda: "{}:{}".format(request.base_url, g.locale))
+                   key=lambda: "{}".format(request.base_url))
+@util.flask.preemptively_cached(data=lambda: dict(path=request.path, base_url=request.url_root) if g.locale else None,
+                                unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"]))
 def localeJs(locale, domain):
 	messages = dict()
 	plural_expr = None

src/octoprint/settings.py

@@ -119,6 +119,9 @@ default_settings = {
 		"diskspace": {
 			"warning": 500 * 1024 * 1024, # 500 MB
 			"critical": 200 * 1024 * 1024, # 200 MB
+		},
+		"preemptiveCache": {
+			"exceptions": []
 		}
 	},
 	"webcam": {
@@ -270,7 +273,8 @@ default_settings = {
 	"devel": {
 		"stylesheet": "css",
 		"cache": {
-			"enabled": True
+			"enabled": True,
+			"preemptive": True
 		},
 		"webassets": {
 			"minify": False,