pingu_98/MrDraw
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'fix/preemptiveUiCaching' into dev/preemptiveUiCachingWithUiPlugins

Browse source 
Conflicts:
	src/octoprint/server/__init__.py
	src/octoprint/server/util/flask.py
	src/octoprint/server/views.py
	src/octoprint/util/__init__.py
This commit is contained in:
Gina Häußge 2015-11-24 14:41:42 +01:00
commit 53a2e6e27e
5 changed files with 199 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/octoprint/server/__init__.py
View file

@ -555,6 +555,9 @@ class Server():
Markdown(app)
preemptive_cache = octoprint.server.util.flask.PreemptiveCache(os.path.join(settings().getBaseFolder("data"), "preemptive_flask_cache.yaml"))
preemptive_cache.attach_to_app(app)
def _setup_i18n(self, app):
global babel
global LOCALES
@ -660,10 +663,18 @@ class Server():
self._register_template_plugins()
def _execute_preemptive_flask_caching(self):
from octoprint.server.util.flask import get_preemptive_cache_data
from werkzeug.test import EnvironBuilder
import time
cache_data = get_preemptive_cache_data()
if not hasattr(app, "preemptive_cache"):
return
# we clean up entries from our preemptive cache settings that haven't been
# accessed longer than server.preemptiveCache.until days
preemptive_cache_timeout = settings().getInt(["server", "preemptiveCache", "until"])
cutoff_timestamp = time.time() + preemptive_cache_timeout * 24 * 60 * 60
cache_data = app.preemptive_cache.clean_all_data(lambda root, entries: filter(lambda entry: "_timestamp" in entry and entry["_timestamp"] <= cutoff_timestamp, entries))
if not cache_data:
return
@ -681,7 +692,8 @@ class Server():
else:
self._logger.info("Preemptively caching {} for {!r}".format(route, kwargs))
builder = EnvironBuilder(**kwargs)
app(builder.get_environ(), lambda *a, **kw: None)
with app.preemptive_cache.disable_timestamp_update():
app(builder.get_environ(), lambda *a, **kw: None)
except:
self._logger.exception("Error while trying to preemptively cache {} for {!r}".format(route, kwargs))

181
src/octoprint/server/util/flask.py
View file

@ -14,6 +14,7 @@ import flask.ext.assets
import webassets.updater
import webassets.utils
import functools
import contextlib
import time
import uuid
import threading
@ -376,75 +377,147 @@ def cache_check_response_headers(response):
return False
_preemptive_flask_cache = "preemptive_flask_cache.yaml"
def preemptively_cached(data, unless=None):
def decorator(f):
@functools.wraps(f)
def decorated_function(*args, **kwargs):
if not (callable(unless) and unless()):
entry_data = data
if callable(entry_data):
entry_data = entry_data()
class PreemptiveCache(object):
if entry_data is not None:
from flask import request
from octoprint.util import atomic_write
import yaml
def __init__(self, cachefile):
self.cachefile = cachefile
data_folder = settings().getBaseFolder("data")
cache_data_file = os.path.join(data_folder, _preemptive_flask_cache)
cache_data = get_preemptive_cache_data()
self._lock = threading.RLock()
self._logger = logging.getLogger(__name__ + "." + self.__class__.__name__)
self._update_timestamp = True
if not request.path in cache_data:
cache_data[request.path] = []
def recorded(self, data, unless=None):
def decorator(f):
@functools.wraps(f)
def decorated_function(*args, **kwargs):
if not (callable(unless) and unless()):
entry_data = data
if callable(entry_data):
entry_data = entry_data()
def strip_ignored(d):
return dict((k, v) for k, v in d.items() if not k.startswith("_"))
if entry_data is not None:
from flask import request
self.add_data(request.path, entry_data)
return f(*args, **kwargs)
return decorated_function
return decorator
def compare(a, b):
return set(strip_ignored(a).items()) == set(strip_ignored(b).items())
@contextlib.contextmanager
def disable_timestamp_update(self):
with self._lock:
self._update_timestamp = False
yield
self._update_timestamp = True
cache_data_for_path = cache_data.get(request.path, [])
if not any(map(lambda entry: compare(entry_data, entry), cache_data_for_path)):
logging.getLogger(__name__).info("Adding {} for {!r} to views to preemptively cache".format(request.path, entry_data))
cache_data[request.path] = cache_data_for_path + [entry_data]
try:
with atomic_write(cache_data_file, "wb", prefix="octoprint-{}-".format(_preemptive_flask_cache[:-len(".yaml")]), suffix=".yaml") as handle:
yaml.safe_dump(cache_data, handle,default_flow_style=False, indent=" ", allow_unicode=True)
except:
logging.getLogger(__name__).exception("Error while writing {}".format(_preemptive_flask_cache))
def clean_all_data(self, cleanup_function):
assert callable(cleanup_function)
return f(*args, **kwargs)
with self._lock:
all_data = self.get_all_data()
for root, entries in all_data.items():
old_count = len(entries)
entries = cleanup_function(root, entries)
if not entries:
del all_data[root]
self._logger.debug("Removed root {} from preemptive cache".format(root))
elif len(entries) < old_count:
all_data[root] = entries
self._logger.debug("Removed {} from preemptive cache for root {}".format(old_count - len(entries), root))
self.set_all_data(all_data)
return decorated_function
return all_data
return decorator
def get_all_data(self):
import yaml
cache_data = None
with self._lock:
try:
with open(self.cachefile, "r") as f:
cache_data = yaml.safe_load(f)
except:
self._logger.exception("Error while reading {}".format(self.cachefile))
def get_preemptive_cache_data(root=None):
import yaml
if cache_data is None:
cache_data = dict()
data_folder = settings().getBaseFolder("data")
cache_data_file = os.path.join(data_folder, _preemptive_flask_cache)
if not os.path.isfile(cache_data_file):
return dict()
cache_data = None
try:
with open(cache_data_file, "r") as f:
cache_data = yaml.safe_load(f)
except:
logging.getLogger(__name__).exception("Error while reading {}".format(_preemptive_flask_cache))
if cache_data is None:
cache_data = dict()
if root:
return cache_data.get(root, dict())
else:
return cache_data
def get_data(self, root):
cache_data = self.get_all_data()
return cache_data.get(root, dict())
def set_all_data(self, data):
from octoprint.util import atomic_write
import yaml
with self._lock:
try:
with atomic_write(self.cachefile, "wb") as handle:
yaml.safe_dump(data, handle,default_flow_style=False, indent=" ", allow_unicode=True)
except:
self._logger.exception("Error while writing {}".format(self.cachefile))
def set_data(self, root, data):
with self._lock:
all_data = self.get_all_data()
all_data[root] = data
self.set_all_data(all_data)
def add_data(self, root, data):
from octoprint.util import dict_filter
def strip_ignored(d):
return dict_filter(d, lambda k, v: not k.startswith("_"))
def compare(a, b):
return set(strip_ignored(a).items()) == set(strip_ignored(b).items())
def split_matched_and_unmatched(entry, entries):
matched = []
unmatched = []
for e in entries:
if compare(e, entry):
matched.append(e)
else:
unmatched.append(e)
return matched, unmatched
with self._lock:
cache_data = self.get_all_data()
if not root in cache_data:
cache_data[root] = []
existing, other = split_matched_and_unmatched(data, cache_data[root])
def get_newest(entries):
result = None
for entry in entries:
if "_timestamp" in entry and (result is None or ("_timestamp" in entry and result["_timestamp"] < entry["_timestamp"])):
result = entry
return result
to_persist = get_newest(existing)
if not to_persist:
import copy
to_persist = copy.deepcopy(data)
to_persist["_timestamp"] = time.time()
self._logger.info("Adding entry for {} and {!r}".format(root, to_persist))
elif self._update_timestamp:
to_persist["_timestamp"] = time.time()
self._logger.debug("Updating timestamp for {} and {!r}".format(root, data))
else:
self._logger.debug("Not updating timestamp for {} and {!r}, currently flagged as disabled".format(root, data))
self.set_data(root, [to_persist] + other)
def attach_to_app(self, app):
app.preemptive_cache = self
def add_non_caching_response_headers(response):
response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0"

26
src/octoprint/server/views.py
View file

@ -73,8 +73,8 @@ def index():
))
# finally decorate our view
return util.flask.preemptively_cached(data=d,
unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"]))(view)
return app.preemptive_cache.recorded(data=d,
unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"]))(view)
def get_cached_view(key, view, additional_key_data=None):
def cache_key():
@ -96,16 +96,16 @@ def index():
for plugin in ui_plugins:
if plugin.will_handle_ui(request):
# plugin claims responsibility, let it render the UI
cached = get_cached_view(plugin._identifier,
plugin.on_ui_render,
plugin.get_ui_additional_key_data_for_cache(request))
preemptively_cached = get_preemptively_cached_view(plugin._identifier,
plugin.on_ui_render,
cached,
plugin.get_ui_data_for_preemptive_caching(request),
plugin.get_ui_additional_request_data_for_preemptive_caching(request))
cached = get_cached_view(plugin._identifier,
preemptively_cached,
plugin.get_ui_additional_key_data_for_cache(request))
response = cached(now, request, render_kwargs)
response = preemptively_cached(now, request, render_kwargs)
if response is not None:
break
@ -131,9 +131,9 @@ def index():
r = util.flask.add_non_caching_response_headers(r)
return r
preemptively_cached = get_preemptively_cached_view("_default", make_default_ui, dict(), dict())
cached = get_cached_view("_default", preemptively_cached)
response = cached()
cached = get_cached_view("_default", make_default_ui)
preemptively_cached = get_preemptively_cached_view("_default", cached, dict(), dict())
response = preemptively_cached()
response.headers["Last-Modified"] = now
@ -554,11 +554,11 @@ def robotsTxt():
@app.route("/i18n/<string:locale>/<string:domain>.js")
@app.preemptive_cache.recorded(data=lambda: dict(path=request.path, base_url=request.url_root),
unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"]))
@util.flask.cached(timeout=-1,
refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values,
key=lambda: "{}".format(request.base_url))
@util.flask.preemptively_cached(data=lambda: dict(path=request.path, base_url=request.url_root) if g.locale else None,
unless=lambda: request.url_root in settings().get(["server", "preemptiveCache", "exceptions"]))
def localeJs(locale, domain):
messages = dict()
plural_expr = None

3
src/octoprint/settings.py
View file

@ -121,7 +121,8 @@ default_settings = {
"critical": 200 * 1024 * 1024, # 200 MB
},
"preemptiveCache": {
"exceptions": []
"exceptions": [],
"until": 7
}
},
"webcam": {

42
src/octoprint/util/__init__.py
View file

@ -577,6 +577,48 @@ class fallback_dict(dict):
return [self.custom] + list(self.fallbacks)
def dict_filter(dictionary, filter_function):
"""
Filters a dictionary with the provided filter_function
Example::
>>> data = dict(key1="value1", key2="value2", other_key="other_value", foo="bar", bar="foo")
>>> dict_filter(data, lambda k, v: k.startswith("key")) == dict(key1="value1", key2="value2")
True
>>> dict_filter(data, lambda k, v: v.startswith("value")) == dict(key1="value1", key2="value2")
True
>>> dict_filter(data, lambda k, v: k == "foo" or v == "foo") == dict(foo="bar", bar="foo")
True
>>> dict_filter(data, lambda k, v: False) == dict()
True
>>> dict_filter(data, lambda k, v: True) == data
True
>>> dict_filter(None, lambda k, v: True)
Traceback (most recent call last):
...
AssertionError
>>> dict_filter(data, None)
Traceback (most recent call last):
...
AssertionError
Arguments:
dictionary (dict): The dictionary to filter
filter_function (callable): The filter function to apply, called with key and
value of an entry in the dictionary, must return ``True`` for values to
keep and ``False`` for values to strip
Returns:
dict: A shallow copy of the provided dictionary, stripped of the key-value-pairs
for which the ``filter_function`` returned ``False``
"""
assert isinstance(dictionary, dict)
assert callable(filter_function)
return dict((k, v) for k, v in dictionary.items() if filter_function(k, v))
class Object(object):
pass