From bccc70632974347168e31c5d6bc0104cd6a485d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 30 Aug 2016 18:03:50 +0200 Subject: [PATCH] First throw at caching of API methods Most caching is left to the client, by utilizing ETag and Last-Modified headers. Where it was easily achievable, an additional server side miniature cache of intermediary results was introduced (e.g. for the files). The regular cached decorator was not used since it targets caching full responses, and the responses in question already contained client request specific data. Caching "one step earlier" allows better usage of the cache here. Also introduced a dependency on the scandir module, to get a bit of a performance boost on os.walk and os.listdir (which have been replaced with scandir.walk and scandir.listdir respectively). See https://github.com/benhoyt/scandir#background on why that made sense. --- setup.py | 3 +- src/octoprint/filemanager/__init__.py | 7 ++ src/octoprint/filemanager/storage.py | 114 ++++++++++++------ src/octoprint/plugin/core.py | 19 +-- .../plugins/announcements/__init__.py | 62 +++++++--- src/octoprint/plugins/cura/static/js/cura.js | 11 +- .../plugins/pluginmanager/__init__.py | 51 +++++--- .../plugins/softwareupdate/__init__.py | 51 ++++++-- .../static/js/softwareupdate.js | 37 +++++- src/octoprint/printer/profile.py | 22 +++- src/octoprint/server/api/__init__.py | 4 +- src/octoprint/server/api/files.py | 93 ++++++++++++-- src/octoprint/server/api/languages.py | 32 ++--- src/octoprint/server/api/log.py | 18 +-- src/octoprint/server/api/printer_profiles.py | 22 +++- src/octoprint/server/api/settings.py | 21 +++- src/octoprint/server/api/slicing.py | 34 +++++- src/octoprint/server/api/timelapse.py | 67 +++++++++- src/octoprint/server/util/__init__.py | 14 +++ src/octoprint/server/util/flask.py | 76 +++++++++++- src/octoprint/settings.py | 12 ++ src/octoprint/slicing/__init__.py | 35 +++++- src/octoprint/static/js/app/client/files.js | 14 ++- src/octoprint/static/js/app/main.js | 12 +- .../static/js/app/viewmodels/files.js | 20 ++- src/octoprint/static/js/app/viewmodels/log.js | 6 +- .../static/js/app/viewmodels/slicing.js | 10 ++ .../templates/sidebar/files_header.jinja2 | 2 +- src/octoprint/timelapse.py | 62 ++++++---- src/octoprint/util/jinja.py | 7 +- 30 files changed, 739 insertions(+), 199 deletions(-) diff --git a/setup.py b/setup.py index c5475d86..49b2cb9c 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,8 @@ INSTALL_REQUIRES = [ "awesome-slugify>=1.6.5,<1.7", "feedparser>=5.2.1,<5.3", "chainmap>=1.0.2,<1.1", - "future>=0.15,<0.16" + "future>=0.15,<0.16", + "scandir>=1.3,<1.4" ] # Additional requirements for optional install options diff --git a/src/octoprint/filemanager/__init__.py b/src/octoprint/filemanager/__init__.py index 57782bc8..c0559040 100644 --- a/src/octoprint/filemanager/__init__.py +++ b/src/octoprint/filemanager/__init__.py @@ -226,6 +226,10 @@ class FileManager(object): return del self._storage_managers[type] + @property + def registered_storages(self): + return list(self._storage_managers.keys()) + @property def slicing_enabled(self): return self._slicing_manager.slicing_enabled @@ -520,6 +524,9 @@ class FileManager(object): def path_in_storage(self, destination, path): return self._storage(destination).path_in_storage(path) + def last_modified(self, destination, path=None, recursive=False): + return self._storage(destination).last_modified(path=path, recursive=recursive) + def _storage(self, destination): if not destination in self._storage_managers: raise NoSuchStorage("No storage configured for destination {destination}".format(**locals())) diff --git a/src/octoprint/filemanager/storage.py b/src/octoprint/filemanager/storage.py index 1a67a1f6..854bd45f 100644 --- a/src/octoprint/filemanager/storage.py +++ b/src/octoprint/filemanager/storage.py @@ -11,6 +11,11 @@ import os import pylru import shutil +try: + from os import scandir, walk +except ImportError: + from scandir import scandir, walk + from octoprint.util import atomic_write from contextlib import contextmanager from copy import deepcopy @@ -39,6 +44,20 @@ class StorageInterface(object): return yield + def last_modified(self, path=None, recursive=False): + """ + Get the last modification date of the specified ``path`` or ``path``'s subtree. + + Args: + path (str or None): Path for which to determine the subtree's last modification date. If left out or + set to None, defatuls to storage root. + recursive (bool): Whether to determine only the date of the specified ``path`` (False, default) or + the whole ``path``'s subtree (True). + + Returns: (float) The last modification date of the indicated subtree + """ + raise NotImplementedError() + def file_in_path(self, path, filepath): """ Returns whether the file indicated by ``file`` is inside ``path`` or not. @@ -453,23 +472,40 @@ class LocalFileStorage(StorageInterface): metadata = self._get_metadata(path) if not metadata: metadata = dict() - for entry in os.listdir(path): - if is_hidden_path(entry) or not octoprint.filemanager.valid_file_type(entry): + for entry in scandir(path): + if is_hidden_path(entry.name) or not octoprint.filemanager.valid_file_type(entry.name): continue - absolute_path = os.path.join(path, entry) - if os.path.isfile(absolute_path): - if not entry in metadata or not isinstance(metadata[entry], dict) or not "analysis" in metadata[entry]: - printer_profile_rels = self.get_link(absolute_path, "printerprofile") + if entry.is_file(): + if not entry.name in metadata or not isinstance(metadata[entry.name], dict) or not "analysis" in metadata[entry.name]: + printer_profile_rels = self.get_link(entry.path, "printerprofile") if printer_profile_rels: printer_profile_id = printer_profile_rels[0]["id"] else: printer_profile_id = None - yield entry, absolute_path, printer_profile_id - elif os.path.isdir(absolute_path): - for sub_entry in self._analysis_backlog_generator(absolute_path): - yield self.join_path(entry, sub_entry[0]), sub_entry[1], sub_entry[2] + yield entry.name, entry.path, printer_profile_id + elif os.path.isdir(entry.path): + for sub_entry in self._analysis_backlog_generator(entry.path): + yield self.join_path(entry.name, sub_entry[0]), sub_entry[1], sub_entry[2] + + def last_modified(self, path=None, recursive=False): + if path is None: + path = self.basefolder + else: + path = os.path.join(self.basefolder, path) + + def last_modified_for_path(p): + metadata = os.path.join(p, ".metadata.yaml") + if os.path.exists(metadata): + return max(os.stat(p).st_mtime, os.stat(metadata).st_mtime) + else: + return os.stat(p).st_mtime + + if recursive: + return max(last_modified_for_path(root) for root, _, _ in walk(path)) + else: + return last_modified_for_path(path) def file_in_path(self, path, filepath): filepath = self.sanitize_path(filepath) @@ -517,10 +553,14 @@ class LocalFileStorage(StorageInterface): if not os.path.exists(folder_path): return - contents = os.listdir(folder_path) - if ".metadata.yaml" in contents: - contents.remove(".metadata.yaml") - if contents and not recursive: + empty = True + for entry in scandir(folder_path): + if entry.name == ".metadata.yaml": + continue + empty = False + break + + if not empty and not recursive: raise StorageError("{name} in {path} is not empty".format(**locals()), code=StorageError.NOT_EMPTY) import shutil @@ -1051,16 +1091,19 @@ class LocalFileStorage(StorageInterface): metadata_dirty = False result = dict() - for entry in os.listdir(path): - if is_hidden_path(entry): + for entry in scandir(path): + if is_hidden_path(entry.name): # no hidden files and folders continue - entry_path = os.path.join(path, entry) - path_in_location = entry if not base else base + entry + entry_name = entry.name + entry_path = entry.path + entry_is_file = entry.is_file() + entry_is_dir = entry.is_dir() + entry_stat = entry.stat() - sanitized = self.sanitize_name(entry) - if sanitized != entry: + sanitized = self.sanitize_name(entry_name) + if sanitized != entry_name: # entry is not sanitized yet, let's take care of that sanitized_path = os.path.join(path, sanitized) sanitized_name, sanitized_ext = os.path.splitext(sanitized) @@ -1075,48 +1118,51 @@ class LocalFileStorage(StorageInterface): shutil.move(entry_path, sanitized_path) self._logger.info("Sanitized \"{}\" to \"{}\"".format(entry_path, sanitized_path)) - entry = sanitized + entry_name = sanitized entry_path = sanitized_path + entry_stat = os.stat(sanitized_path) except: self._logger.exception("Error while trying to rename \"{}\" to \"{}\", ignoring file".format(entry_path, sanitized_path)) continue + path_in_location = entry_name if not base else base + entry_name + # file handling - if os.path.isfile(entry_path): - type_path = octoprint.filemanager.get_file_type(entry) + if entry_is_file: + type_path = octoprint.filemanager.get_file_type(entry_name) if not type_path: # only supported extensions continue else: file_type = type_path[0] - if entry in metadata and isinstance(metadata[entry], dict): - entry_data = metadata[entry] + if entry_name in metadata and isinstance(metadata[entry_name], dict): + entry_data = metadata[entry_name] else: - entry_data = self._add_basic_metadata(path, entry, save=False, metadata=metadata) + entry_data = self._add_basic_metadata(path, entry_name, save=False, metadata=metadata) metadata_dirty = True # TODO extract model hash from source if possible to recreate link - if not filter or filter(entry, entry_data): + if not filter or filter(entry_name, entry_data): # only add files passing the optional filter extended_entry_data = dict() extended_entry_data.update(entry_data) - extended_entry_data["name"] = entry + extended_entry_data["name"] = entry_name extended_entry_data["path"] = path_in_location extended_entry_data["type"] = file_type extended_entry_data["typePath"] = type_path - stat = os.stat(entry_path) + stat = entry_stat if stat: extended_entry_data["size"] = stat.st_size extended_entry_data["date"] = int(stat.st_mtime) - result[entry] = extended_entry_data + result[entry_name] = extended_entry_data # folder recursion - elif os.path.isdir(entry_path): + elif entry_is_dir: entry_data = dict( - name=entry, + name=entry_name, path=path_in_location, type="folder", type_path=["folder"] @@ -1126,7 +1172,7 @@ class LocalFileStorage(StorageInterface): recursive=recursive) entry_data["children"] = sub_result - if not filter or filter(entry, entry_data): + if not filter or filter(entry_name, entry_data): def get_size(): total_size = 0 for element in entry_data["children"].values(): @@ -1141,7 +1187,7 @@ class LocalFileStorage(StorageInterface): if recursive: extended_entry_data["size"] = get_size() - result[entry] = extended_entry_data + result[entry_name] = extended_entry_data # TODO recreate links if we have metadata less entries diff --git a/src/octoprint/plugin/core.py b/src/octoprint/plugin/core.py index 1195087d..d74b1f23 100644 --- a/src/octoprint/plugin/core.py +++ b/src/octoprint/plugin/core.py @@ -35,6 +35,11 @@ import logging import pkg_resources import pkginfo +try: + from os import scandir +except ImportError: + from scandir import scandir + EntryPointOrigin = namedtuple("EntryPointOrigin", "type, entry_point, module_name, package_name, package_version") FolderOrigin = namedtuple("FolderOrigin", "type, folder") @@ -535,13 +540,11 @@ class PluginManager(object): self.logger.warn("Plugin folder {folder} could not be found, skipping it".format(folder=folder)) continue - entries = os.listdir(folder) - for entry in entries: - path = os.path.join(folder, entry) - if os.path.isdir(path) and os.path.isfile(os.path.join(path, "__init__.py")): - key = entry - elif os.path.isfile(path) and entry.endswith(".py"): - key = entry[:-3] # strip off the .py extension + for entry in scandir(folder): + if entry.is_dir() and os.path.isfile(os.path.join(entry.path, "__init__.py")): + key = entry.name + elif entry.is_file() and entry.name.endswith(".py"): + key = entry.name[:-3] # strip off the .py extension else: continue @@ -637,7 +640,7 @@ class PluginManager(object): else: return None except: - self.logger.warn("Could not locate plugin {key}") + self.logger.warn("Could not locate plugin {key}".format(key=key)) return None plugin = self._import_plugin(key, *module, name=name, version=version, summary=summary, author=author, url=url, license=license) diff --git a/src/octoprint/plugins/announcements/__init__.py b/src/octoprint/plugins/announcements/__init__.py index 0eb7f3ae..890e8291 100644 --- a/src/octoprint/plugins/announcements/__init__.py +++ b/src/octoprint/plugins/announcements/__init__.py @@ -19,7 +19,7 @@ import feedparser import flask from octoprint.server import admin_permission -from octoprint.server.util.flask import restricted_access +from octoprint.server.util.flask import restricted_access, with_revalidation_checking, check_etag from flask.ext.babel import gettext class AnnouncementPlugin(octoprint.plugin.AssetPlugin, @@ -100,32 +100,54 @@ class AnnouncementPlugin(octoprint.plugin.AssetPlugin, result = dict() - force = "force" in flask.request.values and flask.request.values["force"] in valid_boolean_trues - - channel_data = self._fetch_all_channels(force=force) - channel_configs = self._get_channel_configs(force=force) + force = flask.request.values.get("force", "false") in valid_boolean_trues enabled = self._settings.get(["enabled_channels"]) forced = self._settings.get(["forced_channels"]) - for key, data in channel_configs.items(): - read_until = channel_configs[key].get("read_until", None) - entries = sorted(self._to_internal_feed(channel_data.get(key, []), read_until=read_until), key=lambda e: e["published"], reverse=True) - unread = len(filter(lambda e: not e["read"], entries)) + channel_configs = self._get_channel_configs(force=force) - if read_until is None and entries: - last = entries[0]["published"] - self._mark_read_until(key, last) + def view(): + channel_data = self._fetch_all_channels(force=force) - result[key] = dict(channel=data["name"], - url=data["url"], - priority=data.get("priority", 2), - enabled=key in enabled or key in forced, - forced=key in forced, - data=entries, - unread=unread) + for key, data in channel_configs.items(): + read_until = channel_configs[key].get("read_until", None) + entries = sorted(self._to_internal_feed(channel_data.get(key, []), read_until=read_until), key=lambda e: e["published"], reverse=True) + unread = len(filter(lambda e: not e["read"], entries)) - return flask.jsonify(result) + if read_until is None and entries: + last = entries[0]["published"] + self._mark_read_until(key, last) + + result[key] = dict(channel=data["name"], + url=data["url"], + priority=data.get("priority", 2), + enabled=key in enabled or key in forced, + forced=key in forced, + data=entries, + unread=unread) + + return flask.jsonify(result) + + def etag(): + import hashlib + hash = hashlib.sha1() + hash.update(repr(sorted(enabled))) + hash.update(repr(sorted(forced))) + + for channel in sorted(channel_configs.keys()): + hash.update(repr(channel_configs[channel])) + channel_data = self._get_channel_data_from_cache(channel, channel_configs[channel]) + hash.update(repr(channel_data)) + + return hash.hexdigest() + + def condition(): + return check_etag(etag()) + + return with_revalidation_checking(etag_factory=lambda *args, **kwargs: etag(), + condition=lambda *args, **kwargs: condition(), + unless=lambda: force)(view)() @octoprint.plugin.BlueprintPlugin.route("/channels/", methods=["POST"]) @restricted_access diff --git a/src/octoprint/plugins/cura/static/js/cura.js b/src/octoprint/plugins/cura/static/js/cura.js index cec56899..4b57e623 100644 --- a/src/octoprint/plugins/cura/static/js/cura.js +++ b/src/octoprint/plugins/cura/static/js/cura.js @@ -188,8 +188,7 @@ $(function() { }; self.requestData = function() { - OctoPrint.slicing.listProfilesForSlicer("cura") - .done(self.fromResponse); + self.slicingViewModel.requestData(); }; self.fromResponse = function(data) { @@ -208,13 +207,19 @@ $(function() { self.onBeforeBinding = function () { self.settings = self.settingsViewModel.settings; - self.requestData(); + //self.requestData(); }; self.onSettingsHidden = function() { self.resetPathTest(); }; + self.onSlicingData = function(data) { + if (data && data.hasOwnProperty("cura") && data.cura.hasOwnProperty("profiles")) { + self.fromResponse(data.cura.profiles); + } + }; + self.resetPathTest = function() { self.pathBroken(false); self.pathOk(false); diff --git a/src/octoprint/plugins/pluginmanager/__init__.py b/src/octoprint/plugins/pluginmanager/__init__.py index c7c523f3..51f002a8 100644 --- a/src/octoprint/plugins/pluginmanager/__init__.py +++ b/src/octoprint/plugins/pluginmanager/__init__.py @@ -10,7 +10,7 @@ import octoprint.plugin import octoprint.plugin.core from octoprint.settings import valid_boolean_trues -from octoprint.server.util.flask import restricted_access +from octoprint.server.util.flask import restricted_access, with_revalidation_checking, check_etag from octoprint.server import admin_permission, VERSION from octoprint.util.pip import LocalPipCaller, UnknownPip @@ -176,26 +176,43 @@ class PluginManagerPlugin(octoprint.plugin.SimpleApiPlugin, if not admin_permission.can(): return make_response("Insufficient rights", 403) - if "refresh_repository" in request.values and request.values["refresh_repository"] in valid_boolean_trues: + refresh_repository = request.values.get("refresh_repository", "false") in valid_boolean_trues + if refresh_repository: self._repository_available = self._refresh_repository() - return jsonify(plugins=self._get_plugins(), - repository=dict( - available=self._repository_available, - plugins=self._repository_plugins - ), - os=self._get_os(), - octoprint=self._get_octoprint_version_string(), - pip=dict( - available=self._pip_caller.available, - version=self._pip_caller.version_string, - install_dir=self._pip_caller.install_dir, - use_user=self._pip_caller.use_user, - virtual_env=self._pip_caller.virtual_env, - additional_args=self._settings.get(["pip_args"]), - python=sys.executable + def view(): + return jsonify(plugins=self._get_plugins(), + repository=dict( + available=self._repository_available, + plugins=self._repository_plugins + ), + os=self._get_os(), + octoprint=self._get_octoprint_version_string(), + pip=dict( + available=self._pip_caller.available, + version=self._pip_caller.version_string, + install_dir=self._pip_caller.install_dir, + use_user=self._pip_caller.use_user, + virtual_env=self._pip_caller.virtual_env, + additional_args=self._settings.get(["pip_args"]), + python=sys.executable )) + def etag(): + import hashlib + hash = hashlib.sha1() + hash.update(repr(self._get_plugins())) + hash.update(str(self._repository_available)) + hash.update(repr(self._repository_plugins)) + return hash.hexdigest() + + def condition(): + return check_etag(etag()) + + return with_revalidation_checking(etag_factory=lambda *args, **kwargs: etag(), + condition=lambda *args, **kwargs: condition(), + unless=lambda: refresh_repository)(view)() + def on_api_command(self, command, data): if not admin_permission.can(): return make_response("Insufficient rights", 403) diff --git a/src/octoprint/plugins/softwareupdate/__init__.py b/src/octoprint/plugins/softwareupdate/__init__.py index 3177d495..e5c74367 100644 --- a/src/octoprint/plugins/softwareupdate/__init__.py +++ b/src/octoprint/plugins/softwareupdate/__init__.py @@ -19,7 +19,7 @@ import hashlib from . import version_checks, updaters, exceptions, util, cli -from octoprint.server.util.flask import restricted_access +from octoprint.server.util.flask import restricted_access, with_revalidation_checking, check_etag from octoprint.server import admin_permission, VERSION, REVISION, BRANCH from octoprint.util import dict_merge import octoprint.settings @@ -411,17 +411,46 @@ class SoftwareUpdatePlugin(octoprint.plugin.BlueprintPlugin, else: check_targets = None - if "force" in flask.request.values and flask.request.values["force"] in octoprint.settings.valid_boolean_trues: - force = True - else: - force = False + force = flask.request.values.get("force", "false") in octoprint.settings.valid_boolean_trues - try: - information, update_available, update_possible = self.get_current_versions(check_targets=check_targets, force=force) - return flask.jsonify(dict(status="updatePossible" if update_available and update_possible else "updateAvailable" if update_available else "current", - information=information)) - except exceptions.ConfigurationInvalid as e: - flask.make_response("Update not properly configured, can't proceed: %s" % e.message, 500) + def view(): + try: + information, update_available, update_possible = self.get_current_versions(check_targets=check_targets, force=force) + return flask.jsonify(dict(status="updatePossible" if update_available and update_possible else "updateAvailable" if update_available else "current", + information=information)) + except exceptions.ConfigurationInvalid as e: + return flask.make_response("Update not properly configured, can't proceed: %s" % e.message, 500) + + def etag(): + checks = self._get_configured_checks() + + targets = check_targets + if targets is None: + targets = checks.keys() + + import hashlib + hash = hashlib.sha1() + + targets = sorted(targets) + for target in targets: + current_hash = self._get_check_hash(checks.get(target, dict())) + if target in self._version_cache and not force: + data = self._version_cache[target] + hash.update(current_hash) + hash.update(str(data["timestamp"] + self._version_cache_ttl >= time.time() > data["timestamp"])) + hash.update(repr(data["information"])) + hash.update(str(data["available"])) + hash.update(str(data["possible"])) + + hash.update(",".join(targets)) + return hash.hexdigest() + + def condition(): + return check_etag(etag()) + + return with_revalidation_checking(etag_factory=lambda *args, **kwargs: etag(), + condition=lambda *args, **kwargs: condition(), + unless=lambda: force)(view)() @octoprint.plugin.BlueprintPlugin.route("/update", methods=["POST"]) diff --git a/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js b/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js index 0a627986..294c65d4 100644 --- a/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js +++ b/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js @@ -12,25 +12,46 @@ var updateUrl = url + "update"; exports.checkEntries = function(entries, force, opts) { + if (arguments.length == 1 && _.isObject(arguments[0])) { + var params = arguments[0]; + entries = params.entries; + force = params.force; + opts = params.opts; + } + entries = entries || []; if (typeof entries == "string") { entries = [entries]; } - var data = { - force: !!force - }; + var data = {}; + if (!!force) { + data.force = true; + } if (entries && entries.length) { - data["check"] = entries.join(",") + data.check = entries.join(","); } return OctoPrint.getWithQuery(checkUrl, data, opts); }; exports.check = function(force, opts) { - return exports.checkEntries([], force, opts); + if (arguments.length == 1 && _.isObject(arguments[0])) { + var params = arguments[0]; + force = params.force; + opts = params.opts; + } + + return exports.checkEntries({entries: [], force: force, opts: opts}); }; exports.update = function(entries, force, opts) { + if (arguments.length == 1 && _.isObject(arguments[0])) { + var params = arguments[0]; + entries = params.entries; + force = params.force; + opts = params.opts; + } + entries = entries || []; if (typeof entries == "string") { entries = [entries]; @@ -44,6 +65,12 @@ }; exports.updateAll = function(force, opts) { + if (arguments.length == 1 && _.isObject(arguments[0])) { + var params = arguments[0]; + force = params.force; + opts = params.opts; + } + var data = { force: !!force }; diff --git a/src/octoprint/printer/profile.py b/src/octoprint/printer/profile.py index aec77a14..1e7a618a 100644 --- a/src/octoprint/printer/profile.py +++ b/src/octoprint/printer/profile.py @@ -11,6 +11,11 @@ import copy import re import logging +try: + from os import scandir +except ImportError: + from scandir import scandir + from octoprint.settings import settings from octoprint.util import dict_merge, dict_sanitize, dict_contains_keys, is_hidden_path @@ -242,6 +247,12 @@ class PrinterProfileManager(object): def profile_count(self): return len(self._load_all_identifiers()) + @property + def last_modified(self): + dates = [os.stat(self._folder).st_mtime] + dates += [entry.stat().st_mtime for entry in scandir(self._folder) if entry.name.endswith(".profile")] + return max(dates) + def get_default(self): default = settings().get(["printerProfiles", "default"]) if default is not None and self.exists(default): @@ -297,16 +308,15 @@ class PrinterProfileManager(object): def _load_all_identifiers(self): results = dict(_default=None) - for entry in os.listdir(self._folder): - if is_hidden_path(entry) or not entry.endswith(".profile") or entry == "_default.profile": + for entry in scandir(self._folder): + if is_hidden_path(entry.name) or not entry.name.endswith(".profile") or entry.name == "_default.profile": continue - path = os.path.join(self._folder, entry) - if not os.path.isfile(path): + if not entry.is_file(): continue - identifier = entry[:-len(".profile")] - results[identifier] = path + identifier = entry.name[:-len(".profile")] + results[identifier] = entry.path return results def _load_from_path(self, path): diff --git a/src/octoprint/server/api/__init__.py b/src/octoprint/server/api/__init__.py index c0cee7c8..41b5a060 100644 --- a/src/octoprint/server/api/__init__.py +++ b/src/octoprint/server/api/__init__.py @@ -19,7 +19,7 @@ import octoprint.server import octoprint.plugin from octoprint.server import admin_permission, NO_CONTENT from octoprint.settings import settings as s, valid_boolean_trues -from octoprint.server.util import noCachingResponseHandler, apiKeyRequestHandler, corsResponseHandler +from octoprint.server.util import noCachingExceptGetResponseHandler, apiKeyRequestHandler, corsResponseHandler from octoprint.server.util.flask import restricted_access, get_json_command_from_request, passive_login @@ -43,7 +43,7 @@ from . import system as api_system VERSION = "0.1" -api.after_request(noCachingResponseHandler) +api.after_request(noCachingExceptGetResponseHandler) api.before_request(apiKeyRequestHandler) api.after_request(corsResponseHandler) diff --git a/src/octoprint/server/api/files.py b/src/octoprint/server/api/files.py index 1cc259ba..b0c5de9e 100644 --- a/src/octoprint/server/api/files.py +++ b/src/octoprint/server/api/files.py @@ -10,7 +10,7 @@ from flask import request, jsonify, make_response, url_for from octoprint.filemanager.destinations import FileDestinations from octoprint.settings import settings, valid_boolean_trues from octoprint.server import printer, fileManager, slicingManager, eventManager, NO_CONTENT -from octoprint.server.util.flask import restricted_access, get_json_command_from_request +from octoprint.server.util.flask import restricted_access, get_json_command_from_request, with_revalidation_checking from octoprint.server.api import api from octoprint.events import Events import octoprint.filemanager @@ -19,15 +19,77 @@ import octoprint.filemanager.storage import octoprint.slicing import psutil +import hashlib +import logging +import threading #~~ GCODE file handling +_file_cache = dict() +_file_cache_mutex = threading.RLock() + +def _clear_file_cache(): + with _file_cache_mutex: + _file_cache.clear() + +def _create_lastmodified(path, recursive): + if path.endswith("/api/files"): + # all storages involved + lms = [0] + for storage in fileManager.registered_storages: + try: + lms.append(fileManager.last_modified(storage, recursive=recursive)) + except: + logging.getLogger(__name__).exception("There was an error retrieving the last modified data from storage {}".format(storage)) + lms.append(None) + + if filter(lambda x: x is None, lms): + # we return None if ANY of the involved storages returned None + return None + + # if we reach this point, we return the maximum of all dates + return max(lms) + + elif path.endswith("/files/local"): + # only local storage involved + try: + return fileManager.last_modified(FileDestinations.LOCAL, recursive=recursive) + except: + logging.getLogger(__name__).exception("There was an error retrieving the last modified data from storage {}".format(FileDestinations.LOCAL)) + return None + + else: + return None + + +def _create_etag(path, recursive, lm=None): + if lm is None: + lm = _create_lastmodified(path) + + if lm is None: + return None + + hash = hashlib.sha1() + hash.update(str(lm)) + hash.update(str(recursive)) + return hash.hexdigest() + @api.route("/files", methods=["GET"]) +@with_revalidation_checking(etag_factory=lambda lm=None: _create_etag(request.path, + request.values.get("recursive", False), + lm=lm), + lastmodified_factory=lambda: _create_lastmodified(request.path, + request.values.get("recursive", False)), + unless=lambda: request.values.get("force", False) or request.values.get("_refresh", False)) def readGcodeFiles(): - filter = "filter" in request.values and request.values["recursive"] in valid_boolean_trues - recursive = "recursive" in request.values and request.values["recursive"] in valid_boolean_trues + filter = request.values.get("filter", "false") in valid_boolean_trues + recursive = request.values.get("recursive", "false") in valid_boolean_trues + force = request.values.get("force", "false") in valid_boolean_trues + + if force: + _clear_file_cache() files = _getFileList(FileDestinations.LOCAL, filter=filter, recursive=recursive) files.extend(_getFileList(FileDestinations.SDCARD)) @@ -37,13 +99,25 @@ def readGcodeFiles(): @api.route("/files/", methods=["GET"]) +@with_revalidation_checking(etag_factory=lambda lm=None: _create_etag(request.path, + request.values.get("recursive", False), + lm=lm), + lastmodified_factory=lambda: _create_lastmodified(request.path, + request.values.get("recursive", False)), + unless=lambda: request.values.get("force", False) or request.values.get("_refresh", False)) def readGcodeFilesForOrigin(origin): if origin not in [FileDestinations.LOCAL, FileDestinations.SDCARD]: return make_response("Unknown origin: %s" % origin, 404) - recursive = False - if "recursive" in request.values: - recursive = request.values["recursive"] in valid_boolean_trues + recursive = request.values.get("recursive", "false") in valid_boolean_trues + force = request.values.get("force", "false") in valid_boolean_trues + + if force: + with _file_cache_mutex: + try: + del _file_cache[origin] + except KeyError: + pass files = _getFileList(origin, recursive=recursive) @@ -89,7 +163,12 @@ def _getFileList(origin, path=None, filter=None, recursive=False): if filter: filter_func = lambda entry, entry_data: octoprint.filemanager.valid_file_type(entry, type=filter) - files = fileManager.list_files(origin, path=path, filter=filter_func, recursive=recursive)[origin].values() + with _file_cache_mutex: + files, lastmodified = _file_cache.get("{}:{}:{}:{}".format(origin, path, recursive, filter), ([], None)) + if lastmodified is None or lastmodified < fileManager.last_modified(origin, path=path, recursive=recursive): + files = fileManager.list_files(origin, path=path, filter=filter_func, recursive=recursive)[origin].values() + lastmodified = fileManager.last_modified(origin, path=path, recursive=recursive) + _file_cache["{}:{}:{}:{}".format(origin, path, recursive, filter)] = (files, lastmodified) def analyse_recursively(files, path=None): if path is None: diff --git a/src/octoprint/server/api/languages.py b/src/octoprint/server/api/languages.py index 9b06284a..19ae5ce4 100644 --- a/src/octoprint/server/api/languages.py +++ b/src/octoprint/server/api/languages.py @@ -9,6 +9,11 @@ import os import tarfile import zipfile +try: + from os import scandir +except ImportError: + from scandir import scandir + from collections import defaultdict from flask import request, jsonify, make_response @@ -33,10 +38,8 @@ def getInstalledLanguagePacks(): core_packs = [] plugin_packs = defaultdict(lambda: dict(identifier=None, display=None, languages=[])) - for folder in os.listdir(translation_folder): - path = os.path.join(translation_folder, folder) - - if not os.path.isdir(path): + for entry in scandir(translation_folder): + if not entry.is_dir(): continue def load_meta(path, locale): @@ -61,24 +64,23 @@ def getInstalledLanguagePacks(): meta["locale_english"] = l.english_name return meta - if folder == "_plugins": - for plugin_folder in os.listdir(path): - plugin_path = os.path.join(path, plugin_folder) - if not os.path.isdir(plugin_path): + if entry.name == "_plugins": + for plugin_entry in scandir(entry.path): + if not plugin_entry.is_dir(): continue - if not plugin_folder in plugin_manager().plugins: + if not plugin_entry.name in plugin_manager().plugins: continue - plugin_info = plugin_manager().plugins[plugin_folder] + plugin_info = plugin_manager().plugins[plugin_entry.name] - plugin_packs[plugin_folder]["identifier"] = plugin_folder - plugin_packs[plugin_folder]["display"] = plugin_info.name + plugin_packs[plugin_entry.name]["identifier"] = plugin_entry.name + plugin_packs[plugin_entry.name]["display"] = plugin_info.name - for language_folder in os.listdir(plugin_path): - plugin_packs[plugin_folder]["languages"].append(load_meta(os.path.join(plugin_path, language_folder), language_folder)) + for language_entry in scandir(plugin_entry.path): + plugin_packs[plugin_entry.name]["languages"].append(load_meta(language_entry.path, language_entry.name)) else: - core_packs.append(load_meta(os.path.join(translation_folder, folder), folder)) + core_packs.append(load_meta(entry.path, entry.name)) result = dict(_core=dict(identifier="_core", display="Core", languages=core_packs)) result.update(plugin_packs) diff --git a/src/octoprint/server/api/log.py b/src/octoprint/server/api/log.py index 7f668c77..3a35e4ce 100644 --- a/src/octoprint/server/api/log.py +++ b/src/octoprint/server/api/log.py @@ -5,7 +5,12 @@ __author__ = "Marc Hannappel Salandora" __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' __copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" + import os +try: + from os import scandir +except ImportError: + from scandir import scandir from flask import request, jsonify, url_for, make_response from werkzeug.utils import secure_filename @@ -52,15 +57,14 @@ def deleteLog(filename): def _getLogFiles(): files = [] basedir = settings().getBaseFolder("logs") - for osFile in os.listdir(basedir): - statResult = os.stat(os.path.join(basedir, osFile)) + for entry in scandir(basedir): files.append({ - "name": osFile, - "date": int(statResult.st_mtime), - "size": statResult.st_size, + "name": entry.name, + "date": int(entry.stat().st_mtime), + "size": entry.stat().st_size, "refs": { - "resource": url_for(".downloadLog", filename=osFile, _external=True), - "download": url_for("index", _external=True) + "downloads/logs/" + osFile + "resource": url_for(".downloadLog", filename=entry.name, _external=True), + "download": url_for("index", _external=True) + "downloads/logs/" + entry.name } }) diff --git a/src/octoprint/server/api/printer_profiles.py b/src/octoprint/server/api/printer_profiles.py index bce3b145..3b13ee2f 100644 --- a/src/octoprint/server/api/printer_profiles.py +++ b/src/octoprint/server/api/printer_profiles.py @@ -11,15 +11,33 @@ import copy from flask import jsonify, make_response, request, url_for from werkzeug.exceptions import BadRequest -from octoprint.server.api import api, NO_CONTENT -from octoprint.server.util.flask import restricted_access +from octoprint.server.api import api, NO_CONTENT, valid_boolean_trues +from octoprint.server.util.flask import restricted_access, with_revalidation_checking from octoprint.util import dict_merge from octoprint.server import printerProfileManager from octoprint.printer.profile import InvalidProfileError, CouldNotOverwriteError, SaveError +def _lastmodified(): + return printerProfileManager.last_modified + + +def _etag(lm=None): + if lm is None: + lm = _lastmodified() + + import hashlib + hash = hashlib.sha1() + hash.update(str(lm)) + hash.update(repr(printerProfileManager.get_default())) + return hash.hexdigest() + + @api.route("/printerprofiles", methods=["GET"]) +@with_revalidation_checking(etag_factory=_etag, + lastmodified_factory=_lastmodified, + unless=lambda: request.values.get("force", "false") in valid_boolean_trues) def printerProfilesList(): all_profiles = printerProfileManager.get_all() return jsonify(dict(profiles=_convert_profiles(all_profiles))) diff --git a/src/octoprint/server/api/settings.py b/src/octoprint/server/api/settings.py index be74a5ba..c60e0c51 100644 --- a/src/octoprint/server/api/settings.py +++ b/src/octoprint/server/api/settings.py @@ -11,19 +11,36 @@ from flask import request, jsonify, make_response from werkzeug.exceptions import BadRequest from octoprint.events import eventManager, Events -from octoprint.settings import settings +from octoprint.settings import settings, valid_boolean_trues from octoprint.server import admin_permission, printer from octoprint.server.api import api -from octoprint.server.util.flask import restricted_access +from octoprint.server.util.flask import restricted_access, with_revalidation_checking import octoprint.plugin import octoprint.util #~~ settings +def _lastmodified(): + return settings().last_modified + +def _etag(lm=None): + if lm is None: + lm = _lastmodified() + + connection_options = printer.__class__.get_connection_options() + + import hashlib + hash = hashlib.sha1() + hash.update(str(lm)) + hash.update(repr(connection_options)) + return hash.hexdigest() @api.route("/settings", methods=["GET"]) +@with_revalidation_checking(etag_factory=_etag, + lastmodified_factory=_lastmodified, + unless=lambda: request.values.get("force", "false") in valid_boolean_trues) def getSettings(): logger = logging.getLogger(__name__) diff --git a/src/octoprint/server/api/slicing.py b/src/octoprint/server/api/slicing.py index 0d6d6611..3aa8a33d 100644 --- a/src/octoprint/server/api/slicing.py +++ b/src/octoprint/server/api/slicing.py @@ -9,7 +9,7 @@ from flask import request, jsonify, make_response, url_for from werkzeug.exceptions import BadRequest from octoprint.server import slicingManager -from octoprint.server.util.flask import restricted_access +from octoprint.server.util.flask import restricted_access, with_revalidation_checking from octoprint.server.api import api, NO_CONTENT from octoprint.settings import settings as s, valid_boolean_trues @@ -17,7 +17,39 @@ from octoprint.settings import settings as s, valid_boolean_trues from octoprint.slicing import UnknownSlicer, SlicerNotConfigured, ProfileAlreadyExists, UnknownProfile, CouldNotDeleteProfile +def _lastmodified(configured): + if configured: + slicers = slicingManager.configured_slicers + else: + slicers = slicingManager.registered_slicers + + lms = [0] + for slicer in slicers: + lms.append(slicingManager.profiles_last_modified(slicer)) + + return max(lms) + + +def _etag(configured, lm=None): + if lm is None: + lm = _lastmodified(configured) + + import hashlib + hash = hashlib.sha1() + hash.update(str(lm)) + + if configured: + hash.update(repr(sorted(slicingManager.configured_slicers))) + else: + hash.update(repr(sorted(slicingManager.registered_slicers))) + + return hash.hexdigest() + + @api.route("/slicing", methods=["GET"]) +@with_revalidation_checking(etag_factory=lambda lm=None: _etag(request.values.get("configured", "false") in valid_boolean_trues, lm=lm), + lastmodified_factory=lambda: _lastmodified(request.values.get("configured", "false") in valid_boolean_trues), + unless=lambda: request.values.get("force", "false") in valid_boolean_trues) def slicingListAll(): from octoprint.filemanager import get_extensions diff --git a/src/octoprint/server/api/timelapse.py b/src/octoprint/server/api/timelapse.py index eaf00639..ad6079b1 100644 --- a/src/octoprint/server/api/timelapse.py +++ b/src/octoprint/server/api/timelapse.py @@ -6,6 +6,7 @@ __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agp __copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" import os +import threading from flask import request, jsonify, url_for, make_response from werkzeug.utils import secure_filename @@ -15,7 +16,7 @@ import octoprint.util as util from octoprint.settings import settings, valid_boolean_trues from octoprint.server import admin_permission, printer -from octoprint.server.util.flask import redirect_to_tornado, restricted_access, get_json_command_from_request +from octoprint.server.util.flask import redirect_to_tornado, restricted_access, get_json_command_from_request, with_revalidation_checking from octoprint.server.api import api from octoprint.server import NO_CONTENT @@ -23,8 +24,36 @@ from octoprint.server import NO_CONTENT #~~ timelapse handling +_timelapse_cache_finished = [] +_timelapse_cache_finished_lastmodified = None +_timelapse_cache_unrendered = [] +_timelapse_cache_unrendered_lastmodified = None +_timelapse_cache_mutex = threading.RLock() + +def _lastmodified(unrendered): + lm_finished = octoprint.timelapse.last_modified_finished() + if unrendered: + lm_unrendered = octoprint.timelapse.last_modified_unrendered() + + if lm_finished is None or lm_unrendered is None: + return None + return max(lm_finished, lm_unrendered) + return lm_finished + +def _etag(unrendered, lm=None): + if lm is None: + lm = _lastmodified(unrendered) + + import hashlib + hash = hashlib.sha1() + hash.update(str(lm)) + + return hash.hexdigest() @api.route("/timelapse", methods=["GET"]) +@with_revalidation_checking(etag_factory=lambda lm=None: _etag(request.values.get("unrendered", "false") in valid_boolean_trues, lm=lm), + lastmodified_factory=lambda: _lastmodified(request.values.get("unrendered", "false") in valid_boolean_trues), + unless=lambda: request.values.get("force", "false") in valid_boolean_trues) def getTimelapseData(): timelapse = octoprint.timelapse.current @@ -41,15 +70,41 @@ def getTimelapseData(): else: config = dict(type="off") - files = octoprint.timelapse.get_finished_timelapses() + force = request.values.get("force", "false") in valid_boolean_trues + unrendered = request.values.get("unrendered", "false") in valid_boolean_trues + + global _timelapse_cache_finished_lastmodified, _timelapse_cache_finished, _timelapse_cache_unrendered_lastmodified, _timelapse_cache_unrendered + with _timelapse_cache_mutex: + current_lastmodified_finished = octoprint.timelapse.last_modified_finished() + current_lastmodified_unrendered = octoprint.timelapse.last_modified_unrendered() + + if not force and _timelapse_cache_finished_lastmodified == current_lastmodified_finished: + files = _timelapse_cache_finished + else: + files = octoprint.timelapse.get_finished_timelapses() + _timelapse_cache_finished = files + _timelapse_cache_finished_lastmodified = current_lastmodified_finished + + unrendered_files = [] + if unrendered: + if not force and _timelapse_cache_unrendered_lastmodified == current_lastmodified_unrendered: + unrendered_files = _timelapse_cache_unrendered + else: + unrendered_files = octoprint.timelapse.get_unrendered_timelapses() + _timelapse_cache_unrendered = unrendered_files + _timelapse_cache_unrendered_lastmodified = current_lastmodified_unrendered + + finished_list = [] for f in files: - f["url"] = url_for("index") + "downloads/timelapse/" + f["name"] + output = dict(f) + output["url"] = url_for("index") + "downloads/timelapse/" + f["name"] + finished_list.append(output) result = dict(config=config, - files=files) + files=finished_list) - if "unrendered" in request.values and request.values["unrendered"] in valid_boolean_trues: - result.update(unrendered=octoprint.timelapse.get_unrendered_timelapses()) + if unrendered: + result.update(unrendered=unrendered_files) return jsonify(result) diff --git a/src/octoprint/server/util/__init__.py b/src/octoprint/server/util/__init__.py index c6416d81..368f8ea6 100644 --- a/src/octoprint/server/util/__init__.py +++ b/src/octoprint/server/util/__init__.py @@ -92,6 +92,20 @@ def noCachingResponseHandler(resp): return flask.add_non_caching_response_headers(resp) +def noCachingExceptGetResponseHandler(resp): + """ + ``after_request`` handler for blueprints which shall set no caching headers + on their responses to any requests that are not sent with method ``GET``. + + See :func:`noCachingResponseHandler`. + """ + + if _flask.request.method == "GET": + return flask.add_no_max_age_response_headers(resp) + else: + return flask.add_non_caching_response_headers(resp) + + def optionsAllowOrigin(request): """ Shortcut for request handling for CORS OPTIONS requests to set CORS headers. diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index 98a7b4e0..045166c8 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -29,6 +29,10 @@ import octoprint.plugin from werkzeug.contrib.cache import BaseCache +try: + from os import scandir, walk +except ImportError: + from scandir import scandir, walk #~~ monkey patching @@ -52,12 +56,12 @@ def enable_additional_translations(default_locale="en", additional_folders=None) if not os.path.isdir(dirname): return [] result = [] - for folder in os.listdir(dirname): - locale_dir = os.path.join(dirname, folder, 'LC_MESSAGES') + for entry in scandir(dirname): + locale_dir = os.path.join(entry.path, 'LC_MESSAGES') if not os.path.isdir(locale_dir): continue - if filter(lambda x: x.endswith('.mo'), os.listdir(locale_dir)): - result.append(Locale.parse(folder)) + if filter(lambda x: x.name.endswith('.mo'), scandir(locale_dir)): + result.append(Locale.parse(entry.name)) if not result: result.append(Locale.parse(self._default_locale)) return result @@ -622,6 +626,61 @@ def conditional(condition, met): return decorator +def with_revalidation_checking(etag_factory=None, + lastmodified_factory=None, + condition=None, + unless=None): + if etag_factory is None: + def etag_factory(lm=None): + return None + + if lastmodified_factory is None: + def lastmodified_factory(): + return None + + if condition is None: + def condition(lm=None, etag=None): + if lm is None: + lm = lastmodified_factory() + + if etag is None: + etag = etag_factory(lm=lm) + + return check_lastmodified(lm) and check_etag(etag) + + if unless is None: + def unless(): + return False + + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + lm = lastmodified_factory() + etag = etag_factory(lm) + + if condition(lm, etag) and not unless(): + return make_response("Not Modified", 304) + + # generate response + response = f(*args, **kwargs) + + # set etag header if not already set + if etag and response.get_etag()[0] is None: + response.set_etag(etag) + + # set last modified header if not already set + if lm and response.headers.get("Last-Modified", None) is None: + if not isinstance(lm, basestring): + from werkzeug.http import http_date + lm = http_date(lm) + response.headers["Last-Modified"] = lm + + response = add_no_max_age_response_headers(response) + return response + return decorated_function + return decorator + + def check_etag(etag): return flask.request.method in ("GET", "HEAD") and \ flask.request.if_none_match is not None and \ @@ -629,6 +688,10 @@ def check_etag(etag): def check_lastmodified(lastmodified): + if isinstance(lastmodified, float): + from datetime import datetime + lastmodified = datetime.fromtimestamp(lastmodified).replace(microsecond=0) + return flask.request.method in ("GET", "HEAD") and \ flask.request.if_modified_since is not None and \ lastmodified >= flask.request.if_modified_since @@ -641,6 +704,11 @@ def add_non_caching_response_headers(response): return response +def add_no_max_age_response_headers(response): + response.headers["Cache-Control"] = "max-age=0" + return response + + #~~ access validators for use with tornado diff --git a/src/octoprint/settings.py b/src/octoprint/settings.py index 10611203..c8722c1e 100644 --- a/src/octoprint/settings.py +++ b/src/octoprint/settings.py @@ -31,6 +31,8 @@ import logging import re import uuid import copy +import time + from builtins import bytes try: @@ -527,6 +529,7 @@ class Settings(object): self._config = None self._dirty = False + self._dirty_time = 0 self._mtime = None self._get_preprocessors = dict( @@ -767,6 +770,10 @@ class Settings(object): stat = os.stat(self._configfile) return stat.st_mtime + @property + def last_modified_or_made_dirty(self): + return max(self.last_modified, self._dirty_time) + #~~ load and save def load(self, migrate=False): @@ -1270,6 +1277,7 @@ class Settings(object): try: chain.del_by_path(path) self._dirty = True + self._dirty_time = time.time() except KeyError: if error_on_path: raise NoSuchSettingsPath() @@ -1321,6 +1329,7 @@ class Settings(object): try: chain.del_by_path(path) self._dirty = True + self._dirty_time = time.time() except KeyError: if error_on_path: raise NoSuchSettingsPath() @@ -1331,6 +1340,7 @@ class Settings(object): else: chain.set_by_path(path, value) self._dirty = True + self._dirty_time = time.time() def setInt(self, path, value, **kwargs): if value is None: @@ -1377,11 +1387,13 @@ class Settings(object): if not self._config["folder"]: del self._config["folder"] self._dirty = True + self._dirty_time = time.time() elif (path != currentPath and path != defaultPath) or force: if not "folder" in self._config.keys(): self._config["folder"] = {} self._config["folder"][type] = path self._dirty = True + self._dirty_time = time.time() def saveScript(self, script_type, name, script): script_folder = self.getBaseFolder("scripts") diff --git a/src/octoprint/slicing/__init__.py b/src/octoprint/slicing/__init__.py index 22682e1b..a1546c0a 100644 --- a/src/octoprint/slicing/__init__.py +++ b/src/octoprint/slicing/__init__.py @@ -20,6 +20,12 @@ __copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms import os + +try: + from os import scandir +except ImportError: + from scandir import scandir + import octoprint.plugin import octoprint.events import octoprint.util @@ -543,17 +549,34 @@ class SlicingManager(object): profiles = dict() slicer_profile_path = self.get_slicer_profile_path(slicer) - for entry in os.listdir(slicer_profile_path): - if not entry.endswith(".profile") or octoprint.util.is_hidden_path(entry): + for entry in scandir(slicer_profile_path): + if not entry.name.endswith(".profile") or octoprint.util.is_hidden_path(entry.name): # we are only interested in profiles and no hidden files continue - path = os.path.join(slicer_profile_path, entry) - profile_name = entry[:-len(".profile")] - - profiles[profile_name] = self._load_profile_from_path(slicer, path, require_configured=require_configured) + profile_name = entry.name[:-len(".profile")] + profiles[profile_name] = self._load_profile_from_path(slicer, entry.path, require_configured=require_configured) return profiles + def profiles_last_modified(self, slicer): + """ + Retrieves the last modification date of ``slicer``'s profiles. + + Args: + slicer (str): the slicer for which to retrieve the last modification date + + Returns: + (float) the time stamp of the last modification of the slicer's profiles + """ + + if not slicer in self.registered_slicers: + raise UnknownSlicer(slicer) + + slicer_profile_path = self.get_slicer_profile_path(slicer) + lms = [os.stat(slicer_profile_path).st_mtime] + lms += [os.stat(entry.path).st_mtime for entry in scandir(slicer_profile_path) if entry.name.endswith(".profile")] + return max(lms) + def get_slicer_profile_path(self, slicer): """ Retrieves the path where the profiles for slicer ``slicer`` are stored. diff --git a/src/octoprint/static/js/app/client/files.js b/src/octoprint/static/js/app/client/files.js index 902a3c18..25f542f3 100644 --- a/src/octoprint/static/js/app/client/files.js +++ b/src/octoprint/static/js/app/client/files.js @@ -50,9 +50,19 @@ OctoPrint.files = { get: getEntry, - list: function (recursively, opts) { + list: function (recursively, force, opts) { recursively = recursively || false; - return OctoPrint.getWithQuery(url, {recursive: recursively}, opts) + force = force || false; + + var query = {}; + if (recursively) { + query.recursive = recursively; + } + if (force) { + query.force = force; + } + + return OctoPrint.getWithQuery(url, query, opts) .done(preProcessList); }, diff --git a/src/octoprint/static/js/app/main.js b/src/octoprint/static/js/app/main.js index e3d9e8b8..d21cdd5f 100644 --- a/src/octoprint/static/js/app/main.js +++ b/src/octoprint/static/js/app/main.js @@ -95,9 +95,15 @@ $(function() { // work around a stupid iOS6 bug where ajax requests get cached and only work once, as described at // http://stackoverflow.com/questions/12506897/is-safari-on-ios-6-caching-ajax-results - $.ajaxSetup({ - type: 'POST', - headers: { "cache-control": "no-cache" } + $.ajaxPrefilter(function(options, originalOptions, jqXHR) { + if (options.type != "GET") { + var headers; + if (options.hasOwnProperty("headers")) { + options.headers["Cache-Control"] = "no-cache"; + } else { + options.headers = { "Cache-Control": "no-cache" }; + } + } }); // send the current UI API key with any request diff --git a/src/octoprint/static/js/app/viewmodels/files.js b/src/octoprint/static/js/app/viewmodels/files.js index 750c7dec..620b8102 100644 --- a/src/octoprint/static/js/app/viewmodels/files.js +++ b/src/octoprint/static/js/app/viewmodels/files.js @@ -203,15 +203,25 @@ $(function() { self._filenameToFocus = undefined; self._locationToFocus = undefined; self._switchToPath = undefined; - self.requestData = function(filenameToFocus, locationToFocus, switchToPath) { - self._filenameToFocus = self._filenameToFocus || filenameToFocus; - self._locationToFocus = self._locationToFocus || locationToFocus; - self._switchToPath = self._switchToPath || switchToPath; + self.requestData = function(filenameToFocus, locationToFocus, switchToPath, force) { + if (arguments.length == 1 && _.isObject(arguments[0])) { + var params = arguments[0]; + self._filenameToFocus = self._filenameToFocus || params.filenameToFocus; + self._locationToFocus = self._locationToFocus || params.locationToFocus; + self._switchToPath = self._switchToPath || params.switchToPath; + force = params.force || false; + } else { + self._filenameToFocus = self._filenameToFocus || filenameToFocus; + self._locationToFocus = self._locationToFocus || locationToFocus; + self._switchToPath = self._switchToPath || switchToPath; + force = force || false; + } + if (self._otherRequestInProgress !== undefined) { return self._otherRequestInProgress } - return self._otherRequestInProgress = OctoPrint.files.list(true) + return self._otherRequestInProgress = OctoPrint.files.list(true, force) .done(function(response) { self.fromResponse(response, self._filenameToFocus, self._locationToFocus, self._switchToPath); }) diff --git a/src/octoprint/static/js/app/viewmodels/log.js b/src/octoprint/static/js/app/viewmodels/log.js index 872eb288..5f2bedbd 100644 --- a/src/octoprint/static/js/app/viewmodels/log.js +++ b/src/octoprint/static/js/app/viewmodels/log.js @@ -53,10 +53,8 @@ $(function() { .done(self.requestData); }; - self.onUserLoggedIn = function(user) { - if (user.admin) { - self.requestData(); - } + self.onSettingsShown = function() { + self.requestData(); }; } diff --git a/src/octoprint/static/js/app/viewmodels/slicing.js b/src/octoprint/static/js/app/viewmodels/slicing.js index 25f4f7f5..8e39b506 100644 --- a/src/octoprint/static/js/app/viewmodels/slicing.js +++ b/src/octoprint/static/js/app/viewmodels/slicing.js @@ -23,6 +23,8 @@ $(function() { self.profiles = ko.observableArray(); self.printerProfile = ko.observable(); + self.allViewModels = undefined; + self.slicersForFile = function(file) { if (file === undefined) { return []; @@ -208,6 +210,10 @@ $(function() { }); self.defaultSlicer = selectedSlicer; + + if (self.allViewModels) { + callViewModels(self.allViewModels, "onSlicingData", [data]); + } }; self.slice = function() { @@ -260,6 +266,10 @@ $(function() { self.onEventSettingsUpdated = function(payload) { self.requestData(); }; + + self.onAllBound = function(allViewModels) { + self.allViewModels = allViewModels; + }; } OCTOPRINT_VIEWMODELS.push([ diff --git a/src/octoprint/templates/sidebar/files_header.jinja2 b/src/octoprint/templates/sidebar/files_header.jinja2 index cb30d1b7..a5955083 100644 --- a/src/octoprint/templates/sidebar/files_header.jinja2 +++ b/src/octoprint/templates/sidebar/files_header.jinja2 @@ -29,7 +29,7 @@
- +
diff --git a/src/octoprint/timelapse.py b/src/octoprint/timelapse.py index 0d78e5ab..42777224 100644 --- a/src/octoprint/timelapse.py +++ b/src/octoprint/timelapse.py @@ -27,6 +27,12 @@ import collections import re +try: + from os import scandir, walk +except ImportError: + from scandir import scandir, walk + + # currently configured timelapse current = None @@ -67,18 +73,25 @@ def _extract_prefix(filename): return filename[:pos] +def last_modified_finished(): + return os.stat(settings().getBaseFolder("timelapse")).st_mtime + + +def last_modified_unrendered(): + return os.stat(settings().getBaseFolder("timelapse_tmp")).st_mtime + + def get_finished_timelapses(): files = [] basedir = settings().getBaseFolder("timelapse") - for osFile in os.listdir(basedir): - if not fnmatch.fnmatch(osFile, "*.mp[g4]"): + for entry in scandir(basedir): + if not fnmatch.fnmatch(entry.name, "*.mp[g4]"): continue - statResult = os.stat(os.path.join(basedir, osFile)) files.append({ - "name": osFile, - "size": util.get_formatted_size(statResult.st_size), - "bytes": statResult.st_size, - "date": util.get_formatted_datetime(datetime.datetime.fromtimestamp(statResult.st_ctime)) + "name": entry.name, + "size": util.get_formatted_size(entry.stat().st_size), + "bytes": entry.stat().st_size, + "date": util.get_formatted_datetime(datetime.datetime.fromtimestamp(entry.stat().st_ctime)) }) return files @@ -92,19 +105,18 @@ def get_unrendered_timelapses(): basedir = settings().getBaseFolder("timelapse_tmp") jobs = collections.defaultdict(lambda: dict(count=0, size=None, bytes=0, date=None, timestamp=None)) - for osFile in os.listdir(basedir): - if not fnmatch.fnmatch(osFile, "*.jpg"): + for entry in scandir(basedir): + if not fnmatch.fnmatch(entry.name, "*.jpg"): continue - prefix = _extract_prefix(osFile) + prefix = _extract_prefix(entry.name) if prefix is None: continue - statResult = os.stat(os.path.join(basedir, osFile)) jobs[prefix]["count"] += 1 - jobs[prefix]["bytes"] += statResult.st_size - if jobs[prefix]["timestamp"] is None or statResult.st_ctime < jobs[prefix]["timestamp"]: - jobs[prefix]["timestamp"] = statResult.st_ctime + jobs[prefix]["bytes"] += entry.stat().st_size + if jobs[prefix]["timestamp"] is None or entry.stat().st_ctime < jobs[prefix]["timestamp"]: + jobs[prefix]["timestamp"] = entry.stat().st_ctime with _job_lock: global current_render_job @@ -130,13 +142,13 @@ def delete_unrendered_timelapse(name): basedir = settings().getBaseFolder("timelapse_tmp") with _cleanup_lock: - for filename in os.listdir(basedir): + for entry in scandir(basedir): try: - if fnmatch.fnmatch(filename, "{}*.jpg".format(name)): - os.remove(os.path.join(basedir, filename)) + if fnmatch.fnmatch(entry.name, "{}*.jpg".format(name)): + os.remove(entry.path) except: if logging.getLogger(__name__).isEnabledFor(logging.DEBUG): - logging.getLogger(__name__).exception("Error while processing file {} during cleanup".format(filename)) + logging.getLogger(__name__).exception("Error while processing file {} during cleanup".format(entry.name)) def render_unrendered_timelapse(name, gcode=None, postfix=None, fps=25): @@ -167,26 +179,24 @@ def delete_old_unrendered_timelapses(): prefixes_to_clean = [] with _cleanup_lock: - for filename in os.listdir(basedir): + for entry in scandir(basedir): try: - path = os.path.join(basedir, filename) - - prefix = _extract_prefix(filename) + prefix = _extract_prefix(entry.name) if prefix is None: # might be an old tmp_00000.jpg kinda frame. we can't # render those easily anymore, so delete that stuff - if _old_capture_format_re.match(filename): - os.remove(path) + if _old_capture_format_re.match(entry.name): + os.remove(entry.path) continue if prefix in prefixes_to_clean: continue - if os.path.getmtime(path) < cutoff: + if os.path.getmtime(entry.path) < cutoff: prefixes_to_clean.append(prefix) except: if logging.getLogger(__name__).isEnabledFor(logging.DEBUG): - logging.getLogger(__name__).exception("Error while processing file {} during cleanup".format(filename)) + logging.getLogger(__name__).exception("Error while processing file {} during cleanup".format(entry.name)) for prefix in prefixes_to_clean: delete_unrendered_timelapse(prefix) diff --git a/src/octoprint/util/jinja.py b/src/octoprint/util/jinja.py index ba55d58c..3b851961 100644 --- a/src/octoprint/util/jinja.py +++ b/src/octoprint/util/jinja.py @@ -7,6 +7,11 @@ __copyright__ = "Copyright (C) 2015 The OctoPrint Project - Released under terms import logging import os +try: + from os import scandir, walk +except ImportError: + from scandir import scandir, walk + from jinja2 import nodes from jinja2.ext import Extension from jinja2.loaders import FileSystemLoader, PrefixLoader, ChoiceLoader, \ @@ -90,7 +95,7 @@ class SelectedFilesLoader(BaseLoader): def get_all_template_paths(loader): def walk_folder(folder): files = [] - walk_dir = os.walk(folder, followlinks=True) + walk_dir = walk(folder, followlinks=True) for dirpath, dirnames, filenames in walk_dir: for filename in filenames: path = os.path.join(dirpath, filename)