From 28738a517972733dfe6d58cdf99de854500a5e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 18 Feb 2015 17:16:37 +0100 Subject: [PATCH] More documentation and unit tests --- docs/development/index.rst | 10 + docs/development/interface/filemanager.rst | 35 ++ docs/development/interface/index.rst | 11 + docs/development/interface/plugin.rst | 8 + docs/development/interface/printer.rst | 8 + docs/index.rst | 1 + src/octoprint/filemanager/__init__.py | 12 +- src/octoprint/filemanager/storage.py | 581 +++++++++++------- src/octoprint/server/api/files.py | 6 +- .../static/js/app/viewmodels/files.js | 2 +- tests/filemanager/test_localstorage.py | 74 +++ 11 files changed, 510 insertions(+), 238 deletions(-) create mode 100644 docs/development/index.rst create mode 100644 docs/development/interface/filemanager.rst create mode 100644 docs/development/interface/index.rst create mode 100644 docs/development/interface/plugin.rst create mode 100644 docs/development/interface/printer.rst diff --git a/docs/development/index.rst b/docs/development/index.rst new file mode 100644 index 00000000..9edfcb67 --- /dev/null +++ b/docs/development/index.rst @@ -0,0 +1,10 @@ +.. _sec-development: + +########### +Development +########### + +.. toctree:: + :maxdepth: 3 + + interface/index.rst \ No newline at end of file diff --git a/docs/development/interface/filemanager.rst b/docs/development/interface/filemanager.rst new file mode 100644 index 00000000..fd856844 --- /dev/null +++ b/docs/development/interface/filemanager.rst @@ -0,0 +1,35 @@ +.. _sec-development-interface-filemanager: + +``octoprint.filemanager`` +------------------------- + +.. automodule:: octoprint.filemanager + :members: + :undoc-members: + +.. _sec-development-interface-filemanager-analysis: + +``octoprint.filemanager.analysis`` +---------------------------------- + +.. automodule:: octoprint.filemanager.analysis + :members: + :undoc-members: + +.. _sec-development-interface-filemanager-destinations: + +``octoprint.filemanager.destinations`` +-------------------------------------- + +.. automodule:: octoprint.filemanager.destinations + :members: + :undoc-members: + +.. _sec-development-interface-filemanager-storage: + +``octoprint.filemanager.storage`` +--------------------------------- + +.. automodule:: octoprint.filemanager.storage + :members: StorageInterface, LocalFileStorage + diff --git a/docs/development/interface/index.rst b/docs/development/interface/index.rst new file mode 100644 index 00000000..6c4866c2 --- /dev/null +++ b/docs/development/interface/index.rst @@ -0,0 +1,11 @@ +.. _sec-development-interface: + +Interfaces +========== + +.. toctree:: + :maxdepth: 3 + + filemanager.rst + plugin.rst + printer.rst diff --git a/docs/development/interface/plugin.rst b/docs/development/interface/plugin.rst new file mode 100644 index 00000000..1b659481 --- /dev/null +++ b/docs/development/interface/plugin.rst @@ -0,0 +1,8 @@ +.. _sec-development-interface-plugin: + +``octoprint.plugin`` +-------------------- + +.. automodule:: octoprint.plugin + :members: plugin_manager, plugin_settings, call_plugin, PluginSettings + :undoc-members: diff --git a/docs/development/interface/printer.rst b/docs/development/interface/printer.rst new file mode 100644 index 00000000..7d6ed019 --- /dev/null +++ b/docs/development/interface/printer.rst @@ -0,0 +1,8 @@ +.. _sec-development-interface-plugin: + +``octoprint.printer`` +--------------------- + +.. automodule:: octoprint.printer + :members: Printer + :undoc-members: diff --git a/docs/index.rst b/docs/index.rst index 3375a90c..5243f634 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,4 +19,5 @@ Contents api/index.rst events/index.rst plugins/index.rst + development/index.rst diff --git a/src/octoprint/filemanager/__init__.py b/src/octoprint/filemanager/__init__.py index 9c65a67a..85e1931c 100644 --- a/src/octoprint/filemanager/__init__.py +++ b/src/octoprint/filemanager/__init__.py @@ -157,7 +157,7 @@ class FileManager(object): def slice(self, slicer_name, source_location, source_path, dest_location, dest_path, position=None, profile=None, printer_profile_id=None, overrides=None, callback=None, callback_args=None): - absolute_source_path = self.get_absolute_path(source_location, source_path) + absolute_source_path = self.path_on_disk(source_location, source_path) def stlProcessed(source_location, source_path, tmp_path, dest_location, dest_path, start_time, printer_profile_id, callback, callback_args, _error=None, _cancelled=False, _analysis=None): try: @@ -291,7 +291,7 @@ class FileManager(object): printer_profile = self._printer_profile_manager.get_current_or_default() file_path = self._storage(destination).add_file(path, file_object, links=links, printer_profile=printer_profile, allow_overwrite=allow_overwrite) - absolute_path = self._storage(destination).get_absolute_path(file_path) + absolute_path = self._storage(destination).path_on_disk(file_path) if analysis is None: file_type = get_file_type(absolute_path) @@ -343,8 +343,8 @@ class FileManager(object): def remove_additional_metadata(self, destination, path, key): self._storage(destination).remove_additional_metadata(path, key) - def get_absolute_path(self, destination, path): - return self._storage(destination).get_absolute_path(path) + def path_on_disk(self, destination, path): + return self._storage(destination).path_on_disk(path) def sanitize(self, destination, path): return self._storage(destination).sanitize(path) @@ -361,8 +361,8 @@ class FileManager(object): def join_path(self, destination, *path): return self._storage(destination).join_path(*path) - def rel_path(self, destination, path): - return self._storage(destination).rel_path(path) + def path_in_storage(self, destination, path): + return self._storage(destination).path_in_storage(path) def _storage(self, destination): if not destination in self._storage_managers: diff --git a/src/octoprint/filemanager/storage.py b/src/octoprint/filemanager/storage.py index b2cf6142..331446ce 100644 --- a/src/octoprint/filemanager/storage.py +++ b/src/octoprint/filemanager/storage.py @@ -15,73 +15,289 @@ import octoprint.filemanager from octoprint.util import safeRename - class StorageInterface(object): + """ + Interface of storage adapters for OctoPrint. + """ + @property def analysis_backlog(self): + """ + Get an iterator over all items stored in the storage that need to be analysed by the :class:`~octoprint.filemanager.AnalysisQueue`. + + The yielded elements are expected as storage specific absolute paths to the respective files. Don't forget + to recurse into folders if your storage adapter supports those. + + :return: an iterator yielding all un-analysed files in the storage + """ # empty generator pattern, yield is intentionally unreachable return yield def file_exists(self, path): + """ + Returns whether the file indicated by ``path`` exists or not. + :param string path: the path to check for existence + :return: ``True`` if the file exists, ``False`` otherwise + """ raise NotImplementedError() def list_files(self, path=None, filter=None, recursive=True): + """ + List all files in storage starting at ``path``. If ``recursive`` is set to True (the default), also dives into + subfolders. + + An optional filter function can be supplied which will be called with a file name and file data and which has + to return True if the file is to be included in the result or False if not. + + The data structure of the returned result will be a dictionary mapping from file names to entry data. File nodes + will contain their metadata here, folder nodes will contain their contained files and folders. Example:: + + { + "some_folder": { + "type": "folder", + "children": { + "some_sub_folder": { + "type": "folder", + "children": { ... } + }, + "some_file.gcode": { + "type": "machinecode", + "hash": "", + "links": [ ... ], + ... + }, + ... + } + "test.gcode": { + "type": "machinecode", + "hash": "", + "links": [...], + ... + }, + "test.stl": { + "type": "model", + "hash": "", + "links": [...], + ... + }, + ... + } + + :param string path: base path from which to recursively list all files, optional, if not supplied listing will start + from root of base folder + :param function filter: a filter that matches the files that are to be returned, may be left out in which case no + filtering will take place + :param bool recursive: will also step into sub folders for building the complete list if set to True + :return: a dictionary mapping entry names to entry data that represents the whole file list + """ raise NotImplementedError() def add_folder(self, path, ignore_existing=True): + """ + Adds a folder as ``path``. The ``path`` will be sanitized. + + :param string path: the path of the new folder + :param bool ignore_existing: if set to True, no error will be raised if the folder to be added already exists + :return: the sanitized name of the new folder to be used for future references to the folder + """ raise NotImplementedError() def remove_folder(self, path, recursive=True): + """ + Removes the folder at ``path``. + + :param string path: the path of the folder to remove + :param bool recursive: if set to True, contained folders and files will also be removed, otherwise and error will + be raised if the folder is not empty (apart from ``.metadata.yaml``) when it's to be removed + """ raise NotImplementedError() def add_file(self, path, file_object, printer_profile=None, links=None, allow_overwrite=False): + """ + Adds the file ``file_object`` as ``path`` + + :param string path: the file's new path, will be sanitized + :param object file_object: a file object that provides a ``save`` method which will be called with the destination path + where the object should then store its contents + :param object printer_profile: the printer profile associated with this file (if any) + :param list links: any links to add with the file + :param bool allow_overwrite: if set to True no error will be raised if the file already exists and the existing file + and its metadata will just be silently overwritten + :return: the sanitized name of the file to be used for future references to it + """ raise NotImplementedError() def remove_file(self, path): + """ + Removes the file at ``path``. Will also take care of deleting the corresponding entries + in the metadata and deleting all links pointing to the file. + + :param string path: path of the file to remove + """ raise NotImplementedError() def get_metadata(self, path): + """ + Retrieves the metadata for the file ``path``. + + :param path: virtual path to the file for which to retrieve the metadata + :return: the metadata associated with the file + """ raise NotImplementedError() def add_link(self, path, rel, data): + """ + Adds a link of relation ``rel`` to file ``path`` with the given ``data``. + + The following relation types are currently supported: + + * ``model``: adds a link to a model from which the file was created/sliced, expected additional data is the ``name`` + and optionally the ``hash`` of the file to link to. If the link can be resolved against another file on the + current ``path``, not only will it be added to the links of ``name`` but a reverse link of type ``machinecode`` + refering to ``name`` and its hash will also be added to the linked ``model`` file + * ``machinecode``: adds a link to a file containing machine code created from the current file (model), expected + additional data is the ``name`` and optionally the ``hash`` of the file to link to. If the link can be resolved + against another file on the current ``path``, not only will it be added to the links of ``name`` but a reverse + link of type ``model`` refering to ``name`` and its hash will also be added to the linked ``model`` file. + * ``web``: adds a location on the web associated with this file (e.g. a website where to download a model), + expected additional data is a ``href`` attribute holding the website's URL and optionally a ``retrieved`` + attribute describing when the content was retrieved + + Note that adding ``model`` links to files identifying as models or ``machinecode`` links to files identifying + as machine code will be refused. + + :param path: path of the file for which to add a link + :param rel: type of relation of the link to add (currently ``model``, ``machinecode`` and ``web`` are supported) + :param data: additional data of the link to add + """ raise NotImplementedError() def remove_link(self, path, rel, data): + """ + Removes the link consisting of ``rel`` and ``data`` from file ``name`` on ``path``. + + :param path: path of the file from which to remove the link + :param rel: type of relation of the link to remove (currently ``model``, ``machinecode`` and ``web`` are supported) + :param data: additional data of the link to remove, must match existing link + """ raise NotImplementedError() def set_additional_metadata(self, path, key, data, overwrite=False, merge=False): + """ + Adds additional metadata to the metadata of ``path``. Metadata in ``data`` will be saved under ``key``. + + If ``overwrite`` is set and ``key`` already exists in ``name``'s metadata, the current value will be overwritten. + + If ``merge`` is set and ``key`` already exists and both ``data`` and the existing data under ``key`` are dictionaries, + the two dictionaries will be merged recursively. + + :param path: the virtual path to the file for which to add additional metadata + :param key: key of metadata to add + :param data: metadata to add + :param overwrite: if True and ``key`` already exists, it will be overwritten + :param merge: if True and ``key`` already exists and both ``data`` and the existing data are dictionaries, they + will be merged + """ raise NotImplementedError() def remove_additional_metadata(self, path, key): + """ + Removes additional metadata under ``key`` for ``name`` on ``path`` + + :param path: the virtual path to the file for which to remove the metadata under ``key`` + :param key: the key to remove + """ raise NotImplementedError() def sanitize(self, path): + """ + Sanitizes the given ``path``, stripping it of all invalid characters. The ``path`` may consist of both + folder and file name, the underlying implementation must separate those if necessary and sanitize individually. + + :param string path: the path to sanitize + :return: a 2-tuple containing the sanitized path and file name + """ raise NotImplementedError() def sanitize_path(self, path): + """ + Sanitizes the given folder-only ``path``, stripping it of all invalid characters. + :param string path: the path to sanitize + :return: the sanitized path + """ raise NotImplementedError() def sanitize_name(self, name): + """ + Sanitizes the given file ``name``, stripping it of all invalid characters. + :param string name: the file name to sanitize + :return: the sanitized name + """ raise NotImplementedError() def split_path(self, path): + """ + Split ``path`` into base directory and file name. + :param path: the path to split + :return: a tuple (base directory, file name) + """ raise NotImplementedError() def join_path(self, *path): + """ + Join path elements together + :param path: path elements to join + :return: joined representation of the path to be usable as fully qualified path for further operations + """ raise NotImplementedError() - def rel_path(self, path): + def path_on_disk(self, path): + """ + Retrieves the path on disk for ``path``. + + Note: if the storage is not on disk and there exists no path on disk to refer to it, this method should + raise an :class:`io.UnsupportedOperation` + + Opposite of :func:`path_in_storage`. + + :param string path: the virtual path for which to retrieve the path on disk + :return: the path on disk to ``path`` + """ raise NotImplementedError() - def get_absolute_path(self, path): + def path_in_storage(self, path): + """ + Retrieves the equivalent in the storage adapter for ``path``. + + Opposite of :func:`path_on_disk`. + + :param string path: the path for which to retrieve the storage path + :return: the path in storage to ``path`` + """ raise NotImplementedError() + class LocalFileStorage(StorageInterface): + """ + The ``LocalFileStorage`` is a storage implementation which holds all files, folders and metadata on disk. + + Metadata is managed inside ``.metadata.yaml`` files in the respective folders, indexed by the sanitized filenames + stored within the folder. Metadata access is managed through an LRU cache to minimize access overhead. + + This storage type implements :func:`path_on_disk`. + """ def __init__(self, basefolder, create=False): + """ + Initializes a ``LocalFileStorage`` instance under the given ``basefolder``, creating the necessary folder + if necessary and ``create`` is set to ``True``. + + :param string basefolder: the path to the folder under which to create the storage + :param bool create: ``True`` if the folder should be created if it doesn't exist yet, ``False`` otherwise + """ self._logger = logging.getLogger(__name__) self.basefolder = os.path.realpath(os.path.abspath(basefolder)) @@ -131,55 +347,6 @@ class LocalFileStorage(StorageInterface): return os.path.exists(file_path) and os.path.isfile(file_path) def list_files(self, path=None, filter=None, recursive=True): - """ - List all files in storage starting at ``path``. If ``recursive`` is set to True (the default), also dives into - subfolders. - - An optional filter function can be supplied which will be called with a file name and file data and which has - to return True if the file is to be included in the result or False if not. - - The data structure of the returned result will be a dictionary mapping from file names to entry data. File nodes - will contain their metadata here, folder nodes will contain their contained files and folders. Example:: - - { - "some_folder": { - "type": "folder", - "children": { - "some_sub_folder": { - "type": "folder", - "children": { ... } - }, - "some_file.gcode": { - "type": "machinecode", - "hash": "", - "links": [ ... ], - ... - }, - ... - } - "test.gcode": { - "type": "machinecode", - "hash": "", - "links": [...], - ... - }, - "test.stl": { - "type": "model", - "hash": "", - "links": [...], - ... - }, - ... - } - - :param path: base path from which to recursively list all files, optional, if not supplied listing will start - from root of base folder - :param filter: a filter that matches the files that are to be returned, may be left out in which case no - filtering will take place - :param recursive: will also step into sub folders for building the complete list if set to True - :return: a dictionary mapping entry names to entry data that represents the whole file list - """ - if path: path = self.sanitize_path(path) else: @@ -187,14 +354,6 @@ class LocalFileStorage(StorageInterface): return self._list_folder(path, filter=filter, recursive=recursive) def add_folder(self, path, ignore_existing=True): - """ - Adds a folder as ``path``. The ``path`` will be sanitized. - - :param path: the path of the new folder - :param ignore_existing: if set to True, no error will be raised if the folder to be added already exists - :return: the sanitized name of the new folder to be used for future references to the folder - """ - path, name = self.sanitize(path) folder_path = os.path.join(path, name) @@ -204,17 +363,9 @@ class LocalFileStorage(StorageInterface): else: os.mkdir(folder_path) - return self.rel_path((path, name)) + return self.path_in_storage((path, name)) def remove_folder(self, path, recursive=True): - """ - Removes the folder at ``path``. - - :param path: the path of the folder to remove - :param recursive: if set to True, contained folders and files will also be removed, otherwise and error will - be raised if the folder is not empty (apart from ``.metadata.yaml``) when it's to be removed - """ - path, name = self.sanitize(path) folder_path = os.path.join(path, name) @@ -231,19 +382,6 @@ class LocalFileStorage(StorageInterface): shutil.rmtree(folder_path) def add_file(self, path, file_object, printer_profile=None, links=None, allow_overwrite=False): - """ - Adds the file ``file_object`` as ``path`` - - :param path: the file's new path, will be sanitized - :param file_object: a file object that provides a ``save`` method which will be called with the destination path - where the object should then store its contents - :param printer_profile: the printer profile associated with this file (if any) - :param links: any links to add with the file - :param allow_overwrite: if set to True no error will be raised if the file already exists and the existing file - and its metadata will just be silently overwritten - :return: the sanitized name of the file to be used for future references to it - """ - path, name = self.sanitize(path) if not octoprint.filemanager.valid_file_type(name): raise RuntimeError("{name} is an unrecognized file type".format(**locals())) @@ -284,16 +422,9 @@ class LocalFileStorage(StorageInterface): self._add_links(name, path, links) - return self.rel_path((path, name)) + return self.path_in_storage((path, name)) def remove_file(self, path): - """ - Removes the file at ``path``. Will also take care of deleting the corresponding entries - in the metadata and deleting all links pointing to the file. - - :param path: path of the file to remove - """ - path, name = self.sanitize(path) metadata = self._get_metadata(path) @@ -322,13 +453,6 @@ class LocalFileStorage(StorageInterface): self._save_metadata(path, metadata) def get_metadata(self, path): - """ - Retrieves the metadata for the file ``path``. - - :param path: virtual path to the file for which to retrieve the metadata - :return: the metadata associated with the file - """ - path, name = self.sanitize(path) metadata = self._get_metadata(path) @@ -343,43 +467,10 @@ class LocalFileStorage(StorageInterface): def add_link(self, path, rel, data): - """ - Adds a link of relation ``rel`` to file ``path`` with the given ``data``. - - The following relation types are currently supported: - - * ``model``: adds a link to a model from which the file was created/sliced, expected additional data is the ``name`` - and optionally the ``hash`` of the file to link to. If the link can be resolved against another file on the - current ``path``, not only will it be added to the links of ``name`` but a reverse link of type ``machinecode`` - refering to ``name`` and its hash will also be added to the linked ``model`` file - * ``machinecode``: adds a link to a file containing machine code created from the current file (model), expected - additional data is the ``name`` and optionally the ``hash`` of the file to link to. If the link can be resolved - against another file on the current ``path``, not only will it be added to the links of ``name`` but a reverse - link of type ``model`` refering to ``name`` and its hash will also be added to the linked ``model`` file. - * ``web``: adds a location on the web associated with this file (e.g. a website where to download a model), - expected additional data is a ``href`` attribute holding the website's URL and optionally a ``retrieved`` - attribute describing when the content was retrieved - - Note that adding ``model`` links to files identifying as models or ``machinecode`` links to files identifying - as machine code will be refused. - - :param path: path of the file for which to add a link - :param rel: type of relation of the link to add (currently ``model``, ``machinecode`` and ``web`` are supported) - :param data: additional data of the link to add - """ - path, name = self.sanitize(path) self._add_links(name, path, [(rel, data)]) def remove_link(self, path, rel, data): - """ - Removes the link consisting of ``rel`` and ``data`` from file ``name`` on ``path``. - - :param path: path of the file from which to remove the link - :param rel: type of relation of the link to remove (currently ``model``, ``machinecode`` and ``web`` are supported) - :param data: additional data of the link to remove, must match existing link - """ - path, name = self.sanitize(path) self._remove_links(name, path, [(rel, data)]) @@ -396,22 +487,6 @@ class LocalFileStorage(StorageInterface): self._update_history(name, path, index) def set_additional_metadata(self, path, key, data, overwrite=False, merge=False): - """ - Adds additional metadata to the metadata of ``path``. Metadata in ``data`` will be saved under ``key``. - - If ``overwrite`` is set and ``key`` already exists in ``name``'s metadata, the current value will be overwritten. - - If ``merge`` is set and ``key`` already exists and both ``data`` and the existing data under ``key`` are dictionaries, - the two dictionaries will be merged recursively. - - :param path: the virtual path to the file for which to add additional metadata - :param key: key of metadata to add - :param data: metadata to add - :param overwrite: if True and ``key`` already exists, it will be overwritten - :param merge: if True and ``key`` already exists and both ``data`` and the existing data are dictionaries, they - will be merged - """ - path, name = self.sanitize(path) metadata = self._get_metadata(path) metadata_dirty = False @@ -437,13 +512,6 @@ class LocalFileStorage(StorageInterface): self._save_metadata(path, metadata) def remove_additional_metadata(self, path, key): - """ - Removes additional metadata under ``key`` for ``name`` on ``path`` - - :param path: the virtual path to the file for which to remove the metadata under ``key`` - :param key: the key to remove - """ - path, name = self.sanitize(path) metadata = self._get_metadata(path) @@ -457,13 +525,6 @@ class LocalFileStorage(StorageInterface): self._save_metadata(path, metadata) def split_path(self, path): - """ - Split ``path`` into base directory and file name. - - :param path: the path to split - :return: a tuple (base directory, file name) - """ - split = path.split("/") if len(split) == 1: return "", split[0] @@ -471,21 +532,143 @@ class LocalFileStorage(StorageInterface): return self.join_path(*split[:-1]), split[-1] def join_path(self, *path): - """ - Join path elements together - :param path: path elements to join - :return: joined representation of the path to be usable as fully qualified path for further operations - """ - return "/".join(path) - def get_absolute_path(self, path): + def sanitize(self, path): """ - Retrieves the absolute path on disk for ``path`` + Returns a ``(path, name)`` tuple derived from the provided ``path``. - :param path: the virtual path for which to retrieve the absolute path on disk - :return: the absolute path on disk to ``path`` + ``path`` may be: + * a storage path + * an absolute file system path + * a tuple or list containing all individual path elements + * a string representation of the path + * with or without a file name + + Note that for a ``path`` without a trailing slash the last part will be considered a file name and + hence be returned at second position. If you only need to convert a folder path, be sure to + include a trailing slash for a string ``path`` or an empty last element for a list ``path``. + + Examples:: + + >>> storage = LocalFileStorage("/some/base/folder") + >>> storage.sanitize("some/folder/and/some file.gco") + ("/some/base/folder/some/folder/and", "some_file.gco") + >>> storage.sanitize(("some", "folder", "and", "some file.gco")) + ("/some/base/folder/some/folder/and", "some_file.gco") + >>> storage.sanitize("some file.gco") + ("/some/base/folder", "some_file.gco") + >>> storage.sanitize(("some file.gco",)) + ("/some/base/folder", "some_file.gco") + >>> storage.sanitize("") + ("/some/base/folder", "") + >>> storage.sanitize("some/folder/with/trailing/slash/") + ("/some/base/folder/some/folder/with/trailing/slash", "") + >>> storage.sanitize("some", "folder", "") + ("/some/base/folder/some/folder", "") """ + name = None + if isinstance(path, (str, unicode, basestring)): + if path.startswith(self.basefolder): + path = path[len(self.basefolder):] + path = path.replace(os.path.sep, "/") + path = path.split("/") + if isinstance(path, (list, tuple)): + if len(path) == 1: + name = path[0] + path = "/" + else: + name = path[-1] + path = "/" + self.join_path(*path[:-1]) + if not path: + path = "/" + + name = self.sanitize_name(name) + path = self.sanitize_path(path) + return path, name + + def sanitize_name(self, name): + """ + Raises a :class:`ValueError` for a ``name`` containing ``/`` or ``\``. Otherwise strips any characters from the + given ``name`` that are not any of the ASCII characters, digits, ``-``, ``_``, ``.``, ``(``, ``)`` or space and + replaces and spaces with ``_``. + + Examples:: + + >>> storage = LocalFileStorage("/some/base/folder") + >>> storage.sanitize_name("some_file.gco") + "some_file.gco" + >>> storage.sanitize_name("some_file with (parentheses) and ümläuts and digits 123.gco") + "some_file_with_(parentheses)_and_mluts_and_digits_123.gco" + >>> storage.sanitize_name("pengüino pequeño.stl") + "pengino_pequeo.stl" + >>> storage.sanitize_name("some/folder/still/left.gco") + Traceback (most recent call last): + File "", line 1, in + ValueError: name must not contain / or \ + >>> storage.sanitize_name("also\\no\\backslashes.gco") + Traceback (most recent call last): + File "", line 1, in + ValueError: name must not contain / or \ + """ + if name is None: + return None + + if "/" in name or "\\" in name: + raise ValueError("name must not contain / or \\") + + import string + valid_chars = "-_.() {ascii}{digits}".format(ascii=string.ascii_letters, digits=string.digits) + sanitized_name = ''.join(c for c in name if c in valid_chars) + sanitized_name = sanitized_name.replace(" ", "_") + return sanitized_name + + def sanitize_path(self, path): + """ + Ensures that the on disk representation of ``path`` is located under the configured basefolder. Resolves all + relative path elements (e.g. ``..``) and sanitizes folder names using :func:`sanitize_name`. Final path is the + absolute path including leading ``basefolder`` path. + + Examples:: + + >>> storage = LocalFileStorage("/some/base/folder") + >>> storage.sanitize_path("folder/with/subfolder") + "/some/base/folder/folder/with/subfolder" + >>> storage.sanitize_path("folder/with/subfolder/../other/folder") + "/some/base/folder/folder/with/other/folder" + >>> storage.sanitize_path("/folder/with/leading/slash") + "/some/base/folder/folder/with/leading/slash" + >>> storage.sanitize_path(".folder/with/leading/dot") + "/some/base/folder/folder/with/leading/dot + >>> storage.sanitize_path("../../folder/out/of/the/basefolder") + Traceback (most recent call last): + File "", line 1, in + ValueError: path not contained in base folder: /some/folder/out/of/the/basefolder + """ + if path[0] == "/" or path[0] == ".": + path = path[1:] + path_elements = path.split("/") + joined_path = self.basefolder + for path_element in path_elements: + joined_path = os.path.join(joined_path, self.sanitize_name(path_element)) + path = os.path.realpath(joined_path) + if not path.startswith(self.basefolder): + raise ValueError("path not contained in base folder: {path}".format(**locals())) + return path + + def path_in_storage(self, path): + if isinstance(path, (tuple, list)): + path = self.join_path(*path) + if isinstance(path, (str, unicode, basestring)): + if path.startswith(self.basefolder): + path = path[len(self.basefolder):] + path = path.replace(os.path.sep, "/") + if path.startswith("/"): + path = path[1:] + + return path + + def path_on_disk(self, path): path, name = self.sanitize(path) return os.path.join(path, name) @@ -791,64 +974,6 @@ class LocalFileStorage(StorageInterface): return hash.hexdigest() - def sanitize(self, path): - name = None - if isinstance(path, (str, unicode, basestring)): - if path.startswith(self.basefolder): - path = path[len(self.basefolder):] - path = path.replace(os.path.sep, "/") - path = path.split("/") - if isinstance(path, (list, tuple)): - if len(path) == 1: - name = path[0] - path = "/" - else: - name = path[-1] - path = "/" + self.join_path(*path[:-1]) - if not path: - path = "/" - - name = self.sanitize_name(name) - path = self.sanitize_path(path) - return path, name - - def sanitize_name(self, name): - if name is None: - return None - - if "/" in name or "\\" in name: - raise ValueError("name must not contain / or \\") - - import string - valid_chars = "-_.() {ascii}{digits}".format(ascii=string.ascii_letters, digits=string.digits) - sanitized_name = ''.join(c for c in name if c in valid_chars) - sanitized_name = sanitized_name.replace(" ", "_") - return sanitized_name - - def sanitize_path(self, path): - if path[0] == "/" or path[0] == ".": - path = path[1:] - path_elements = path.split("/") - joined_path = self.basefolder - for path_element in path_elements: - joined_path = os.path.join(joined_path, self.sanitize_name(path_element)) - path = os.path.realpath(joined_path) - if not path.startswith(self.basefolder): - raise ValueError("path not contained in base folder: {path}".format(**locals())) - return path - - def rel_path(self, path): - if isinstance(path, (tuple, list)): - path = self.join_path(*path) - if isinstance(path, (str, unicode, basestring)): - if path.startswith(self.basefolder): - path = path[len(self.basefolder):] - path = path.replace(os.path.sep, "/") - if path.startswith("/"): - path = path[1:] - - return path - def _get_metadata(self, path): if path in self._metadata_cache: return self._metadata_cache[path] diff --git a/src/octoprint/server/api/files.py b/src/octoprint/server/api/files.py index adcbb791..b2e1cc24 100644 --- a/src/octoprint/server/api/files.py +++ b/src/octoprint/server/api/files.py @@ -210,7 +210,7 @@ def uploadGcodeFile(target): filename = added_file done = True else: - filename = fileProcessingFinished(added_file, fileManager.get_absolute_path(FileDestinations.LOCAL, added_file), target) + filename = fileProcessingFinished(added_file, fileManager.path_on_disk(FileDestinations.LOCAL, added_file), target) done = True sdFilename = None @@ -293,7 +293,7 @@ def gcodeFileCommand(filename, target): filenameToSelect = filename sd = True else: - filenameToSelect = fileManager.get_absolute_path(target, filename) + filenameToSelect = fileManager.path_on_disk(target, filename) printer.selectFile(filenameToSelect, sd, printAfterLoading) elif command == "slice": @@ -371,7 +371,7 @@ def gcodeFileCommand(filename, target): filenameToSelect = gcode_name sd = True else: - filenameToSelect = fileManager.get_absolute_path(target, gcode_name) + filenameToSelect = fileManager.path_on_disk(target, gcode_name) printer.selectFile(filenameToSelect, sd, print_after_slicing) ok, result = fileManager.slice(slicer, target, filename, target, gcode_name, diff --git a/src/octoprint/static/js/app/viewmodels/files.js b/src/octoprint/static/js/app/viewmodels/files.js index e9b711d9..73c78be7 100644 --- a/src/octoprint/static/js/app/viewmodels/files.js +++ b/src/octoprint/static/js/app/viewmodels/files.js @@ -67,7 +67,7 @@ function GcodeFilesViewModel(printerStateViewModel, loginStateViewModel, slicing } }, "name", - ["machinecode"], + [], [["sd", "local"], ["machinecode", "model"]], 0 ); diff --git a/tests/filemanager/test_localstorage.py b/tests/filemanager/test_localstorage.py index f8f93830..c722d900 100644 --- a/tests/filemanager/test_localstorage.py +++ b/tests/filemanager/test_localstorage.py @@ -8,6 +8,8 @@ __copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms import unittest import os +from ddt import ddt, unpack, data + import octoprint.filemanager.storage @@ -33,6 +35,7 @@ FILE_BP_CASE_STL = FileWrapper("bp_case.stl") FILE_BP_CASE_GCODE = FileWrapper("bp_case.gcode") FILE_CRAZYRADIO_STL = FileWrapper("crazyradio.stl") +@ddt class LocalStorageTest(unittest.TestCase): def setUp(self): @@ -286,6 +289,77 @@ class LocalStorageTest(unittest.TestCase): self.assertEquals(0, len(gcode_metadata["links"])) self.assertEquals(1, len(stl_metadata["links"])) + @data( + ("some_file.gco", "some_file.gco"), + ("some_file with (parentheses) and ümläuts and digits 123.gco", "some_file_with_(parentheses)_and_mluts_and_digits_123.gco"), + ("pengüino pequeño.stl", "pengino_pequeo.stl") + ) + @unpack + def test_sanitize_name(self, input, expected): + actual = self.storage.sanitize_name(input) + self.assertEquals(expected, actual) + + @data( + "some/folder/still/left.gco", + "also\\no\\backslashes.gco" + ) + def test_sanitize_name_invalid(self, input): + try: + self.storage.sanitize_name(input) + self.fail("expected a ValueError") + except ValueError as e: + self.assertEquals("name must not contain / or \\", e.message) + + @data( + ("folder/with/subfolder", "/folder/with/subfolder"), + ("folder/with/subfolder/../other/folder", "/folder/with/other/folder"), + ("/folder/with/leading/slash", "/folder/with/leading/slash"), + ("folder/with/leading/dot", "/folder/with/leading/dot") + ) + @unpack + def test_sanitize_path(self, input, expected): + actual = self.storage.sanitize_path(input) + self.assertTrue(actual.startswith(self.basefolder)) + self.assertEquals(expected, actual[len(self.basefolder):].replace(os.path.sep, "/")) + + @data( + "../../folder/out/of/the/basefolder", + "some/folder/../../../and/then/back" + ) + def test_sanitize_path_invalid(self, input): + try: + self.storage.sanitize_path(input) + self.fail("expected a ValueError") + except ValueError as e: + self.assertTrue(e.message.startswith("path not contained in base folder: ")) + + @data( + ("some/folder/and/some file.gco", "/some/folder/and", "some_file.gco"), + (("some", "folder", "and", "some file.gco"), "/some/folder/and", "some_file.gco"), + ("some file.gco", "/", "some_file.gco"), + (("some file.gco",), "/", "some_file.gco"), + ("", "/", ""), + ("some/folder/with/trailing/slash/", "/some/folder/with/trailing/slash", ""), + (("some", "folder", ""), "/some/folder", "") + ) + @unpack + def test_sanitize(self, input, expected_path, expected_name): + actual = self.storage.sanitize(input) + self.assertTrue(isinstance(actual, tuple)) + self.assertEquals(2, len(actual)) + + actual_path, actual_name = actual + self.assertTrue(actual_path.startswith(self.basefolder)) + actual_path = actual_path[len(self.basefolder):].replace(os.path.sep, "/") + if not actual_path.startswith("/"): + # if the actual path originally was just the base folder, we just stripped + # away everything, so let's add a / again so the behaviour matches the + # other preprocessing of our test data here + actual_path = "/" + actual_path + + self.assertEquals(expected_path, actual_path) + self.assertEquals(expected_name, actual_name) + def _add_file(self, path, expected_path, file_object, links=None, overwrite=False): sanitized_path = self.storage.add_file(path, file_object, links=links, allow_overwrite=overwrite) split_path = sanitized_path.split("/")