From 4abd97df093131d5651717421eec6d5cc95b68e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 19 Jun 2015 19:19:02 +0200 Subject: [PATCH] File manager will now migrate analysis result from old metadata That should make the initial startup after switching to 1.2.0 way faster. --- src/octoprint/filemanager/__init__.py | 16 ++++- src/octoprint/filemanager/storage.py | 87 +++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 16 deletions(-) diff --git a/src/octoprint/filemanager/__init__.py b/src/octoprint/filemanager/__init__.py index bc8903b7..2fdfeb89 100644 --- a/src/octoprint/filemanager/__init__.py +++ b/src/octoprint/filemanager/__init__.py @@ -136,8 +136,16 @@ class FileManager(object): def initialize(self): self.reload_plugins() - for storage_type, storage_manager in self._storage_managers.items(): - self._determine_analysis_backlog(storage_type, storage_manager) + + def worker(): + self._logger.info("Adding backlog items from all storage types to analysis queue...".format(**locals())) + for storage_type, storage_manager in self._storage_managers.items(): + self._determine_analysis_backlog(storage_type, storage_manager) + + import threading + thread = threading.Thread(target=worker) + thread.daemon = True + thread.start() def reload_plugins(self): self._progress_plugins = octoprint.plugin.plugin_manager().get_implementations(octoprint.plugin.ProgressPlugin) @@ -150,13 +158,15 @@ class FileManager(object): self._slicing_progress_callbacks.remove(callback) def _determine_analysis_backlog(self, storage_type, storage_manager): - self._logger.info("Adding backlog items from {storage_type} to analysis queue".format(**locals())) + counter = 0 for entry, path, printer_profile in storage_manager.analysis_backlog: file_type = get_file_type(path)[-1] # we'll use the default printer profile for the backlog since we don't know better queue_entry = QueueEntry(entry, file_type, storage_type, path, self._printer_profile_manager.get_default()) self._analysis_queue.enqueue(queue_entry, high_priority=False) + counter += 1 + self._logger.info("Added {counter} items from storage type \"{storage_type}\" to analysis queue".format(**locals())) def add_storage(self, storage_type, storage_manager): self._storage_managers[storage_type] = storage_manager diff --git a/src/octoprint/filemanager/storage.py b/src/octoprint/filemanager/storage.py index bd81df0e..6e855cce 100644 --- a/src/octoprint/filemanager/storage.py +++ b/src/octoprint/filemanager/storage.py @@ -309,6 +309,41 @@ class LocalFileStorage(StorageInterface): self._metadata_cache = pylru.lrucache(10) + self._old_metadata = None + self._initialize_metadata() + + def _initialize_metadata(self): + self._logger.info("Initializing the file metadata for {}...".format(self.basefolder)) + + old_metadata_path = os.path.join(self.basefolder, "metadata.yaml") + backup_path = os.path.join(self.basefolder, "metadata.yaml.backup") + + if os.path.exists(old_metadata_path): + # load the old metadata file + try: + with open(old_metadata_path) as f: + import yaml + self._old_metadata = yaml.safe_load(f) + except: + self._logger.exception("Error while loading old metadata file") + + # make sure the metadata is initialized as far as possible + self._list_folder(self.basefolder) + + # rename the old metadata file + self._old_metadata = None + try: + import shutil + shutil.move(old_metadata_path, backup_path) + except: + self._logger.exception("Could not rename old metadata.yaml file") + + else: + # make sure the metadata is initialized as far as possible + self._list_folder(self.basefolder) + + self._logger.info("... file metadata for {} initialized successfully.".format(self.basefolder)) + @property def analysis_backlog(self): for entry in self._analysis_backlog_generator(): @@ -919,12 +954,7 @@ class LocalFileStorage(StorageInterface): if entry in metadata and isinstance(metadata[entry], dict): entry_data = metadata[entry] else: - entry_data = dict( - hash=self._create_hash(entry_path), - links=[], - notes=[] - ) - metadata[entry] = entry_data + entry_data = self._add_basic_metadata(path, entry, save=False, metadata=metadata) metadata_dirty = True # TODO extract model hash from source if possible to recreate link @@ -959,6 +989,31 @@ class LocalFileStorage(StorageInterface): return result + def _add_basic_metadata(self, path, entry, additional_metadata=None, save=True, metadata=None): + if additional_metadata is None: + additional_metadata = dict() + + if metadata is None: + metadata = self._get_metadata(path) + + entry_data = dict( + hash=self._create_hash(os.path.join(path, entry)), + links=[], + notes=[] + ) + + if path == self.basefolder and self._old_metadata is not None and entry in self._old_metadata and "gcodeAnalysis" in self._old_metadata[entry]: + # if there is still old metadata available and that contains an analysis for this file, use it! + entry_data["analysis"] = self._old_metadata[entry]["gcodeAnalysis"] + + entry_data.update(additional_metadata) + metadata[entry] = entry_data + + if save: + self._save_metadata(path, metadata) + + return entry_data + def _create_hash(self, path): import hashlib @@ -993,16 +1048,22 @@ class LocalFileStorage(StorageInterface): def _save_metadata(self, path, metadata): metadata_path = os.path.join(path, ".metadata.yaml") - fh, metadata_temporary_path = tempfile.mkstemp() - os.close(fh) - with self._metadata_lock: try: - with open(metadata_temporary_path, "w") as f: - import yaml - yaml.safe_dump(metadata, stream=f, default_flow_style=False, indent=" ", allow_unicode=True) + import yaml import shutil - shutil.move(metadata_temporary_path, metadata_path) + + file_obj = tempfile.NamedTemporaryFile(delete=False) + try: + yaml.safe_dump(metadata, stream=file_obj, default_flow_style=False, indent=" ", allow_unicode=True) + file_obj.close() + shutil.move(file_obj.name, metadata_path) + finally: + try: + if os.path.exists(file_obj.name): + os.remove(file_obj.name) + except Exception as e: + self._logger.warn("Could not delete file {}: {}".format(file_obj.name, str(e))) except: self._logger.exception("Error while writing .metadata.yaml to {path}".format(**locals())) else: