From 2bc0c4a77f698901dc7179afbaa84e4ff21dfefa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 19 Jul 2017 13:46:43 +0200 Subject: [PATCH] Synchronize pip helper cache access --- src/octoprint/util/pip.py | 211 +++++++++++++++++++------------------- 1 file changed, 108 insertions(+), 103 deletions(-) diff --git a/src/octoprint/util/pip.py b/src/octoprint/util/pip.py index d03e0034..c58c5c02 100644 --- a/src/octoprint/util/pip.py +++ b/src/octoprint/util/pip.py @@ -10,6 +10,7 @@ import sarge import sys import logging import site +import threading import pkg_resources @@ -17,6 +18,7 @@ from .commandline import CommandlineCaller, clean_ansi from octoprint.util import to_unicode _cache = dict(version=dict(), setup=dict()) +_cache_mutex = threading.RLock() class UnknownPip(Exception): pass @@ -173,8 +175,6 @@ class PipCaller(CommandlineCaller): self._logger.error("This version of pip is known to have bugs that make it incompatible with how it needs to be used by OctoPrint. Please upgrade your pip version.") return - self._logger.info("Version of pip is {}".format(version_segment)) - # Now figure out if pip belongs to a virtual environment and if the # default installation directory is writable. # @@ -194,10 +194,6 @@ class PipCaller(CommandlineCaller): self._logger.error("Cannot use pip") return - self._logger.info("pip installs to {}, --user flag needed => {}, virtual env => {}".format(pip_install_dir, - "yes" if pip_user else "no", - "yes" if pip_virtual_env else "no")) - self._command = pip_command self._version = pip_version self._version_string = version_segment @@ -251,122 +247,131 @@ class PipCaller(CommandlineCaller): if isinstance(pip_command_str, list): pip_command_str = " ".join(pip_command_str) - if not self.ignore_cache and pip_command_str in _cache["version"]: - self._logger.debug("Using cached pip version information for {}".format(pip_command_str)) - return _cache["version"][pip_command_str] + with _cache_mutex: + if not self.ignore_cache and pip_command_str in _cache["version"]: + self._logger.debug("Using cached pip version information for {}".format(pip_command_str)) + return _cache["version"][pip_command_str] - sarge_command = self.to_sarge_command(pip_command, "--version") - p = sarge.run(sarge_command, stdout=sarge.Capture(), stderr=sarge.Capture()) + sarge_command = self.to_sarge_command(pip_command, "--version") + p = sarge.run(sarge_command, stdout=sarge.Capture(), stderr=sarge.Capture()) - if p.returncode != 0: - self._logger.warn("Error while trying to run pip --version: {}".format(p.stderr.text)) - return None, None + if p.returncode != 0: + self._logger.warn("Error while trying to run pip --version: {}".format(p.stderr.text)) + return None, None - output = PipCaller._preprocess(p.stdout.text) - # output should look something like this: - # - # pip from () - # - # we'll just split on whitespace and then try to use the second entry + output = PipCaller._preprocess(p.stdout.text) + # output should look something like this: + # + # pip from () + # + # we'll just split on whitespace and then try to use the second entry - if not output.startswith("pip"): - self._logger.warn("pip command returned unparseable output, can't determine version: {}".format(output)) + if not output.startswith("pip"): + self._logger.warn("pip command returned unparseable output, can't determine version: {}".format(output)) - split_output = map(lambda x: x.strip(), output.split()) - if len(split_output) < 2: - self._logger.warn("pip command returned unparseable output, can't determine version: {}".format(output)) + split_output = map(lambda x: x.strip(), output.split()) + if len(split_output) < 2: + self._logger.warn("pip command returned unparseable output, can't determine version: {}".format(output)) - version_segment = split_output[1] + version_segment = split_output[1] - try: - pip_version = pkg_resources.parse_version(version_segment) - except: - self._logger.exception("Error while trying to parse version string from pip command") - return None, None + try: + pip_version = pkg_resources.parse_version(version_segment) + except: + self._logger.exception("Error while trying to parse version string from pip command") + return None, None - result = pip_version, version_segment - _cache["version"][pip_command_str] = result - return result + self._logger.info("Version of pip is {}".format(version_segment)) + + result = pip_version, version_segment + _cache["version"][pip_command_str] = result + return result def _check_pip_setup(self, pip_command): pip_command_str = pip_command if isinstance(pip_command_str, list): pip_command_str = " ".join(pip_command_str) - if not self.ignore_cache and pip_command_str in _cache["setup"]: - self._logger.debug("Using cached pip setup information for {}".format(pip_command_str)) - return _cache["setup"][pip_command_str] + with _cache_mutex: + if not self.ignore_cache and pip_command_str in _cache["setup"]: + self._logger.debug("Using cached pip setup information for {}".format(pip_command_str)) + return _cache["setup"][pip_command_str] - # This is horribly ugly and I'm sorry... - # - # While we can figure out the install directory, if that's writable and if a virtual environment - # is active for pip that belongs to our sys.executable python instance by just checking some - # variables, we can't for stuff like third party software we allow to update via the software - # update plugin. - # - # What we do instead for these situations is try to install (and of course uninstall) the - # testballoon dummy package, which collects that information for us. For pip <= 7 we could - # have the testballoon provide us with the info needed through stdout, if pip was called - # with --verbose anything printed to stdout within setup.py would be output. Pip 8 managed - # to break this mechanism. Any (!) output within setup.py appears to be suppressed now, and - # no combination of --log and multiple --verbose or -v arguments could get it to bring the - # output back. - # - # So here's what we do now instead. Our sarge call sets an environment variable - # "TESTBALLOON_OUTPUT" that points to a temporary file. If the testballoon sees that - # variable set, it opens the file and writes to it the output it so far printed on stdout. - # We then open the file and read in the data that way. - # - # Yeah, I'm not happy with that either. But as long as there's no way to otherwise figure - # out for a generic pip command whether OctoPrint can even install anything with that - # and if so how, well, that's how we'll have to do things. + # This is horribly ugly and I'm sorry... + # + # While we can figure out the install directory, if that's writable and if a virtual environment + # is active for pip that belongs to our sys.executable python instance by just checking some + # variables, we can't for stuff like third party software we allow to update via the software + # update plugin. + # + # What we do instead for these situations is try to install (and of course uninstall) the + # testballoon dummy package, which collects that information for us. For pip <= 7 we could + # have the testballoon provide us with the info needed through stdout, if pip was called + # with --verbose anything printed to stdout within setup.py would be output. Pip 8 managed + # to break this mechanism. Any (!) output within setup.py appears to be suppressed now, and + # no combination of --log and multiple --verbose or -v arguments could get it to bring the + # output back. + # + # So here's what we do now instead. Our sarge call sets an environment variable + # "TESTBALLOON_OUTPUT" that points to a temporary file. If the testballoon sees that + # variable set, it opens the file and writes to it the output it so far printed on stdout. + # We then open the file and read in the data that way. + # + # Yeah, I'm not happy with that either. But as long as there's no way to otherwise figure + # out for a generic pip command whether OctoPrint can even install anything with that + # and if so how, well, that's how we'll have to do things. - import os - testballoon = os.path.join(os.path.realpath(os.path.dirname(__file__)), "piptestballoon") + import os + testballoon = os.path.join(os.path.realpath(os.path.dirname(__file__)), "piptestballoon") - from octoprint.util import temppath - with temppath() as testballoon_output_file: - sarge_command = self.to_sarge_command(pip_command, "install", ".") - try: - # our testballoon is no real package, so this command will fail - that's ok though, - # we only need the output produced within the pip environment - sarge.run(sarge_command, - stdout=sarge.Capture(), - stderr=sarge.Capture(), - cwd=testballoon, - env=dict(TESTBALLOON_OUTPUT=testballoon_output_file)) - except: - self._logger.exception("Error while trying to install testballoon to figure out pip setup") + from octoprint.util import temppath + with temppath() as testballoon_output_file: + sarge_command = self.to_sarge_command(pip_command, "install", ".") + try: + # our testballoon is no real package, so this command will fail - that's ok though, + # we only need the output produced within the pip environment + sarge.run(sarge_command, + stdout=sarge.Capture(), + stderr=sarge.Capture(), + cwd=testballoon, + env=dict(TESTBALLOON_OUTPUT=testballoon_output_file)) + except: + self._logger.exception("Error while trying to install testballoon to figure out pip setup") + return False, False, False, None + + data = dict() + with open(testballoon_output_file) as f: + for line in f: + key, value = line.split("=", 2) + data[key] = value + + install_dir_str = data.get("PIP_INSTALL_DIR", None) + virtual_env_str = data.get("PIP_VIRTUAL_ENV", None) + writable_str = data.get("PIP_WRITABLE", None) + + if install_dir_str is not None and virtual_env_str is not None and writable_str is not None: + install_dir = install_dir_str.strip() + virtual_env = virtual_env_str.strip() == "True" + writable = writable_str.strip() == "True" + + can_use_user_flag = not virtual_env and site.ENABLE_USER_SITE + + ok = writable or can_use_user_flag + user_flag = not writable and can_use_user_flag + + self._logger.info("pip installs to {}, --user flag needed => {}, " + "virtual env => {}".format(install_dir, + "yes" if user_flag else "no", + "yes" if virtual_env else "no")) + + # ok, enable user flag, virtual env yes/no, installation dir + result = ok, user_flag, virtual_env, install_dir + _cache["setup"][pip_command_str] = result + return result + else: + self._logger.debug("Could not detect desired output from testballoon install, got this instead: {!r}".format(data)) return False, False, False, None - data = dict() - with open(testballoon_output_file) as f: - for line in f: - key, value = line.split("=", 2) - data[key] = value - - install_dir_str = data.get("PIP_INSTALL_DIR", None) - virtual_env_str = data.get("PIP_VIRTUAL_ENV", None) - writable_str = data.get("PIP_WRITABLE", None) - - if install_dir_str is not None and virtual_env_str is not None and writable_str is not None: - install_dir = install_dir_str.strip() - virtual_env = virtual_env_str.strip() == "True" - writable = writable_str.strip() == "True" - - can_use_user_flag = not virtual_env and site.ENABLE_USER_SITE - - # ok, enable user flag, virtual env yes/no, installation dir - result = writable or can_use_user_flag, \ - not writable and can_use_user_flag, \ - virtual_env, \ - install_dir - _cache["setup"][pip_command_str] = result - return result - else: - self._logger.debug("Could not detect desired output from testballoon install, got this instead: {!r}".format(data)) - return False, False, False, None - def _preprocess_lines(self, *lines): return map(self._preprocess, lines)