update_manager: pip improvements

Re-implement pip updates using a pinned version.  A version
check is now always done prior to installing python requirements,
updating when necessary.

Signed-off-by:  Eric Callahan <arksine.code@gmail.com>

moonraker/components/update_manager/app_deploy.py

@@ -10,6 +10,7 @@ import pathlib
 import shutil
 import hashlib
 import logging
+import re
 from .base_deploy import BaseDeploy
 
 # Annotation imports
@@ -20,12 +21,16 @@ from typing import (
     Union,
     Dict,
     List,
+    Tuple
 )
 if TYPE_CHECKING:
     from confighelper import ConfigHelper
     from .update_manager import CommandHelper
     from ..machine import Machine
     from ..file_manager.file_manager import FileManager
+    from ..shell_command import ShellCommandFactory as ShellCmd
+
+MIN_PIP_VERSION = (23, 0)
 
 SUPPORTED_CHANNELS = {
     "zip": ["stable", "beta"],
@@ -71,25 +76,37 @@ class AppDeploy(BaseDeploy):
             raise config.error(
                 f"Invalid Channel '{self.channel}' for config "
                 f"section [{config.get_name()}], type: {self.type}")
-        self._verify_path(config, 'path', self.path)
+        self._verify_path(config, 'path', self.path, check_file=False)
         self.executable: Optional[pathlib.Path] = None
-        self.pip_exe: Optional[pathlib.Path] = None
+        self.pip_cmd: Optional[str] = None
+        self.pip_version: Tuple[int, ...] = tuple()
         self.venv_args: Optional[str] = None
         if executable is not None:
             self.executable = pathlib.Path(executable).expanduser()
-            self.pip_exe = self.executable.parent.joinpath("pip")
+            self._verify_path(config, 'env', self.executable, check_exe=True)
-            if not self.pip_exe.exists():
+            # Detect if executable is actually located in a virtualenv
+            # by checking the parent for the activation script
+            act_path = self.executable.parent.joinpath("activate")
+            while not act_path.is_file():
                 if self.executable.is_symlink():
                     self.executable = pathlib.Path(os.readlink(self.executable))
-                    self.pip_exe = self.executable.parent.joinpath("pip")
+                    act_path = self.executable.parent.joinpath("activate")
-                if not self.pip_exe.exists():
+                else:
-                    logging.info(
+                    break
-                        f"Update Manger {self.name}: Unable to locate pip "
+            if act_path.is_file():
-                        "executable")
+                venv_dir = self.executable.parent.parent
-                    self.pip_exe = None
+                self.log_info(f"Detected virtualenv: {venv_dir}")
-            self._verify_path(config, 'env', self.executable)
+                pip_exe = self.executable.parent.joinpath("pip")
+                if pip_exe.is_file():
+                    self.pip_cmd = f"{self.executable} -m pip"
+                else:
+                    self.log_info("Unable to locate pip executable")
+            else:
+                self.log_info(
+                    f"Unable to detect virtualenv at: {executable}"
+                )
+                self.executable = pathlib.Path(executable).expanduser()
             self.venv_args = config.get('venv_args', None)
-
         self.info_tags: List[str] = config.getlist("info_tags", [])
         self.managed_services: List[str] = []
         svc_default = []
@@ -143,19 +160,32 @@ class AppDeploy(BaseDeploy):
 
     async def initialize(self) -> Dict[str, Any]:
         storage = await super().initialize()
-        self.need_channel_update = storage.get('need_channel_upate', False)
+        self.need_channel_update = storage.get("need_channel_update", False)
-        self._is_valid = storage.get('is_valid', False)
+        self._is_valid = storage.get("is_valid", False)
+        self.pip_version = tuple(storage.get("pip_version", []))
+        if self.pip_version:
+            ver_str = ".".join([str(part) for part in self.pip_version])
+            self.log_info(f"Stored pip version: {ver_str}")
         return storage
 
-    def _verify_path(self,
+    def _verify_path(
-                     config: ConfigHelper,
+        self,
-                     option: str,
+        config: ConfigHelper,
-                     file_path: pathlib.Path
+        option: str,
-                     ) -> None:
+        path: pathlib.Path,
-        if not file_path.exists():
+        check_file: bool = True,
-            raise config.error(
+        check_exe: bool = False
-                f"Invalid path for option `{option}` in section "
+    ) -> None:
-                f"[{config.get_name()}]: Path `{file_path}` does not exist")
+        base_msg = (
+            f"Invalid path for option `{option}` in section "
+            f"[{config.get_name()}]: Path `{path}`"
+        )
+        if not path.exists():
+            raise config.error(f"{base_msg} does not exist")
+        if check_file and not path.is_file():
+            raise config.error(f"{base_msg} is not a file")
+        if check_exe and not os.access(path, os.X_OK):
+            raise config.error(f"{base_msg} is not executable")
 
     def check_need_channel_swap(self) -> bool:
         return self.need_channel_update
@@ -234,6 +264,7 @@ class AppDeploy(BaseDeploy):
         storage = super().get_persistent_data()
         storage['is_valid'] = self._is_valid
         storage['need_channel_update'] = self.need_channel_update
+        storage['pip_version'] = list(self.pip_version)
         return storage
 
     async def _get_file_hash(self,
@@ -272,8 +303,9 @@ class AppDeploy(BaseDeploy):
     async def _update_virtualenv(self,
                                  requirements: Union[pathlib.Path, List[str]]
                                  ) -> None:
-        if self.pip_exe is None:
+        if self.pip_cmd is None:
             return
+        await self._update_pip()
         # Update python dependencies
         if isinstance(requirements, pathlib.Path):
             if not requirements.is_file():
@@ -285,20 +317,68 @@ class AppDeploy(BaseDeploy):
             args = " ".join(requirements)
         self.notify_status("Updating python packages...")
         try:
-            # First attempt to update pip
-            # await self.cmd_helper.run_cmd(
-            #     f"{self.pip_exe} install -U pip", timeout=1200., notify=True,
-            #     retries=3)
             await self.cmd_helper.run_cmd(
-                f"{self.pip_exe} install {args}", timeout=1200., notify=True,
+                f"{self.pip_cmd} install {args}", timeout=1200., notify=True,
-                retries=3)
+                retries=3
+            )
         except Exception:
             self.log_exc("Error updating python requirements")
 
-    async def _build_virtualenv(self) -> None:
+    async def _update_pip(self) -> None:
-        if self.pip_exe is None or self.venv_args is None:
+        if self.pip_cmd is None:
             return
-        bin_dir = self.pip_exe.parent
+        update_ver = await self._check_pip_version()
+        if update_ver is None:
+            return
+        cur_vstr = ".".join([str(part) for part in self.pip_version])
+        self.notify_status(
+            f"Updating pip from version {cur_vstr} to {update_ver}..."
+        )
+        try:
+            await self.cmd_helper.run_cmd(
+                f"{self.pip_cmd} install pip=={update_ver}",
+                timeout=1200., notify=True, retries=3
+            )
+        except Exception:
+            self.log_exc("Error updating python pip")
+
+    async def _check_pip_version(self) -> Optional[str]:
+        if self.pip_cmd is None:
+            return None
+        self.notify_status("Checking pip version...")
+        scmd: ShellCmd = self.server.lookup_component("shell_command")
+        try:
+            data: str = await scmd.exec_cmd(
+                f"{self.pip_cmd} --version", timeout=30., retries=3
+            )
+            match = re.match(
+                r"^pip ([0-9.]+) from .+? \(python ([0-9.]+)\)$", data.strip()
+            )
+            if match is None:
+                return None
+            pipver_str: str = match.group(1)
+            pyver_str: str = match.group(2)
+            pipver = tuple([int(part) for part in pipver_str.split(".")])
+            pyver = tuple([int(part) for part in pyver_str.split(".")])
+        except Exception:
+            self.log_exc("Error Getting Pip Version")
+            return None
+        self.pip_version = pipver
+        if not self.pip_version:
+            return None
+        self.log_info(
+            f"Dectected pip version: {pipver_str}, Python {pyver_str}"
+        )
+        if pyver < (3, 7):
+            return None
+        if self.pip_version < MIN_PIP_VERSION:
+            return ".".join([str(ver) for ver in MIN_PIP_VERSION])
+        return None
+
+    async def _build_virtualenv(self) -> None:
+        if self.executable is None or self.venv_args is None:
+            return
+        bin_dir = self.executable.parent
         env_path = bin_dir.parent.resolve()
         self.notify_status(f"Creating virtualenv at: {env_path}...")
         if env_path.exists():
@@ -309,5 +389,5 @@ class AppDeploy(BaseDeploy):
         except Exception:
             self.log_exc(f"Error creating virtualenv")
             return
-        if not self.pip_exe.exists():
+        if not self.executable.exists():
             raise self.log_exc("Failed to create new virtualenv", False)