project: add support for dependency syncing

Use pre-commit to synchronize python requirements between
pyproject.toml and the various requirements files.  In addition,
syncronize the contents of system-dependencies.json with
Moonraker's installer.

Signed-off-by:  Eric Callahan <arksine.code@gmail.com>
This commit is contained in:
Eric Callahan 2024-07-27 16:20:01 -04:00
parent 247640cc27
commit 83371cf135
4 changed files with 195 additions and 3 deletions

13
.pre-commit-config.yaml Normal file
View File

@ -0,0 +1,13 @@
repos:
- repo: local
hooks:
- id: sync-requirements
name: sync python requirements
language: system
entry: python3 scripts/sync_dependencies.py
files: ^pyproject.toml$
- id: sync-os-packages
name: sync packages
language: system
entry: python3 scripts/sync_dependencies.py
files: ^scripts/system-dependencies.json$

View File

@ -53,7 +53,11 @@ changelog = "https://moonraker.readthedocs.io/en/latest/changelog/"
[project.optional-dependencies]
msgspec = ["msgspec>=0.18.4 ; python_version>='3.8'"]
uvloop = ["uvloop>=0.17.0"]
speedups = ["moonraker[msgspec,uvloop]"]
speedups = [
"msgspec>=0.18.4 ; python_version>='3.8'",
"uvloop>=0.17.0"
]
dev = ["pre-commit"]
[tool.pdm.version]
source = "scm"

View File

@ -0,0 +1 @@
pre-commit

174
scripts/sync_dependencies.py Executable file
View File

@ -0,0 +1,174 @@
#! /usr/bin/python3
# Script for syncing package dependencies and python reqs
#
# Copyright (C) 2024 Eric Callahan <arksine.code@gmail.com>
#
# This file may be distributed under the terms of the GNU GPLv3 license
from __future__ import annotations
import argparse
import pathlib
import tomllib
import json
import re
from typing import Dict, List
MAX_LINE_LENGTH = 88
SCRIPTS_PATH = pathlib.Path(__file__).parent
INST_PKG_HEADER = "# *** AUTO GENERATED OS PACKAGE DEPENDENCES START ***"
INST_PKG_FOOTER = "# *** AUTO GENERATED OS PACKAGE DEPENDENCES END ***"
def gen_multline_var(var_name: str, values: List[str], indent: int = 0) -> str:
idt = " " * indent
if not values:
return f'{idt}{var_name}=""'
line_list: List[str] = []
current_line = f"{idt}{var_name}=\"{values.pop(0)}"
for val in values:
if len(current_line) + len(val) + 2 > MAX_LINE_LENGTH:
line_list.append(f'{current_line}"')
current_line = (f"{idt}{var_name}=\"${{{var_name}}} {val}")
else:
current_line += f" {val}"
line_list.append(f'{current_line}"')
return "\n".join(line_list)
def sync_packages() -> int:
inst_script = SCRIPTS_PATH.joinpath("install-moonraker.sh")
sys_deps_file = SCRIPTS_PATH.joinpath("system-dependencies.json")
new_deps: Dict[str, List[str]] = json.loads(sys_deps_file.read_bytes())
# Copy install script in memory.
install_data: List[str] = []
prev_deps: Dict[str, List[str]] = {}
distro_name = ""
skip_data = False
with inst_script.open("r") as inst_file:
for line in inst_file:
cur_line = line.strip()
if not skip_data:
install_data.append(line)
else:
# parse current dependencies
distro_match = re.match(
r"(?:el)?if \[ \$\{DISTRIBUTION\} = \"([a-z0-9._-]+)\" \]; then",
cur_line
)
if distro_match is not None:
distro_name = distro_match.group(1)
prev_deps[distro_name] = []
elif cur_line.startswith("PACKAGES"):
pkgs = cur_line.split("=", maxsplit=1)[1].strip('"')
pkg_list = pkgs.split()
if pkg_list and pkg_list[0] == "${PACKAGES}":
pkg_list.pop(0)
prev_deps[distro_name].extend(pkg_list)
if cur_line == INST_PKG_HEADER:
skip_data = True
elif cur_line == INST_PKG_FOOTER:
skip_data = False
install_data.append(line)
# Check if an update is necessary
if set(prev_deps.keys()) == set(new_deps.keys()):
for distro, pkg_list in prev_deps.items():
new_pkgs = new_deps[distro]
if set(pkg_list) != set(new_pkgs):
break
else:
# Dependencies match, exit
print("System package dependencies match")
return 0
print("Writing new system dependencies to install script...")
with inst_script.open("w+") as inst_file:
# Find and replace old package defs
for line in install_data:
inst_file.write(line)
if line.strip() == INST_PKG_HEADER:
indent_count = len(line) - len(line.lstrip())
idt = " " * indent_count
# Write Package data
first_distro = True
for distro, packages in new_deps.items():
prefix = f"{idt}if" if first_distro else f"{idt}elif"
first_distro = False
inst_file.write(
f'{prefix} [ ${{DISTRIBUTION}} = "{distro}" ]; then\n'
)
pkg_var = gen_multline_var("PACKAGES", packages, indent_count + 4)
inst_file.write(pkg_var)
inst_file.write("\n")
inst_file.write(f"{idt}fi\n")
return 1
def check_reqs_changed(reqs_file: pathlib.Path, new_reqs: List[str]) -> bool:
req_list = []
for requirement in reqs_file.read_text().splitlines():
requirement = requirement.strip()
if not requirement or requirement[0] in ("-", "#"):
continue
req_list.append(requirement)
return set(new_reqs) != set(req_list)
def sync_requirements() -> int:
ret: int = 0
src_path = SCRIPTS_PATH.parent
proj_file = src_path.joinpath("pyproject.toml")
with proj_file.open("rb") as f:
data = tomllib.load(f)
python_deps = data["project"]["dependencies"]
optional_deps = data["project"]["optional-dependencies"]
reqs_path = SCRIPTS_PATH.joinpath("moonraker-requirements.txt")
if check_reqs_changed(reqs_path, python_deps):
print("Syncing Moonraker Python Requirements...")
ret = 1
with reqs_path.open("w+") as req_file:
req_file.write("# Python dependencies for Moonraker\n")
req_file.write("--find-links=python_wheels\n")
for requirement in python_deps:
req_file.write(f"{requirement}\n")
else:
print("Moonraker Python requirements match")
# sync speedups
speedups_path = SCRIPTS_PATH.joinpath("moonraker-speedups.txt")
speedup_deps = optional_deps["speedups"]
if check_reqs_changed(speedups_path, speedup_deps):
print("Syncing speedup requirements...")
ret = 1
with speedups_path.open("w+") as req_file:
for requirement in speedup_deps:
req_file.write(f"{requirement}\n")
else:
print("Speedup sequirements match")
# sync dev dependencies
dev_reqs_path = SCRIPTS_PATH.joinpath("moonraker-dev-reqs.txt")
dev_deps = optional_deps["dev"]
if check_reqs_changed(dev_reqs_path, dev_deps):
print("Syncing dev requirements")
ret = 1
with dev_reqs_path.open("r+") as req_file:
for requirement in dev_deps:
req_file.write(f"{requirement}\n")
else:
print("Dev requirements match")
return ret
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument(
"filename", default="", nargs="?",
help="The name of the dependency file to sync"
)
args = parser.parse_args()
fname: str = args.filename
if not fname:
ret = sync_requirements()
ret += sync_packages()
return 1 if ret > 0 else 0
elif fname == "pyproject.toml":
return sync_requirements()
elif fname == "scripts/system-dependencies.json":
return sync_packages()
return 0
if __name__ == "__main__":
raise SystemExit(main())