339 lines
13 KiB
Python
339 lines
13 KiB
Python
#!/usr/bin/python3
|
|
# Builds zip release files for Moonraker and Klipper
|
|
|
|
import os
|
|
import sys
|
|
import argparse
|
|
import shutil
|
|
import tempfile
|
|
import json
|
|
import pathlib
|
|
import time
|
|
import traceback
|
|
import subprocess
|
|
import re
|
|
from typing import Dict, Any, List, Set, Optional
|
|
|
|
MOONRAKER_PATH = os.path.abspath(
|
|
os.path.join(os.path.dirname(__file__), ".."))
|
|
sys.path.append(os.path.join(MOONRAKER_PATH, "moonraker"))
|
|
from utils import hash_directory, retrieve_git_version # noqa:E402
|
|
|
|
# Dirs and exts to ignore when calculating the repo hash
|
|
IGNORE_DIRS = ["out", "lib", "test", "docs", "__pycache__"]
|
|
IGNORE_EXTS = [".o", ".so", ".pyc", ".pyo", ".pyd", ".yml", ".yaml"]
|
|
|
|
# Files not to include in the source package
|
|
SKIP_FILES = [".gitignore", ".gitattributes", ".readthedocs.yaml",
|
|
"mkdocs.yml", "__pycache__"]
|
|
|
|
RELEASE_URL = "https://api.github.com/repos/Arksine/moonraker/releases"
|
|
GIT_MAX_LOG_CNT = 100
|
|
GIT_LOG_FMT = \
|
|
"sha:%H%x1Dauthor:%an%x1Ddate:%ct%x1Dsubject:%s%x1Dmessage:%b%x1E"
|
|
OWNER_REPOS = {
|
|
'moonraker': "arksine/moonraker",
|
|
'klippy': "klipper3d/klipper"
|
|
}
|
|
INSTALL_SCRIPTS = {
|
|
'klippy': {
|
|
'debian': "install-octopi.sh",
|
|
'arch': "install-arch.sh",
|
|
'centos': "install-centos.sh"
|
|
},
|
|
'moonraker': {
|
|
'debian': "install-moonraker.sh"
|
|
}
|
|
}
|
|
|
|
class CopyIgnore:
|
|
def __init__(self, root_dir: str) -> None:
|
|
self.root_dir = root_dir
|
|
|
|
def __call__(self, dir_path: str, dir_items: List[str]) -> List[str]:
|
|
ignored: List[str] = []
|
|
for item in dir_items:
|
|
if item in SKIP_FILES:
|
|
ignored.append(item)
|
|
elif dir_path == self.root_dir:
|
|
full_path = os.path.join(dir_path, item)
|
|
# Ignore all hidden directories in the root
|
|
if os.path.isdir(full_path) and item[0] == ".":
|
|
ignored.append(item)
|
|
return ignored
|
|
|
|
def search_install_script(data: str,
|
|
regex: str,
|
|
exclude: str
|
|
) -> List[str]:
|
|
items: Set[str] = set()
|
|
lines: List[str] = re.findall(regex, data)
|
|
for line in lines:
|
|
items.update(line.strip().split())
|
|
try:
|
|
items.remove(exclude)
|
|
except KeyError:
|
|
pass
|
|
return list(items)
|
|
|
|
def generate_dependency_info(repo_path: str, app_name: str) -> None:
|
|
inst_scripts = INSTALL_SCRIPTS[app_name]
|
|
package_info: Dict[str, Any] = {}
|
|
for distro, script_name in inst_scripts.items():
|
|
script_path = os.path.join(repo_path, "scripts", script_name)
|
|
script = pathlib.Path(script_path)
|
|
if not script.exists():
|
|
continue
|
|
data = script.read_text()
|
|
packages: List[str] = search_install_script(
|
|
data, r'PKGLIST="(.*)"', "${PKGLIST}")
|
|
package_info[distro] = {'packages': sorted(packages)}
|
|
if distro == "arch":
|
|
aur_packages: List[str] = search_install_script(
|
|
data, r'AURLIST="(.*)"', "${AURLIST}")
|
|
package_info[distro]['aur_packages'] = sorted(aur_packages)
|
|
req_file_name = os.path.join(repo_path, "scripts",
|
|
f"{app_name}-requirements.txt")
|
|
req_file = pathlib.Path(req_file_name)
|
|
python_reqs: List[str] = []
|
|
if req_file.exists():
|
|
req_data = req_file.read_text()
|
|
lines = [line.strip() for line in req_data.split('\n')
|
|
if line.strip()]
|
|
for line in lines:
|
|
comment_idx = line.find('#')
|
|
if comment_idx == 0:
|
|
continue
|
|
if comment_idx > 0:
|
|
line = line[:comment_idx].strip()
|
|
python_reqs.append(line)
|
|
package_info['python'] = sorted(python_reqs)
|
|
dep_file = pathlib.Path(os.path.join(repo_path, ".dependencies"))
|
|
dep_file.write_text(json.dumps(package_info))
|
|
|
|
def clean_repo(path: str) -> None:
|
|
# Obtain version info from "git" program
|
|
prog = ('git', '-C', path, 'clean', '-x', '-f', '-d')
|
|
process = subprocess.Popen(prog, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, cwd=path)
|
|
retcode = process.wait()
|
|
if retcode != 0:
|
|
print(f"Error running git clean: {path}")
|
|
|
|
def get_releases() -> List[Dict[str, Any]]:
|
|
print("Fetching Release List...")
|
|
prog = ('curl', '-H', "Accept: application/vnd.github.v3+json",
|
|
RELEASE_URL)
|
|
process = subprocess.Popen(prog, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
response, err = process.communicate()
|
|
retcode = process.wait()
|
|
if retcode != 0:
|
|
print(f"Release list request returned with code {retcode},"
|
|
f" response:\n{err.decode()}")
|
|
return []
|
|
releases = json.loads(response.decode().strip())
|
|
print(f"Found {len(releases)} releases")
|
|
return releases
|
|
|
|
def get_last_release_info(moonraker_version: str,
|
|
is_beta: bool,
|
|
releases: List[Dict[str, Any]]
|
|
) -> Dict[str, Any]:
|
|
print("Searching for previous release assets...")
|
|
cur_tag, commit_count = moonraker_version.split('-', 2)[:2]
|
|
release_assets = []
|
|
matched_tag: Optional[str] = None
|
|
for release in releases:
|
|
if int(commit_count) != 0:
|
|
# This is build is not being done against a fresh release,
|
|
# return release info from a matching tag
|
|
if release['tag_name'] == cur_tag:
|
|
release_assets = release['assets']
|
|
matched_tag = cur_tag
|
|
break
|
|
else:
|
|
# Get the most recent non-matching tag
|
|
if release['tag_name'] == cur_tag:
|
|
continue
|
|
if is_beta or not release['prerelease']:
|
|
# Get the last tagged release. If we are building a beta,
|
|
# that is the most recent release. Otherwise we should
|
|
# omit pre-releases
|
|
release_assets = release['assets']
|
|
matched_tag = release['tag_name']
|
|
break
|
|
if matched_tag is None:
|
|
print("No matching release found")
|
|
matched_tag = "No Tag"
|
|
else:
|
|
print(f"Found release: {matched_tag}")
|
|
|
|
asset_url: Optional[str] = None
|
|
content_type: str = ""
|
|
for asset in release_assets:
|
|
if asset['name'] == "RELEASE_INFO":
|
|
asset_url = asset['browser_download_url']
|
|
content_type = asset['content_type']
|
|
break
|
|
if asset_url is None:
|
|
print(f"RELEASE_INFO asset not found in release: {matched_tag}")
|
|
return {}
|
|
# This build is prior to a tagged release, so fetch the current tag
|
|
print(f"Release Info Download URL: {asset_url}")
|
|
prog = ('curl', '-L', '-H', f"Accept: {content_type}", asset_url)
|
|
process = subprocess.Popen(prog, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
response, err = process.communicate()
|
|
retcode = process.wait()
|
|
if retcode != 0:
|
|
print("Request for release info failed")
|
|
return {}
|
|
resp = response.decode().strip()
|
|
print(f"Found Info for release {matched_tag}")
|
|
return json.loads(resp)
|
|
|
|
def get_commit_log(path: str,
|
|
release_info: Dict[str, Any]
|
|
) -> List[Dict[str, Any]]:
|
|
print(f"Preparing commit log for {path.split('/')[-1]}")
|
|
start_sha = release_info.get('commit_hash', None)
|
|
prog = ['git', '-C', path, 'log', f'--format={GIT_LOG_FMT}',
|
|
f'--max-count={GIT_MAX_LOG_CNT}']
|
|
if start_sha is not None:
|
|
prog = ['git', '-C', path, 'log', f'{start_sha}..HEAD',
|
|
f'--format={GIT_LOG_FMT}', f'--max-count={GIT_MAX_LOG_CNT}']
|
|
process = subprocess.Popen(prog, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, cwd=path)
|
|
response, err = process.communicate()
|
|
retcode = process.wait()
|
|
if retcode != 0:
|
|
return []
|
|
resp = response.decode().strip()
|
|
commit_log: List[Dict[str, Any]] = []
|
|
for log_entry in resp.split('\x1E'):
|
|
log_entry = log_entry.strip()
|
|
if not log_entry:
|
|
continue
|
|
log_items = [li.strip() for li in log_entry.split('\x1D')
|
|
if li.strip()]
|
|
cbh = [li.split(':', 1) for li in log_items]
|
|
commit_log.append(dict(cbh)) # type: ignore
|
|
print(f"Found {len(commit_log)} commits")
|
|
return commit_log
|
|
|
|
def get_commit_hash(path: str) -> str:
|
|
prog = ('git', '-C', path, 'rev-parse', 'HEAD')
|
|
process = subprocess.Popen(prog, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, cwd=path)
|
|
commit_hash, err = process.communicate()
|
|
retcode = process.wait()
|
|
if retcode == 0:
|
|
return commit_hash.strip().decode()
|
|
raise Exception(f"Failed to get commit hash: {commit_hash.decode()}")
|
|
|
|
def generate_version_info(path: str,
|
|
source_dir: str,
|
|
channel: str,
|
|
release_tag: Optional[str] = None
|
|
) -> Dict[str, Any]:
|
|
print(f"Generating version info: {source_dir}")
|
|
clean_repo(path)
|
|
owner_repo = OWNER_REPOS[source_dir]
|
|
curtime = int(time.time())
|
|
date_str = time.strftime("%Y%m%d", time.gmtime(curtime))
|
|
version = retrieve_git_version(path)
|
|
if release_tag is None:
|
|
release_tag = version.split('-')[0]
|
|
source_hash = hash_directory(path, IGNORE_EXTS, IGNORE_DIRS)
|
|
long_version = f"{version}-moonraker-{date_str}"
|
|
release_info = {
|
|
'git_version': version,
|
|
'long_version': long_version,
|
|
'commit_hash': get_commit_hash(path),
|
|
'source_checksum': source_hash,
|
|
'ignored_exts': IGNORE_EXTS,
|
|
'ignored_dirs': IGNORE_DIRS,
|
|
'build_date': curtime,
|
|
'channel': channel,
|
|
'owner_repo': owner_repo,
|
|
'host_repo': OWNER_REPOS['moonraker'],
|
|
'release_tag': release_tag
|
|
}
|
|
vfile = pathlib.Path(os.path.join(path, source_dir, ".version"))
|
|
vfile.write_text(long_version)
|
|
rfile = pathlib.Path(os.path.join(path, ".release_info"))
|
|
rfile.write_text(json.dumps(release_info))
|
|
generate_dependency_info(path, source_dir)
|
|
return release_info
|
|
|
|
def create_zip(repo_path: str,
|
|
repo_name: str,
|
|
output_path: str
|
|
) -> None:
|
|
print(f"Creating Zip Release: {repo_name}")
|
|
zip_path = os.path.join(output_path, repo_name)
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
dest_path = os.path.join(tmp_dir, repo_name)
|
|
ingore_cb = CopyIgnore(repo_path)
|
|
shutil.copytree(repo_path, dest_path, ignore=ingore_cb)
|
|
shutil.make_archive(zip_path, "zip", root_dir=dest_path)
|
|
|
|
def main() -> None:
|
|
# Parse start arguments
|
|
parser = argparse.ArgumentParser(
|
|
description="Generates zip releases for Moonraker and Klipper")
|
|
parser.add_argument(
|
|
"-k", "--klipper", default="~/klipper",
|
|
metavar='<klipper_path>',
|
|
help="Path to Klipper git repo")
|
|
parser.add_argument(
|
|
"-o", "--output", default=os.path.join(MOONRAKER_PATH, ".dist"),
|
|
metavar='<output_path>', help="Path to output directory")
|
|
parser.add_argument(
|
|
"-b", "--beta", action='store_true',
|
|
help="Tag release as beta")
|
|
args = parser.parse_args()
|
|
kpath: str = os.path.abspath(os.path.expanduser(args.klipper))
|
|
opath: str = os.path.abspath(os.path.expanduser(args.output))
|
|
is_beta: bool = args.beta
|
|
channel = "beta" if is_beta else "stable"
|
|
if not os.path.exists(kpath):
|
|
print(f"Invalid path to Klipper: {kpath}")
|
|
sys.exit(-1)
|
|
if not os.path.exists(opath):
|
|
print(f"Invalid output path: {opath}")
|
|
sys.exit(-1)
|
|
releases = get_releases()
|
|
all_info: Dict[str, Dict[str, Any]] = {}
|
|
try:
|
|
print("Generating Moonraker Zip Distribution...")
|
|
all_info['moonraker'] = generate_version_info(
|
|
MOONRAKER_PATH, "moonraker", channel)
|
|
create_zip(MOONRAKER_PATH, 'moonraker', opath)
|
|
rtag: str = all_info['moonraker']['release_tag']
|
|
print("Generating Klipper Zip Distribution...")
|
|
all_info['klipper'] = generate_version_info(
|
|
kpath, "klippy", channel, rtag)
|
|
create_zip(kpath, 'klipper', opath)
|
|
info_file = pathlib.Path(os.path.join(opath, "RELEASE_INFO"))
|
|
info_file.write_text(json.dumps(all_info))
|
|
last_rinfo = get_last_release_info(
|
|
all_info['moonraker']['git_version'], is_beta, releases)
|
|
commit_log = {}
|
|
commit_log['moonraker'] = get_commit_log(
|
|
MOONRAKER_PATH, last_rinfo.get('moonraker', {}))
|
|
commit_log['klipper'] = get_commit_log(
|
|
kpath, last_rinfo.get('klipper', {}))
|
|
clog_file = pathlib.Path(os.path.join(opath, "COMMIT_LOG"))
|
|
clog_file.write_text(json.dumps(commit_log))
|
|
except Exception:
|
|
print("Error Creating Zip Distribution")
|
|
traceback.print_exc(file=sys.stdout)
|
|
sys.exit(-1)
|
|
print(f"Build Complete. Files are located at '{opath}'")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|