Rewrite build script (#115)

* scripts: build: rewrite build script * ci: use new build script in workflow * docs: new build script usage
2025-10-26 00:36:17 +02:00 · 2024-02-28 19:15:29 +01:00
parent f8399c3226
commit 1e55f2834d
6 changed files with 202 additions and 114 deletions
--- a/.github/workflows/template-image-build-workflow.yaml
+++ b/.github/workflows/template-image-build-workflow.yaml
@@ -52,5 +52,15 @@ jobs:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

+      - name: Install Build requirements
+        run: pip install -r scripts/build/requirements.txt
      - name: Run Build Script
-        run: bash ./scripts/build-images.sh ${{ inputs.app }} ${{ secrets.DOCKERHUB_USERNAME }}/
+        run: |
+          python3 scripts/build/build.py ${{ inputs.app }} \
+            --backfill 3 \
+            --platform linux/amd64 \
+            --platform linux/arm/v6 \
+            --platform linux/arm/v7 \
+            --platform linux/arm64/v8 \
+            --registry docker.io/${GITHUB_REPOSITORY_OWNER} \
+            --push
--- a/.gitignore
+++ b/.gitignore
@@ -3,4 +3,5 @@ klippy-env
 out
 *.bkp
 resonances
-*.tar.gz
+*.tar.gz
+venv
--- a/scripts/build-images.sh
+++ b/scripts/build-images.sh
@@ -1,112 +0,0 @@
-#!/bin/bash
-
-## Setup for building multiplatform images
-##
-## apt install qemu-user-static
-## docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-## docker buildx create --use --name cross
-## docker buildx inspect --bootstrap
-## docker buildx build --platform linux/amd64,linux/arm/v7 -t octoprint:latest --target run .
-
-set -e
-
-app=${1}
-registry=${2}
-
-# Set build parameters
-platform="linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8"
-dockerfile=docker/${app}/Dockerfile
-context=$(echo -n ${dockerfile} | rev | cut -f2- -d'/' | rev)
-
-# Get get versioning info from upstream repo
-## Set up directories
-pwd=$(pwd)
-tmp=$(mktemp -d)
-## Get upstream repo from Dockerfile
-source=$(grep "ARG REPO" ${dockerfile} | sed -r 's/.*REPO=(.*)$/\1/g')
-## Clone repo
-git clone ${source} ${tmp} > /dev/null
-## enter repo directory and get infos
-cd ${tmp}
-upstream_version=$(git describe --tags)
-upstream_tags=($(git tag -l --sort='v:refname' | tail -n3))
-## Return to previous directory and remove tmp
-cd ${pwd}
-rm -rf ${tmp}
-
-# Set label Values
-label_date=$(date --rfc-3339=seconds)
-if [ "${CI}" == "true" ]; then
-  label_prind_version="${GITHUB_SHA}"
-  label_author="${GITHUB_REPOSITORY_OWNER}"
-  label_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}"
-  label_doc="${label_url}/blob/${GITHUB_SHA}/docker/${app}/README.md"
-  label_src="${label_url}/blob/${GITHUB_SHA}/docker/${app}"
-else
-  label_prind_version="$(git rev-parse HEAD)"
-  label_author="$(whoami)"
-  label_url="local"
-  label_doc="local"
-  label_src="local"
-fi
-
-# Colorful output
-function log {
-  echo -e "\033[0;36m## ${1} \033[0m"
-}
-
-## Explicitly build Targets, except 'build'
-for target in $(grep "FROM .* as" ${dockerfile} | sed -r 's/.*FROM.*as (.*)/\1/g' | grep -v build); do
-
-  ## Append Target to Tag unless it is 'run'
-  if [ "${target}" != "run" ]; then
-    tag_extra="-${target}"
-  fi
-
-  ## latest
-  if docker buildx imagetools inspect ${registry}${app}:${upstream_version}${tag_extra} > /dev/null; then
-    log "## Image ${registry}${app}:${upstream_version}${tag_extra} already exists, nothing to do."
-  else
-    log "## Building latest Image ${registry}${app}:${upstream_version}${tag_extra}"
-    docker buildx build \
-      --build-arg VERSION=${upstream_sha} \
-      --platform ${platform} \
-      --tag ${registry}${app}:${upstream_version}${tag_extra} \
-      --tag ${registry}${app}:latest${tag_extra} \
-      --label org.prind.version=${label_prind_version} \
-      --label org.prind.image.created="${label_date}" \
-      --label org.prind.image.authors="${label_author}" \
-      --label org.prind.image.url="${label_url}" \
-      --label org.prind.image.documentation="${label_doc}" \
-      --label org.prind.image.source="${label_src}" \
-      --label org.prind.image.version="${upstream_version}" \
-      --target ${target} \
-      --push \
-      ${context}
-  fi
-
-  ## Tags
-  for tag in ${upstream_tags[@]}; do
-    if docker buildx imagetools inspect ${registry}${app}:${tag}${tag_extra} > /dev/null; then
-      log "## Image ${registry}${app}:${tag}${tag_extra} already exists, nothing to do."
-    else
-      log "## Building Image for tagged release ${registry}${app}:${tag}${tag_extra}"
-      docker buildx build \
-        --build-arg VERSION=${tag} \
-        --platform ${platform} \
-        --tag ${registry}${app}:${tag}${tag_extra} \
-        --label org.prind.version=${label_prind_version} \
-        --label org.prind.image.created="${label_date}" \
-        --label org.prind.image.authors="${label_author}" \
-        --label org.prind.image.url="${label_url}" \
-        --label org.prind.image.documentation="${label_doc}" \
-        --label org.prind.image.source="${label_src}" \
-        --label org.prind.image.version="${tag}" \
-        --target ${target} \
-        --push \
-        ${context}
-    fi
-  done
-
-  unset tag_extra
-done
--- a/scripts/build/README.md
+++ b/scripts/build/README.md
@@ -0,0 +1,40 @@
+# build.py
+This script is used to build multi platform images provided by prind and upload them to the registry. 
+
+## Local usage
+### Multi-Platform Requirements
+To build multi-platform images on your local machine, run the follwing commands to set up qemu and a docker buildx instance. This is not necessary if you only want to build images for the platform you're currently running on.
+
+```bash
+apt install qemu-user-static
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+docker buildx create --use --name cross
+docker buildx inspect --bootstrap
+```
+### Running the build script
+Set up a venv and install requirements.  
+All commands are run from the root of the repository
+```bash
+python3 -m venv venv
+venv/bin/pip install -r scripts/build/requirements.txt
+```
+
+Usage description:
+```bash
+usage: Build [-h] [--backfill BACKFILL] [--registry REGISTRY] [--platform PLATFORM] [--push] [--dry-run] [--force]
+             app
+
+Build container images for prind
+
+positional arguments:
+  app                  App to build. Directory must be located at ./docker/<app>
+
+options:
+  -h, --help           show this help message and exit
+  --backfill BACKFILL  Number of latest upstream git tags to build images for [default: 3]
+  --registry REGISTRY  Where to push images to, /<app> will be appended
+  --platform PLATFORM  Platform to build for. Repeat to build a multi-platform image [default: linux/amd64]
+  --push               Push image to registry [default: False]
+  --dry-run            Do not actually build images [default: False]
+  --force              Build images even though they exist in the registry [default: False]
+```
--- a/scripts/build/build.py
+++ b/scripts/build/build.py
@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+
+## Prerequisites
+## * user has read access to upstream repo
+## * docker login for each repo was executed
+## * qemu has been set up
+## * docker buildx instance has been created and bootstrapped
+
+import os
+import re
+import git
+import getpass
+import logging
+import argparse
+import tempfile
+
+from datetime import datetime, timezone
+from python_on_whales import docker
+
+# Parse arguments
+parser = argparse.ArgumentParser(
+  prog="Build",
+  description="Build container images for prind"
+)
+parser.add_argument("app",help="App to build. Directory must be located at ./docker/<app>")
+parser.add_argument("--backfill",type=int,default=3,help="Number of latest upstream git tags to build images for [default: 3]")
+parser.add_argument("--registry",help="Where to push images to, /<app> will be appended")
+parser.add_argument("--platform",action="append",default=["linux/amd64"],help="Platform to build for. Repeat to build a multi-platform image [default: linux/amd64]")
+parser.add_argument("--push",action="store_true",default=False,help="Push image to registry [default: False]")
+parser.add_argument("--dry-run",action="store_true",default=False,help="Do not actually build images [default: False]")
+parser.add_argument("--force",action="store_true",default=False,help="Build images even though they exist in the registry [default: False]")
+args = parser.parse_args()
+
+#---
+# Set up logging
+logger = logging.getLogger('prind')
+logger.setLevel(logging.DEBUG)
+
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ch = logging.StreamHandler()
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+#---
+# static definitions
+context = "docker/" + args.app
+dockerfile = context + "/Dockerfile"
+build = {
+  "upstream": None,
+  "targets": [],
+  "versions": {},
+  "labels": {
+    "org.prind.version": os.environ.get("GITHUB_SHA",(git.Repo(search_parent_directories=True)).head.object.hexsha),
+    "org.prind.image.created": datetime.now(timezone.utc).astimezone().isoformat(),
+    "org.prind.image.authors": os.environ.get("GITHUB_REPOSITORY_OWNER",getpass.getuser()),
+    "org.prind.image.url": "{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}".format(**os.environ) if "GITHUB_REPOSITORY" in os.environ else "local",
+    "org.prind.image.documentation": ("{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/blob/{GITHUB_SHA}/docker/" + args.app + "/README.md").format(**os.environ) if "GITHUB_REPOSITORY" in os.environ else "local",
+    "org.prind.image.source": ("{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/blob/{GITHUB_SHA}/docker/" + args.app).format(**os.environ) if "GITHUB_REPOSITORY" in os.environ else "local",
+  }
+}
+
+#---
+# extract info from dockerfile
+logger.info("Reading " + dockerfile)
+with open(dockerfile) as file:
+  for line in file:
+
+    # upstream repository url
+    repo = re.findall(r'ARG REPO.*', line)
+    if repo:
+      build["upstream"] = repo[0].split('=')[1]
+
+    # build targets
+    target = re.findall(r'FROM .* as .*', line)
+    if target:
+      if not "build" in target[0]:
+        build["targets"].append(target[0].split(' as ')[-1])
+
+logger.info("Found upstream repository: " + build["upstream"])
+logger.info("Found docker targets: " + str(build["targets"]))
+
+#---
+# extract info from upstream
+logger.info("Cloning Upstream repository")
+tmp = tempfile.TemporaryDirectory()
+upstream_repo = git.Repo.clone_from(build["upstream"], tmp.name)
+
+logger.info("Generating Versions from Upstream repository")
+## latest
+latest_version = upstream_repo.git.describe("--tags")
+build["versions"][latest_version] = { "latest": True }
+
+## tags
+upstream_repo_sorted_tags = upstream_repo.git.tag("-l", "--sort=v:refname").split('\n')
+for i in range(1,args.backfill+1):
+  tag = upstream_repo_sorted_tags[-abs(i)]
+  if tag not in build.keys():
+    build["versions"][tag] = { "latest": False }
+
+tmp.cleanup()
+logger.info("Found versions: " + str(build["versions"]))
+
+#---
+# Build all targets for all versions
+for version in build["versions"].keys():
+  for target in build["targets"]:
+
+    # Create list of docker tags
+    docker_image = "/".join(filter(None, (args.registry, args.app)))
+    tags = [
+      docker_image + ":" + (version if target == "run" else '-'.join([version, target])),
+      *(docker_image + (":latest" if target == "run" else '-'.join([":latest", target])) for _i in range(1) if build["versions"][version]["latest"]),
+    ]
+
+    try:
+      if args.force:
+        logger.warning("Build is forced")
+        raise
+      else:
+        # Check if the image already exists
+        docker.buildx.imagetools.inspect(tags[0])
+        logger.info("Image " + tags[0] + " exists, nothing to to.")
+    except:
+      if args.dry_run:
+        logger.debug("[dry-run] Would build " + tags[0])
+      else:
+        # Build if image does not exist
+        logger.info("Building " + tags[0])
+        stream = (
+          docker.buildx.build(
+            # Build specific
+            context_path = context,
+            build_args = {"VERSION": version},
+            platforms = args.platform,
+            target = target,
+            push = args.push,
+            tags = tags,
+            labels = {
+              **build["labels"],
+              "org.prind.image.version": version
+            },
+            stream_logs = True
+          )
+        )
+
+        for line in stream:
+          logger.info("BUILD: " + line.strip())
--- a/scripts/build/requirements.txt
+++ b/scripts/build/requirements.txt
@@ -0,0 +1,2 @@
+GitPython==3.1.42
+python-on-whales==0.69.0