A-Team_Digital_Solutions/build_manifests
Watch
2
0
Fork
You've already forked build_manifests
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

merge into: A-Team_Digital_Solutions:main
Branches Tags
A-Team_Digital_Solutions:main
A-Team_Digital_Solutions:shared-jenkinsfile
A-Team_Digital_Solutions:ofrp-12.1-build-toml
A-Team_Digital_Solutions:twrp-12.1-genevn
A-Team_Digital_Solutions:ofrp-12.1
A-Team_Digital_Solutions:twrp-12.1
A-Team_Digital_Solutions:derpfest-17.0
...
pull from: A-Team_Digital_Solutions:shared-jenkinsfile
Branches Tags
A-Team_Digital_Solutions:shared-jenkinsfile
A-Team_Digital_Solutions:ofrp-12.1-build-toml
A-Team_Digital_Solutions:twrp-12.1-genevn
A-Team_Digital_Solutions:ofrp-12.1
A-Team_Digital_Solutions:twrp-12.1
A-Team_Digital_Solutions:derpfest-17.0
A-Team_Digital_Solutions:main
Sign in to create a new pull request.
Author SHA1 Message Date
Nicholas Andrew
6cebf1a5a6 Jenkinsfile: tolerate unregistered parameters on a job's first run 
A new job's first run has no PORTAL_REQUEST_ID/PORTAL_USER/CHANGELOG yet,
which tripped set -u in Preflight. Default them to empty everywhere.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
 2026-10-11 00:52:02 -04:00
Nicholas Andrew
823de71f1d Add the shared recovery build pipeline 
Jenkinsfile generalizes the OFRP-12.1 job for every base branch: one job
per branch named after it (cms-<branch> jobs target the cms test portal),
base settings from the branch build.toml (validated by
pipeline/build_config.py), the same preflight, warm rollback, manifest
sync, build, rename, checksum, Garage upload and portal registration
stages, plus portal_request_id for builds started from the portal and a
warm-base lock shared by a base's live and test jobs. README documents
the device XMLs, the pipeline and build.toml.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
 2026-10-11 00:21:42 -04:00
3 changed files with 1326 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

971
Jenkinsfile vendored Normal file
View file

@ -0,0 +1,971 @@
// Shared A-Team recovery build pipeline.
//
// One Jenkins job per build_manifests branch, named after the branch
// (ofrp-12.1, twrp-12.1, ...), using "Pipeline script from SCM" on this file
// (branch main). Base settings come from build.toml on the job's branch; the
// device comes from <DEVICE>.xml on that branch.
//
// Jobs named cms-<branch> build the same base for the cms test portal
// (test bucket, test credentials, ateam-cms.gotadell.com).
def JOB = env.JOB_BASE_NAME
def IS_TEST_JOB = JOB.startsWith('cms-')
def BASE_BRANCH = IS_TEST_JOB ? JOB.substring(4) : JOB
if (!(BASE_BRANCH ==~ /[a-z0-9][a-z0-9._-]*/)) {
error "Job name '${JOB}' must be a build_manifests branch name (optionally prefixed with cms-)"
}
def TARGET = IS_TEST_JOB ? [
bucket : 'ateam-cms-staging',
s3Credentials : 'garage-s3-cms',
portalCredentials : 'portal-ingest-token-cms',
portalUrl : 'https://ateam-cms.gotadell.com/api/jenkins/build-complete/',
releaseRoot : '/storage/jenkins-artifacts/staging-cms',
] : [
bucket : 'ateam-staging',
s3Credentials : 'garage-s3',
portalCredentials : 'portal-ingest-token',
portalUrl : 'https://ateam.gotadell.com/api/jenkins/build-complete/',
releaseRoot : '/storage/jenkins-artifacts/staging',
]
pipeline {
agent {
label 'buildBox'
}
options {
// The repo is checked out inside Preflight, once the per-build TMPDIR exists
skipDefaultCheckout()
timestamps()
disableConcurrentBuilds()
// A base's live and cms- jobs share one ZFS dataset; never roll it back under each other.
lock(resource: "warm-base:${BASE_BRANCH}")
timeout(time: 6, unit: 'HOURS')
buildDiscarder(logRotator(numToKeepStr: '20'))
}
parameters {
// Device list = every .xml on this job's build_manifests branch (Active Choices plugin)
activeChoice(
name: 'DEVICE',
description: 'Device codename. Builds from <DEVICE>.xml on the manifest branch.',
choiceType: 'PT_SINGLE_SELECT',
filterable: false,
script: groovyScript(
script: [
classpath: [],
sandbox: false,
script: """
import groovy.json.JsonSlurper
def api = 'https://ateam.gotadell.com/git/api/v1/repos/A-Team_Digital_Solutions/build_manifests/contents?ref=${BASE_BRANCH}'
def conn = new URL(api).openConnection()
conn.connectTimeout = 5000
conn.readTimeout = 5000
return new JsonSlurper().parse(conn.inputStream)
.findAll { it.type == 'file' && it.name.endsWith('.xml') }
.collect { it.name[0..-5] }
.sort()
"""
],
fallbackScript: [
classpath: [],
sandbox: false,
script: 'return ["ERROR-could-not-list-manifests"]'
]
)
)
text(
name: 'CHANGELOG',
defaultValue: '',
description: 'Testing notes displayed in the A-Team Portal (empty = changelog from build.toml).'
)
string(
name: 'PORTAL_REQUEST_ID',
defaultValue: '',
description: 'Set by the portal "Start build" button. Empty for builds started here (owner-only approval).'
)
string(
name: 'PORTAL_USER',
defaultValue: '',
description: 'Portal user who started the build (informational; set by the portal).'
)
string(
name: 'MANIFEST_REF',
defaultValue: '',
description: 'cms- test jobs only: build_manifests ref to read build.toml and the device XML from (default: the job branch).'
)
}
environment {
DEVICE = "${params.DEVICE}"
MANIFEST_URL = 'ssh://git@ateam.gotadell.com/A-Team_Digital_Solutions/build_manifests.git'
MANIFEST_BRANCH = "${BASE_BRANCH}"
MANIFEST_REF = "${params.MANIFEST_REF ?: BASE_BRANCH}"
MANIFEST_FILE = "${params.DEVICE}.xml"
TEST_JOB = "${IS_TEST_JOB ? '1' : ''}"
TMPDIR = "/build/tmp/jenkins/${JOB}-${params.DEVICE}-${BUILD_NUMBER}"
BUILD_CONFIG = "/build/tmp/jenkins/${JOB}-${params.DEVICE}-${BUILD_NUMBER}/build-config.env"
RELEASE_ROOT = "${TARGET.releaseRoot}"
// Match the env the warm out/ was built with, or soong regenerates everything
// (OUT_DIR comes from build.toml: <source_root>/out)
CC_WRAPPER = '/usr/bin/ccache'
CCACHE_EXEC = '/usr/bin/ccache'
CCACHE_DIR = '/build/ccache'
USE_CCACHE = '1'
ALLOW_MISSING_DEPENDENCIES = 'true'
S3_ENDPOINT_URL = 'http://10.0.55.47:9000'
S3_REGION = 'garage'
S3_BUCKET = "${TARGET.bucket}"
PORTAL_URL = "${TARGET.portalUrl}"
}
stages {
stage('Preflight') {
steps {
sh '''#!/usr/bin/env bash
set -Eeuo pipefail
[[ "$DEVICE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: bad DEVICE '$DEVICE'"; exit 1; }
[[ -z "${PORTAL_REQUEST_ID:-}" || "${PORTAL_REQUEST_ID:-}" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] \
|| { echo "ERROR: bad PORTAL_REQUEST_ID"; exit 1; }
if [ -z "$TEST_JOB" ] && [ "$MANIFEST_REF" != "$MANIFEST_BRANCH" ]; then
echo "ERROR: MANIFEST_REF is only allowed on cms- test jobs"
exit 1
fi
[[ "$MANIFEST_REF" =~ ^[A-Za-z0-9._/-]+$ ]] || { echo "ERROR: bad MANIFEST_REF"; exit 1; }
echo "===== REQUIRED TOOLS ====="
for TOOL in \
git \
repo \
python3 \
zfs \
ccache \
aws \
jq \
curl \
md5sum \
sha256sum \
unzip
do
command -v "$TOOL"
done
echo
echo "===== VERIFY CLEAN PER-BUILD PATHS ====="
test ! -e "$TMPDIR"
install -d -m 2775 "$TMPDIR"
'''
// Jenkinsfile + pipeline/build_config.py from build_manifests main
checkout scm
sh '''#!/usr/bin/env bash
set -Eeuo pipefail
echo "===== FETCH BUILD MANIFEST ====="
MANIFEST_CHECKOUT="$TMPDIR/build_manifests"
git clone \
--depth 1 \
--branch "$MANIFEST_REF" \
--single-branch \
"$MANIFEST_URL" \
"$MANIFEST_CHECKOUT"
test -s "$MANIFEST_CHECKOUT/$MANIFEST_FILE" \
|| { echo "ERROR: $MANIFEST_FILE not found on $MANIFEST_REF"; exit 1; }
echo
echo "===== RESOLVE build.toml FOR $DEVICE ====="
python3 "$WORKSPACE/pipeline/build_config.py" \
"$MANIFEST_CHECKOUT/build.toml" \
"$DEVICE" \
> "$BUILD_CONFIG"
source "$BUILD_CONFIG"
{
printf 'RELEASE_DIR=%q\\n' "$RELEASE_ROOT/$PROJECT/$DEVICE/$PROJECT-$DEVICE-$BUILD_NUMBER"
printf 'STAGING_PREFIX=%q\\n' "$PROJECT/$DEVICE/build-$BUILD_NUMBER/"
} >> "$BUILD_CONFIG"
cat "$BUILD_CONFIG"
source "$BUILD_CONFIG"
echo
echo "===== BUILD IDENTIFICATION ====="
echo "Target: $([ -n "$TEST_JOB" ] && echo "cms test portal" || echo "live portal")"
echo "Project: $PROJECT_NAME $VERSION"
echo "Device: $DEVICE"
echo "Manifest: $MANIFEST_URL ($MANIFEST_REF/$MANIFEST_FILE)"
echo "Warm base: $WARM_SNAPSHOT"
echo "Lunch: $LUNCH_TARGET"
echo "Portal prefix: $STAGING_PREFIX"
echo "Portal request: ${PORTAL_REQUEST_ID:-none} ${PORTAL_USER:+(by $PORTAL_USER)}"
echo
echo "===== VERIFY WARM SNAPSHOT ====="
zfs list -H -t snapshot -o name,creation "$WARM_SNAPSHOT"
zfs allow "$ZFS_DATASET"
MOUNTPOINT="$(zfs get -H -o value mountpoint "$ZFS_DATASET")"
test "$MOUNTPOINT" = "$SOURCE_ROOT" \
|| { echo "ERROR: $ZFS_DATASET is mounted at $MOUNTPOINT, build.toml says $SOURCE_ROOT"; exit 1; }
echo
echo "===== PREPARE RELEASE DIRECTORY ====="
test ! -e "$RELEASE_DIR"
install -d -m 2775 "$RELEASE_DIR"
echo "Temporary: $TMPDIR"
echo "Release staging: $RELEASE_DIR"
'''
}
}
stage('Restore Warm Base') {
steps {
sh '''#!/usr/bin/env bash
set -Eeuo pipefail
source "$BUILD_CONFIG"
echo "===== ROLLBACK TO WARM ====="
zfs rollback "$WARM_SNAPSHOT"
echo
echo "===== VERIFY WARM BASE ====="
cd "$SOURCE_ROOT"
test -d .repo
test -d out
for P in $WARM_CHECK; do
test -e "$P" || { echo "ERROR: $P missing from warm base"; exit 1; }
done
if compgen -G "device/*/$DEVICE" >/dev/null; then
echo "ERROR: device tree for $DEVICE already present in warm base"
exit 1
fi
test -z "$(ls -A out/target/product 2>/dev/null)"
test -z "$(ls -A .repo/local_manifests 2>/dev/null)"
echo
echo "===== VERIFY JENKINS CAN WRITE THE TREE ====="
test -w "$SOURCE_ROOT"
test -w "$SOURCE_ROOT/out"
test -w "$SOURCE_ROOT/.repo"
echo "Source: $SOURCE_ROOT"
'''
}
}
stage('Sync Device Trees From Manifest') {
steps {
sh '''#!/usr/bin/env bash
set -Eeuo pipefail
source "$BUILD_CONFIG"
MANIFEST_CHECKOUT="$TMPDIR/build_manifests"
MANIFEST_COMMIT="$(git -C "$MANIFEST_CHECKOUT" rev-parse HEAD)"
printf '%s\\n' "$MANIFEST_COMMIT" > "$RELEASE_DIR/manifest-commit.txt"
echo "Manifest commit: $MANIFEST_COMMIT"
cd "$SOURCE_ROOT"
mkdir -p .repo/local_manifests
install -m 0644 \
"$MANIFEST_CHECKOUT/$MANIFEST_FILE" \
".repo/local_manifests/$MANIFEST_FILE"
echo
echo "===== RESOLVE BRANCH NAMES (case-insensitive) ====="
python3 - ".repo/local_manifests/$MANIFEST_FILE" <<'PYEOF'
import sys, subprocess
import xml.etree.ElementTree as ET
path = sys.argv[1]
tree = ET.parse(path)
root = tree.getroot()
remotes = {r.get("name"): r.get("fetch", "") for r in root.iter("remote")}
default = root.find("default")
dflt = lambda k: default.get(k) if default is not None else None
changed = False
for p in root.iter("project"):
name = p.get("name")
rev = p.get("revision") or dflt("revision")
# leave tags, full refs and commit hashes alone
if not rev or rev.startswith("refs/") or len(rev) == 40:
continue
fetch = remotes.get(p.get("remote") or dflt("remote"), "")
if "://" not in fetch and "@" not in fetch:
continue
url = fetch + ("" if fetch.endswith(("/", ":")) else "/") + name
out = subprocess.run(["git", "ls-remote", "--heads", url],
capture_output=True, text=True, check=True).stdout
heads = [l.split("refs/heads/", 1)[1] for l in out.splitlines() if "refs/heads/" in l]
if rev in heads:
print(f"{name}: {rev}")
continue
match = [h for h in heads if h.lower() == rev.lower()]
if len(match) != 1:
sys.exit(f"ERROR: {name}: no branch '{rev}'. Branches: {', '.join(heads) or 'none'}")
print(f"{name}: {match[0]} (manifest says {rev})")
p.set("revision", match[0])
changed = True
if changed:
tree.write(path, xml_declaration=True, encoding="UTF-8")
PYEOF
echo
echo "===== PROJECTS IN $MANIFEST_FILE ====="
mapfile -t PATHS < <(
python3 - ".repo/local_manifests/$MANIFEST_FILE" <<'PYEOF'
import sys
import xml.etree.ElementTree as ET
for p in ET.parse(sys.argv[1]).getroot().iter("project"):
print(p.get("path") or p.get("name"))
PYEOF
)
test "${#PATHS[@]}" -ge 1
printf ' %s\\n' "${PATHS[@]}"
echo
echo "===== DEVICE INFO FROM $MANIFEST_FILE ====="
python3 - ".repo/local_manifests/$MANIFEST_FILE" > "$RELEASE_DIR/device-info.env" <<'PYEOF'
import sys, shlex
import xml.etree.ElementTree as ET
root = ET.parse(sys.argv[1]).getroot()
remotes = {r.get("name"): r.get("fetch", "") for r in root.iter("remote")}
default = root.find("default")
dflt = lambda k: default.get(k, "") if default is not None else ""
out = lambda k, v: print(f"{k}={shlex.quote(v or '')}")
# The device tree is the project carrying the ateam.* annotations
trees = [p for p in root.iter("project")
if any(a.get("name", "").startswith("ateam.") for a in p.iter("annotation"))]
if len(trees) != 1:
sys.exit(f"ERROR: expected ateam.* annotations on exactly one project, found {len(trees)}")
p = trees[0]
for a in p.iter("annotation"):
n = a.get("name", "")
if n.startswith("ateam."):
out(n[6:].upper(), a.get("value"))
name = p.get("name")
rev = p.get("revision") or dflt("revision")
fetch = remotes.get(p.get("remote") or dflt("remote"), "")
host = fetch.split("@")[-1].split("://")[-1].strip("/").split("/")[0].split(":")[0]
if host == "ateam.gotadell.com":
url = f"https://ateam.gotadell.com/git/{name}/src/branch/{rev}"
elif host == "github.com":
url = f"https://github.com/{name}/tree/{rev}"
else:
url = f"{fetch.rstrip('/')}/{name}"
out("TREE_PATH", p.get("path") or name)
out("TREE_URL", url)
PYEOF
cat "$RELEASE_DIR/device-info.env"
source "$RELEASE_DIR/device-info.env"
for VAR in MODEL DEVICE_NAME MANUFACTURER PARTITION TREE_PATH TREE_URL; do
test -n "${!VAR:-}" || { echo "ERROR: $VAR missing (check ateam.* annotations in $MANIFEST_FILE)"; exit 1; }
done
case "$PARTITION" in
recovery|boot|vendor_boot) ;;
*) echo "ERROR: unknown ateam.partition '$PARTITION'"; exit 1 ;;
esac
echo
echo "===== SYNC ONLY MANIFEST PROJECTS ====="
repo sync \
-c \
--no-tags \
--no-clone-bundle \
--force-sync \
-j8 \
"${PATHS[@]}"
test -d "$TREE_PATH/.git" || test -e "$TREE_PATH/.git"
echo
echo "===== RECORD SYNCED COMMITS ====="
for P in "${PATHS[@]}"; do
printf '%s %s\\n' "$P" "$(git -C "$P" rev-parse HEAD)"
done | tee "$RELEASE_DIR/synced-projects.txt"
TREE_COMMIT="$(git -C "$TREE_PATH" rev-parse HEAD)"
TREE_SUBJECT="$(git -C "$TREE_PATH" log -1 --pretty=%s)"
printf '%s\\n' "$TREE_COMMIT" > "$RELEASE_DIR/device-tree-commit.txt"
printf '%s\\n' "$TREE_SUBJECT" > "$RELEASE_DIR/device-tree-subject.txt"
echo
echo "Tree commit: $TREE_COMMIT"
echo "Tree subject: $TREE_SUBJECT"
'''
}
}
stage('Configure Ccache') {
steps {
sh '''#!/usr/bin/env bash
set -Eeuo pipefail
mkdir -p "$CCACHE_DIR"
test -d "$CCACHE_DIR"
test -w "$CCACHE_DIR"
ccache -M 100G
ccache -o compression=true
ccache -z
ccache -s
'''
}
}
stage('Lunch') {
steps {
sh '''#!/usr/bin/env bash
set -o pipefail
source "$BUILD_CONFIG" || exit 1
cd "$SOURCE_ROOT" || exit 1
export OUT_DIR
export CC_WRAPPER
export CCACHE_EXEC
export TMPDIR
export TMP="$TMPDIR"
export TEMP="$TMPDIR"
export CCACHE_DIR
export USE_CCACHE
export ALLOW_MISSING_DEPENDENCIES
for VAR in $BUILD_ENV_NAMES; do export "$VAR"; done
source build/envsetup.sh
type lunch >/dev/null 2>&1 || exit 1
type mka >/dev/null 2>&1 || exit 1
lunch "$LUNCH_TARGET" || exit 1
echo
echo "===== BUILD VARIABLES ====="
echo -n "TARGET_PRODUCT: "
get_build_var TARGET_PRODUCT || exit 1
echo -n "TARGET_DEVICE: "
get_build_var TARGET_DEVICE || exit 1
echo -n "PRODUCT_OUT: "
get_build_var PRODUCT_OUT || exit 1
'''
}
}
stage('Install Clean and Build') {
steps {
sh '''#!/usr/bin/env bash
set -o pipefail
source "$BUILD_CONFIG" || exit 1
cd "$SOURCE_ROOT" || exit 1
export OUT_DIR
export CC_WRAPPER
export CCACHE_EXEC
export TMPDIR
export TMP="$TMPDIR"
export TEMP="$TMPDIR"
export CCACHE_DIR
export USE_CCACHE
export ALLOW_MISSING_DEPENDENCIES
for VAR in $BUILD_ENV_NAMES; do export "$VAR"; done
source build/envsetup.sh
lunch "$LUNCH_TARGET" || exit 1
echo "===== INSTALL CLEAN ====="
m installclean
INSTALLCLEAN_STATUS=$?
if [ "$INSTALLCLEAN_STATUS" -ne 0 ]; then
echo "ERROR: installclean failed with status $INSTALLCLEAN_STATUS"
exit "$INSTALLCLEAN_STATUS"
fi
source "$RELEASE_DIR/device-info.env" || exit 1
case "$PARTITION" in
recovery) BUILD_TARGET=recoveryimage ;;
boot) BUILD_TARGET=bootimage ;;
vendor_boot) BUILD_TARGET=vendorbootimage ;;
*) echo "ERROR: unknown partition '$PARTITION'"; exit 1 ;;
esac
echo
echo "===== BUILD $PROJECT_NAME ($MAKE_EXTRA $BUILD_TARGET) ====="
START_TIME="$(date +%s)"
# MAKE_EXTRA is validated by build_config.py (plain words only)
mka $MAKE_EXTRA "$BUILD_TARGET"
BUILD_STATUS=$?
END_TIME="$(date +%s)"
ELAPSED=$((END_TIME - START_TIME))
if [ "$BUILD_STATUS" -ne 0 ]; then
echo "ERROR: $PROJECT_NAME build failed with status $BUILD_STATUS"
exit "$BUILD_STATUS"
fi
PRODUCT_OUT="$(get_build_var PRODUCT_OUT)" || exit 1
echo
echo "===== BUILD TIME ====="
printf "$PROJECT_NAME $DEVICE build: %d:%02d (mm:ss)\\n" \
"$((ELAPSED / 60))" \
"$((ELAPSED % 60))"
echo
echo "===== PRODUCT OUTPUT ====="
cd "$PRODUCT_OUT" || exit 1
for EXT in $OUTPUT_EXTS; do
GLOB_VAR="OUTPUT_${EXT^^}_GLOB"
find . -maxdepth 1 -type f \\( -name "${!GLOB_VAR}" -o -name "${!GLOB_VAR}.*" \\) \
-printf '%f %s bytes\\n'
done | sort
'''
}
}
stage('Collect and Rename Artifacts') {
steps {
sh '''#!/usr/bin/env bash
set -Eeuo pipefail
source "$BUILD_CONFIG"
source "$RELEASE_DIR/device-info.env"
PRODUCT_OUT="$OUT_DIR/target/product/$DEVICE"
cd "$PRODUCT_OUT"
BASE="${RELEASE_TAG}-${RELEASE_VER}_A-Team_${DEVICE}_${MODEL}-$(TZ=America/New_York date +%y%m%d)"
: > "$RELEASE_DIR/deliverables.txt"
for EXT in $OUTPUT_EXTS; do
GLOB_VAR="OUTPUT_${EXT^^}_GLOB"
CHECKSUM_VAR="OUTPUT_${EXT^^}_CHECKSUM"
mapfile -t FOUND < <(find . -maxdepth 1 -type f -name "${!GLOB_VAR}" -printf '%f\\n')
if [ "${#FOUND[@]}" -ne 1 ]; then
echo "ERROR: Expected exactly one .$EXT matching '${!GLOB_VAR}', found ${#FOUND[@]}."
printf '%s\\n' "${FOUND[@]}"
exit 1
fi
FILE="${FOUND[0]}"
case "${!CHECKSUM_VAR}" in
md5)
test -s "$FILE.md5"
EXPECTED="$(awk 'NR == 1 {print $1}' "$FILE.md5")"
ACTUAL="$(md5sum "$FILE" | awk '{print $1}')"
test "$EXPECTED" = "$ACTUAL"
echo "$FILE: native MD5 OK"
;;
sha256)
test -s "$FILE.sha256"
EXPECTED="$(awk 'NR == 1 {print $1}' "$FILE.sha256")"
ACTUAL="$(sha256sum "$FILE" | awk '{print $1}')"
test "$EXPECTED" = "$ACTUAL"
echo "$FILE: native SHA-256 OK"
;;
none)
;;
esac
install -m 0644 "$FILE" "$RELEASE_DIR/$BASE.$EXT"
( cd "$RELEASE_DIR" && sha256sum "$BASE.$EXT" > "$BASE.$EXT.sha256" )
printf '%s\\n%s\\n' "$BASE.$EXT" "$BASE.$EXT.sha256" >> "$RELEASE_DIR/deliverables.txt"
done
echo
echo "===== RENAMED TO A-TEAM SCHEME ====="
ls -lh "$RELEASE_DIR"
'''
}
}
stage('Verify Deliverables') {
steps {
sh '''#!/usr/bin/env bash
set -Eeuo pipefail
source "$BUILD_CONFIG"
cd "$RELEASE_DIR"
mapfile -t FILES < deliverables.txt
EXPECTED_COUNT=$(( $(wc -w <<< "$OUTPUT_EXTS") * 2 ))
test "${#FILES[@]}" -eq "$EXPECTED_COUNT"
for FILE in "${FILES[@]}"; do
test -s "$FILE"
ls -lh "$FILE"
done
echo
echo "===== VERIFY SHA-256 FILES ====="
for FILE in "${FILES[@]}"; do
case "$FILE" in *.sha256) sha256sum -c "$FILE" ;; esac
done
for EXT in $OUTPUT_EXTS; do
TEST_ZIP_VAR="OUTPUT_${EXT^^}_TEST_ZIP"
if [ -n "${!TEST_ZIP_VAR}" ]; then
echo
echo "===== VERIFY .$EXT ARCHIVE ====="
for FILE in "${FILES[@]}"; do
case "$FILE" in *."$EXT") unzip -t "$FILE" ;; esac
done
fi
done
echo
echo "===== CREATE INTERNAL SHA-256 MANIFEST ====="
sha256sum "${FILES[@]}" > artifact-manifest.sha256
test "$(wc -l < artifact-manifest.sha256)" -eq "$EXPECTED_COUNT"
cat artifact-manifest.sha256
'''
}
}
stage('Upload to Garage') {
steps {
withCredentials([
usernamePassword(
credentialsId: TARGET.s3Credentials,
usernameVariable: 'AWS_ACCESS_KEY_ID',
passwordVariable: 'AWS_SECRET_ACCESS_KEY'
)
]) {
sh '''#!/usr/bin/env bash
set -Eeuo pipefail
set +x
source "$BUILD_CONFIG"
export AWS_DEFAULT_REGION="$S3_REGION"
export AWS_REGION="$S3_REGION"
export AWS_EC2_METADATA_DISABLED=true
cd "$RELEASE_DIR"
mapfile -t FILES < deliverables.txt
FILES+=("artifact-manifest.sha256")
test "${#FILES[@]}" -eq $(( $(wc -w <<< "$OUTPUT_EXTS") * 2 + 1 ))
echo "===== UPLOAD GARAGE ARTIFACTS ====="
echo "Bucket: $S3_BUCKET"
echo "Prefix: $STAGING_PREFIX"
for FILE in "${FILES[@]}"; do
echo "Uploading: $FILE"
aws \
--endpoint-url "$S3_ENDPOINT_URL" \
s3api put-object \
--bucket "$S3_BUCKET" \
--key "${STAGING_PREFIX}${FILE}" \
--body "$FILE" \
>/dev/null
done
echo
echo "GARAGE UPLOAD COMPLETE"
'''
}
}
}
stage('Verify Garage Objects') {
steps {
withCredentials([
usernamePassword(
credentialsId: TARGET.s3Credentials,
usernameVariable: 'AWS_ACCESS_KEY_ID',
passwordVariable: 'AWS_SECRET_ACCESS_KEY'
)
]) {
sh '''#!/usr/bin/env bash
set -Eeuo pipefail
set +x
source "$BUILD_CONFIG"
export AWS_DEFAULT_REGION="$S3_REGION"
export AWS_REGION="$S3_REGION"
export AWS_EC2_METADATA_DISABLED=true
cd "$RELEASE_DIR"
mapfile -t FILES < deliverables.txt
FILES+=("artifact-manifest.sha256")
echo "===== VERIFY REMOTE OBJECT SIZES ====="
for FILE in "${FILES[@]}"; do
LOCAL_SIZE="$(stat -c '%s' "$FILE")"
REMOTE_SIZE="$(
aws \
--endpoint-url "$S3_ENDPOINT_URL" \
s3api head-object \
--bucket "$S3_BUCKET" \
--key "${STAGING_PREFIX}${FILE}" \
--query ContentLength \
--output text
)"
printf '%-80s local=%s remote=%s\\n' \
"$FILE" \
"$LOCAL_SIZE" \
"$REMOTE_SIZE"
test "$LOCAL_SIZE" = "$REMOTE_SIZE"
done
REMOTE_MANIFEST="$(mktemp)"
trap 'rm -f "$REMOTE_MANIFEST"' EXIT
aws \
--endpoint-url "$S3_ENDPOINT_URL" \
s3api get-object \
--bucket "$S3_BUCKET" \
--key "${STAGING_PREFIX}artifact-manifest.sha256" \
"$REMOTE_MANIFEST" \
>/dev/null
cmp artifact-manifest.sha256 "$REMOTE_MANIFEST"
echo
echo "GARAGE VERIFICATION PASSED"
'''
}
}
}
stage('Register Portal Testing Build') {
steps {
withCredentials([
string(
credentialsId: TARGET.portalCredentials,
variable: 'PORTAL_TOKEN'
)
]) {
sh '''#!/usr/bin/env bash
set -Eeuo pipefail
set +x
source "$BUILD_CONFIG"
TREE_COMMIT="$(cat "$RELEASE_DIR/device-tree-commit.txt")"
TREE_SUBJECT="$(cat "$RELEASE_DIR/device-tree-subject.txt")"
source "$RELEASE_DIR/device-info.env"
PAYLOAD_FILE="$(mktemp)"
RESPONSE_FILE="$(mktemp)"
cleanup_files() {
rm -f "$PAYLOAD_FILE" "$RESPONSE_FILE"
}
trap cleanup_files EXIT
jq -n \
--arg project "$PROJECT" \
--arg project_name "$PROJECT_NAME" \
--arg device "$DEVICE" \
--arg device_name "$DEVICE_NAME ($MODEL)" \
--arg manufacturer "$MANUFACTURER" \
--arg version "$VERSION" \
--argjson build_number "$BUILD_NUMBER" \
--arg staging_prefix "$STAGING_PREFIX" \
--arg source_branch "$MANIFEST_BRANCH" \
--arg jenkins_url "$BUILD_URL" \
--arg tree_url "$TREE_URL" \
--arg tree_commit "$TREE_COMMIT" \
--arg tree_subject "$TREE_SUBJECT" \
--arg operator_changelog "${CHANGELOG:-$CHANGELOG_DEFAULT}" \
--arg portal_request_id "${PORTAL_REQUEST_ID:-}" \
'{
project: $project,
project_name: $project_name,
device: $device,
device_name: $device_name,
manufacturer: $manufacturer,
version: $version,
build_number: $build_number,
staging_prefix: $staging_prefix,
source_branch: $source_branch,
jenkins_url: $jenkins_url,
portal_request_id: $portal_request_id,
changelog: (
$operator_changelog
+ "\\n\\nDevice tree: "
+ $tree_url
+ "\\nCommit: "
+ $tree_commit
+ "\\nCommit subject: "
+ $tree_subject
)
}' \
> "$PAYLOAD_FILE"
HTTP_CODE="$(
curl \
-sS \
--connect-timeout 15 \
--max-time 120 \
-o "$RESPONSE_FILE" \
-w '%{http_code}' \
-H "X-Jenkins-Token: $PORTAL_TOKEN" \
-H 'Content-Type: application/json' \
--data-binary "@$PAYLOAD_FILE" \
"$PORTAL_URL"
)"
echo "===== PORTAL RESPONSE ====="
echo "HTTP=$HTTP_CODE"
jq . "$RESPONSE_FILE" \
2>/dev/null \
|| cat "$RESPONSE_FILE"
case "$HTTP_CODE" in
200|201)
;;
*)
echo "ERROR: Portal registration failed."
exit 1
;;
esac
if jq -e '.error' "$RESPONSE_FILE" >/dev/null 2>&1; then
echo "ERROR: Portal returned an error."
exit 1
fi
if [ -n "${PORTAL_REQUEST_ID:-}" ] && ! jq -e '.started_by' "$RESPONSE_FILE" >/dev/null 2>&1; then
echo "WARNING: portal did not attribute this build: $(jq -r '.attribution_warning // empty' "$RESPONSE_FILE")"
fi
echo
echo "PORTAL TESTING REGISTRATION PASSED"
'''
}
}
}
stage('Ccache Results') {
steps {
sh '''#!/usr/bin/env bash
set -Eeuo pipefail
ccache -s
'''
}
}
}
post {
success {
sh '''#!/usr/bin/env bash
set -Eeuo pipefail
source "$BUILD_CONFIG"
echo "===== SUCCESSFUL BUILD CLEANUP ====="
rm -rf "$TMPDIR"
echo
echo "===== RETURN BASE TO WARM ====="
zfs rollback "$WARM_SNAPSHOT"
echo
echo "Warm base restored: $WARM_SNAPSHOT"
echo "Local release files retained at:"
echo "$RELEASE_DIR"
echo
echo "Portal status: Testing"
echo "Garage prefix: $STAGING_PREFIX"
'''
echo "${BASE_BRANCH} ${env.DEVICE} build uploaded and registered for testing."
}
failure {
echo "${BASE_BRANCH} ${env.DEVICE} pipeline failed. Source tree left as-is for diagnosis (next run rolls back to @warm)."
sh '''#!/usr/bin/env bash
set +e
source "$BUILD_CONFIG" 2>/dev/null
echo "===== RETAINED FAILURE PATHS ====="
echo "Source + out: ${SOURCE_ROOT:-not resolved}"
echo "Temporary files: $TMPDIR"
echo "Release staging: ${RELEASE_DIR:-not created}"
echo "Garage prefix: ${STAGING_PREFIX:-not resolved}"
'''
}
aborted {
echo "${BASE_BRANCH} ${env.DEVICE} pipeline was aborted. Diagnostic paths were retained."
}
always {
echo "${BASE_BRANCH} ${env.DEVICE} pipeline finished."
}
}
}

163
README.md
View file

@ -1,3 +1,164 @@
# build_manifests
device manifests for a-team digital solutions
Device manifests for the A-Team Jenkins build pipelines.
Each base branch has one Jenkins job (named after the branch) running the shared `Jenkinsfile` from `main`. It rolls the base's warm source tree back to `@warm`, drops one device XML into `.repo/local_manifests/`, syncs only the projects listed in it, reads the device info from its annotations and the base settings from the branch's `build.toml`.
## Layout
**One branch per build base, one file per device.**
```
ofrp-12.1/ genevn.xml milanf.xml rtwo.xml ...
twrp-12.1/ genevn.xml ...
aera-16.0/ genevn.xml milanf.xml ...
```
A job points at a branch + file, e.g. `ofrp-12.1` / `genevn.xml`.
## Device XML
A standard repo local manifest, plus `ateam.*` annotations on the device tree project.
```xml
<?xml version="1.0" encoding="UTF-8"?>
<manifest>
<remote name="ateam" fetch="ssh://git@ateam.gotadell.com/" />
<project path="device/motorola/genevn"
name="A-Team_Digital_Solutions/recovery_device_motorola_genevn"
remote="ateam" revision="ofrp-12.1">
<annotation name="ateam.model" value="XT2315"/>
<annotation name="ateam.device_name" value="Moto G Stylus 5G 2023"/>
<annotation name="ateam.manufacturer" value="Motorola"/>
<annotation name="ateam.partition" value="recovery"/>
</project>
</manifest>
```
### Annotations
All four are required; the build stops if one is missing.
| Annotation | Used for | Example |
|---|---|---|
| `ateam.model` | Release file name, portal device name | `XT2315` |
| `ateam.device_name` | Portal device name | `Moto G Stylus 5G 2023` |
| `ateam.manufacturer` | Portal | `Motorola` |
| `ateam.partition` | Build target and flash target | `recovery` |
`ateam.partition` must be one of:
| Value | Builds |
|---|---|
| `recovery` | `recoveryimage` |
| `boot` | `bootimage` |
| `vendor_boot` | `vendorbootimage` |
The portal shows the device as `<device_name> (<model>)`.
### Projects
- List the device tree plus anything else the device needs (kernel, vendor, common trees).
- Every project listed gets synced; nothing else in the base tree is touched.
- Any git source works. Add another `<remote>` for GitHub or elsewhere; the device tree doesn't have to live on A-Team Forgejo.
- Pin `revision` to the branch for that base (e.g. `ofrp-12.1`).
- Don't add projects that override the base's own repos. Device-specific fixes go in the device trees.
## Release naming
Builds come out as:
```
<RECOVERY>-<ver>_A-Team_<codename>_<model>-<YYMMDD>.<img|zip>
```
plus a `.sha256` for each, e.g. `OFRP-12.1_A-Team_genevn_XT2315-261009.zip`.
## Adding a device
1. Clone and switch to the base branch:
```
git clone ssh://git@ateam.gotadell.com/A-Team_Digital_Solutions/build_manifests.git
cd build_manifests
git checkout ofrp-12.1
```
2. Copy an existing XML to `<codename>.xml` and edit the projects and annotations.
3. Commit and push.
4. That's it: the branch's Jenkins job (named after the branch, e.g. `ofrp-12.1`) lists the new XML in its
`DEVICE` dropdown, and the portal picks it up on its next device sync.
## Adding a base
1. Create a new branch named `<base>-<version>` (e.g. `shrp-12.1`) and add a device XML for each device built on it.
Each base has its own copy of a device's XML, so update model or name changes on every branch that has that device.
2. Add a `build.toml` (see the README on `main`) describing the base: portal project, release naming, ZFS dataset,
lunch target and release files.
3. Prepare the warm source tree on buildBox and snapshot it as `<dataset>@warm`.
4. Create a Jenkins job named exactly like the branch, using *Pipeline script from SCM* on `build_manifests`
branch `main`, script path `Jenkinsfile`.
5. Map the portal project to the job in the portal admin.
## Build pipeline
`Jenkinsfile` on `main` is the build pipeline for every base. Each base branch has one Jenkins job named exactly
like the branch (`ofrp-12.1`, `twrp-12.1`, ...), set to *Pipeline script from SCM* on this repo, branch `main`,
script path `Jenkinsfile`. A job named `cms-<branch>` builds the same base for the cms test portal (test bucket,
test credentials, `ateam-cms.gotadell.com`).
Per run the job:
1. checks the device codename, resolves `build.toml` for that device (`pipeline/build_config.py`) and checks the
warm snapshot and its mountpoint;
2. rolls the base back to `<zfs_dataset>@warm` and checks the warm base is clean (no device tree for the device,
empty `out/target/product`, empty `.repo/local_manifests`);
3. copies `<DEVICE>.xml` into `.repo/local_manifests/`, fixes branch-name case, reads the `ateam.*` annotations and
syncs only the listed projects;
4. runs `lunch`, `m installclean` and `mka <make_extra> <partition target>`;
5. picks the release files from `out/target/product/<codename>/`, verifies their native checksums, renames them to
the A-Team scheme with a `.sha256` each, uploads them to Garage and registers the build with the portal as
testing (including the portal request id when the portal started it);
6. rolls the base back to `@warm` again (on failure the tree is left for diagnosis; the next run rolls it back).
A base's live and `cms-` jobs share one ZFS dataset and hold the `warm-base:<branch>` lock (Lockable Resources
plugin) for the whole run.
Parameters: `DEVICE` (lists the branch's XMLs), `CHANGELOG`, `PORTAL_REQUEST_ID` and `PORTAL_USER` (set by the
portal's *Start build*), and `MANIFEST_REF` (cms- test jobs only, to build from an unmerged branch).
## build.toml
Every base branch has a `build.toml` next to the device XMLs. It is data only: unknown keys, wrong types or
placeholders other than `{codename}` stop the build.
```toml
[base]
project = "orangefox" # portal project slug
project_name = "OrangeFox"
version = "R12.1" # version shown in the portal
release_tag = "OFRP" # <release_tag>-<release_ver>_A-Team_<codename>_<model>-<YYMMDD>
release_ver = "12.1"
source_root = "/build/ofox-12.1" # mountpoint of zfs_dataset; out/ lives inside it
zfs_dataset = "build/ofox-12.1" # rolled back to @warm before and after every build
warm_check = ["vendor/recovery"] # must exist in the warm base
lunch_target = "twrp_{codename}-eng"
make_extra = ["adbd"] # built together with the ateam.partition target
changelog = "Automated OrangeFox testing build."
[base.env] # extra environment for lunch and the build
FOX_BUILD_DEVICE = "{codename}"
[outputs.img] # release files in out/target/product/<codename>/, exactly one match each
glob = "OrangeFox-*.img"
checksum = "md5" # native sidecar to verify: md5 | sha256 | none
[outputs.zip]
glob = "OrangeFox-*.zip"
checksum = "md5"
test_zip = true
[devices.sycamore_row_5G] # optional per-device overrides: lunch_target, make_extra, env, outputs
lunch_target = "twrp_sycamore_row_5G-eng"
```
Each `[outputs.<ext>]` becomes `<release name>.<ext>` plus `<release name>.<ext>.sha256`.

193
pipeline/build_config.py Normal file
View file

@ -0,0 +1,193 @@
#!/usr/bin/env python3
"""
Resolve a base's build.toml for one device and print shell assignments.
build_config.py <build.toml> <codename>
Used by the shared Jenkinsfile. build.toml is data only: unknown keys, wrong
types or unexpected placeholders stop the build instead of being ignored.
"""
from __future__ import annotations
import re
import shlex
import sys
import tomllib
BASE_KEYS = {
"project": str,
"project_name": str,
"version": str,
"release_tag": str,
"release_ver": str,
"source_root": str,
"zfs_dataset": str,
"warm_check": list,
"lunch_target": str,
"make_extra": list,
"changelog": str,
"env": dict,
}
REQUIRED_BASE = BASE_KEYS.keys() - {"warm_check", "make_extra", "changelog", "env"}
DEVICE_KEYS = {"lunch_target", "make_extra", "env", "outputs"}
OUTPUT_KEYS = {"glob": str, "checksum": str, "test_zip": bool}
CHECKSUMS = {"md5", "sha256", "none"}
CODENAME = re.compile(r"^[A-Za-z0-9_]+$")
SLUG = re.compile(r"^[a-z0-9][a-z0-9-]*$")
TOKEN = re.compile(r"^[A-Za-z0-9_.+-]+$")
ENV_NAME = re.compile(r"^[A-Z][A-Z0-9_]*$")
EXT = re.compile(r"^[a-z0-9]+$")
PLACEHOLDER = re.compile(r"\{([^}]*)\}")
def fail(message: str) -> None:
sys.exit(f"ERROR: build.toml: {message}")
def expand(value: str, where: str, codename: str) -> str:
for name in PLACEHOLDER.findall(value):
if name != "codename":
fail(f"{where}: unknown placeholder {{{name}}} (only {{codename}} is allowed)")
return value.replace("{codename}", codename)
def check_tokens(values, where: str) -> list[str]:
if not all(isinstance(v, str) and TOKEN.match(v) for v in values):
fail(f"{where}: entries must be plain words (letters, digits, _ . + -)")
return list(values)
def check_outputs(outputs, where: str, partial: bool = False) -> dict:
if not isinstance(outputs, dict) or not outputs:
fail(f"{where}: needs at least one [outputs.<ext>] table")
for ext, spec in outputs.items():
if not EXT.match(ext):
fail(f"{where}.{ext}: extension must be lowercase letters/digits")
if not isinstance(spec, dict):
fail(f"{where}.{ext}: must be a table")
for key, value in spec.items():
if key not in OUTPUT_KEYS:
fail(f"{where}.{ext}: unknown key '{key}'")
if not isinstance(value, OUTPUT_KEYS[key]):
fail(f"{where}.{ext}.{key}: wrong type")
if "glob" not in spec and not partial:
fail(f"{where}.{ext}: 'glob' is required")
if "/" in spec.get("glob", ""):
fail(f"{where}.{ext}.glob: must be a file name pattern, not a path")
if spec.get("checksum", "none") not in CHECKSUMS:
fail(f"{where}.{ext}.checksum: one of {', '.join(sorted(CHECKSUMS))}")
return outputs
def resolve(config: dict, codename: str) -> dict[str, str]:
unknown = config.keys() - {"base", "outputs", "devices"}
if unknown:
fail(f"unknown section(s): {', '.join(sorted(unknown))}")
base = config.get("base")
if not isinstance(base, dict):
fail("[base] section is missing")
for key, value in base.items():
if key not in BASE_KEYS:
fail(f"[base]: unknown key '{key}'")
if not isinstance(value, BASE_KEYS[key]):
fail(f"[base].{key}: wrong type")
missing = REQUIRED_BASE - base.keys()
if missing:
fail(f"[base]: missing {', '.join(sorted(missing))}")
if not SLUG.match(base["project"]):
fail("[base].project: lowercase portal project slug expected")
if not base["source_root"].startswith("/"):
fail("[base].source_root: must be an absolute path")
for key in ("release_tag", "release_ver", "version", "zfs_dataset"):
if not re.match(r"^[A-Za-z0-9_.+/-]+$", base[key]):
fail(f"[base].{key}: unexpected characters")
outputs = check_outputs(config.get("outputs"), "[outputs]")
lunch = base["lunch_target"]
make_extra = base.get("make_extra", [])
env = dict(base.get("env", {}))
devices = config.get("devices", {})
if not isinstance(devices, dict):
fail("[devices] must be a table of [devices.<codename>]")
override = devices.get(codename, {})
for key in override:
if key not in DEVICE_KEYS:
fail(f"[devices.{codename}]: unknown key '{key}'")
lunch = override.get("lunch_target", lunch)
make_extra = override.get("make_extra", make_extra)
env.update(override.get("env", {}))
if "outputs" in override:
merged = {ext: dict(spec) for ext, spec in outputs.items()}
for ext, spec in check_outputs(override["outputs"], f"[devices.{codename}.outputs]", partial=True).items():
merged.setdefault(ext, {}).update(spec)
outputs = check_outputs(merged, f"[devices.{codename}] outputs")
for name, value in env.items():
if not ENV_NAME.match(name) or not isinstance(value, str):
fail(f"env.{name}: names must be UPPER_CASE and values strings")
warm_check = base.get("warm_check", [])
for path in warm_check:
if (
not isinstance(path, str)
or path.startswith("/")
or ".." in path.split("/")
or not re.match(r"^[A-Za-z0-9_./-]+$", path)
):
fail(f"[base].warm_check: '{path}' must be a relative path inside the tree")
result = {
"PROJECT": base["project"],
"PROJECT_NAME": base["project_name"],
"VERSION": base["version"],
"RELEASE_TAG": base["release_tag"],
"RELEASE_VER": base["release_ver"],
"SOURCE_ROOT": base["source_root"].rstrip("/"),
"OUT_DIR": base["source_root"].rstrip("/") + "/out",
"ZFS_DATASET": base["zfs_dataset"],
"WARM_SNAPSHOT": base["zfs_dataset"] + "@warm",
"WARM_CHECK": " ".join(warm_check),
"LUNCH_TARGET": expand(lunch, "lunch_target", codename),
"MAKE_EXTRA": " ".join(check_tokens(make_extra, "make_extra")),
"CHANGELOG_DEFAULT": base.get("changelog", f"Automated {base['project_name']} testing build."),
"BUILD_ENV_NAMES": " ".join(sorted(env)),
"OUTPUT_EXTS": " ".join(sorted(outputs)),
}
if not TOKEN.match(result["LUNCH_TARGET"]):
fail("lunch_target: unexpected characters")
for name, value in env.items():
result[name] = expand(value, f"env.{name}", codename)
for ext, spec in outputs.items():
key = ext.upper()
result[f"OUTPUT_{key}_GLOB"] = expand(spec["glob"], f"outputs.{ext}.glob", codename)
result[f"OUTPUT_{key}_CHECKSUM"] = spec.get("checksum", "none")
result[f"OUTPUT_{key}_TEST_ZIP"] = "1" if spec.get("test_zip") else ""
return result
def main() -> None:
if len(sys.argv) != 3:
sys.exit(__doc__)
path, codename = sys.argv[1], sys.argv[2]
if not CODENAME.match(codename):
fail(f"bad codename '{codename}'")
try:
with open(path, "rb") as handle:
config = tomllib.load(handle)
except FileNotFoundError:
fail(f"{path} not found (every base branch needs a build.toml)")
except tomllib.TOMLDecodeError as error:
fail(f"invalid TOML: {error}")
for name, value in resolve(config, codename).items():
print(f"{name}={shlex.quote(value)}")
if __name__ == "__main__":
main()