823de71f1d
Jenkinsfile generalizes the OFRP-12.1 job for every base branch: one job per branch named after it (cms-<branch> jobs target the cms test portal), base settings from the branch build.toml (validated by pipeline/build_config.py), the same preflight, warm rollback, manifest sync, build, rename, checksum, Garage upload and portal registration stages, plus portal_request_id for builds started from the portal and a warm-base lock shared by a base's live and test jobs. README documents the device XMLs, the pipeline and build.toml. Co-Authored-By: Claude Opus 5.5 <[email protected]>
971 lines
36 KiB
Groovy
971 lines
36 KiB
Groovy
// Shared A-Team recovery build pipeline.
|
|
//
|
|
// One Jenkins job per build_manifests branch, named after the branch
|
|
// (ofrp-12.1, twrp-12.1, ...), using "Pipeline script from SCM" on this file
|
|
// (branch main). Base settings come from build.toml on the job's branch; the
|
|
// device comes from <DEVICE>.xml on that branch.
|
|
//
|
|
// Jobs named cms-<branch> build the same base for the cms test portal
|
|
// (test bucket, test credentials, ateam-cms.gotadell.com).
|
|
|
|
def JOB = env.JOB_BASE_NAME
|
|
def IS_TEST_JOB = JOB.startsWith('cms-')
|
|
def BASE_BRANCH = IS_TEST_JOB ? JOB.substring(4) : JOB
|
|
|
|
if (!(BASE_BRANCH ==~ /[a-z0-9][a-z0-9._-]*/)) {
|
|
error "Job name '${JOB}' must be a build_manifests branch name (optionally prefixed with cms-)"
|
|
}
|
|
|
|
def TARGET = IS_TEST_JOB ? [
|
|
bucket : 'ateam-cms-staging',
|
|
s3Credentials : 'garage-s3-cms',
|
|
portalCredentials : 'portal-ingest-token-cms',
|
|
portalUrl : 'https://ateam-cms.gotadell.com/api/jenkins/build-complete/',
|
|
releaseRoot : '/storage/jenkins-artifacts/staging-cms',
|
|
] : [
|
|
bucket : 'ateam-staging',
|
|
s3Credentials : 'garage-s3',
|
|
portalCredentials : 'portal-ingest-token',
|
|
portalUrl : 'https://ateam.gotadell.com/api/jenkins/build-complete/',
|
|
releaseRoot : '/storage/jenkins-artifacts/staging',
|
|
]
|
|
|
|
pipeline {
|
|
agent {
|
|
label 'buildBox'
|
|
}
|
|
|
|
options {
|
|
// The repo is checked out inside Preflight, once the per-build TMPDIR exists
|
|
skipDefaultCheckout()
|
|
timestamps()
|
|
disableConcurrentBuilds()
|
|
// A base's live and cms- jobs share one ZFS dataset; never roll it back under each other.
|
|
lock(resource: "warm-base:${BASE_BRANCH}")
|
|
timeout(time: 6, unit: 'HOURS')
|
|
buildDiscarder(logRotator(numToKeepStr: '20'))
|
|
}
|
|
|
|
parameters {
|
|
// Device list = every .xml on this job's build_manifests branch (Active Choices plugin)
|
|
activeChoice(
|
|
name: 'DEVICE',
|
|
description: 'Device codename. Builds from <DEVICE>.xml on the manifest branch.',
|
|
choiceType: 'PT_SINGLE_SELECT',
|
|
filterable: false,
|
|
script: groovyScript(
|
|
script: [
|
|
classpath: [],
|
|
sandbox: false,
|
|
script: """
|
|
import groovy.json.JsonSlurper
|
|
|
|
def api = 'https://ateam.gotadell.com/git/api/v1/repos/A-Team_Digital_Solutions/build_manifests/contents?ref=${BASE_BRANCH}'
|
|
def conn = new URL(api).openConnection()
|
|
conn.connectTimeout = 5000
|
|
conn.readTimeout = 5000
|
|
|
|
return new JsonSlurper().parse(conn.inputStream)
|
|
.findAll { it.type == 'file' && it.name.endsWith('.xml') }
|
|
.collect { it.name[0..-5] }
|
|
.sort()
|
|
"""
|
|
],
|
|
fallbackScript: [
|
|
classpath: [],
|
|
sandbox: false,
|
|
script: 'return ["ERROR-could-not-list-manifests"]'
|
|
]
|
|
)
|
|
)
|
|
text(
|
|
name: 'CHANGELOG',
|
|
defaultValue: '',
|
|
description: 'Testing notes displayed in the A-Team Portal (empty = changelog from build.toml).'
|
|
)
|
|
string(
|
|
name: 'PORTAL_REQUEST_ID',
|
|
defaultValue: '',
|
|
description: 'Set by the portal "Start build" button. Empty for builds started here (owner-only approval).'
|
|
)
|
|
string(
|
|
name: 'PORTAL_USER',
|
|
defaultValue: '',
|
|
description: 'Portal user who started the build (informational; set by the portal).'
|
|
)
|
|
string(
|
|
name: 'MANIFEST_REF',
|
|
defaultValue: '',
|
|
description: 'cms- test jobs only: build_manifests ref to read build.toml and the device XML from (default: the job branch).'
|
|
)
|
|
}
|
|
|
|
environment {
|
|
DEVICE = "${params.DEVICE}"
|
|
MANIFEST_URL = 'ssh://[email protected]/A-Team_Digital_Solutions/build_manifests.git'
|
|
MANIFEST_BRANCH = "${BASE_BRANCH}"
|
|
MANIFEST_REF = "${params.MANIFEST_REF ?: BASE_BRANCH}"
|
|
MANIFEST_FILE = "${params.DEVICE}.xml"
|
|
TEST_JOB = "${IS_TEST_JOB ? '1' : ''}"
|
|
|
|
TMPDIR = "/build/tmp/jenkins/${JOB}-${params.DEVICE}-${BUILD_NUMBER}"
|
|
BUILD_CONFIG = "/build/tmp/jenkins/${JOB}-${params.DEVICE}-${BUILD_NUMBER}/build-config.env"
|
|
RELEASE_ROOT = "${TARGET.releaseRoot}"
|
|
|
|
// Match the env the warm out/ was built with, or soong regenerates everything
|
|
// (OUT_DIR comes from build.toml: <source_root>/out)
|
|
CC_WRAPPER = '/usr/bin/ccache'
|
|
CCACHE_EXEC = '/usr/bin/ccache'
|
|
CCACHE_DIR = '/build/ccache'
|
|
USE_CCACHE = '1'
|
|
ALLOW_MISSING_DEPENDENCIES = 'true'
|
|
|
|
S3_ENDPOINT_URL = 'http://10.0.55.47:9000'
|
|
S3_REGION = 'garage'
|
|
S3_BUCKET = "${TARGET.bucket}"
|
|
|
|
PORTAL_URL = "${TARGET.portalUrl}"
|
|
}
|
|
|
|
stages {
|
|
stage('Preflight') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
[[ "$DEVICE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: bad DEVICE '$DEVICE'"; exit 1; }
|
|
[[ -z "$PORTAL_REQUEST_ID" || "$PORTAL_REQUEST_ID" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] \
|
|
|| { echo "ERROR: bad PORTAL_REQUEST_ID"; exit 1; }
|
|
if [ -z "$TEST_JOB" ] && [ "$MANIFEST_REF" != "$MANIFEST_BRANCH" ]; then
|
|
echo "ERROR: MANIFEST_REF is only allowed on cms- test jobs"
|
|
exit 1
|
|
fi
|
|
[[ "$MANIFEST_REF" =~ ^[A-Za-z0-9._/-]+$ ]] || { echo "ERROR: bad MANIFEST_REF"; exit 1; }
|
|
|
|
echo "===== REQUIRED TOOLS ====="
|
|
|
|
for TOOL in \
|
|
git \
|
|
repo \
|
|
python3 \
|
|
zfs \
|
|
ccache \
|
|
aws \
|
|
jq \
|
|
curl \
|
|
md5sum \
|
|
sha256sum \
|
|
unzip
|
|
do
|
|
command -v "$TOOL"
|
|
done
|
|
|
|
echo
|
|
echo "===== VERIFY CLEAN PER-BUILD PATHS ====="
|
|
|
|
test ! -e "$TMPDIR"
|
|
install -d -m 2775 "$TMPDIR"
|
|
'''
|
|
|
|
// Jenkinsfile + pipeline/build_config.py from build_manifests main
|
|
checkout scm
|
|
|
|
sh '''#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
echo "===== FETCH BUILD MANIFEST ====="
|
|
|
|
MANIFEST_CHECKOUT="$TMPDIR/build_manifests"
|
|
|
|
git clone \
|
|
--depth 1 \
|
|
--branch "$MANIFEST_REF" \
|
|
--single-branch \
|
|
"$MANIFEST_URL" \
|
|
"$MANIFEST_CHECKOUT"
|
|
|
|
test -s "$MANIFEST_CHECKOUT/$MANIFEST_FILE" \
|
|
|| { echo "ERROR: $MANIFEST_FILE not found on $MANIFEST_REF"; exit 1; }
|
|
|
|
echo
|
|
echo "===== RESOLVE build.toml FOR $DEVICE ====="
|
|
|
|
python3 "$WORKSPACE/pipeline/build_config.py" \
|
|
"$MANIFEST_CHECKOUT/build.toml" \
|
|
"$DEVICE" \
|
|
> "$BUILD_CONFIG"
|
|
|
|
source "$BUILD_CONFIG"
|
|
|
|
{
|
|
printf 'RELEASE_DIR=%q\\n' "$RELEASE_ROOT/$PROJECT/$DEVICE/$PROJECT-$DEVICE-$BUILD_NUMBER"
|
|
printf 'STAGING_PREFIX=%q\\n' "$PROJECT/$DEVICE/build-$BUILD_NUMBER/"
|
|
} >> "$BUILD_CONFIG"
|
|
|
|
cat "$BUILD_CONFIG"
|
|
source "$BUILD_CONFIG"
|
|
|
|
echo
|
|
echo "===== BUILD IDENTIFICATION ====="
|
|
echo "Target: $([ -n "$TEST_JOB" ] && echo "cms test portal" || echo "live portal")"
|
|
echo "Project: $PROJECT_NAME $VERSION"
|
|
echo "Device: $DEVICE"
|
|
echo "Manifest: $MANIFEST_URL ($MANIFEST_REF/$MANIFEST_FILE)"
|
|
echo "Warm base: $WARM_SNAPSHOT"
|
|
echo "Lunch: $LUNCH_TARGET"
|
|
echo "Portal prefix: $STAGING_PREFIX"
|
|
echo "Portal request: ${PORTAL_REQUEST_ID:-none} ${PORTAL_USER:+(by $PORTAL_USER)}"
|
|
|
|
echo
|
|
echo "===== VERIFY WARM SNAPSHOT ====="
|
|
|
|
zfs list -H -t snapshot -o name,creation "$WARM_SNAPSHOT"
|
|
zfs allow "$ZFS_DATASET"
|
|
MOUNTPOINT="$(zfs get -H -o value mountpoint "$ZFS_DATASET")"
|
|
test "$MOUNTPOINT" = "$SOURCE_ROOT" \
|
|
|| { echo "ERROR: $ZFS_DATASET is mounted at $MOUNTPOINT, build.toml says $SOURCE_ROOT"; exit 1; }
|
|
|
|
echo
|
|
echo "===== PREPARE RELEASE DIRECTORY ====="
|
|
|
|
test ! -e "$RELEASE_DIR"
|
|
install -d -m 2775 "$RELEASE_DIR"
|
|
|
|
echo "Temporary: $TMPDIR"
|
|
echo "Release staging: $RELEASE_DIR"
|
|
'''
|
|
}
|
|
}
|
|
|
|
stage('Restore Warm Base') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
source "$BUILD_CONFIG"
|
|
|
|
echo "===== ROLLBACK TO WARM ====="
|
|
|
|
zfs rollback "$WARM_SNAPSHOT"
|
|
|
|
echo
|
|
echo "===== VERIFY WARM BASE ====="
|
|
|
|
cd "$SOURCE_ROOT"
|
|
|
|
test -d .repo
|
|
test -d out
|
|
for P in $WARM_CHECK; do
|
|
test -e "$P" || { echo "ERROR: $P missing from warm base"; exit 1; }
|
|
done
|
|
if compgen -G "device/*/$DEVICE" >/dev/null; then
|
|
echo "ERROR: device tree for $DEVICE already present in warm base"
|
|
exit 1
|
|
fi
|
|
test -z "$(ls -A out/target/product 2>/dev/null)"
|
|
test -z "$(ls -A .repo/local_manifests 2>/dev/null)"
|
|
|
|
echo
|
|
echo "===== VERIFY JENKINS CAN WRITE THE TREE ====="
|
|
|
|
test -w "$SOURCE_ROOT"
|
|
test -w "$SOURCE_ROOT/out"
|
|
test -w "$SOURCE_ROOT/.repo"
|
|
|
|
echo "Source: $SOURCE_ROOT"
|
|
'''
|
|
}
|
|
}
|
|
|
|
stage('Sync Device Trees From Manifest') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
source "$BUILD_CONFIG"
|
|
|
|
MANIFEST_CHECKOUT="$TMPDIR/build_manifests"
|
|
|
|
MANIFEST_COMMIT="$(git -C "$MANIFEST_CHECKOUT" rev-parse HEAD)"
|
|
printf '%s\\n' "$MANIFEST_COMMIT" > "$RELEASE_DIR/manifest-commit.txt"
|
|
echo "Manifest commit: $MANIFEST_COMMIT"
|
|
|
|
cd "$SOURCE_ROOT"
|
|
|
|
mkdir -p .repo/local_manifests
|
|
install -m 0644 \
|
|
"$MANIFEST_CHECKOUT/$MANIFEST_FILE" \
|
|
".repo/local_manifests/$MANIFEST_FILE"
|
|
|
|
echo
|
|
echo "===== RESOLVE BRANCH NAMES (case-insensitive) ====="
|
|
|
|
python3 - ".repo/local_manifests/$MANIFEST_FILE" <<'PYEOF'
|
|
import sys, subprocess
|
|
import xml.etree.ElementTree as ET
|
|
|
|
path = sys.argv[1]
|
|
tree = ET.parse(path)
|
|
root = tree.getroot()
|
|
remotes = {r.get("name"): r.get("fetch", "") for r in root.iter("remote")}
|
|
default = root.find("default")
|
|
dflt = lambda k: default.get(k) if default is not None else None
|
|
changed = False
|
|
|
|
for p in root.iter("project"):
|
|
name = p.get("name")
|
|
rev = p.get("revision") or dflt("revision")
|
|
# leave tags, full refs and commit hashes alone
|
|
if not rev or rev.startswith("refs/") or len(rev) == 40:
|
|
continue
|
|
fetch = remotes.get(p.get("remote") or dflt("remote"), "")
|
|
if "://" not in fetch and "@" not in fetch:
|
|
continue
|
|
url = fetch + ("" if fetch.endswith(("/", ":")) else "/") + name
|
|
|
|
out = subprocess.run(["git", "ls-remote", "--heads", url],
|
|
capture_output=True, text=True, check=True).stdout
|
|
heads = [l.split("refs/heads/", 1)[1] for l in out.splitlines() if "refs/heads/" in l]
|
|
|
|
if rev in heads:
|
|
print(f"{name}: {rev}")
|
|
continue
|
|
match = [h for h in heads if h.lower() == rev.lower()]
|
|
if len(match) != 1:
|
|
sys.exit(f"ERROR: {name}: no branch '{rev}'. Branches: {', '.join(heads) or 'none'}")
|
|
print(f"{name}: {match[0]} (manifest says {rev})")
|
|
p.set("revision", match[0])
|
|
changed = True
|
|
|
|
if changed:
|
|
tree.write(path, xml_declaration=True, encoding="UTF-8")
|
|
PYEOF
|
|
|
|
echo
|
|
echo "===== PROJECTS IN $MANIFEST_FILE ====="
|
|
|
|
mapfile -t PATHS < <(
|
|
python3 - ".repo/local_manifests/$MANIFEST_FILE" <<'PYEOF'
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
for p in ET.parse(sys.argv[1]).getroot().iter("project"):
|
|
print(p.get("path") or p.get("name"))
|
|
PYEOF
|
|
)
|
|
|
|
test "${#PATHS[@]}" -ge 1
|
|
printf ' %s\\n' "${PATHS[@]}"
|
|
|
|
echo
|
|
echo "===== DEVICE INFO FROM $MANIFEST_FILE ====="
|
|
|
|
python3 - ".repo/local_manifests/$MANIFEST_FILE" > "$RELEASE_DIR/device-info.env" <<'PYEOF'
|
|
import sys, shlex
|
|
import xml.etree.ElementTree as ET
|
|
|
|
root = ET.parse(sys.argv[1]).getroot()
|
|
remotes = {r.get("name"): r.get("fetch", "") for r in root.iter("remote")}
|
|
default = root.find("default")
|
|
dflt = lambda k: default.get(k, "") if default is not None else ""
|
|
out = lambda k, v: print(f"{k}={shlex.quote(v or '')}")
|
|
|
|
# The device tree is the project carrying the ateam.* annotations
|
|
trees = [p for p in root.iter("project")
|
|
if any(a.get("name", "").startswith("ateam.") for a in p.iter("annotation"))]
|
|
if len(trees) != 1:
|
|
sys.exit(f"ERROR: expected ateam.* annotations on exactly one project, found {len(trees)}")
|
|
p = trees[0]
|
|
|
|
for a in p.iter("annotation"):
|
|
n = a.get("name", "")
|
|
if n.startswith("ateam."):
|
|
out(n[6:].upper(), a.get("value"))
|
|
|
|
name = p.get("name")
|
|
rev = p.get("revision") or dflt("revision")
|
|
fetch = remotes.get(p.get("remote") or dflt("remote"), "")
|
|
host = fetch.split("@")[-1].split("://")[-1].strip("/").split("/")[0].split(":")[0]
|
|
|
|
if host == "ateam.gotadell.com":
|
|
url = f"https://ateam.gotadell.com/git/{name}/src/branch/{rev}"
|
|
elif host == "github.com":
|
|
url = f"https://github.com/{name}/tree/{rev}"
|
|
else:
|
|
url = f"{fetch.rstrip('/')}/{name}"
|
|
|
|
out("TREE_PATH", p.get("path") or name)
|
|
out("TREE_URL", url)
|
|
PYEOF
|
|
|
|
cat "$RELEASE_DIR/device-info.env"
|
|
source "$RELEASE_DIR/device-info.env"
|
|
|
|
for VAR in MODEL DEVICE_NAME MANUFACTURER PARTITION TREE_PATH TREE_URL; do
|
|
test -n "${!VAR:-}" || { echo "ERROR: $VAR missing (check ateam.* annotations in $MANIFEST_FILE)"; exit 1; }
|
|
done
|
|
|
|
case "$PARTITION" in
|
|
recovery|boot|vendor_boot) ;;
|
|
*) echo "ERROR: unknown ateam.partition '$PARTITION'"; exit 1 ;;
|
|
esac
|
|
|
|
echo
|
|
echo "===== SYNC ONLY MANIFEST PROJECTS ====="
|
|
|
|
repo sync \
|
|
-c \
|
|
--no-tags \
|
|
--no-clone-bundle \
|
|
--force-sync \
|
|
-j8 \
|
|
"${PATHS[@]}"
|
|
|
|
test -d "$TREE_PATH/.git" || test -e "$TREE_PATH/.git"
|
|
|
|
echo
|
|
echo "===== RECORD SYNCED COMMITS ====="
|
|
|
|
for P in "${PATHS[@]}"; do
|
|
printf '%s %s\\n' "$P" "$(git -C "$P" rev-parse HEAD)"
|
|
done | tee "$RELEASE_DIR/synced-projects.txt"
|
|
|
|
TREE_COMMIT="$(git -C "$TREE_PATH" rev-parse HEAD)"
|
|
TREE_SUBJECT="$(git -C "$TREE_PATH" log -1 --pretty=%s)"
|
|
|
|
printf '%s\\n' "$TREE_COMMIT" > "$RELEASE_DIR/device-tree-commit.txt"
|
|
printf '%s\\n' "$TREE_SUBJECT" > "$RELEASE_DIR/device-tree-subject.txt"
|
|
|
|
echo
|
|
echo "Tree commit: $TREE_COMMIT"
|
|
echo "Tree subject: $TREE_SUBJECT"
|
|
'''
|
|
}
|
|
}
|
|
|
|
stage('Configure Ccache') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
mkdir -p "$CCACHE_DIR"
|
|
test -d "$CCACHE_DIR"
|
|
test -w "$CCACHE_DIR"
|
|
|
|
ccache -M 100G
|
|
ccache -o compression=true
|
|
ccache -z
|
|
ccache -s
|
|
'''
|
|
}
|
|
}
|
|
|
|
stage('Lunch') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -o pipefail
|
|
source "$BUILD_CONFIG" || exit 1
|
|
|
|
cd "$SOURCE_ROOT" || exit 1
|
|
|
|
export OUT_DIR
|
|
export CC_WRAPPER
|
|
export CCACHE_EXEC
|
|
|
|
export TMPDIR
|
|
export TMP="$TMPDIR"
|
|
export TEMP="$TMPDIR"
|
|
export CCACHE_DIR
|
|
export USE_CCACHE
|
|
export ALLOW_MISSING_DEPENDENCIES
|
|
for VAR in $BUILD_ENV_NAMES; do export "$VAR"; done
|
|
|
|
source build/envsetup.sh
|
|
|
|
type lunch >/dev/null 2>&1 || exit 1
|
|
type mka >/dev/null 2>&1 || exit 1
|
|
|
|
lunch "$LUNCH_TARGET" || exit 1
|
|
|
|
echo
|
|
echo "===== BUILD VARIABLES ====="
|
|
|
|
echo -n "TARGET_PRODUCT: "
|
|
get_build_var TARGET_PRODUCT || exit 1
|
|
|
|
echo -n "TARGET_DEVICE: "
|
|
get_build_var TARGET_DEVICE || exit 1
|
|
|
|
echo -n "PRODUCT_OUT: "
|
|
get_build_var PRODUCT_OUT || exit 1
|
|
'''
|
|
}
|
|
}
|
|
|
|
stage('Install Clean and Build') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -o pipefail
|
|
source "$BUILD_CONFIG" || exit 1
|
|
|
|
cd "$SOURCE_ROOT" || exit 1
|
|
|
|
export OUT_DIR
|
|
export CC_WRAPPER
|
|
export CCACHE_EXEC
|
|
|
|
export TMPDIR
|
|
export TMP="$TMPDIR"
|
|
export TEMP="$TMPDIR"
|
|
export CCACHE_DIR
|
|
export USE_CCACHE
|
|
export ALLOW_MISSING_DEPENDENCIES
|
|
for VAR in $BUILD_ENV_NAMES; do export "$VAR"; done
|
|
|
|
source build/envsetup.sh
|
|
lunch "$LUNCH_TARGET" || exit 1
|
|
|
|
echo "===== INSTALL CLEAN ====="
|
|
|
|
m installclean
|
|
INSTALLCLEAN_STATUS=$?
|
|
|
|
if [ "$INSTALLCLEAN_STATUS" -ne 0 ]; then
|
|
echo "ERROR: installclean failed with status $INSTALLCLEAN_STATUS"
|
|
exit "$INSTALLCLEAN_STATUS"
|
|
fi
|
|
|
|
source "$RELEASE_DIR/device-info.env" || exit 1
|
|
|
|
case "$PARTITION" in
|
|
recovery) BUILD_TARGET=recoveryimage ;;
|
|
boot) BUILD_TARGET=bootimage ;;
|
|
vendor_boot) BUILD_TARGET=vendorbootimage ;;
|
|
*) echo "ERROR: unknown partition '$PARTITION'"; exit 1 ;;
|
|
esac
|
|
|
|
echo
|
|
echo "===== BUILD $PROJECT_NAME ($MAKE_EXTRA $BUILD_TARGET) ====="
|
|
|
|
START_TIME="$(date +%s)"
|
|
|
|
# MAKE_EXTRA is validated by build_config.py (plain words only)
|
|
mka $MAKE_EXTRA "$BUILD_TARGET"
|
|
BUILD_STATUS=$?
|
|
|
|
END_TIME="$(date +%s)"
|
|
ELAPSED=$((END_TIME - START_TIME))
|
|
|
|
if [ "$BUILD_STATUS" -ne 0 ]; then
|
|
echo "ERROR: $PROJECT_NAME build failed with status $BUILD_STATUS"
|
|
exit "$BUILD_STATUS"
|
|
fi
|
|
|
|
PRODUCT_OUT="$(get_build_var PRODUCT_OUT)" || exit 1
|
|
|
|
echo
|
|
echo "===== BUILD TIME ====="
|
|
|
|
printf "$PROJECT_NAME $DEVICE build: %d:%02d (mm:ss)\\n" \
|
|
"$((ELAPSED / 60))" \
|
|
"$((ELAPSED % 60))"
|
|
|
|
echo
|
|
echo "===== PRODUCT OUTPUT ====="
|
|
|
|
cd "$PRODUCT_OUT" || exit 1
|
|
for EXT in $OUTPUT_EXTS; do
|
|
GLOB_VAR="OUTPUT_${EXT^^}_GLOB"
|
|
find . -maxdepth 1 -type f \\( -name "${!GLOB_VAR}" -o -name "${!GLOB_VAR}.*" \\) \
|
|
-printf '%f %s bytes\\n'
|
|
done | sort
|
|
'''
|
|
}
|
|
}
|
|
|
|
stage('Collect and Rename Artifacts') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
source "$BUILD_CONFIG"
|
|
source "$RELEASE_DIR/device-info.env"
|
|
|
|
PRODUCT_OUT="$OUT_DIR/target/product/$DEVICE"
|
|
cd "$PRODUCT_OUT"
|
|
|
|
BASE="${RELEASE_TAG}-${RELEASE_VER}_A-Team_${DEVICE}_${MODEL}-$(TZ=America/New_York date +%y%m%d)"
|
|
: > "$RELEASE_DIR/deliverables.txt"
|
|
|
|
for EXT in $OUTPUT_EXTS; do
|
|
GLOB_VAR="OUTPUT_${EXT^^}_GLOB"
|
|
CHECKSUM_VAR="OUTPUT_${EXT^^}_CHECKSUM"
|
|
|
|
mapfile -t FOUND < <(find . -maxdepth 1 -type f -name "${!GLOB_VAR}" -printf '%f\\n')
|
|
|
|
if [ "${#FOUND[@]}" -ne 1 ]; then
|
|
echo "ERROR: Expected exactly one .$EXT matching '${!GLOB_VAR}', found ${#FOUND[@]}."
|
|
printf '%s\\n' "${FOUND[@]}"
|
|
exit 1
|
|
fi
|
|
FILE="${FOUND[0]}"
|
|
|
|
case "${!CHECKSUM_VAR}" in
|
|
md5)
|
|
test -s "$FILE.md5"
|
|
EXPECTED="$(awk 'NR == 1 {print $1}' "$FILE.md5")"
|
|
ACTUAL="$(md5sum "$FILE" | awk '{print $1}')"
|
|
test "$EXPECTED" = "$ACTUAL"
|
|
echo "$FILE: native MD5 OK"
|
|
;;
|
|
sha256)
|
|
test -s "$FILE.sha256"
|
|
EXPECTED="$(awk 'NR == 1 {print $1}' "$FILE.sha256")"
|
|
ACTUAL="$(sha256sum "$FILE" | awk '{print $1}')"
|
|
test "$EXPECTED" = "$ACTUAL"
|
|
echo "$FILE: native SHA-256 OK"
|
|
;;
|
|
none)
|
|
;;
|
|
esac
|
|
|
|
install -m 0644 "$FILE" "$RELEASE_DIR/$BASE.$EXT"
|
|
( cd "$RELEASE_DIR" && sha256sum "$BASE.$EXT" > "$BASE.$EXT.sha256" )
|
|
printf '%s\\n%s\\n' "$BASE.$EXT" "$BASE.$EXT.sha256" >> "$RELEASE_DIR/deliverables.txt"
|
|
done
|
|
|
|
echo
|
|
echo "===== RENAMED TO A-TEAM SCHEME ====="
|
|
ls -lh "$RELEASE_DIR"
|
|
'''
|
|
}
|
|
}
|
|
|
|
stage('Verify Deliverables') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
source "$BUILD_CONFIG"
|
|
|
|
cd "$RELEASE_DIR"
|
|
|
|
mapfile -t FILES < deliverables.txt
|
|
EXPECTED_COUNT=$(( $(wc -w <<< "$OUTPUT_EXTS") * 2 ))
|
|
test "${#FILES[@]}" -eq "$EXPECTED_COUNT"
|
|
|
|
for FILE in "${FILES[@]}"; do
|
|
test -s "$FILE"
|
|
ls -lh "$FILE"
|
|
done
|
|
|
|
echo
|
|
echo "===== VERIFY SHA-256 FILES ====="
|
|
|
|
for FILE in "${FILES[@]}"; do
|
|
case "$FILE" in *.sha256) sha256sum -c "$FILE" ;; esac
|
|
done
|
|
|
|
for EXT in $OUTPUT_EXTS; do
|
|
TEST_ZIP_VAR="OUTPUT_${EXT^^}_TEST_ZIP"
|
|
if [ -n "${!TEST_ZIP_VAR}" ]; then
|
|
echo
|
|
echo "===== VERIFY .$EXT ARCHIVE ====="
|
|
for FILE in "${FILES[@]}"; do
|
|
case "$FILE" in *."$EXT") unzip -t "$FILE" ;; esac
|
|
done
|
|
fi
|
|
done
|
|
|
|
echo
|
|
echo "===== CREATE INTERNAL SHA-256 MANIFEST ====="
|
|
|
|
sha256sum "${FILES[@]}" > artifact-manifest.sha256
|
|
|
|
test "$(wc -l < artifact-manifest.sha256)" -eq "$EXPECTED_COUNT"
|
|
cat artifact-manifest.sha256
|
|
'''
|
|
}
|
|
}
|
|
|
|
stage('Upload to Garage') {
|
|
steps {
|
|
withCredentials([
|
|
usernamePassword(
|
|
credentialsId: TARGET.s3Credentials,
|
|
usernameVariable: 'AWS_ACCESS_KEY_ID',
|
|
passwordVariable: 'AWS_SECRET_ACCESS_KEY'
|
|
)
|
|
]) {
|
|
sh '''#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
set +x
|
|
source "$BUILD_CONFIG"
|
|
|
|
export AWS_DEFAULT_REGION="$S3_REGION"
|
|
export AWS_REGION="$S3_REGION"
|
|
export AWS_EC2_METADATA_DISABLED=true
|
|
|
|
cd "$RELEASE_DIR"
|
|
|
|
mapfile -t FILES < deliverables.txt
|
|
FILES+=("artifact-manifest.sha256")
|
|
|
|
test "${#FILES[@]}" -eq $(( $(wc -w <<< "$OUTPUT_EXTS") * 2 + 1 ))
|
|
|
|
echo "===== UPLOAD GARAGE ARTIFACTS ====="
|
|
echo "Bucket: $S3_BUCKET"
|
|
echo "Prefix: $STAGING_PREFIX"
|
|
|
|
for FILE in "${FILES[@]}"; do
|
|
echo "Uploading: $FILE"
|
|
|
|
aws \
|
|
--endpoint-url "$S3_ENDPOINT_URL" \
|
|
s3api put-object \
|
|
--bucket "$S3_BUCKET" \
|
|
--key "${STAGING_PREFIX}${FILE}" \
|
|
--body "$FILE" \
|
|
>/dev/null
|
|
done
|
|
|
|
echo
|
|
echo "GARAGE UPLOAD COMPLETE"
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Verify Garage Objects') {
|
|
steps {
|
|
withCredentials([
|
|
usernamePassword(
|
|
credentialsId: TARGET.s3Credentials,
|
|
usernameVariable: 'AWS_ACCESS_KEY_ID',
|
|
passwordVariable: 'AWS_SECRET_ACCESS_KEY'
|
|
)
|
|
]) {
|
|
sh '''#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
set +x
|
|
source "$BUILD_CONFIG"
|
|
|
|
export AWS_DEFAULT_REGION="$S3_REGION"
|
|
export AWS_REGION="$S3_REGION"
|
|
export AWS_EC2_METADATA_DISABLED=true
|
|
|
|
cd "$RELEASE_DIR"
|
|
|
|
mapfile -t FILES < deliverables.txt
|
|
FILES+=("artifact-manifest.sha256")
|
|
|
|
echo "===== VERIFY REMOTE OBJECT SIZES ====="
|
|
|
|
for FILE in "${FILES[@]}"; do
|
|
LOCAL_SIZE="$(stat -c '%s' "$FILE")"
|
|
|
|
REMOTE_SIZE="$(
|
|
aws \
|
|
--endpoint-url "$S3_ENDPOINT_URL" \
|
|
s3api head-object \
|
|
--bucket "$S3_BUCKET" \
|
|
--key "${STAGING_PREFIX}${FILE}" \
|
|
--query ContentLength \
|
|
--output text
|
|
)"
|
|
|
|
printf '%-80s local=%s remote=%s\\n' \
|
|
"$FILE" \
|
|
"$LOCAL_SIZE" \
|
|
"$REMOTE_SIZE"
|
|
|
|
test "$LOCAL_SIZE" = "$REMOTE_SIZE"
|
|
done
|
|
|
|
REMOTE_MANIFEST="$(mktemp)"
|
|
trap 'rm -f "$REMOTE_MANIFEST"' EXIT
|
|
|
|
aws \
|
|
--endpoint-url "$S3_ENDPOINT_URL" \
|
|
s3api get-object \
|
|
--bucket "$S3_BUCKET" \
|
|
--key "${STAGING_PREFIX}artifact-manifest.sha256" \
|
|
"$REMOTE_MANIFEST" \
|
|
>/dev/null
|
|
|
|
cmp artifact-manifest.sha256 "$REMOTE_MANIFEST"
|
|
|
|
echo
|
|
echo "GARAGE VERIFICATION PASSED"
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Register Portal Testing Build') {
|
|
steps {
|
|
withCredentials([
|
|
string(
|
|
credentialsId: TARGET.portalCredentials,
|
|
variable: 'PORTAL_TOKEN'
|
|
)
|
|
]) {
|
|
sh '''#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
set +x
|
|
source "$BUILD_CONFIG"
|
|
|
|
TREE_COMMIT="$(cat "$RELEASE_DIR/device-tree-commit.txt")"
|
|
TREE_SUBJECT="$(cat "$RELEASE_DIR/device-tree-subject.txt")"
|
|
source "$RELEASE_DIR/device-info.env"
|
|
|
|
PAYLOAD_FILE="$(mktemp)"
|
|
RESPONSE_FILE="$(mktemp)"
|
|
|
|
cleanup_files() {
|
|
rm -f "$PAYLOAD_FILE" "$RESPONSE_FILE"
|
|
}
|
|
|
|
trap cleanup_files EXIT
|
|
|
|
jq -n \
|
|
--arg project "$PROJECT" \
|
|
--arg project_name "$PROJECT_NAME" \
|
|
--arg device "$DEVICE" \
|
|
--arg device_name "$DEVICE_NAME ($MODEL)" \
|
|
--arg manufacturer "$MANUFACTURER" \
|
|
--arg version "$VERSION" \
|
|
--argjson build_number "$BUILD_NUMBER" \
|
|
--arg staging_prefix "$STAGING_PREFIX" \
|
|
--arg source_branch "$MANIFEST_BRANCH" \
|
|
--arg jenkins_url "$BUILD_URL" \
|
|
--arg tree_url "$TREE_URL" \
|
|
--arg tree_commit "$TREE_COMMIT" \
|
|
--arg tree_subject "$TREE_SUBJECT" \
|
|
--arg operator_changelog "${CHANGELOG:-$CHANGELOG_DEFAULT}" \
|
|
--arg portal_request_id "$PORTAL_REQUEST_ID" \
|
|
'{
|
|
project: $project,
|
|
project_name: $project_name,
|
|
device: $device,
|
|
device_name: $device_name,
|
|
manufacturer: $manufacturer,
|
|
version: $version,
|
|
build_number: $build_number,
|
|
staging_prefix: $staging_prefix,
|
|
source_branch: $source_branch,
|
|
jenkins_url: $jenkins_url,
|
|
portal_request_id: $portal_request_id,
|
|
changelog: (
|
|
$operator_changelog
|
|
+ "\\n\\nDevice tree: "
|
|
+ $tree_url
|
|
+ "\\nCommit: "
|
|
+ $tree_commit
|
|
+ "\\nCommit subject: "
|
|
+ $tree_subject
|
|
)
|
|
}' \
|
|
> "$PAYLOAD_FILE"
|
|
|
|
HTTP_CODE="$(
|
|
curl \
|
|
-sS \
|
|
--connect-timeout 15 \
|
|
--max-time 120 \
|
|
-o "$RESPONSE_FILE" \
|
|
-w '%{http_code}' \
|
|
-H "X-Jenkins-Token: $PORTAL_TOKEN" \
|
|
-H 'Content-Type: application/json' \
|
|
--data-binary "@$PAYLOAD_FILE" \
|
|
"$PORTAL_URL"
|
|
)"
|
|
|
|
echo "===== PORTAL RESPONSE ====="
|
|
echo "HTTP=$HTTP_CODE"
|
|
|
|
jq . "$RESPONSE_FILE" \
|
|
2>/dev/null \
|
|
|| cat "$RESPONSE_FILE"
|
|
|
|
case "$HTTP_CODE" in
|
|
200|201)
|
|
;;
|
|
*)
|
|
echo "ERROR: Portal registration failed."
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if jq -e '.error' "$RESPONSE_FILE" >/dev/null 2>&1; then
|
|
echo "ERROR: Portal returned an error."
|
|
exit 1
|
|
fi
|
|
|
|
if [ -n "$PORTAL_REQUEST_ID" ] && ! jq -e '.started_by' "$RESPONSE_FILE" >/dev/null 2>&1; then
|
|
echo "WARNING: portal did not attribute this build: $(jq -r '.attribution_warning // empty' "$RESPONSE_FILE")"
|
|
fi
|
|
|
|
echo
|
|
echo "PORTAL TESTING REGISTRATION PASSED"
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Ccache Results') {
|
|
steps {
|
|
sh '''#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
ccache -s
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
|
|
post {
|
|
success {
|
|
sh '''#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
source "$BUILD_CONFIG"
|
|
|
|
echo "===== SUCCESSFUL BUILD CLEANUP ====="
|
|
|
|
rm -rf "$TMPDIR"
|
|
|
|
echo
|
|
echo "===== RETURN BASE TO WARM ====="
|
|
|
|
zfs rollback "$WARM_SNAPSHOT"
|
|
|
|
echo
|
|
echo "Warm base restored: $WARM_SNAPSHOT"
|
|
echo "Local release files retained at:"
|
|
echo "$RELEASE_DIR"
|
|
echo
|
|
echo "Portal status: Testing"
|
|
echo "Garage prefix: $STAGING_PREFIX"
|
|
'''
|
|
|
|
echo "${BASE_BRANCH} ${env.DEVICE} build uploaded and registered for testing."
|
|
}
|
|
|
|
failure {
|
|
echo "${BASE_BRANCH} ${env.DEVICE} pipeline failed. Source tree left as-is for diagnosis (next run rolls back to @warm)."
|
|
|
|
sh '''#!/usr/bin/env bash
|
|
set +e
|
|
source "$BUILD_CONFIG" 2>/dev/null
|
|
|
|
echo "===== RETAINED FAILURE PATHS ====="
|
|
echo "Source + out: ${SOURCE_ROOT:-not resolved}"
|
|
echo "Temporary files: $TMPDIR"
|
|
echo "Release staging: ${RELEASE_DIR:-not created}"
|
|
echo "Garage prefix: ${STAGING_PREFIX:-not resolved}"
|
|
'''
|
|
}
|
|
|
|
aborted {
|
|
echo "${BASE_BRANCH} ${env.DEVICE} pipeline was aborted. Diagnostic paths were retained."
|
|
}
|
|
|
|
always {
|
|
echo "${BASE_BRANCH} ${env.DEVICE} pipeline finished."
|
|
}
|
|
}
|
|
}
|