diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index e2da9d8..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,971 +0,0 @@ -// Shared A-Team recovery build pipeline. -// -// One Jenkins job per build_manifests branch, named after the branch -// (ofrp-12.1, twrp-12.1, ...), using "Pipeline script from SCM" on this file -// (branch main). Base settings come from build.toml on the job's branch; the -// device comes from .xml on that branch. -// -// Jobs named cms- build the same base for the cms test portal -// (test bucket, test credentials, ateam-cms.gotadell.com). - -def JOB = env.JOB_BASE_NAME -def IS_TEST_JOB = JOB.startsWith('cms-') -def BASE_BRANCH = IS_TEST_JOB ? JOB.substring(4) : JOB - -if (!(BASE_BRANCH ==~ /[a-z0-9][a-z0-9._-]*/)) { - error "Job name '${JOB}' must be a build_manifests branch name (optionally prefixed with cms-)" -} - -def TARGET = IS_TEST_JOB ? [ - bucket : 'ateam-cms-staging', - s3Credentials : 'garage-s3-cms', - portalCredentials : 'portal-ingest-token-cms', - portalUrl : 'https://ateam-cms.gotadell.com/api/jenkins/build-complete/', - releaseRoot : '/storage/jenkins-artifacts/staging-cms', -] : [ - bucket : 'ateam-staging', - s3Credentials : 'garage-s3', - portalCredentials : 'portal-ingest-token', - portalUrl : 'https://ateam.gotadell.com/api/jenkins/build-complete/', - releaseRoot : '/storage/jenkins-artifacts/staging', -] - -pipeline { - agent { - label 'buildBox' - } - - options { - // The repo is checked out inside Preflight, once the per-build TMPDIR exists - skipDefaultCheckout() - timestamps() - disableConcurrentBuilds() - // A base's live and cms- jobs share one ZFS dataset; never roll it back under each other. - lock(resource: "warm-base:${BASE_BRANCH}") - timeout(time: 6, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20')) - } - - parameters { - // Device list = every .xml on this job's build_manifests branch (Active Choices plugin) - activeChoice( - name: 'DEVICE', - description: 'Device codename. Builds from .xml on the manifest branch.', - choiceType: 'PT_SINGLE_SELECT', - filterable: false, - script: groovyScript( - script: [ - classpath: [], - sandbox: false, - script: """ - import groovy.json.JsonSlurper - - def api = 'https://ateam.gotadell.com/git/api/v1/repos/A-Team_Digital_Solutions/build_manifests/contents?ref=${BASE_BRANCH}' - def conn = new URL(api).openConnection() - conn.connectTimeout = 5000 - conn.readTimeout = 5000 - - return new JsonSlurper().parse(conn.inputStream) - .findAll { it.type == 'file' && it.name.endsWith('.xml') } - .collect { it.name[0..-5] } - .sort() - """ - ], - fallbackScript: [ - classpath: [], - sandbox: false, - script: 'return ["ERROR-could-not-list-manifests"]' - ] - ) - ) - text( - name: 'CHANGELOG', - defaultValue: '', - description: 'Testing notes displayed in the A-Team Portal (empty = changelog from build.toml).' - ) - string( - name: 'PORTAL_REQUEST_ID', - defaultValue: '', - description: 'Set by the portal "Start build" button. Empty for builds started here (owner-only approval).' - ) - string( - name: 'PORTAL_USER', - defaultValue: '', - description: 'Portal user who started the build (informational; set by the portal).' - ) - string( - name: 'MANIFEST_REF', - defaultValue: '', - description: 'cms- test jobs only: build_manifests ref to read build.toml and the device XML from (default: the job branch).' - ) - } - - environment { - DEVICE = "${params.DEVICE}" - MANIFEST_URL = 'ssh://git@ateam.gotadell.com/A-Team_Digital_Solutions/build_manifests.git' - MANIFEST_BRANCH = "${BASE_BRANCH}" - MANIFEST_REF = "${params.MANIFEST_REF ?: BASE_BRANCH}" - MANIFEST_FILE = "${params.DEVICE}.xml" - TEST_JOB = "${IS_TEST_JOB ? '1' : ''}" - - TMPDIR = "/build/tmp/jenkins/${JOB}-${params.DEVICE}-${BUILD_NUMBER}" - BUILD_CONFIG = "/build/tmp/jenkins/${JOB}-${params.DEVICE}-${BUILD_NUMBER}/build-config.env" - RELEASE_ROOT = "${TARGET.releaseRoot}" - - // Match the env the warm out/ was built with, or soong regenerates everything - // (OUT_DIR comes from build.toml: /out) - CC_WRAPPER = '/usr/bin/ccache' - CCACHE_EXEC = '/usr/bin/ccache' - CCACHE_DIR = '/build/ccache' - USE_CCACHE = '1' - ALLOW_MISSING_DEPENDENCIES = 'true' - - S3_ENDPOINT_URL = 'http://10.0.55.47:9000' - S3_REGION = 'garage' - S3_BUCKET = "${TARGET.bucket}" - - PORTAL_URL = "${TARGET.portalUrl}" - } - - stages { - stage('Preflight') { - steps { - sh '''#!/usr/bin/env bash - set -Eeuo pipefail - - [[ "$DEVICE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: bad DEVICE '$DEVICE'"; exit 1; } - [[ -z "${PORTAL_REQUEST_ID:-}" || "${PORTAL_REQUEST_ID:-}" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] \ - || { echo "ERROR: bad PORTAL_REQUEST_ID"; exit 1; } - if [ -z "$TEST_JOB" ] && [ "$MANIFEST_REF" != "$MANIFEST_BRANCH" ]; then - echo "ERROR: MANIFEST_REF is only allowed on cms- test jobs" - exit 1 - fi - [[ "$MANIFEST_REF" =~ ^[A-Za-z0-9._/-]+$ ]] || { echo "ERROR: bad MANIFEST_REF"; exit 1; } - - echo "===== REQUIRED TOOLS =====" - - for TOOL in \ - git \ - repo \ - python3 \ - zfs \ - ccache \ - aws \ - jq \ - curl \ - md5sum \ - sha256sum \ - unzip - do - command -v "$TOOL" - done - - echo - echo "===== VERIFY CLEAN PER-BUILD PATHS =====" - - test ! -e "$TMPDIR" - install -d -m 2775 "$TMPDIR" - ''' - - // Jenkinsfile + pipeline/build_config.py from build_manifests main - checkout scm - - sh '''#!/usr/bin/env bash - set -Eeuo pipefail - - echo "===== FETCH BUILD MANIFEST =====" - - MANIFEST_CHECKOUT="$TMPDIR/build_manifests" - - git clone \ - --depth 1 \ - --branch "$MANIFEST_REF" \ - --single-branch \ - "$MANIFEST_URL" \ - "$MANIFEST_CHECKOUT" - - test -s "$MANIFEST_CHECKOUT/$MANIFEST_FILE" \ - || { echo "ERROR: $MANIFEST_FILE not found on $MANIFEST_REF"; exit 1; } - - echo - echo "===== RESOLVE build.toml FOR $DEVICE =====" - - python3 "$WORKSPACE/pipeline/build_config.py" \ - "$MANIFEST_CHECKOUT/build.toml" \ - "$DEVICE" \ - > "$BUILD_CONFIG" - - source "$BUILD_CONFIG" - - { - printf 'RELEASE_DIR=%q\\n' "$RELEASE_ROOT/$PROJECT/$DEVICE/$PROJECT-$DEVICE-$BUILD_NUMBER" - printf 'STAGING_PREFIX=%q\\n' "$PROJECT/$DEVICE/build-$BUILD_NUMBER/" - } >> "$BUILD_CONFIG" - - cat "$BUILD_CONFIG" - source "$BUILD_CONFIG" - - echo - echo "===== BUILD IDENTIFICATION =====" - echo "Target: $([ -n "$TEST_JOB" ] && echo "cms test portal" || echo "live portal")" - echo "Project: $PROJECT_NAME $VERSION" - echo "Device: $DEVICE" - echo "Manifest: $MANIFEST_URL ($MANIFEST_REF/$MANIFEST_FILE)" - echo "Warm base: $WARM_SNAPSHOT" - echo "Lunch: $LUNCH_TARGET" - echo "Portal prefix: $STAGING_PREFIX" - echo "Portal request: ${PORTAL_REQUEST_ID:-none} ${PORTAL_USER:+(by $PORTAL_USER)}" - - echo - echo "===== VERIFY WARM SNAPSHOT =====" - - zfs list -H -t snapshot -o name,creation "$WARM_SNAPSHOT" - zfs allow "$ZFS_DATASET" - MOUNTPOINT="$(zfs get -H -o value mountpoint "$ZFS_DATASET")" - test "$MOUNTPOINT" = "$SOURCE_ROOT" \ - || { echo "ERROR: $ZFS_DATASET is mounted at $MOUNTPOINT, build.toml says $SOURCE_ROOT"; exit 1; } - - echo - echo "===== PREPARE RELEASE DIRECTORY =====" - - test ! -e "$RELEASE_DIR" - install -d -m 2775 "$RELEASE_DIR" - - echo "Temporary: $TMPDIR" - echo "Release staging: $RELEASE_DIR" - ''' - } - } - - stage('Restore Warm Base') { - steps { - sh '''#!/usr/bin/env bash - set -Eeuo pipefail - source "$BUILD_CONFIG" - - echo "===== ROLLBACK TO WARM =====" - - zfs rollback "$WARM_SNAPSHOT" - - echo - echo "===== VERIFY WARM BASE =====" - - cd "$SOURCE_ROOT" - - test -d .repo - test -d out - for P in $WARM_CHECK; do - test -e "$P" || { echo "ERROR: $P missing from warm base"; exit 1; } - done - if compgen -G "device/*/$DEVICE" >/dev/null; then - echo "ERROR: device tree for $DEVICE already present in warm base" - exit 1 - fi - test -z "$(ls -A out/target/product 2>/dev/null)" - test -z "$(ls -A .repo/local_manifests 2>/dev/null)" - - echo - echo "===== VERIFY JENKINS CAN WRITE THE TREE =====" - - test -w "$SOURCE_ROOT" - test -w "$SOURCE_ROOT/out" - test -w "$SOURCE_ROOT/.repo" - - echo "Source: $SOURCE_ROOT" - ''' - } - } - - stage('Sync Device Trees From Manifest') { - steps { - sh '''#!/usr/bin/env bash - set -Eeuo pipefail - source "$BUILD_CONFIG" - - MANIFEST_CHECKOUT="$TMPDIR/build_manifests" - - MANIFEST_COMMIT="$(git -C "$MANIFEST_CHECKOUT" rev-parse HEAD)" - printf '%s\\n' "$MANIFEST_COMMIT" > "$RELEASE_DIR/manifest-commit.txt" - echo "Manifest commit: $MANIFEST_COMMIT" - - cd "$SOURCE_ROOT" - - mkdir -p .repo/local_manifests - install -m 0644 \ - "$MANIFEST_CHECKOUT/$MANIFEST_FILE" \ - ".repo/local_manifests/$MANIFEST_FILE" - - echo - echo "===== RESOLVE BRANCH NAMES (case-insensitive) =====" - - python3 - ".repo/local_manifests/$MANIFEST_FILE" <<'PYEOF' -import sys, subprocess -import xml.etree.ElementTree as ET - -path = sys.argv[1] -tree = ET.parse(path) -root = tree.getroot() -remotes = {r.get("name"): r.get("fetch", "") for r in root.iter("remote")} -default = root.find("default") -dflt = lambda k: default.get(k) if default is not None else None -changed = False - -for p in root.iter("project"): - name = p.get("name") - rev = p.get("revision") or dflt("revision") - # leave tags, full refs and commit hashes alone - if not rev or rev.startswith("refs/") or len(rev) == 40: - continue - fetch = remotes.get(p.get("remote") or dflt("remote"), "") - if "://" not in fetch and "@" not in fetch: - continue - url = fetch + ("" if fetch.endswith(("/", ":")) else "/") + name - - out = subprocess.run(["git", "ls-remote", "--heads", url], - capture_output=True, text=True, check=True).stdout - heads = [l.split("refs/heads/", 1)[1] for l in out.splitlines() if "refs/heads/" in l] - - if rev in heads: - print(f"{name}: {rev}") - continue - match = [h for h in heads if h.lower() == rev.lower()] - if len(match) != 1: - sys.exit(f"ERROR: {name}: no branch '{rev}'. Branches: {', '.join(heads) or 'none'}") - print(f"{name}: {match[0]} (manifest says {rev})") - p.set("revision", match[0]) - changed = True - -if changed: - tree.write(path, xml_declaration=True, encoding="UTF-8") -PYEOF - - echo - echo "===== PROJECTS IN $MANIFEST_FILE =====" - - mapfile -t PATHS < <( - python3 - ".repo/local_manifests/$MANIFEST_FILE" <<'PYEOF' -import sys -import xml.etree.ElementTree as ET -for p in ET.parse(sys.argv[1]).getroot().iter("project"): - print(p.get("path") or p.get("name")) -PYEOF - ) - - test "${#PATHS[@]}" -ge 1 - printf ' %s\\n' "${PATHS[@]}" - - echo - echo "===== DEVICE INFO FROM $MANIFEST_FILE =====" - - python3 - ".repo/local_manifests/$MANIFEST_FILE" > "$RELEASE_DIR/device-info.env" <<'PYEOF' -import sys, shlex -import xml.etree.ElementTree as ET - -root = ET.parse(sys.argv[1]).getroot() -remotes = {r.get("name"): r.get("fetch", "") for r in root.iter("remote")} -default = root.find("default") -dflt = lambda k: default.get(k, "") if default is not None else "" -out = lambda k, v: print(f"{k}={shlex.quote(v or '')}") - -# The device tree is the project carrying the ateam.* annotations -trees = [p for p in root.iter("project") - if any(a.get("name", "").startswith("ateam.") for a in p.iter("annotation"))] -if len(trees) != 1: - sys.exit(f"ERROR: expected ateam.* annotations on exactly one project, found {len(trees)}") -p = trees[0] - -for a in p.iter("annotation"): - n = a.get("name", "") - if n.startswith("ateam."): - out(n[6:].upper(), a.get("value")) - -name = p.get("name") -rev = p.get("revision") or dflt("revision") -fetch = remotes.get(p.get("remote") or dflt("remote"), "") -host = fetch.split("@")[-1].split("://")[-1].strip("/").split("/")[0].split(":")[0] - -if host == "ateam.gotadell.com": - url = f"https://ateam.gotadell.com/git/{name}/src/branch/{rev}" -elif host == "github.com": - url = f"https://github.com/{name}/tree/{rev}" -else: - url = f"{fetch.rstrip('/')}/{name}" - -out("TREE_PATH", p.get("path") or name) -out("TREE_URL", url) -PYEOF - - cat "$RELEASE_DIR/device-info.env" - source "$RELEASE_DIR/device-info.env" - - for VAR in MODEL DEVICE_NAME MANUFACTURER PARTITION TREE_PATH TREE_URL; do - test -n "${!VAR:-}" || { echo "ERROR: $VAR missing (check ateam.* annotations in $MANIFEST_FILE)"; exit 1; } - done - - case "$PARTITION" in - recovery|boot|vendor_boot) ;; - *) echo "ERROR: unknown ateam.partition '$PARTITION'"; exit 1 ;; - esac - - echo - echo "===== SYNC ONLY MANIFEST PROJECTS =====" - - repo sync \ - -c \ - --no-tags \ - --no-clone-bundle \ - --force-sync \ - -j8 \ - "${PATHS[@]}" - - test -d "$TREE_PATH/.git" || test -e "$TREE_PATH/.git" - - echo - echo "===== RECORD SYNCED COMMITS =====" - - for P in "${PATHS[@]}"; do - printf '%s %s\\n' "$P" "$(git -C "$P" rev-parse HEAD)" - done | tee "$RELEASE_DIR/synced-projects.txt" - - TREE_COMMIT="$(git -C "$TREE_PATH" rev-parse HEAD)" - TREE_SUBJECT="$(git -C "$TREE_PATH" log -1 --pretty=%s)" - - printf '%s\\n' "$TREE_COMMIT" > "$RELEASE_DIR/device-tree-commit.txt" - printf '%s\\n' "$TREE_SUBJECT" > "$RELEASE_DIR/device-tree-subject.txt" - - echo - echo "Tree commit: $TREE_COMMIT" - echo "Tree subject: $TREE_SUBJECT" - ''' - } - } - - stage('Configure Ccache') { - steps { - sh '''#!/usr/bin/env bash - set -Eeuo pipefail - - mkdir -p "$CCACHE_DIR" - test -d "$CCACHE_DIR" - test -w "$CCACHE_DIR" - - ccache -M 100G - ccache -o compression=true - ccache -z - ccache -s - ''' - } - } - - stage('Lunch') { - steps { - sh '''#!/usr/bin/env bash - set -o pipefail - source "$BUILD_CONFIG" || exit 1 - - cd "$SOURCE_ROOT" || exit 1 - - export OUT_DIR - export CC_WRAPPER - export CCACHE_EXEC - - export TMPDIR - export TMP="$TMPDIR" - export TEMP="$TMPDIR" - export CCACHE_DIR - export USE_CCACHE - export ALLOW_MISSING_DEPENDENCIES - for VAR in $BUILD_ENV_NAMES; do export "$VAR"; done - - source build/envsetup.sh - - type lunch >/dev/null 2>&1 || exit 1 - type mka >/dev/null 2>&1 || exit 1 - - lunch "$LUNCH_TARGET" || exit 1 - - echo - echo "===== BUILD VARIABLES =====" - - echo -n "TARGET_PRODUCT: " - get_build_var TARGET_PRODUCT || exit 1 - - echo -n "TARGET_DEVICE: " - get_build_var TARGET_DEVICE || exit 1 - - echo -n "PRODUCT_OUT: " - get_build_var PRODUCT_OUT || exit 1 - ''' - } - } - - stage('Install Clean and Build') { - steps { - sh '''#!/usr/bin/env bash - set -o pipefail - source "$BUILD_CONFIG" || exit 1 - - cd "$SOURCE_ROOT" || exit 1 - - export OUT_DIR - export CC_WRAPPER - export CCACHE_EXEC - - export TMPDIR - export TMP="$TMPDIR" - export TEMP="$TMPDIR" - export CCACHE_DIR - export USE_CCACHE - export ALLOW_MISSING_DEPENDENCIES - for VAR in $BUILD_ENV_NAMES; do export "$VAR"; done - - source build/envsetup.sh - lunch "$LUNCH_TARGET" || exit 1 - - echo "===== INSTALL CLEAN =====" - - m installclean - INSTALLCLEAN_STATUS=$? - - if [ "$INSTALLCLEAN_STATUS" -ne 0 ]; then - echo "ERROR: installclean failed with status $INSTALLCLEAN_STATUS" - exit "$INSTALLCLEAN_STATUS" - fi - - source "$RELEASE_DIR/device-info.env" || exit 1 - - case "$PARTITION" in - recovery) BUILD_TARGET=recoveryimage ;; - boot) BUILD_TARGET=bootimage ;; - vendor_boot) BUILD_TARGET=vendorbootimage ;; - *) echo "ERROR: unknown partition '$PARTITION'"; exit 1 ;; - esac - - echo - echo "===== BUILD $PROJECT_NAME ($MAKE_EXTRA $BUILD_TARGET) =====" - - START_TIME="$(date +%s)" - - # MAKE_EXTRA is validated by build_config.py (plain words only) - mka $MAKE_EXTRA "$BUILD_TARGET" - BUILD_STATUS=$? - - END_TIME="$(date +%s)" - ELAPSED=$((END_TIME - START_TIME)) - - if [ "$BUILD_STATUS" -ne 0 ]; then - echo "ERROR: $PROJECT_NAME build failed with status $BUILD_STATUS" - exit "$BUILD_STATUS" - fi - - PRODUCT_OUT="$(get_build_var PRODUCT_OUT)" || exit 1 - - echo - echo "===== BUILD TIME =====" - - printf "$PROJECT_NAME $DEVICE build: %d:%02d (mm:ss)\\n" \ - "$((ELAPSED / 60))" \ - "$((ELAPSED % 60))" - - echo - echo "===== PRODUCT OUTPUT =====" - - cd "$PRODUCT_OUT" || exit 1 - for EXT in $OUTPUT_EXTS; do - GLOB_VAR="OUTPUT_${EXT^^}_GLOB" - find . -maxdepth 1 -type f \\( -name "${!GLOB_VAR}" -o -name "${!GLOB_VAR}.*" \\) \ - -printf '%f %s bytes\\n' - done | sort - ''' - } - } - - stage('Collect and Rename Artifacts') { - steps { - sh '''#!/usr/bin/env bash - set -Eeuo pipefail - source "$BUILD_CONFIG" - source "$RELEASE_DIR/device-info.env" - - PRODUCT_OUT="$OUT_DIR/target/product/$DEVICE" - cd "$PRODUCT_OUT" - - BASE="${RELEASE_TAG}-${RELEASE_VER}_A-Team_${DEVICE}_${MODEL}-$(TZ=America/New_York date +%y%m%d)" - : > "$RELEASE_DIR/deliverables.txt" - - for EXT in $OUTPUT_EXTS; do - GLOB_VAR="OUTPUT_${EXT^^}_GLOB" - CHECKSUM_VAR="OUTPUT_${EXT^^}_CHECKSUM" - - mapfile -t FOUND < <(find . -maxdepth 1 -type f -name "${!GLOB_VAR}" -printf '%f\\n') - - if [ "${#FOUND[@]}" -ne 1 ]; then - echo "ERROR: Expected exactly one .$EXT matching '${!GLOB_VAR}', found ${#FOUND[@]}." - printf '%s\\n' "${FOUND[@]}" - exit 1 - fi - FILE="${FOUND[0]}" - - case "${!CHECKSUM_VAR}" in - md5) - test -s "$FILE.md5" - EXPECTED="$(awk 'NR == 1 {print $1}' "$FILE.md5")" - ACTUAL="$(md5sum "$FILE" | awk '{print $1}')" - test "$EXPECTED" = "$ACTUAL" - echo "$FILE: native MD5 OK" - ;; - sha256) - test -s "$FILE.sha256" - EXPECTED="$(awk 'NR == 1 {print $1}' "$FILE.sha256")" - ACTUAL="$(sha256sum "$FILE" | awk '{print $1}')" - test "$EXPECTED" = "$ACTUAL" - echo "$FILE: native SHA-256 OK" - ;; - none) - ;; - esac - - install -m 0644 "$FILE" "$RELEASE_DIR/$BASE.$EXT" - ( cd "$RELEASE_DIR" && sha256sum "$BASE.$EXT" > "$BASE.$EXT.sha256" ) - printf '%s\\n%s\\n' "$BASE.$EXT" "$BASE.$EXT.sha256" >> "$RELEASE_DIR/deliverables.txt" - done - - echo - echo "===== RENAMED TO A-TEAM SCHEME =====" - ls -lh "$RELEASE_DIR" - ''' - } - } - - stage('Verify Deliverables') { - steps { - sh '''#!/usr/bin/env bash - set -Eeuo pipefail - source "$BUILD_CONFIG" - - cd "$RELEASE_DIR" - - mapfile -t FILES < deliverables.txt - EXPECTED_COUNT=$(( $(wc -w <<< "$OUTPUT_EXTS") * 2 )) - test "${#FILES[@]}" -eq "$EXPECTED_COUNT" - - for FILE in "${FILES[@]}"; do - test -s "$FILE" - ls -lh "$FILE" - done - - echo - echo "===== VERIFY SHA-256 FILES =====" - - for FILE in "${FILES[@]}"; do - case "$FILE" in *.sha256) sha256sum -c "$FILE" ;; esac - done - - for EXT in $OUTPUT_EXTS; do - TEST_ZIP_VAR="OUTPUT_${EXT^^}_TEST_ZIP" - if [ -n "${!TEST_ZIP_VAR}" ]; then - echo - echo "===== VERIFY .$EXT ARCHIVE =====" - for FILE in "${FILES[@]}"; do - case "$FILE" in *."$EXT") unzip -t "$FILE" ;; esac - done - fi - done - - echo - echo "===== CREATE INTERNAL SHA-256 MANIFEST =====" - - sha256sum "${FILES[@]}" > artifact-manifest.sha256 - - test "$(wc -l < artifact-manifest.sha256)" -eq "$EXPECTED_COUNT" - cat artifact-manifest.sha256 - ''' - } - } - - stage('Upload to Garage') { - steps { - withCredentials([ - usernamePassword( - credentialsId: TARGET.s3Credentials, - usernameVariable: 'AWS_ACCESS_KEY_ID', - passwordVariable: 'AWS_SECRET_ACCESS_KEY' - ) - ]) { - sh '''#!/usr/bin/env bash - set -Eeuo pipefail - set +x - source "$BUILD_CONFIG" - - export AWS_DEFAULT_REGION="$S3_REGION" - export AWS_REGION="$S3_REGION" - export AWS_EC2_METADATA_DISABLED=true - - cd "$RELEASE_DIR" - - mapfile -t FILES < deliverables.txt - FILES+=("artifact-manifest.sha256") - - test "${#FILES[@]}" -eq $(( $(wc -w <<< "$OUTPUT_EXTS") * 2 + 1 )) - - echo "===== UPLOAD GARAGE ARTIFACTS =====" - echo "Bucket: $S3_BUCKET" - echo "Prefix: $STAGING_PREFIX" - - for FILE in "${FILES[@]}"; do - echo "Uploading: $FILE" - - aws \ - --endpoint-url "$S3_ENDPOINT_URL" \ - s3api put-object \ - --bucket "$S3_BUCKET" \ - --key "${STAGING_PREFIX}${FILE}" \ - --body "$FILE" \ - >/dev/null - done - - echo - echo "GARAGE UPLOAD COMPLETE" - ''' - } - } - } - - stage('Verify Garage Objects') { - steps { - withCredentials([ - usernamePassword( - credentialsId: TARGET.s3Credentials, - usernameVariable: 'AWS_ACCESS_KEY_ID', - passwordVariable: 'AWS_SECRET_ACCESS_KEY' - ) - ]) { - sh '''#!/usr/bin/env bash - set -Eeuo pipefail - set +x - source "$BUILD_CONFIG" - - export AWS_DEFAULT_REGION="$S3_REGION" - export AWS_REGION="$S3_REGION" - export AWS_EC2_METADATA_DISABLED=true - - cd "$RELEASE_DIR" - - mapfile -t FILES < deliverables.txt - FILES+=("artifact-manifest.sha256") - - echo "===== VERIFY REMOTE OBJECT SIZES =====" - - for FILE in "${FILES[@]}"; do - LOCAL_SIZE="$(stat -c '%s' "$FILE")" - - REMOTE_SIZE="$( - aws \ - --endpoint-url "$S3_ENDPOINT_URL" \ - s3api head-object \ - --bucket "$S3_BUCKET" \ - --key "${STAGING_PREFIX}${FILE}" \ - --query ContentLength \ - --output text - )" - - printf '%-80s local=%s remote=%s\\n' \ - "$FILE" \ - "$LOCAL_SIZE" \ - "$REMOTE_SIZE" - - test "$LOCAL_SIZE" = "$REMOTE_SIZE" - done - - REMOTE_MANIFEST="$(mktemp)" - trap 'rm -f "$REMOTE_MANIFEST"' EXIT - - aws \ - --endpoint-url "$S3_ENDPOINT_URL" \ - s3api get-object \ - --bucket "$S3_BUCKET" \ - --key "${STAGING_PREFIX}artifact-manifest.sha256" \ - "$REMOTE_MANIFEST" \ - >/dev/null - - cmp artifact-manifest.sha256 "$REMOTE_MANIFEST" - - echo - echo "GARAGE VERIFICATION PASSED" - ''' - } - } - } - - stage('Register Portal Testing Build') { - steps { - withCredentials([ - string( - credentialsId: TARGET.portalCredentials, - variable: 'PORTAL_TOKEN' - ) - ]) { - sh '''#!/usr/bin/env bash - set -Eeuo pipefail - set +x - source "$BUILD_CONFIG" - - TREE_COMMIT="$(cat "$RELEASE_DIR/device-tree-commit.txt")" - TREE_SUBJECT="$(cat "$RELEASE_DIR/device-tree-subject.txt")" - source "$RELEASE_DIR/device-info.env" - - PAYLOAD_FILE="$(mktemp)" - RESPONSE_FILE="$(mktemp)" - - cleanup_files() { - rm -f "$PAYLOAD_FILE" "$RESPONSE_FILE" - } - - trap cleanup_files EXIT - - jq -n \ - --arg project "$PROJECT" \ - --arg project_name "$PROJECT_NAME" \ - --arg device "$DEVICE" \ - --arg device_name "$DEVICE_NAME ($MODEL)" \ - --arg manufacturer "$MANUFACTURER" \ - --arg version "$VERSION" \ - --argjson build_number "$BUILD_NUMBER" \ - --arg staging_prefix "$STAGING_PREFIX" \ - --arg source_branch "$MANIFEST_BRANCH" \ - --arg jenkins_url "$BUILD_URL" \ - --arg tree_url "$TREE_URL" \ - --arg tree_commit "$TREE_COMMIT" \ - --arg tree_subject "$TREE_SUBJECT" \ - --arg operator_changelog "${CHANGELOG:-$CHANGELOG_DEFAULT}" \ - --arg portal_request_id "${PORTAL_REQUEST_ID:-}" \ - '{ - project: $project, - project_name: $project_name, - device: $device, - device_name: $device_name, - manufacturer: $manufacturer, - version: $version, - build_number: $build_number, - staging_prefix: $staging_prefix, - source_branch: $source_branch, - jenkins_url: $jenkins_url, - portal_request_id: $portal_request_id, - changelog: ( - $operator_changelog - + "\\n\\nDevice tree: " - + $tree_url - + "\\nCommit: " - + $tree_commit - + "\\nCommit subject: " - + $tree_subject - ) - }' \ - > "$PAYLOAD_FILE" - - HTTP_CODE="$( - curl \ - -sS \ - --connect-timeout 15 \ - --max-time 120 \ - -o "$RESPONSE_FILE" \ - -w '%{http_code}' \ - -H "X-Jenkins-Token: $PORTAL_TOKEN" \ - -H 'Content-Type: application/json' \ - --data-binary "@$PAYLOAD_FILE" \ - "$PORTAL_URL" - )" - - echo "===== PORTAL RESPONSE =====" - echo "HTTP=$HTTP_CODE" - - jq . "$RESPONSE_FILE" \ - 2>/dev/null \ - || cat "$RESPONSE_FILE" - - case "$HTTP_CODE" in - 200|201) - ;; - *) - echo "ERROR: Portal registration failed." - exit 1 - ;; - esac - - if jq -e '.error' "$RESPONSE_FILE" >/dev/null 2>&1; then - echo "ERROR: Portal returned an error." - exit 1 - fi - - if [ -n "${PORTAL_REQUEST_ID:-}" ] && ! jq -e '.started_by' "$RESPONSE_FILE" >/dev/null 2>&1; then - echo "WARNING: portal did not attribute this build: $(jq -r '.attribution_warning // empty' "$RESPONSE_FILE")" - fi - - echo - echo "PORTAL TESTING REGISTRATION PASSED" - ''' - } - } - } - - stage('Ccache Results') { - steps { - sh '''#!/usr/bin/env bash - set -Eeuo pipefail - ccache -s - ''' - } - } - } - - post { - success { - sh '''#!/usr/bin/env bash - set -Eeuo pipefail - source "$BUILD_CONFIG" - - echo "===== SUCCESSFUL BUILD CLEANUP =====" - - rm -rf "$TMPDIR" - - echo - echo "===== RETURN BASE TO WARM =====" - - zfs rollback "$WARM_SNAPSHOT" - - echo - echo "Warm base restored: $WARM_SNAPSHOT" - echo "Local release files retained at:" - echo "$RELEASE_DIR" - echo - echo "Portal status: Testing" - echo "Garage prefix: $STAGING_PREFIX" - ''' - - echo "${BASE_BRANCH} ${env.DEVICE} build uploaded and registered for testing." - } - - failure { - echo "${BASE_BRANCH} ${env.DEVICE} pipeline failed. Source tree left as-is for diagnosis (next run rolls back to @warm)." - - sh '''#!/usr/bin/env bash - set +e - source "$BUILD_CONFIG" 2>/dev/null - - echo "===== RETAINED FAILURE PATHS =====" - echo "Source + out: ${SOURCE_ROOT:-not resolved}" - echo "Temporary files: $TMPDIR" - echo "Release staging: ${RELEASE_DIR:-not created}" - echo "Garage prefix: ${STAGING_PREFIX:-not resolved}" - ''' - } - - aborted { - echo "${BASE_BRANCH} ${env.DEVICE} pipeline was aborted. Diagnostic paths were retained." - } - - always { - echo "${BASE_BRANCH} ${env.DEVICE} pipeline finished." - } - } -} diff --git a/README.md b/README.md index d1f3446..2002c2d 100644 --- a/README.md +++ b/README.md @@ -1,164 +1,3 @@ # build_manifests -Device manifests for the A-Team Jenkins build pipelines. - -Each base branch has one Jenkins job (named after the branch) running the shared `Jenkinsfile` from `main`. It rolls the base's warm source tree back to `@warm`, drops one device XML into `.repo/local_manifests/`, syncs only the projects listed in it, reads the device info from its annotations and the base settings from the branch's `build.toml`. - -## Layout - -**One branch per build base, one file per device.** - -``` -ofrp-12.1/ genevn.xml milanf.xml rtwo.xml ... -twrp-12.1/ genevn.xml ... -aera-16.0/ genevn.xml milanf.xml ... -``` - -A job points at a branch + file, e.g. `ofrp-12.1` / `genevn.xml`. - -## Device XML - -A standard repo local manifest, plus `ateam.*` annotations on the device tree project. - -```xml - - - - - - - - - - - -``` - -### Annotations - -All four are required; the build stops if one is missing. - -| Annotation | Used for | Example | -|---|---|---| -| `ateam.model` | Release file name, portal device name | `XT2315` | -| `ateam.device_name` | Portal device name | `Moto G Stylus 5G 2023` | -| `ateam.manufacturer` | Portal | `Motorola` | -| `ateam.partition` | Build target and flash target | `recovery` | - -`ateam.partition` must be one of: - -| Value | Builds | -|---|---| -| `recovery` | `recoveryimage` | -| `boot` | `bootimage` | -| `vendor_boot` | `vendorbootimage` | - -The portal shows the device as ` ()`. - -### Projects - -- List the device tree plus anything else the device needs (kernel, vendor, common trees). -- Every project listed gets synced; nothing else in the base tree is touched. -- Any git source works. Add another `` for GitHub or elsewhere; the device tree doesn't have to live on A-Team Forgejo. -- Pin `revision` to the branch for that base (e.g. `ofrp-12.1`). -- Don't add projects that override the base's own repos. Device-specific fixes go in the device trees. - -## Release naming - -Builds come out as: - -``` --_A-Team__-. -``` - -plus a `.sha256` for each, e.g. `OFRP-12.1_A-Team_genevn_XT2315-261009.zip`. - -## Adding a device - -1. Clone and switch to the base branch: - ``` - git clone ssh://git@ateam.gotadell.com/A-Team_Digital_Solutions/build_manifests.git - cd build_manifests - git checkout ofrp-12.1 - ``` -2. Copy an existing XML to `.xml` and edit the projects and annotations. -3. Commit and push. -4. That's it: the branch's Jenkins job (named after the branch, e.g. `ofrp-12.1`) lists the new XML in its - `DEVICE` dropdown, and the portal picks it up on its next device sync. - -## Adding a base - -1. Create a new branch named `-` (e.g. `shrp-12.1`) and add a device XML for each device built on it. - Each base has its own copy of a device's XML, so update model or name changes on every branch that has that device. -2. Add a `build.toml` (see the README on `main`) describing the base: portal project, release naming, ZFS dataset, - lunch target and release files. -3. Prepare the warm source tree on buildBox and snapshot it as `@warm`. -4. Create a Jenkins job named exactly like the branch, using *Pipeline script from SCM* on `build_manifests` - branch `main`, script path `Jenkinsfile`. -5. Map the portal project to the job in the portal admin. - -## Build pipeline - -`Jenkinsfile` on `main` is the build pipeline for every base. Each base branch has one Jenkins job named exactly -like the branch (`ofrp-12.1`, `twrp-12.1`, ...), set to *Pipeline script from SCM* on this repo, branch `main`, -script path `Jenkinsfile`. A job named `cms-` builds the same base for the cms test portal (test bucket, -test credentials, `ateam-cms.gotadell.com`). - -Per run the job: - -1. checks the device codename, resolves `build.toml` for that device (`pipeline/build_config.py`) and checks the - warm snapshot and its mountpoint; -2. rolls the base back to `@warm` and checks the warm base is clean (no device tree for the device, - empty `out/target/product`, empty `.repo/local_manifests`); -3. copies `.xml` into `.repo/local_manifests/`, fixes branch-name case, reads the `ateam.*` annotations and - syncs only the listed projects; -4. runs `lunch`, `m installclean` and `mka `; -5. picks the release files from `out/target/product//`, verifies their native checksums, renames them to - the A-Team scheme with a `.sha256` each, uploads them to Garage and registers the build with the portal as - testing (including the portal request id when the portal started it); -6. rolls the base back to `@warm` again (on failure the tree is left for diagnosis; the next run rolls it back). - -A base's live and `cms-` jobs share one ZFS dataset and hold the `warm-base:` lock (Lockable Resources -plugin) for the whole run. - -Parameters: `DEVICE` (lists the branch's XMLs), `CHANGELOG`, `PORTAL_REQUEST_ID` and `PORTAL_USER` (set by the -portal's *Start build*), and `MANIFEST_REF` (cms- test jobs only, to build from an unmerged branch). - -## build.toml - -Every base branch has a `build.toml` next to the device XMLs. It is data only: unknown keys, wrong types or -placeholders other than `{codename}` stop the build. - -```toml -[base] -project = "orangefox" # portal project slug -project_name = "OrangeFox" -version = "R12.1" # version shown in the portal -release_tag = "OFRP" # -_A-Team__- -release_ver = "12.1" -source_root = "/build/ofox-12.1" # mountpoint of zfs_dataset; out/ lives inside it -zfs_dataset = "build/ofox-12.1" # rolled back to @warm before and after every build -warm_check = ["vendor/recovery"] # must exist in the warm base -lunch_target = "twrp_{codename}-eng" -make_extra = ["adbd"] # built together with the ateam.partition target -changelog = "Automated OrangeFox testing build." - -[base.env] # extra environment for lunch and the build -FOX_BUILD_DEVICE = "{codename}" - -[outputs.img] # release files in out/target/product//, exactly one match each -glob = "OrangeFox-*.img" -checksum = "md5" # native sidecar to verify: md5 | sha256 | none - -[outputs.zip] -glob = "OrangeFox-*.zip" -checksum = "md5" -test_zip = true - -[devices.sycamore_row_5G] # optional per-device overrides: lunch_target, make_extra, env, outputs -lunch_target = "twrp_sycamore_row_5G-eng" -``` - -Each `[outputs.]` becomes `.` plus `..sha256`. +device manifests for a-team digital solutions \ No newline at end of file diff --git a/pipeline/build_config.py b/pipeline/build_config.py deleted file mode 100644 index f90f72a..0000000 --- a/pipeline/build_config.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -""" -Resolve a base's build.toml for one device and print shell assignments. - - build_config.py - -Used by the shared Jenkinsfile. build.toml is data only: unknown keys, wrong -types or unexpected placeholders stop the build instead of being ignored. -""" - -from __future__ import annotations - -import re -import shlex -import sys -import tomllib - -BASE_KEYS = { - "project": str, - "project_name": str, - "version": str, - "release_tag": str, - "release_ver": str, - "source_root": str, - "zfs_dataset": str, - "warm_check": list, - "lunch_target": str, - "make_extra": list, - "changelog": str, - "env": dict, -} -REQUIRED_BASE = BASE_KEYS.keys() - {"warm_check", "make_extra", "changelog", "env"} -DEVICE_KEYS = {"lunch_target", "make_extra", "env", "outputs"} -OUTPUT_KEYS = {"glob": str, "checksum": str, "test_zip": bool} -CHECKSUMS = {"md5", "sha256", "none"} - -CODENAME = re.compile(r"^[A-Za-z0-9_]+$") -SLUG = re.compile(r"^[a-z0-9][a-z0-9-]*$") -TOKEN = re.compile(r"^[A-Za-z0-9_.+-]+$") -ENV_NAME = re.compile(r"^[A-Z][A-Z0-9_]*$") -EXT = re.compile(r"^[a-z0-9]+$") -PLACEHOLDER = re.compile(r"\{([^}]*)\}") - - -def fail(message: str) -> None: - sys.exit(f"ERROR: build.toml: {message}") - - -def expand(value: str, where: str, codename: str) -> str: - for name in PLACEHOLDER.findall(value): - if name != "codename": - fail(f"{where}: unknown placeholder {{{name}}} (only {{codename}} is allowed)") - return value.replace("{codename}", codename) - - -def check_tokens(values, where: str) -> list[str]: - if not all(isinstance(v, str) and TOKEN.match(v) for v in values): - fail(f"{where}: entries must be plain words (letters, digits, _ . + -)") - return list(values) - - -def check_outputs(outputs, where: str, partial: bool = False) -> dict: - if not isinstance(outputs, dict) or not outputs: - fail(f"{where}: needs at least one [outputs.] table") - for ext, spec in outputs.items(): - if not EXT.match(ext): - fail(f"{where}.{ext}: extension must be lowercase letters/digits") - if not isinstance(spec, dict): - fail(f"{where}.{ext}: must be a table") - for key, value in spec.items(): - if key not in OUTPUT_KEYS: - fail(f"{where}.{ext}: unknown key '{key}'") - if not isinstance(value, OUTPUT_KEYS[key]): - fail(f"{where}.{ext}.{key}: wrong type") - if "glob" not in spec and not partial: - fail(f"{where}.{ext}: 'glob' is required") - if "/" in spec.get("glob", ""): - fail(f"{where}.{ext}.glob: must be a file name pattern, not a path") - if spec.get("checksum", "none") not in CHECKSUMS: - fail(f"{where}.{ext}.checksum: one of {', '.join(sorted(CHECKSUMS))}") - return outputs - - -def resolve(config: dict, codename: str) -> dict[str, str]: - unknown = config.keys() - {"base", "outputs", "devices"} - if unknown: - fail(f"unknown section(s): {', '.join(sorted(unknown))}") - - base = config.get("base") - if not isinstance(base, dict): - fail("[base] section is missing") - for key, value in base.items(): - if key not in BASE_KEYS: - fail(f"[base]: unknown key '{key}'") - if not isinstance(value, BASE_KEYS[key]): - fail(f"[base].{key}: wrong type") - missing = REQUIRED_BASE - base.keys() - if missing: - fail(f"[base]: missing {', '.join(sorted(missing))}") - - if not SLUG.match(base["project"]): - fail("[base].project: lowercase portal project slug expected") - if not base["source_root"].startswith("/"): - fail("[base].source_root: must be an absolute path") - for key in ("release_tag", "release_ver", "version", "zfs_dataset"): - if not re.match(r"^[A-Za-z0-9_.+/-]+$", base[key]): - fail(f"[base].{key}: unexpected characters") - - outputs = check_outputs(config.get("outputs"), "[outputs]") - lunch = base["lunch_target"] - make_extra = base.get("make_extra", []) - env = dict(base.get("env", {})) - - devices = config.get("devices", {}) - if not isinstance(devices, dict): - fail("[devices] must be a table of [devices.]") - override = devices.get(codename, {}) - for key in override: - if key not in DEVICE_KEYS: - fail(f"[devices.{codename}]: unknown key '{key}'") - lunch = override.get("lunch_target", lunch) - make_extra = override.get("make_extra", make_extra) - env.update(override.get("env", {})) - if "outputs" in override: - merged = {ext: dict(spec) for ext, spec in outputs.items()} - for ext, spec in check_outputs(override["outputs"], f"[devices.{codename}.outputs]", partial=True).items(): - merged.setdefault(ext, {}).update(spec) - outputs = check_outputs(merged, f"[devices.{codename}] outputs") - - for name, value in env.items(): - if not ENV_NAME.match(name) or not isinstance(value, str): - fail(f"env.{name}: names must be UPPER_CASE and values strings") - - warm_check = base.get("warm_check", []) - for path in warm_check: - if ( - not isinstance(path, str) - or path.startswith("/") - or ".." in path.split("/") - or not re.match(r"^[A-Za-z0-9_./-]+$", path) - ): - fail(f"[base].warm_check: '{path}' must be a relative path inside the tree") - - result = { - "PROJECT": base["project"], - "PROJECT_NAME": base["project_name"], - "VERSION": base["version"], - "RELEASE_TAG": base["release_tag"], - "RELEASE_VER": base["release_ver"], - "SOURCE_ROOT": base["source_root"].rstrip("/"), - "OUT_DIR": base["source_root"].rstrip("/") + "/out", - "ZFS_DATASET": base["zfs_dataset"], - "WARM_SNAPSHOT": base["zfs_dataset"] + "@warm", - "WARM_CHECK": " ".join(warm_check), - "LUNCH_TARGET": expand(lunch, "lunch_target", codename), - "MAKE_EXTRA": " ".join(check_tokens(make_extra, "make_extra")), - "CHANGELOG_DEFAULT": base.get("changelog", f"Automated {base['project_name']} testing build."), - "BUILD_ENV_NAMES": " ".join(sorted(env)), - "OUTPUT_EXTS": " ".join(sorted(outputs)), - } - if not TOKEN.match(result["LUNCH_TARGET"]): - fail("lunch_target: unexpected characters") - for name, value in env.items(): - result[name] = expand(value, f"env.{name}", codename) - for ext, spec in outputs.items(): - key = ext.upper() - result[f"OUTPUT_{key}_GLOB"] = expand(spec["glob"], f"outputs.{ext}.glob", codename) - result[f"OUTPUT_{key}_CHECKSUM"] = spec.get("checksum", "none") - result[f"OUTPUT_{key}_TEST_ZIP"] = "1" if spec.get("test_zip") else "" - - return result - - -def main() -> None: - if len(sys.argv) != 3: - sys.exit(__doc__) - path, codename = sys.argv[1], sys.argv[2] - if not CODENAME.match(codename): - fail(f"bad codename '{codename}'") - try: - with open(path, "rb") as handle: - config = tomllib.load(handle) - except FileNotFoundError: - fail(f"{path} not found (every base branch needs a build.toml)") - except tomllib.TOMLDecodeError as error: - fail(f"invalid TOML: {error}") - - for name, value in resolve(config, codename).items(): - print(f"{name}={shlex.quote(value)}") - - -if __name__ == "__main__": - main()