// Shared A-Team recovery build pipeline.
//
// One Jenkins job per build_manifests branch, named after the branch
// (ofrp-12.1, twrp-12.1, ...), using "Pipeline script from SCM" on this file
// (branch main). Base settings come from build.toml on the job's branch; the
// device comes from <DEVICE>.xml on that branch.
//
// Jobs named cms-<branch> build the same base for the cms test portal
// (test bucket, test credentials, ateam-cms.gotadell.com).

def JOB = env.JOB_BASE_NAME
def IS_TEST_JOB = JOB.startsWith('cms-')
def BASE_BRANCH = IS_TEST_JOB ? JOB.substring(4) : JOB

if (!(BASE_BRANCH ==~ /[a-z0-9][a-z0-9._-]*/)) {
    error "Job name '${JOB}' must be a build_manifests branch name (optionally prefixed with cms-)"
}

def TARGET = IS_TEST_JOB ? [
    bucket            : 'ateam-cms-staging',
    s3Credentials     : 'garage-s3-cms',
    portalCredentials : 'portal-ingest-token-cms',
    portalUrl         : 'https://ateam-cms.gotadell.com/api/jenkins/build-complete/',
    releaseRoot       : '/storage/jenkins-artifacts/staging-cms',
] : [
    bucket            : 'ateam-staging',
    s3Credentials     : 'garage-s3',
    portalCredentials : 'portal-ingest-token',
    portalUrl         : 'https://ateam.gotadell.com/api/jenkins/build-complete/',
    releaseRoot       : '/storage/jenkins-artifacts/staging',
]

pipeline {
    agent {
        label 'buildBox'
    }

    options {
        // The repo is checked out inside Preflight, once the per-build TMPDIR exists
        skipDefaultCheckout()
        timestamps()
        disableConcurrentBuilds()
        // A base's live and cms- jobs share one ZFS dataset; never roll it back under each other.
        lock(resource: "warm-base:${BASE_BRANCH}")
        timeout(time: 6, unit: 'HOURS')
        buildDiscarder(logRotator(numToKeepStr: '20'))
    }

    parameters {
        // Device list = every .xml on this job's build_manifests branch (Active Choices plugin)
        activeChoice(
            name: 'DEVICE',
            description: 'Device codename. Builds from <DEVICE>.xml on the manifest branch.',
            choiceType: 'PT_SINGLE_SELECT',
            filterable: false,
            script: groovyScript(
                script: [
                    classpath: [],
                    sandbox: false,
                    script: """
                        import groovy.json.JsonSlurper

                        def api = 'https://ateam.gotadell.com/git/api/v1/repos/A-Team_Digital_Solutions/build_manifests/contents?ref=${BASE_BRANCH}'
                        def conn = new URL(api).openConnection()
                        conn.connectTimeout = 5000
                        conn.readTimeout = 5000

                        return new JsonSlurper().parse(conn.inputStream)
                            .findAll { it.type == 'file' && it.name.endsWith('.xml') }
                            .collect { it.name[0..-5] }
                            .sort()
                    """
                ],
                fallbackScript: [
                    classpath: [],
                    sandbox: false,
                    script: 'return ["ERROR-could-not-list-manifests"]'
                ]
            )
        )
        text(
            name: 'CHANGELOG',
            defaultValue: '',
            description: 'Testing notes displayed in the A-Team Portal (empty = changelog from build.toml).'
        )
        string(
            name: 'PORTAL_REQUEST_ID',
            defaultValue: '',
            description: 'Set by the portal "Start build" button. Empty for builds started here (owner-only approval).'
        )
        string(
            name: 'PORTAL_USER',
            defaultValue: '',
            description: 'Portal user who started the build (informational; set by the portal).'
        )
        string(
            name: 'MANIFEST_REF',
            defaultValue: '',
            description: 'cms- test jobs only: build_manifests ref to read build.toml and the device XML from (default: the job branch).'
        )
    }

    environment {
        DEVICE           = "${params.DEVICE}"
        MANIFEST_URL     = 'ssh://git@ateam.gotadell.com/A-Team_Digital_Solutions/build_manifests.git'
        MANIFEST_BRANCH  = "${BASE_BRANCH}"
        MANIFEST_REF     = "${params.MANIFEST_REF ?: BASE_BRANCH}"
        MANIFEST_FILE    = "${params.DEVICE}.xml"
        TEST_JOB         = "${IS_TEST_JOB ? '1' : ''}"

        TMPDIR           = "/build/tmp/jenkins/${JOB}-${params.DEVICE}-${BUILD_NUMBER}"
        BUILD_CONFIG     = "/build/tmp/jenkins/${JOB}-${params.DEVICE}-${BUILD_NUMBER}/build-config.env"
        RELEASE_ROOT     = "${TARGET.releaseRoot}"

        // Match the env the warm out/ was built with, or soong regenerates everything
        // (OUT_DIR comes from build.toml: <source_root>/out)
        CC_WRAPPER       = '/usr/bin/ccache'
        CCACHE_EXEC      = '/usr/bin/ccache'
        CCACHE_DIR       = '/build/ccache'
        USE_CCACHE       = '1'
        ALLOW_MISSING_DEPENDENCIES = 'true'

        S3_ENDPOINT_URL  = 'http://10.0.55.47:9000'
        S3_REGION        = 'garage'
        S3_BUCKET        = "${TARGET.bucket}"

        PORTAL_URL       = "${TARGET.portalUrl}"
    }

    stages {
        stage('Preflight') {
            steps {
                sh '''#!/usr/bin/env bash
                    set -Eeuo pipefail

                    [[ "$DEVICE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: bad DEVICE '$DEVICE'"; exit 1; }
                    [[ -z "$PORTAL_REQUEST_ID" || "$PORTAL_REQUEST_ID" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] \
                      || { echo "ERROR: bad PORTAL_REQUEST_ID"; exit 1; }
                    if [ -z "$TEST_JOB" ] && [ "$MANIFEST_REF" != "$MANIFEST_BRANCH" ]; then
                        echo "ERROR: MANIFEST_REF is only allowed on cms- test jobs"
                        exit 1
                    fi
                    [[ "$MANIFEST_REF" =~ ^[A-Za-z0-9._/-]+$ ]] || { echo "ERROR: bad MANIFEST_REF"; exit 1; }

                    echo "===== REQUIRED TOOLS ====="

                    for TOOL in \
                        git \
                        repo \
                        python3 \
                        zfs \
                        ccache \
                        aws \
                        jq \
                        curl \
                        md5sum \
                        sha256sum \
                        unzip
                    do
                        command -v "$TOOL"
                    done

                    echo
                    echo "===== VERIFY CLEAN PER-BUILD PATHS ====="

                    test ! -e "$TMPDIR"
                    install -d -m 2775 "$TMPDIR"
                '''

                // Jenkinsfile + pipeline/build_config.py from build_manifests main
                checkout scm

                sh '''#!/usr/bin/env bash
                    set -Eeuo pipefail

                    echo "===== FETCH BUILD MANIFEST ====="

                    MANIFEST_CHECKOUT="$TMPDIR/build_manifests"

                    git clone \
                      --depth 1 \
                      --branch "$MANIFEST_REF" \
                      --single-branch \
                      "$MANIFEST_URL" \
                      "$MANIFEST_CHECKOUT"

                    test -s "$MANIFEST_CHECKOUT/$MANIFEST_FILE" \
                      || { echo "ERROR: $MANIFEST_FILE not found on $MANIFEST_REF"; exit 1; }

                    echo
                    echo "===== RESOLVE build.toml FOR $DEVICE ====="

                    python3 "$WORKSPACE/pipeline/build_config.py" \
                      "$MANIFEST_CHECKOUT/build.toml" \
                      "$DEVICE" \
                      > "$BUILD_CONFIG"

                    source "$BUILD_CONFIG"

                    {
                        printf 'RELEASE_DIR=%q\\n' "$RELEASE_ROOT/$PROJECT/$DEVICE/$PROJECT-$DEVICE-$BUILD_NUMBER"
                        printf 'STAGING_PREFIX=%q\\n' "$PROJECT/$DEVICE/build-$BUILD_NUMBER/"
                    } >> "$BUILD_CONFIG"

                    cat "$BUILD_CONFIG"
                    source "$BUILD_CONFIG"

                    echo
                    echo "===== BUILD IDENTIFICATION ====="
                    echo "Target: $([ -n "$TEST_JOB" ] && echo "cms test portal" || echo "live portal")"
                    echo "Project: $PROJECT_NAME $VERSION"
                    echo "Device: $DEVICE"
                    echo "Manifest: $MANIFEST_URL ($MANIFEST_REF/$MANIFEST_FILE)"
                    echo "Warm base: $WARM_SNAPSHOT"
                    echo "Lunch: $LUNCH_TARGET"
                    echo "Portal prefix: $STAGING_PREFIX"
                    echo "Portal request: ${PORTAL_REQUEST_ID:-none} ${PORTAL_USER:+(by $PORTAL_USER)}"

                    echo
                    echo "===== VERIFY WARM SNAPSHOT ====="

                    zfs list -H -t snapshot -o name,creation "$WARM_SNAPSHOT"
                    zfs allow "$ZFS_DATASET"
                    MOUNTPOINT="$(zfs get -H -o value mountpoint "$ZFS_DATASET")"
                    test "$MOUNTPOINT" = "$SOURCE_ROOT" \
                      || { echo "ERROR: $ZFS_DATASET is mounted at $MOUNTPOINT, build.toml says $SOURCE_ROOT"; exit 1; }

                    echo
                    echo "===== PREPARE RELEASE DIRECTORY ====="

                    test ! -e "$RELEASE_DIR"
                    install -d -m 2775 "$RELEASE_DIR"

                    echo "Temporary: $TMPDIR"
                    echo "Release staging: $RELEASE_DIR"
                '''
            }
        }

        stage('Restore Warm Base') {
            steps {
                sh '''#!/usr/bin/env bash
                    set -Eeuo pipefail
                    source "$BUILD_CONFIG"

                    echo "===== ROLLBACK TO WARM ====="

                    zfs rollback "$WARM_SNAPSHOT"

                    echo
                    echo "===== VERIFY WARM BASE ====="

                    cd "$SOURCE_ROOT"

                    test -d .repo
                    test -d out
                    for P in $WARM_CHECK; do
                        test -e "$P" || { echo "ERROR: $P missing from warm base"; exit 1; }
                    done
                    if compgen -G "device/*/$DEVICE" >/dev/null; then
                        echo "ERROR: device tree for $DEVICE already present in warm base"
                        exit 1
                    fi
                    test -z "$(ls -A out/target/product 2>/dev/null)"
                    test -z "$(ls -A .repo/local_manifests 2>/dev/null)"

                    echo
                    echo "===== VERIFY JENKINS CAN WRITE THE TREE ====="

                    test -w "$SOURCE_ROOT"
                    test -w "$SOURCE_ROOT/out"
                    test -w "$SOURCE_ROOT/.repo"

                    echo "Source: $SOURCE_ROOT"
                '''
            }
        }

        stage('Sync Device Trees From Manifest') {
            steps {
                sh '''#!/usr/bin/env bash
                    set -Eeuo pipefail
                    source "$BUILD_CONFIG"

                    MANIFEST_CHECKOUT="$TMPDIR/build_manifests"

                    MANIFEST_COMMIT="$(git -C "$MANIFEST_CHECKOUT" rev-parse HEAD)"
                    printf '%s\\n' "$MANIFEST_COMMIT" > "$RELEASE_DIR/manifest-commit.txt"
                    echo "Manifest commit: $MANIFEST_COMMIT"

                    cd "$SOURCE_ROOT"

                    mkdir -p .repo/local_manifests
                    install -m 0644 \
                      "$MANIFEST_CHECKOUT/$MANIFEST_FILE" \
                      ".repo/local_manifests/$MANIFEST_FILE"

                    echo
                    echo "===== RESOLVE BRANCH NAMES (case-insensitive) ====="

                    python3 - ".repo/local_manifests/$MANIFEST_FILE" <<'PYEOF'
import sys, subprocess
import xml.etree.ElementTree as ET

path = sys.argv[1]
tree = ET.parse(path)
root = tree.getroot()
remotes = {r.get("name"): r.get("fetch", "") for r in root.iter("remote")}
default = root.find("default")
dflt = lambda k: default.get(k) if default is not None else None
changed = False

for p in root.iter("project"):
    name = p.get("name")
    rev = p.get("revision") or dflt("revision")
    # leave tags, full refs and commit hashes alone
    if not rev or rev.startswith("refs/") or len(rev) == 40:
        continue
    fetch = remotes.get(p.get("remote") or dflt("remote"), "")
    if "://" not in fetch and "@" not in fetch:
        continue
    url = fetch + ("" if fetch.endswith(("/", ":")) else "/") + name

    out = subprocess.run(["git", "ls-remote", "--heads", url],
                         capture_output=True, text=True, check=True).stdout
    heads = [l.split("refs/heads/", 1)[1] for l in out.splitlines() if "refs/heads/" in l]

    if rev in heads:
        print(f"{name}: {rev}")
        continue
    match = [h for h in heads if h.lower() == rev.lower()]
    if len(match) != 1:
        sys.exit(f"ERROR: {name}: no branch '{rev}'. Branches: {', '.join(heads) or 'none'}")
    print(f"{name}: {match[0]}  (manifest says {rev})")
    p.set("revision", match[0])
    changed = True

if changed:
    tree.write(path, xml_declaration=True, encoding="UTF-8")
PYEOF

                    echo
                    echo "===== PROJECTS IN $MANIFEST_FILE ====="

                    mapfile -t PATHS < <(
                      python3 - ".repo/local_manifests/$MANIFEST_FILE" <<'PYEOF'
import sys
import xml.etree.ElementTree as ET
for p in ET.parse(sys.argv[1]).getroot().iter("project"):
    print(p.get("path") or p.get("name"))
PYEOF
                    )

                    test "${#PATHS[@]}" -ge 1
                    printf '  %s\\n' "${PATHS[@]}"

                    echo
                    echo "===== DEVICE INFO FROM $MANIFEST_FILE ====="

                    python3 - ".repo/local_manifests/$MANIFEST_FILE" > "$RELEASE_DIR/device-info.env" <<'PYEOF'
import sys, shlex
import xml.etree.ElementTree as ET

root = ET.parse(sys.argv[1]).getroot()
remotes = {r.get("name"): r.get("fetch", "") for r in root.iter("remote")}
default = root.find("default")
dflt = lambda k: default.get(k, "") if default is not None else ""
out = lambda k, v: print(f"{k}={shlex.quote(v or '')}")

# The device tree is the project carrying the ateam.* annotations
trees = [p for p in root.iter("project")
         if any(a.get("name", "").startswith("ateam.") for a in p.iter("annotation"))]
if len(trees) != 1:
    sys.exit(f"ERROR: expected ateam.* annotations on exactly one project, found {len(trees)}")
p = trees[0]

for a in p.iter("annotation"):
    n = a.get("name", "")
    if n.startswith("ateam."):
        out(n[6:].upper(), a.get("value"))

name = p.get("name")
rev  = p.get("revision") or dflt("revision")
fetch = remotes.get(p.get("remote") or dflt("remote"), "")
host = fetch.split("@")[-1].split("://")[-1].strip("/").split("/")[0].split(":")[0]

if host == "ateam.gotadell.com":
    url = f"https://ateam.gotadell.com/git/{name}/src/branch/{rev}"
elif host == "github.com":
    url = f"https://github.com/{name}/tree/{rev}"
else:
    url = f"{fetch.rstrip('/')}/{name}"

out("TREE_PATH", p.get("path") or name)
out("TREE_URL", url)
PYEOF

                    cat "$RELEASE_DIR/device-info.env"
                    source "$RELEASE_DIR/device-info.env"

                    for VAR in MODEL DEVICE_NAME MANUFACTURER PARTITION TREE_PATH TREE_URL; do
                        test -n "${!VAR:-}" || { echo "ERROR: $VAR missing (check ateam.* annotations in $MANIFEST_FILE)"; exit 1; }
                    done

                    case "$PARTITION" in
                        recovery|boot|vendor_boot) ;;
                        *) echo "ERROR: unknown ateam.partition '$PARTITION'"; exit 1 ;;
                    esac

                    echo
                    echo "===== SYNC ONLY MANIFEST PROJECTS ====="

                    repo sync \
                      -c \
                      --no-tags \
                      --no-clone-bundle \
                      --force-sync \
                      -j8 \
                      "${PATHS[@]}"

                    test -d "$TREE_PATH/.git" || test -e "$TREE_PATH/.git"

                    echo
                    echo "===== RECORD SYNCED COMMITS ====="

                    for P in "${PATHS[@]}"; do
                        printf '%s %s\\n' "$P" "$(git -C "$P" rev-parse HEAD)"
                    done | tee "$RELEASE_DIR/synced-projects.txt"

                    TREE_COMMIT="$(git -C "$TREE_PATH" rev-parse HEAD)"
                    TREE_SUBJECT="$(git -C "$TREE_PATH" log -1 --pretty=%s)"

                    printf '%s\\n' "$TREE_COMMIT"  > "$RELEASE_DIR/device-tree-commit.txt"
                    printf '%s\\n' "$TREE_SUBJECT" > "$RELEASE_DIR/device-tree-subject.txt"

                    echo
                    echo "Tree commit: $TREE_COMMIT"
                    echo "Tree subject: $TREE_SUBJECT"
                '''
            }
        }

        stage('Configure Ccache') {
            steps {
                sh '''#!/usr/bin/env bash
                    set -Eeuo pipefail

                    mkdir -p "$CCACHE_DIR"
                    test -d "$CCACHE_DIR"
                    test -w "$CCACHE_DIR"

                    ccache -M 100G
                    ccache -o compression=true
                    ccache -z
                    ccache -s
                '''
            }
        }

        stage('Lunch') {
            steps {
                sh '''#!/usr/bin/env bash
                    set -o pipefail
                    source "$BUILD_CONFIG" || exit 1

                    cd "$SOURCE_ROOT" || exit 1

                    export OUT_DIR
                    export CC_WRAPPER
                    export CCACHE_EXEC

                    export TMPDIR
                    export TMP="$TMPDIR"
                    export TEMP="$TMPDIR"
                    export CCACHE_DIR
                    export USE_CCACHE
                    export ALLOW_MISSING_DEPENDENCIES
                    for VAR in $BUILD_ENV_NAMES; do export "$VAR"; done

                    source build/envsetup.sh

                    type lunch >/dev/null 2>&1 || exit 1
                    type mka >/dev/null 2>&1 || exit 1

                    lunch "$LUNCH_TARGET" || exit 1

                    echo
                    echo "===== BUILD VARIABLES ====="

                    echo -n "TARGET_PRODUCT: "
                    get_build_var TARGET_PRODUCT || exit 1

                    echo -n "TARGET_DEVICE: "
                    get_build_var TARGET_DEVICE || exit 1

                    echo -n "PRODUCT_OUT: "
                    get_build_var PRODUCT_OUT || exit 1
                '''
            }
        }

        stage('Install Clean and Build') {
            steps {
                sh '''#!/usr/bin/env bash
                    set -o pipefail
                    source "$BUILD_CONFIG" || exit 1

                    cd "$SOURCE_ROOT" || exit 1

                    export OUT_DIR
                    export CC_WRAPPER
                    export CCACHE_EXEC

                    export TMPDIR
                    export TMP="$TMPDIR"
                    export TEMP="$TMPDIR"
                    export CCACHE_DIR
                    export USE_CCACHE
                    export ALLOW_MISSING_DEPENDENCIES
                    for VAR in $BUILD_ENV_NAMES; do export "$VAR"; done

                    source build/envsetup.sh
                    lunch "$LUNCH_TARGET" || exit 1

                    echo "===== INSTALL CLEAN ====="

                    m installclean
                    INSTALLCLEAN_STATUS=$?

                    if [ "$INSTALLCLEAN_STATUS" -ne 0 ]; then
                        echo "ERROR: installclean failed with status $INSTALLCLEAN_STATUS"
                        exit "$INSTALLCLEAN_STATUS"
                    fi

                    source "$RELEASE_DIR/device-info.env" || exit 1

                    case "$PARTITION" in
                        recovery)    BUILD_TARGET=recoveryimage ;;
                        boot)        BUILD_TARGET=bootimage ;;
                        vendor_boot) BUILD_TARGET=vendorbootimage ;;
                        *) echo "ERROR: unknown partition '$PARTITION'"; exit 1 ;;
                    esac

                    echo
                    echo "===== BUILD $PROJECT_NAME ($MAKE_EXTRA $BUILD_TARGET) ====="

                    START_TIME="$(date +%s)"

                    # MAKE_EXTRA is validated by build_config.py (plain words only)
                    mka $MAKE_EXTRA "$BUILD_TARGET"
                    BUILD_STATUS=$?

                    END_TIME="$(date +%s)"
                    ELAPSED=$((END_TIME - START_TIME))

                    if [ "$BUILD_STATUS" -ne 0 ]; then
                        echo "ERROR: $PROJECT_NAME build failed with status $BUILD_STATUS"
                        exit "$BUILD_STATUS"
                    fi

                    PRODUCT_OUT="$(get_build_var PRODUCT_OUT)" || exit 1

                    echo
                    echo "===== BUILD TIME ====="

                    printf "$PROJECT_NAME $DEVICE build: %d:%02d (mm:ss)\\n" \
                      "$((ELAPSED / 60))" \
                      "$((ELAPSED % 60))"

                    echo
                    echo "===== PRODUCT OUTPUT ====="

                    cd "$PRODUCT_OUT" || exit 1
                    for EXT in $OUTPUT_EXTS; do
                        GLOB_VAR="OUTPUT_${EXT^^}_GLOB"
                        find . -maxdepth 1 -type f \\( -name "${!GLOB_VAR}" -o -name "${!GLOB_VAR}.*" \\) \
                          -printf '%f  %s bytes\\n'
                    done | sort
                '''
            }
        }

        stage('Collect and Rename Artifacts') {
            steps {
                sh '''#!/usr/bin/env bash
                    set -Eeuo pipefail
                    source "$BUILD_CONFIG"
                    source "$RELEASE_DIR/device-info.env"

                    PRODUCT_OUT="$OUT_DIR/target/product/$DEVICE"
                    cd "$PRODUCT_OUT"

                    BASE="${RELEASE_TAG}-${RELEASE_VER}_A-Team_${DEVICE}_${MODEL}-$(TZ=America/New_York date +%y%m%d)"
                    : > "$RELEASE_DIR/deliverables.txt"

                    for EXT in $OUTPUT_EXTS; do
                        GLOB_VAR="OUTPUT_${EXT^^}_GLOB"
                        CHECKSUM_VAR="OUTPUT_${EXT^^}_CHECKSUM"

                        mapfile -t FOUND < <(find . -maxdepth 1 -type f -name "${!GLOB_VAR}" -printf '%f\\n')

                        if [ "${#FOUND[@]}" -ne 1 ]; then
                            echo "ERROR: Expected exactly one .$EXT matching '${!GLOB_VAR}', found ${#FOUND[@]}."
                            printf '%s\\n' "${FOUND[@]}"
                            exit 1
                        fi
                        FILE="${FOUND[0]}"

                        case "${!CHECKSUM_VAR}" in
                            md5)
                                test -s "$FILE.md5"
                                EXPECTED="$(awk 'NR == 1 {print $1}' "$FILE.md5")"
                                ACTUAL="$(md5sum "$FILE" | awk '{print $1}')"
                                test "$EXPECTED" = "$ACTUAL"
                                echo "$FILE: native MD5 OK"
                                ;;
                            sha256)
                                test -s "$FILE.sha256"
                                EXPECTED="$(awk 'NR == 1 {print $1}' "$FILE.sha256")"
                                ACTUAL="$(sha256sum "$FILE" | awk '{print $1}')"
                                test "$EXPECTED" = "$ACTUAL"
                                echo "$FILE: native SHA-256 OK"
                                ;;
                            none)
                                ;;
                        esac

                        install -m 0644 "$FILE" "$RELEASE_DIR/$BASE.$EXT"
                        ( cd "$RELEASE_DIR" && sha256sum "$BASE.$EXT" > "$BASE.$EXT.sha256" )
                        printf '%s\\n%s\\n' "$BASE.$EXT" "$BASE.$EXT.sha256" >> "$RELEASE_DIR/deliverables.txt"
                    done

                    echo
                    echo "===== RENAMED TO A-TEAM SCHEME ====="
                    ls -lh "$RELEASE_DIR"
                '''
            }
        }

        stage('Verify Deliverables') {
            steps {
                sh '''#!/usr/bin/env bash
                    set -Eeuo pipefail
                    source "$BUILD_CONFIG"

                    cd "$RELEASE_DIR"

                    mapfile -t FILES < deliverables.txt
                    EXPECTED_COUNT=$(( $(wc -w <<< "$OUTPUT_EXTS") * 2 ))
                    test "${#FILES[@]}" -eq "$EXPECTED_COUNT"

                    for FILE in "${FILES[@]}"; do
                        test -s "$FILE"
                        ls -lh "$FILE"
                    done

                    echo
                    echo "===== VERIFY SHA-256 FILES ====="

                    for FILE in "${FILES[@]}"; do
                        case "$FILE" in *.sha256) sha256sum -c "$FILE" ;; esac
                    done

                    for EXT in $OUTPUT_EXTS; do
                        TEST_ZIP_VAR="OUTPUT_${EXT^^}_TEST_ZIP"
                        if [ -n "${!TEST_ZIP_VAR}" ]; then
                            echo
                            echo "===== VERIFY .$EXT ARCHIVE ====="
                            for FILE in "${FILES[@]}"; do
                                case "$FILE" in *."$EXT") unzip -t "$FILE" ;; esac
                            done
                        fi
                    done

                    echo
                    echo "===== CREATE INTERNAL SHA-256 MANIFEST ====="

                    sha256sum "${FILES[@]}" > artifact-manifest.sha256

                    test "$(wc -l < artifact-manifest.sha256)" -eq "$EXPECTED_COUNT"
                    cat artifact-manifest.sha256
                '''
            }
        }

        stage('Upload to Garage') {
            steps {
                withCredentials([
                    usernamePassword(
                        credentialsId: TARGET.s3Credentials,
                        usernameVariable: 'AWS_ACCESS_KEY_ID',
                        passwordVariable: 'AWS_SECRET_ACCESS_KEY'
                    )
                ]) {
                    sh '''#!/usr/bin/env bash
                        set -Eeuo pipefail
                        set +x
                        source "$BUILD_CONFIG"

                        export AWS_DEFAULT_REGION="$S3_REGION"
                        export AWS_REGION="$S3_REGION"
                        export AWS_EC2_METADATA_DISABLED=true

                        cd "$RELEASE_DIR"

                        mapfile -t FILES < deliverables.txt
                        FILES+=("artifact-manifest.sha256")

                        test "${#FILES[@]}" -eq $(( $(wc -w <<< "$OUTPUT_EXTS") * 2 + 1 ))

                        echo "===== UPLOAD GARAGE ARTIFACTS ====="
                        echo "Bucket: $S3_BUCKET"
                        echo "Prefix: $STAGING_PREFIX"

                        for FILE in "${FILES[@]}"; do
                            echo "Uploading: $FILE"

                            aws \
                              --endpoint-url "$S3_ENDPOINT_URL" \
                              s3api put-object \
                              --bucket "$S3_BUCKET" \
                              --key "${STAGING_PREFIX}${FILE}" \
                              --body "$FILE" \
                              >/dev/null
                        done

                        echo
                        echo "GARAGE UPLOAD COMPLETE"
                    '''
                }
            }
        }

        stage('Verify Garage Objects') {
            steps {
                withCredentials([
                    usernamePassword(
                        credentialsId: TARGET.s3Credentials,
                        usernameVariable: 'AWS_ACCESS_KEY_ID',
                        passwordVariable: 'AWS_SECRET_ACCESS_KEY'
                    )
                ]) {
                    sh '''#!/usr/bin/env bash
                        set -Eeuo pipefail
                        set +x
                        source "$BUILD_CONFIG"

                        export AWS_DEFAULT_REGION="$S3_REGION"
                        export AWS_REGION="$S3_REGION"
                        export AWS_EC2_METADATA_DISABLED=true

                        cd "$RELEASE_DIR"

                        mapfile -t FILES < deliverables.txt
                        FILES+=("artifact-manifest.sha256")

                        echo "===== VERIFY REMOTE OBJECT SIZES ====="

                        for FILE in "${FILES[@]}"; do
                            LOCAL_SIZE="$(stat -c '%s' "$FILE")"

                            REMOTE_SIZE="$(
                              aws \
                                --endpoint-url "$S3_ENDPOINT_URL" \
                                s3api head-object \
                                --bucket "$S3_BUCKET" \
                                --key "${STAGING_PREFIX}${FILE}" \
                                --query ContentLength \
                                --output text
                            )"

                            printf '%-80s local=%s remote=%s\\n' \
                              "$FILE" \
                              "$LOCAL_SIZE" \
                              "$REMOTE_SIZE"

                            test "$LOCAL_SIZE" = "$REMOTE_SIZE"
                        done

                        REMOTE_MANIFEST="$(mktemp)"
                        trap 'rm -f "$REMOTE_MANIFEST"' EXIT

                        aws \
                          --endpoint-url "$S3_ENDPOINT_URL" \
                          s3api get-object \
                          --bucket "$S3_BUCKET" \
                          --key "${STAGING_PREFIX}artifact-manifest.sha256" \
                          "$REMOTE_MANIFEST" \
                          >/dev/null

                        cmp artifact-manifest.sha256 "$REMOTE_MANIFEST"

                        echo
                        echo "GARAGE VERIFICATION PASSED"
                    '''
                }
            }
        }

        stage('Register Portal Testing Build') {
            steps {
                withCredentials([
                    string(
                        credentialsId: TARGET.portalCredentials,
                        variable: 'PORTAL_TOKEN'
                    )
                ]) {
                    sh '''#!/usr/bin/env bash
                        set -Eeuo pipefail
                        set +x
                        source "$BUILD_CONFIG"

                        TREE_COMMIT="$(cat "$RELEASE_DIR/device-tree-commit.txt")"
                        TREE_SUBJECT="$(cat "$RELEASE_DIR/device-tree-subject.txt")"
                        source "$RELEASE_DIR/device-info.env"

                        PAYLOAD_FILE="$(mktemp)"
                        RESPONSE_FILE="$(mktemp)"

                        cleanup_files() {
                            rm -f "$PAYLOAD_FILE" "$RESPONSE_FILE"
                        }

                        trap cleanup_files EXIT

                        jq -n \
                          --arg project "$PROJECT" \
                          --arg project_name "$PROJECT_NAME" \
                          --arg device "$DEVICE" \
                          --arg device_name "$DEVICE_NAME ($MODEL)" \
                          --arg manufacturer "$MANUFACTURER" \
                          --arg version "$VERSION" \
                          --argjson build_number "$BUILD_NUMBER" \
                          --arg staging_prefix "$STAGING_PREFIX" \
                          --arg source_branch "$MANIFEST_BRANCH" \
                          --arg jenkins_url "$BUILD_URL" \
                          --arg tree_url "$TREE_URL" \
                          --arg tree_commit "$TREE_COMMIT" \
                          --arg tree_subject "$TREE_SUBJECT" \
                          --arg operator_changelog "${CHANGELOG:-$CHANGELOG_DEFAULT}" \
                          --arg portal_request_id "$PORTAL_REQUEST_ID" \
                          '{
                              project: $project,
                              project_name: $project_name,
                              device: $device,
                              device_name: $device_name,
                              manufacturer: $manufacturer,
                              version: $version,
                              build_number: $build_number,
                              staging_prefix: $staging_prefix,
                              source_branch: $source_branch,
                              jenkins_url: $jenkins_url,
                              portal_request_id: $portal_request_id,
                              changelog: (
                                  $operator_changelog
                                  + "\\n\\nDevice tree: "
                                  + $tree_url
                                  + "\\nCommit: "
                                  + $tree_commit
                                  + "\\nCommit subject: "
                                  + $tree_subject
                              )
                          }' \
                          > "$PAYLOAD_FILE"

                        HTTP_CODE="$(
                          curl \
                            -sS \
                            --connect-timeout 15 \
                            --max-time 120 \
                            -o "$RESPONSE_FILE" \
                            -w '%{http_code}' \
                            -H "X-Jenkins-Token: $PORTAL_TOKEN" \
                            -H 'Content-Type: application/json' \
                            --data-binary "@$PAYLOAD_FILE" \
                            "$PORTAL_URL"
                        )"

                        echo "===== PORTAL RESPONSE ====="
                        echo "HTTP=$HTTP_CODE"

                        jq . "$RESPONSE_FILE" \
                          2>/dev/null \
                          || cat "$RESPONSE_FILE"

                        case "$HTTP_CODE" in
                            200|201)
                                ;;
                            *)
                                echo "ERROR: Portal registration failed."
                                exit 1
                                ;;
                        esac

                        if jq -e '.error' "$RESPONSE_FILE" >/dev/null 2>&1; then
                            echo "ERROR: Portal returned an error."
                            exit 1
                        fi

                        if [ -n "$PORTAL_REQUEST_ID" ] && ! jq -e '.started_by' "$RESPONSE_FILE" >/dev/null 2>&1; then
                            echo "WARNING: portal did not attribute this build: $(jq -r '.attribution_warning // empty' "$RESPONSE_FILE")"
                        fi

                        echo
                        echo "PORTAL TESTING REGISTRATION PASSED"
                    '''
                }
            }
        }

        stage('Ccache Results') {
            steps {
                sh '''#!/usr/bin/env bash
                    set -Eeuo pipefail
                    ccache -s
                '''
            }
        }
    }

    post {
        success {
            sh '''#!/usr/bin/env bash
                set -Eeuo pipefail
                source "$BUILD_CONFIG"

                echo "===== SUCCESSFUL BUILD CLEANUP ====="

                rm -rf "$TMPDIR"

                echo
                echo "===== RETURN BASE TO WARM ====="

                zfs rollback "$WARM_SNAPSHOT"

                echo
                echo "Warm base restored: $WARM_SNAPSHOT"
                echo "Local release files retained at:"
                echo "$RELEASE_DIR"
                echo
                echo "Portal status: Testing"
                echo "Garage prefix: $STAGING_PREFIX"
            '''

            echo "${BASE_BRANCH} ${env.DEVICE} build uploaded and registered for testing."
        }

        failure {
            echo "${BASE_BRANCH} ${env.DEVICE} pipeline failed. Source tree left as-is for diagnosis (next run rolls back to @warm)."

            sh '''#!/usr/bin/env bash
                set +e
                source "$BUILD_CONFIG" 2>/dev/null

                echo "===== RETAINED FAILURE PATHS ====="
                echo "Source + out: ${SOURCE_ROOT:-not resolved}"
                echo "Temporary files: $TMPDIR"
                echo "Release staging: ${RELEASE_DIR:-not created}"
                echo "Garage prefix: ${STAGING_PREFIX:-not resolved}"
            '''
        }

        aborted {
            echo "${BASE_BRANCH} ${env.DEVICE} pipeline was aborted. Diagnostic paths were retained."
        }

        always {
            echo "${BASE_BRANCH} ${env.DEVICE} pipeline finished."
        }
    }
}