name: Build cache on: workflow_dispatch: workflow_call: inputs: variant: required: true type: string rootfscache_version: required: true type: string secrets: PAT1: required: true GPG_KEY1: required: true GPG_PASSPHRASE1: required: true GPG_KEY2: required: true GPG_PASSPHRASE2: required: true SCRIPTS_ACCESS_TOKEN: required: true SSH_KEY_TORRENTS: required: true KNOWN_HOSTS_UPLOAD: required: true jobs: Prepare: name: "Make build list" runs-on: [Linux] outputs: runner: ${{steps.list_dirs.outputs.runner}} board: ${{steps.list_dirs.outputs.board}} release: ${{steps.list_dirs.outputs.release}} variant: ${{steps.list_dirs.outputs.variant}} desktop: ${{steps.list_dirs.outputs.desktop}} appgroup: ${{steps.list_dirs.outputs.appgroup}} rootfscache_version: ${{steps.list_dirs.outputs.rootfscache_version}} steps: - name: Runner prepare uses: armbian/actions/runner-prepare@main - name: Checkout build repository uses: actions/checkout@v3.1.0 with: fetch-depth: 10 repository: armbian/build path: build clean: false - name: Prepare matrix id: list_dirs run: | CHUNK=${{ inputs.variant }} # desktop uefi-x86 VAR=$(echo $CHUNK | cut -d":" -f1) ARC=$(echo $CHUNK | cut -d":" -f2) #echo ::set-output name=board::$(echo $ARC) || true echo "board=$(echo $ARC)" >> $GITHUB_OUTPUT ROOTFSCACHE_VERSION=${{ inputs.rootfscache_version }} [[ -z "$ROOTFSCACHE_VERSION" ]] && ROOTFSCACHE_VERSION=$(date --utc +"%Y%m%d") #echo ::set-output name=rootfscache_version::$(echo $rootfscache_version) echo "rootfscache_version=$ROOTFSCACHE_VERSION" >> $GITHUB_OUTPUT # list all but remove particular here #echo ::set-output name=release::$(for x in $(find build/config/distributions -mindepth 1 -type d | cut -d"/" -f4 | sed '/buster\|focal/d'); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true echo "release=$(for x in $(find build/config/distributions -mindepth 1 -type d | cut -d"/" -f4 | sed '/buster\|focal/d'); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT if [[ $ARC == uefi-riscv64 ]]; then #echo ::set-output name=release::$(for value in jammy kinetic; do echo $value; done | jq -cnR '[inputs | select(length>0)]' | jq) || true echo "release=$(for value in jammy kinetic; do echo $value; done | jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT fi #echo ::set-output name=desktop::$(for x in $(echo "none"); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true echo "desktop=$(for x in $(echo "none"); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT #echo ::set-output name=appgroup::$(for x in $(echo "none"); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true echo "appgroup=$(for x in $(echo "none"); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT if [[ $VAR == desktop* ]]; then #echo ::set-output name=desktop::$(for x in $(find build/config/desktop/*/environments/ -mindepth 1 -maxdepth 1 -type d | grep -v "_" | cut -d"/" -f6 | sort | uniq | sed '/deepin\|enlightenment\|i3-wm\|lxde\|xmonad\|budgie\|kde-plasma\|mate/d'); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true echo "desktop=$(for x in $(find build/config/desktop/*/environments/ -mindepth 1 -maxdepth 1 -type d | grep -v "_" | cut -d"/" -f6 | sort | uniq | sed '/deepin\|enlightenment\|i3-wm\|lxde\|xmonad\|budgie\|kde-plasma\|mate/d'); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT if [[ $ARC == tinkerboard ]]; then #echo ::set-output name=desktop::$(for x in $(find build/config/desktop/*/environments/ -mindepth 1 -maxdepth 1 -type d '!' -exec test -e "{}/only_for" ';' -print | xargs -I{} grep -rw -e 'supported' {} | cut -d"/" -f6 | sort | uniq); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true echo "desktop=$(for x in $(find build/config/desktop/*/environments/ -mindepth 1 -maxdepth 1 -type d '!' -exec test -e "{}/only_for" ';' -print | xargs -I{} grep -rw -e 'supported' {} | cut -d"/" -f6 | sort | uniq); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT fi #echo ::set-output name=appgroup::$(for x in $(seq -w 01 04); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true echo "appgroup=$(for x in $(seq -w 01 05); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT if [[ $ARC == uefi-riscv64 ]]; then #echo ::set-output name=desktop::$(echo "xfce" |jq -cnR '[inputs | select(length>0)]' | jq) || true echo "desktop=$(echo "xfce" |jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT #echo ::set-output name=appgroup::$(echo "05" |jq -cnR '[inputs | select(length>0)]' | jq) || true echo "appgroup=$(echo "04" |jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT fi fi # passing minimal, server or desktop #echo ::set-output name=variant::$(for x in $(echo $VAR); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true echo "variant=$(for x in $(echo $VAR); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT # Let's choose runners wisely #echo ::set-output name=runner::$(echo 'ubuntu-latest') || true # default echo "runner=ubuntu-latest" >> $GITHUB_OUTPUT || true # As we currently don't have many ARM runners and until signing is not done elsewhere, comment them #[[ $VAR != desktop && ($ARC == tinkerboard || $ARC == uefi-arm64) ]] && echo ::set-output name=runner::$(echo 'aarch64') || true [[ $VAR != desktop && ($ARC == tinkerboard || $ARC == uefi-arm64) ]] && echo "runner=aarch64" >> $GITHUB_OUTPUT || true #[[ $ARC == uefi-riscv64 ]] && echo ::set-output name=runner::$(echo 'small') || true [[ $ARC == uefi-riscv64 ]] && echo "runner=small" >> $GITHUB_OUTPUT || true # This later will become obsolete #[[ $VAR == desktop && $ARC != uefi-x86 ]] && echo ::set-output name=runner::$(echo 'small') || true [[ $VAR == desktop && $ARC != uefi-x86 ]] && echo "runner=small" >> $GITHUB_OUTPUT || true #[[ $VAR == desktop && $ARC == uefi-x86 ]] && echo ::set-output name=runner::$(echo 'X64') || true [[ $VAR == desktop && $ARC == uefi-x86 ]] && echo "runner=X64" >> $GITHUB_OUTPUT || true image: needs: Prepare strategy: max-parallel: 32 fail-fast: false matrix: release: ${{fromJson(needs.Prepare.outputs.release)}} variant: ${{fromJson(needs.Prepare.outputs.variant)}} desktop: ${{fromJson(needs.Prepare.outputs.desktop)}} appgroup: ${{fromJson(needs.Prepare.outputs.appgroup)}} runs-on: ["${{ needs.Prepare.outputs.runner }}"] steps: - name: Runner prepare uses: armbian/actions/runner-prepare@main - name: Checkout repository uses: actions/checkout@v3.1.0 with: repository: armbian/build fetch-depth: 1 path: build clean: false - name: Checkout support scripts uses: actions/checkout@v3.1.0 with: repository: armbian/scripts fetch-depth: 1 path: scripts clean: false - name: Copy templates run: | mkdir -p build/userpatches sudo cp scripts/configs/* build/userpatches/ - name: Pull Docker image run: | [[ -z $(command -v docker) ]] && sudo apt-get -yy install docker containerd docker.io sudo docker kill $(sudo docker ps -q) 2>/dev/null || true sudo docker image rm $(sudo docker images | grep -v latest"-$(dpkg --print-architecture)" | grep armbian | awk 'NR>1 {print $3}') 2> /dev/null || true sudo docker pull ghcr.io/armbian/build:latest"-$(dpkg --print-architecture)" - name: Install SSH key for storage uses: shimataro/ssh-key-action@v2 with: key: ${{ secrets.SSH_KEY_TORRENTS }} known_hosts: ${{ secrets.KNOWN_HOSTS_UPLOAD }} if_key_exists: replace - name: Mount upload folders run: | sudo mountpoint -q build/cache/rootfs && sudo fusermount -u build/cache/rootfs || true sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true sudo apt-get -y -qq install sshfs sudo mkdir -p /root/.ssh/ sudo cp ~/.ssh/* /root/.ssh/ sudo mkdir -p build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }} || true sudo mkdir -p build/cache/rootfs || true ls -l build/cache/rootfs/ echo "Clean folder" sudo find build/cache/rootfs/ -maxdepth 1 -mindepth 1 -exec sudo rm -fr -- {} + if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" || $(curl -s http://ifconfig.me) == "46.19.33.90" ]]; then echo "Mounting NFS share" sudo mount nas:/tank/armbian/users.armbian.com/upload/rootfs build/cache/rootfs.upload || true else echo "Mounting SSH share" sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o nonempty,allow_other || true fi # remove true when all runners are on jammy sudo df - name: Generate rootfs run: | if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then #CUSTOM_UBUNTU_MIRROR="si.archive.ubuntu.com/ubuntu" CUSTOM_UBUNTU_MIRROR="us.archive.ubuntu.com/ubuntu" else CUSTOM_UBUNTU_MIRROR="us.archive.ubuntu.com/ubuntu" fi DESKTOP_APPGROUPS=${{ matrix.appgroup }} case "$DESKTOP_APPGROUPS" in "01") DESKTOP_APPGROUPS_SELECTED="3dsupport" ;; "02") DESKTOP_APPGROUPS_SELECTED="3dsupport browsers" ;; "03") DESKTOP_APPGROUPS_SELECTED="3dsupport browsers chat desktop_tools editors email internet multimedia office programming remote_desktop" ;; "04") DESKTOP_APPGROUPS_SELECTED="browsers chat desktop_tools editors email internet multimedia office programming remote_desktop" ;; *) DESKTOP_APPGROUPS_SELECTED="" ;; esac echo $HOME cd build BUILD_DESKTOP="no" BUILD_MINIMAL="no" DESKTOP_ENVIRONMENT="${{ matrix.desktop }}" [[ "${{ matrix.desktop }}" == "none" ]] && DESKTOP_ENVIRONMENT="" [[ -n "$DESKTOP_ENVIRONMENT" ]] && BUILD_DESKTOP="yes" [[ "${{ matrix.variant }}" == minimal ]] && BUILD_MINIMAL="yes" #GPG_PASS="${{ secrets.GPG_PASSPHRASE1 }}" sudo sed -i "s/-it --rm/-i --rm/" userpatches/config-docker.conf [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes ./compile.sh docker \ BETA="yes" \ ROOT_FS_CREATE_ONLY="yes" \ ROOT_FS_CREATE_VERSION=${{ needs.Prepare.outputs.rootfscache_version }} \ RELEASE="${{ matrix.release }}" \ KERNEL_ONLY="no" \ SKIP_EXTERNAL_TOOLCHAINS="yes" \ IGNORE_UPDATES="yes" \ SYNC_CLOCK="no" \ DESKTOP_ENVIRONMENT_CONFIG_NAME="config_base" \ BUILD_DESKTOP="$BUILD_DESKTOP" \ BUILD_MINIMAL="$BUILD_MINIMAL" \ DESKTOP_ENVIRONMENT="$DESKTOP_ENVIRONMENT" \ KERNEL_CONFIGURE="no" \ BOARD="${{ needs.Prepare.outputs.board }}" \ OFFLINE_WORK="yes" \ BRANCH="current" \ USE_TORRENT="yes" \ CUSTOM_UBUNTU_MIRROR="${CUSTOM_UBUNTU_MIRROR}" \ USE_MAINLINE_GOOGLE_MIRROR="yes" \ REPOSITORY_INSTALL="u-boot,kernel,bsp,armbian-desktop,armbian-config,armbian-firmware" \ DESKTOP_APPGROUPS_SELECTED="$DESKTOP_APPGROUPS_SELECTED" \ EXPERT="yes" cd .. sudo chown -R $USER:$USER build/cache/rootfs || true FOLDER=$(echo ${{ inputs.variant }} | sed "s/://" ) # clean it rm -rf $FOLDER echo "UPLOAD=$FOLDER" >> $GITHUB_ENV # pass value via artefacts mkdir -p $FOLDER echo ${{ needs.Prepare.outputs.rootfscache_version }} > $FOLDER/version touch $FOLDER/$(ls -1 build/cache/rootfs/*.zst | awk -F/ '{print $NF}') - name: Upload rootfs archive and packages list uses: ncipollo/release-action@v1 if: ${{ github.repository_owner == 'Armbian' }} with: artifacts: "build/cache/rootfs/*.zst,build/cache/rootfs/*.list" artifactErrorsFailBuild: true tag: "${{ needs.Prepare.outputs.rootfscache_version }}" omitBody: true omitName: true allowUpdates: true token: ${{ secrets.PAT1 }} - name: Upload build list uses: actions/upload-artifact@v3 with: path: ${{ env.UPLOAD }} name: ${{ env.UPLOAD }} if-no-files-found: ignore - name: Upload to NAS if: ${{ github.repository_owner == 'Armbian' }} run: | sudo mkdir -p build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }} [[ ! -d build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }} ]] && sudo chown -R 1008.1002 build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }} # Do it twice, then you may fail sudo rsync -ahv --no-o --no-g --no-perms --delete --progress build/cache/rootfs/* build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }} || true sudo rsync -ahv --no-o --no-g --no-perms --delete --progress build/cache/rootfs/* build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }} - name: Unmount folders if: ${{ github.repository_owner == 'Armbian' }} run: | sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true sign-list: needs: image name: "Signing on powerful servers" runs-on: [Linux] outputs: files: ${{steps.sign-list.outputs.files}} steps: - name: Read run: | FOLDER=$(echo ${{ inputs.variant }} | sed "s/://" ) echo "UPLOAD=$FOLDER" >> $GITHUB_ENV rm -rf $FOLDER - name: Runner prepare if: ${{ github.repository_owner == 'Armbian' }} uses: armbian/actions/runner-prepare@main - name: Download changes uses: actions/download-artifact@v3 with: name: ${{ env.UPLOAD }} path: ${{ env.UPLOAD }} - name: Checkout repository if: ${{ github.repository_owner == 'Armbian' }} uses: actions/checkout@v3.1.0 with: repository: armbian/build fetch-depth: 1 path: build clean: false - name: Unmount folders if: ${{ github.repository_owner == 'Armbian' }} run: | if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" || $(curl -s http://ifconfig.me) == "46.19.33.90" ]]; then echo "Mounting NFS share" sudo mount nas:/tank/armbian/users.armbian.com/upload/rootfs build/cache/rootfs.upload || true else echo "Mounting SSH share" sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o nonempty,allow_other || true fi - name: Signining and torrent creation id: sign-list run: | MATRIX=$(ls -1 ${{ env.UPLOAD }}/*.zst | awk -F/ '{print $NF}') #echo ::set-output name=files::$(for x in $(echo "${MATRIX}"); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) echo "files=$(for x in $(echo "${MATRIX}"); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT - name: Unmount folders if: ${{ github.repository_owner == 'Armbian' }} run: | sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true sign: needs: sign-list #if: ${{ needs.sign-list.outputs.matrix != '[]' && needs.sign-list.outputs.matrix != '' }} runs-on: [big,igor] strategy: max-parallel: 16 fail-fast: false matrix: node: ${{fromJson(needs.sign-list.outputs.files)}} steps: - name: Read run: | echo "UPLOAD=$(echo ${{ inputs.variant }} | sed "s/://" )" >> $GITHUB_ENV - name: Runner prepare uses: armbian/actions/runner-prepare@main - name: Download changes uses: actions/download-artifact@v3 with: name: ${{ env.UPLOAD }} path: ${{ env.UPLOAD }} - name: Checkout tracker lists uses: actions/checkout@v3.1.0 with: fetch-depth: 1 repository: ngosang/trackerslist path: trackerslist - name: Checkout repository if: ${{ github.repository_owner == 'Armbian' }} uses: actions/checkout@v3.1.0 with: repository: armbian/build fetch-depth: 1 path: build clean: false - name: Import GPG key uses: crazy-max/ghaction-import-gpg@v5 with: gpg_private_key: ${{ secrets.GPG_KEY1 }} passphrase: ${{ secrets.GPG_PASSPHRASE1 }} workdir: build git_user_signingkey: true - name: Mount folders if: ${{ github.repository_owner == 'Armbian' }} run: | sudo mountpoint -q build/cache/rootfs && sudo fusermount -u build/cache/rootfs || true sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true sudo apt-get -y -qq install sshfs sudo mkdir -p /root/.ssh/ sudo cp ~/.ssh/* /root/.ssh/ sudo mkdir -p build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }} || true sudo mkdir -p build/cache/rootfs || true ls -l build/cache/rootfs/ echo "Clean folder" sudo find build/cache/rootfs/ -maxdepth 1 -mindepth 1 -exec sudo rm -fr -- {} + if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" || $(curl -s http://ifconfig.me) == "46.19.33.90" ]]; then echo "Mounting NFS share" sudo mount nas:/tank/armbian/users.armbian.com/upload/rootfs build/cache/rootfs.upload else echo "Mounting SSH share" sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o nonempty,allow_other || true fi - name: Signining and torrent creation if: ${{ github.repository_owner == 'Armbian' }} run: | # exit if keys were not loaded [[ -z $(gpg -K) ]] && exit 1 sudo apt-get -y -qq install jq buildtorrent UPLOAD=${{ env.UPLOAD }} ROOTFSCACHE_VERSION=$(cat $UPLOAD/version) # cleaning rm -rf $UPLOAD echo "ROOTFSCACHE_VERSION=$ROOTFSCACHE_VERSION" >> $GITHUB_ENV FILE=${{ matrix.node }} ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | sed '/^$/d' | shuf -n 1) TRACKERS=$(cat trackerslist/trackers_best_ip.txt | sed '/^\s*$/d' | while read line; do printf ",""${line}"; done | cut -c 2-) WEBSEEDS="--webseeds="https://github.com/armbian/cache/releases/download/$ROOTFSCACHE_VERSION/$FILE,"$(curl -s https://cache.armbian.com/mirrors | jq -r '.'default' | .[] | values' | sed -e 's/$/rootfs\/'$ROOTFSCACHE_VERSION'\/'$FILE'/' | tr '\n' , | sed 's/.$//')" cd build/cache/rootfs.upload/$ROOTFSCACHE_VERSION echo "Generate torrent" sudo --preserve-env buildtorrent $WEBSEEDS --announce=$ANNOUNCE --announcelist=$TRACKERS $FILE -c "Armbian rootfs cache" ${FILE}.torrent echo "Sign" FOLDER=$(mktemp -d) echo ${{ secrets.GPG_PASSPHRASE1 }} | gpg --quiet --armor --batch --yes --passphrase-fd 0 --detach-sign --pinentry-mode loopback --output ${FOLDER}/${FILE}.asc ${FILE} sudo mv ${FOLDER}/${FILE}.asc . echo "Verify" zstd -t ${FILE} if [[ "$(cat ${FILE}.asc | gpg --list-packets --no-keyring | grep -oE '[0-9A-F]{40}')" == "DF00FAF1C577104B50BF1D0093D6889F9F0E78D5" ]]; then echo "Signature is O.K." else exit 1 fi - name: Upload signature and torrent uses: ncipollo/release-action@v1 if: ${{ github.repository_owner == 'Armbian' }} with: artifacts: "build/cache/rootfs.upload/${{ env.ROOTFSCACHE_VERSION }}/${{ matrix.node }}.asc,build/cache/rootfs.upload/${{ env.ROOTFSCACHE_VERSION }}/${{ matrix.node }}.torrent" artifactErrorsFailBuild: true tag: "${{ env.ROOTFSCACHE_VERSION }}" omitBody: true omitName: true allowUpdates: true token: ${{ secrets.PAT1 }} - name: Unmount folders if: ${{ github.repository_owner == 'Armbian' }} run: | sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true - name: Runner cleanup uses: armbian/actions/runner-prepare@main