armbian/scripts
0
0
Fork 0
mirror of https://github.com/armbian/scripts.git synced 2026-01-06 10:32:48 -08:00
Code Issues Packages Projects Releases Wiki Activity
Files
devel2
scripts/.github/workflows/update-cache.yml
Igor Pečovnik c533fd1a05 Update update-cache.yml
2022-09-09 16:32:50 +02:00

493 lines
20 KiB
YAML
Raw Permalink Blame History

name: Build cache
on:
workflow_dispatch:
workflow_call:
inputs:
variant:
required: true
type: string
rootfscache_version:
required: true
type: string
secrets:
PAT1:
required: true
GPG_KEY1:
required: true
GPG_PASSPHRASE1:
required: true
GPG_KEY2:
required: true
GPG_PASSPHRASE2:
required: true
SCRIPTS_ACCESS_TOKEN:
required: true
SSH_KEY_TORRENTS:
required: true
KNOWN_HOSTS_UPLOAD:
required: true
jobs:
Prepare:
name: "Make build list"
runs-on: [Linux]
outputs:
runner: ${{steps.list_dirs.outputs.runner}}
board: ${{steps.list_dirs.outputs.board}}
release: ${{steps.list_dirs.outputs.release}}
variant: ${{steps.list_dirs.outputs.variant}}
desktop: ${{steps.list_dirs.outputs.desktop}}
appgroup: ${{steps.list_dirs.outputs.appgroup}}
rootfscache_version: ${{steps.list_dirs.outputs.rootfscache_version}}
steps:
- name: Runner prepare
uses: armbian/actions/runner-prepare@main
- name: Checkout build repository
uses: actions/checkout@v3
with:
fetch-depth: 10
repository: armbian/build
path: build
clean: false
- name: Prepare matrix
id: list_dirs
run: |
CHUNK=${{ inputs.variant }} # desktop uefi-x86
VAR=$(echo $CHUNK | cut -d":" -f1)
ARC=$(echo $CHUNK | cut -d":" -f2)
echo ::set-output name=board::$(echo $ARC) || true
ROOTFSCACHE_VERSION=${{ inputs.rootfscache_version }}
[[ -z "$ROOTFSCACHE_VERSION" ]] && ROOTFSCACHE_VERSION=$(date --utc +"%Y%m%d")
echo ::set-output name=rootfscache_version::$(echo $ROOTFSCACHE_VERSION) || true
echo "$VAR $ARC"
#echo ::set-output name=release::$(for x in $(grep -rw build/config/distributions/*/ -e 'supported' | cut -d"/" -f4 | sed '/buster\|sid/d'); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true
# list all but remove particular here
echo ::set-output name=release::$(for x in $(find build/config/distributions -mindepth 1 -type d | cut -d"/" -f4 | sed '/buster\|focal/d'); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true
echo ::set-output name=desktop::$(for x in $(echo "none"); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true
echo ::set-output name=appgroup::$(for x in $(echo "none"); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true
if [[ $VAR == desktop* ]]; then
#echo ::set-output name=desktop::$(for x in $(grep -rw build/config/desktop/*/environments/*/support -e 'supported' | cut -d"/" -f6 | sort | uniq); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true
echo ::set-output name=desktop::$(for x in $(find build/config/desktop/*/environments/ -mindepth 1 -maxdepth 1 -type d | cut -d"/" -f6 | sort | uniq | sed '/deepin\|enlightenment\|i3-wm\|lxde\|xmonad/d'); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true
if [[ $ARC == tinkerboard ]]; then
echo ::set-output name=desktop::$(for x in $(find build/config/desktop/*/environments/ -mindepth 1 -maxdepth 1 -type d '!' -exec test -e "{}/only_for" ';' -print | xargs -I{} grep -rw -e 'supported' {} | cut -d"/" -f6 | sort | uniq); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true
fi
echo ::set-output name=appgroup::$(for x in $(seq -w 01 04); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true
fi
# passing minimal, server or desktop
echo ::set-output name=variant::$(for x in $(echo $VAR); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) || true
# Let's choose runners wisely
echo ::set-output name=runner::$(echo 'ubuntu-latest') || true # default
# As we currently don't have many ARM runners and until signing is not done elsewhere, comment them
[[ $VAR != desktop && ($ARC == tinkerboard || $ARC == uefi-arm64) ]] && echo ::set-output name=runner::$(echo 'aarch64') || true
# This later will become obsolete
[[ $VAR == desktop && $ARC != uefi-x86 ]] && echo ::set-output name=runner::$(echo 'small') || true
[[ $VAR == desktop && $ARC == uefi-x86 ]] && echo ::set-output name=runner::$(echo 'X64') || true
image:
needs: Prepare
strategy:
max-parallel: 32
fail-fast: false
matrix:
release: ${{fromJson(needs.Prepare.outputs.release)}}
variant: ${{fromJson(needs.Prepare.outputs.variant)}}
desktop: ${{fromJson(needs.Prepare.outputs.desktop)}}
appgroup: ${{fromJson(needs.Prepare.outputs.appgroup)}}
runs-on: ["${{ needs.Prepare.outputs.runner }}"]
steps:
- name: Runner prepare
uses: armbian/actions/runner-prepare@main
- name: Checkout repository
uses: actions/checkout@v3
with:
repository: armbian/build
fetch-depth: 1
path: build
clean: false
- name: Checkout support scripts
uses: actions/checkout@v3
with:
repository: armbian/scripts
fetch-depth: 1
path: scripts
clean: false
- name: Copy templates
run: |
mkdir -p build/userpatches
sudo cp scripts/configs/* build/userpatches/
- name: Pull Docker image
run: |
[[ -z $(command -v docker) ]] && sudo apt-get -yy install docker containerd docker.io
sudo docker kill $(sudo docker ps -q) 2>/dev/null || true
sudo docker image rm $(sudo docker images | grep -v $(cat build/VERSION | sed 's/trunk.*/trunk/')"-$(dpkg --print-architecture)" | grep armbian | awk 'NR>1 {print $3}') 2> /dev/null || true
sudo docker pull ghcr.io/armbian/build:$(cat build/VERSION | cut -d"." -f1-2)"-$(dpkg --print-architecture)"
- name: Install SSH key for storage
uses: shimataro/ssh-key-action@v2
with:
key: ${{ secrets.SSH_KEY_TORRENTS }}
known_hosts: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
if_key_exists: replace
- name: Mount upload folders
run: |
sudo mountpoint -q build/cache/rootfs && sudo fusermount -u build/cache/rootfs || true
sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
sudo apt-get -y -qq install sshfs
sudo mkdir -p /root/.ssh/
sudo cp ~/.ssh/* /root/.ssh/
sudo mkdir -p build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }} || true
sudo mkdir -p build/cache/rootfs || true
ls -l build/cache/rootfs/
echo "Clean folder"
sudo find build/cache/rootfs/ -maxdepth 1 -mindepth 1 -exec sudo rm -fr -- {} +
if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" || $(curl -s http://ifconfig.me) == "46.19.33.90" ]]; then
echo "Mounting NFS share"
sudo mount nas:/tank/armbian/users.armbian.com/upload/rootfs build/cache/rootfs.upload || true
else
echo "Mounting SSH share"
sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true
sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o nonempty,allow_other || true
fi
# remove true when all runners are on jammy
sudo df
- name: Generate rootfs
run: |
if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then
#CUSTOM_UBUNTU_MIRROR="si.archive.ubuntu.com/ubuntu"
CUSTOM_UBUNTU_MIRROR="us.archive.ubuntu.com/ubuntu"
else
CUSTOM_UBUNTU_MIRROR="us.archive.ubuntu.com/ubuntu"
fi
DESKTOP_APPGROUPS=${{ matrix.appgroup }}
case "$DESKTOP_APPGROUPS" in
"01")
DESKTOP_APPGROUPS_SELECTED="3dsupport"
;;
"02")
DESKTOP_APPGROUPS_SELECTED="3dsupport browsers"
;;
"03")
DESKTOP_APPGROUPS_SELECTED="3dsupport browsers chat desktop_tools editors email internet multimedia office programming remote_desktop"
;;
*)
DESKTOP_APPGROUPS_SELECTED=""
;;
esac
echo $HOME
cd build
BUILD_DESKTOP="no"
BUILD_MINIMAL="no"
DESKTOP_ENVIRONMENT="${{ matrix.desktop }}"
[[ "${{ matrix.desktop }}" == "none" ]] && DESKTOP_ENVIRONMENT=""
[[ -n "$DESKTOP_ENVIRONMENT" ]] && BUILD_DESKTOP="yes"
[[ "${{ matrix.variant }}" == minimal ]] && BUILD_MINIMAL="yes"
#GPG_PASS="${{ secrets.GPG_PASSPHRASE1 }}"
sudo sed -i "s/-it --rm/-i --rm/" userpatches/config-docker.conf
[[ ! -f .ignore_changes ]] && sudo touch .ignore_changes
./compile.sh docker \
BETA="yes" \
ROOT_FS_CREATE_ONLY="yes" \
ROOT_FS_CREATE_VERSION=${{ needs.Prepare.outputs.rootfscache_version }} \
RELEASE="${{ matrix.release }}" \
KERNEL_ONLY="no" \
SKIP_EXTERNAL_TOOLCHAINS="yes" \
IGNORE_UPDATES="yes" \
SYNC_CLOCK="no" \
DESKTOP_ENVIRONMENT_CONFIG_NAME="config_base" \
BUILD_DESKTOP="$BUILD_DESKTOP" \
BUILD_MINIMAL="$BUILD_MINIMAL" \
DESKTOP_ENVIRONMENT="$DESKTOP_ENVIRONMENT" \
KERNEL_CONFIGURE="no" \
BOARD="${{ needs.Prepare.outputs.board }}" \
OFFLINE_WORK="yes" \
BRANCH="current" \
USE_TORRENT="yes" \
CUSTOM_UBUNTU_MIRROR="${CUSTOM_UBUNTU_MIRROR}" \
USE_MAINLINE_GOOGLE_MIRROR="yes" \
REPOSITORY_INSTALL="u-boot,kernel,bsp,armbian-desktop,armbian-config,armbian-firmware" \
DESKTOP_APPGROUPS_SELECTED="$DESKTOP_APPGROUPS_SELECTED" \
EXPERT="yes"
cd ..
sudo chown -R $USER:$USER build/cache/rootfs || true
FOLDER=$(echo ${{ inputs.variant }} | sed "s/://" )
# clean it
rm -rf $FOLDER
echo "UPLOAD=$FOLDER" >> $GITHUB_ENV
# pass value via artefacts
mkdir -p $FOLDER
echo ${{ needs.Prepare.outputs.rootfscache_version }} > $FOLDER/version
touch $FOLDER/$(ls -1 build/cache/rootfs/*.zst | awk -F/ '{print $NF}')
- name: Upload rootfs archive and packages list
uses: ncipollo/release-action@v1
if: ${{ github.repository_owner == 'Armbian' }}
with:
artifacts: "build/cache/rootfs/*.zst,build/cache/rootfs/*.list"
artifactErrorsFailBuild: true
tag: "${{ needs.Prepare.outputs.rootfscache_version }}"
omitBody: true
omitName: true
allowUpdates: true
token: ${{ secrets.PAT1 }}
- name: Upload build list
uses: actions/upload-artifact@v3
with:
path: ${{ env.UPLOAD }}
name: ${{ env.UPLOAD }}
if-no-files-found: ignore
- name: Upload to NAS
if: ${{ github.repository_owner == 'Armbian' }}
run: |
sudo mkdir -p build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }}
[[ ! -d build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }} ]] && sudo chown -R 1008.1002 build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }}
# Do it twice, then you may fail
sudo rsync -ahv --no-o --no-g --no-perms --delete --progress build/cache/rootfs/* build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }} || true
sudo rsync -ahv --no-o --no-g --no-perms --delete --progress build/cache/rootfs/* build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }}
- name: Unmount folders
if: ${{ github.repository_owner == 'Armbian' }}
run: |
sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
sign-list:
needs: image
name: "Signing on powerful servers"
runs-on: [Linux]
outputs:
files: ${{steps.sign-list.outputs.files}}
steps:
- name: Read
run: |
FOLDER=$(echo ${{ inputs.variant }} | sed "s/://" )
echo "UPLOAD=$FOLDER" >> $GITHUB_ENV
rm -rf $FOLDER
- name: Runner prepare
if: ${{ github.repository_owner == 'Armbian' }}
uses: armbian/actions/runner-prepare@main
- name: Download changes
uses: actions/download-artifact@v3
with:
name: ${{ env.UPLOAD }}
path: ${{ env.UPLOAD }}
- name: Checkout repository
if: ${{ github.repository_owner == 'Armbian' }}
uses: actions/checkout@v3
with:
repository: armbian/build
fetch-depth: 1
path: build
clean: false
- name: Unmount folders
if: ${{ github.repository_owner == 'Armbian' }}
run: |
if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" || $(curl -s http://ifconfig.me) == "46.19.33.90" ]]; then
echo "Mounting NFS share"
sudo mount nas:/tank/armbian/users.armbian.com/upload/rootfs build/cache/rootfs.upload || true
else
echo "Mounting SSH share"
sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true
sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o nonempty,allow_other || true
fi
- name: Signining and torrent creation
id: sign-list
run: |
MATRIX=$(ls -1 ${{ env.UPLOAD }}/*.zst | awk -F/ '{print $NF}')
echo ::set-output name=files::$(for x in $(echo "${MATRIX}"); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq)
- name: Unmount folders
if: ${{ github.repository_owner == 'Armbian' }}
run: |
sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
sign:
needs: sign-list
#if: ${{ needs.sign-list.outputs.matrix != '[]' && needs.sign-list.outputs.matrix != '' }}
runs-on: [big,igor]
strategy:
max-parallel: 16
fail-fast: false
matrix:
node: ${{fromJson(needs.sign-list.outputs.files)}}
steps:
- name: Read
run: |
echo "UPLOAD=$(echo ${{ inputs.variant }} | sed "s/://" )" >> $GITHUB_ENV
- name: Runner prepare
uses: armbian/actions/runner-prepare@main
- name: Download changes
uses: actions/download-artifact@v3
with:
name: ${{ env.UPLOAD }}
path: ${{ env.UPLOAD }}
- name: Checkout tracker lists
uses: actions/checkout@v3
with:
fetch-depth: 1
repository: ngosang/trackerslist
path: trackerslist
- name: Checkout repository
if: ${{ github.repository_owner == 'Armbian' }}
uses: actions/checkout@v3
with:
repository: armbian/build
fetch-depth: 1
path: build
clean: false
- name: Import GPG key
uses: crazy-max/ghaction-import-gpg@v5
with:
gpg_private_key: ${{ secrets.GPG_KEY1 }}
passphrase: ${{ secrets.GPG_PASSPHRASE1 }}
workdir: build
git_user_signingkey: true
- name: Mount folders
if: ${{ github.repository_owner == 'Armbian' }}
run: |
sudo mountpoint -q build/cache/rootfs && sudo fusermount -u build/cache/rootfs || true
sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
sudo apt-get -y -qq install sshfs
sudo mkdir -p /root/.ssh/
sudo cp ~/.ssh/* /root/.ssh/
sudo mkdir -p build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }} || true
sudo mkdir -p build/cache/rootfs || true
ls -l build/cache/rootfs/
echo "Clean folder"
sudo find build/cache/rootfs/ -maxdepth 1 -mindepth 1 -exec sudo rm -fr -- {} +
if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" || $(curl -s http://ifconfig.me) == "46.19.33.90" ]]; then
echo "Mounting NFS share"
sudo mount nas:/tank/armbian/users.armbian.com/upload/rootfs build/cache/rootfs.upload
else
echo "Mounting SSH share"
sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true
sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o nonempty,allow_other || true
fi
- name: Signining and torrent creation
if: ${{ github.repository_owner == 'Armbian' }}
run: |
# exit if keys were not loaded
[[ -z $(gpg -K) ]] && exit 1
sudo apt-get -y -qq install jq buildtorrent
UPLOAD=${{ env.UPLOAD }}
ROOTFSCACHE_VERSION=$(cat $UPLOAD/version)
# cleaning
rm -rf $UPLOAD
echo "ROOTFSCACHE_VERSION=$ROOTFSCACHE_VERSION" >> $GITHUB_ENV
FILE=${{ matrix.node }}
ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | sed '/^$/d' | shuf -n 1)
TRACKERS=$(cat trackerslist/trackers_best_ip.txt | sed '/^\s*$/d' | while read line; do printf ",""${line}"; done | cut -c 2-)
WEBSEEDS="--webseeds="https://github.com/armbian/cache/releases/download/$ROOTFSCACHE_VERSION/$FILE,"$(curl -s https://cache.armbian.com/mirrors | jq -r '.'default' | .[] | values' | sed -e 's/$/rootfs\/'$ROOTFSCACHE_VERSION'\/'$FILE'/' | tr '\n' , | sed 's/.$//')"
cd build/cache/rootfs.upload/$ROOTFSCACHE_VERSION
echo "Generate torrent"
sudo --preserve-env buildtorrent $WEBSEEDS --announce=$ANNOUNCE --announcelist=$TRACKERS $FILE -c "Armbian rootfs cache" ${FILE}.torrent
echo "Sign"
FOLDER=$(mktemp -d)
echo ${{ secrets.GPG_PASSPHRASE1 }} | gpg --quiet --armor --batch --yes --passphrase-fd 0 --detach-sign --pinentry-mode loopback --output ${FOLDER}/${FILE}.asc ${FILE}
sudo mv ${FOLDER}/${FILE}.asc .
echo "Verify"
zstd -t ${FILE}
if [[ "$(cat ${FILE}.asc | gpg --list-packets --no-keyring | grep -oE '[0-9A-F]{40}')" == "DF00FAF1C577104B50BF1D0093D6889F9F0E78D5" ]]; then
echo "Signature is O.K."
else
exit 1
fi
- name: Upload signature and torrent
uses: ncipollo/release-action@v1
if: ${{ github.repository_owner == 'Armbian' }}
with:
artifacts: "build/cache/rootfs.upload/${{ env.ROOTFSCACHE_VERSION }}/${{ matrix.node }}.asc,build/cache/rootfs.upload/${{ env.ROOTFSCACHE_VERSION }}/${{ matrix.node }}.torrent"
artifactErrorsFailBuild: true
tag: "${{ env.ROOTFSCACHE_VERSION }}"
omitBody: true
omitName: true
allowUpdates: true
token: ${{ secrets.PAT1 }}
- name: Unmount folders
if: ${{ github.repository_owner == 'Armbian' }}
run: |
sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
- name: Runner cleanup
uses: armbian/actions/runner-prepare@main
Reference in New Issue View Git Blame Copy Permalink