mirror of
https://github.com/armbian/scripts.git
synced 2026-01-06 10:32:48 -08:00
455 lines
18 KiB
YAML
455 lines
18 KiB
YAML
name: Build cache
|
|
on:
|
|
workflow_dispatch:
|
|
workflow_call:
|
|
inputs:
|
|
variant:
|
|
required: true
|
|
type: string
|
|
rootfscache_version:
|
|
required: true
|
|
type: string
|
|
runner:
|
|
required: true
|
|
type: string
|
|
secrets:
|
|
PAT1:
|
|
required: true
|
|
GPG_KEY1:
|
|
required: true
|
|
GPG_PASSPHRASE1:
|
|
required: true
|
|
GPG_KEY2:
|
|
required: true
|
|
GPG_PASSPHRASE2:
|
|
required: true
|
|
SCRIPTS_ACCESS_TOKEN:
|
|
required: true
|
|
SSH_KEY_TORRENTS:
|
|
required: true
|
|
KNOWN_HOSTS_UPLOAD:
|
|
required: true
|
|
|
|
jobs:
|
|
|
|
Prepare:
|
|
|
|
name: "Make build list for ${{ inputs.rootfscache_version }}"
|
|
runs-on: [Linux]
|
|
outputs:
|
|
|
|
runner: ${{steps.list_dirs.outputs.runner}}
|
|
board: ${{steps.list_dirs.outputs.board}}
|
|
release: ${{steps.list_dirs.outputs.release}}
|
|
variant: ${{steps.list_dirs.outputs.variant}}
|
|
desktop: ${{steps.list_dirs.outputs.desktop}}
|
|
appgroup: ${{steps.list_dirs.outputs.appgroup}}
|
|
rootfscache_version: ${{steps.list_dirs.outputs.rootfscache_version}}
|
|
|
|
steps:
|
|
|
|
- name: Runner prepare
|
|
uses: armbian/actions/runner-prepare@main
|
|
|
|
- name: Checkout build repository
|
|
uses: actions/checkout@v3.3.0
|
|
with:
|
|
fetch-depth: 10
|
|
repository: armbian/build
|
|
path: build
|
|
ref: armbian-next
|
|
clean: false
|
|
|
|
- name: Prepare matrix
|
|
id: list_dirs
|
|
run: |
|
|
|
|
CHUNK=${{ inputs.variant }}
|
|
VAR=$(echo $CHUNK | cut -d":" -f1)
|
|
ARC=$(echo $CHUNK | cut -d":" -f2)
|
|
|
|
echo "board=$(echo $ARC)" >> $GITHUB_OUTPUT
|
|
|
|
ROOTFSCACHE_VERSION=${{ inputs.rootfscache_version }}
|
|
[[ -z "$ROOTFSCACHE_VERSION" ]] && ROOTFSCACHE_VERSION=$(date --utc +"%Y%m%d")
|
|
echo "rootfscache_version=$ROOTFSCACHE_VERSION" >> $GITHUB_OUTPUT
|
|
|
|
# list all but remove particular here
|
|
echo "release=$(for x in $(find build/config/distributions -mindepth 1 -type d | cut -d"/" -f4 | sed '/buster\|focal/d'); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT
|
|
|
|
if [[ $ARC == uefi-riscv64 ]]; then
|
|
echo "release=$(for value in jammy kinetic; do echo $value; done | jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
echo "desktop=$(for x in $(echo "none"); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT
|
|
echo "appgroup=$(for x in $(echo "none"); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT
|
|
|
|
if [[ $VAR == desktop* ]]; then
|
|
echo "desktop=$(for x in $(find build/config/desktop/*/environments/ -mindepth 1 -maxdepth 1 -type d | grep -v "_" | cut -d"/" -f6 | sort | uniq | sed '/deepin\|enlightenment\|i3-wm\|lxde\|xmonad\|budgie\|kde-plasma\|mate/d'); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT
|
|
if [[ $ARC == tinkerboard ]]; then
|
|
echo "desktop=$(for x in $(find build/config/desktop/*/environments/ -mindepth 1 -maxdepth 1 -type d '!' -exec test -e "{}/only_for" ';' -print | xargs -I{} grep -rw -e 'supported' {} | cut -d"/" -f6 | sort | uniq); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT
|
|
fi
|
|
echo "appgroup=$(for x in $(seq -w 01 05); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT
|
|
if [[ $ARC == uefi-riscv64 ]]; then
|
|
echo "desktop=$(echo "xfce" |jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT
|
|
echo "appgroup=$(echo "05" |jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT
|
|
fi
|
|
fi
|
|
|
|
# passing minimal, server or desktop
|
|
echo "variant=$(for x in $(echo $VAR); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT
|
|
|
|
# Let's choose runners wisely
|
|
echo "runner=ubuntu-latest" >> $GITHUB_OUTPUT || true
|
|
|
|
# As we currently don't have many ARM runners and until signing is not done elsewhere, comment them
|
|
[[ $VAR != desktop && ($ARC == tinkerboard || $ARC == uefi-arm64) ]] && echo "runner=aarch64" >> $GITHUB_OUTPUT || true
|
|
[[ $ARC == uefi-riscv64 ]] && echo "runner=small" >> $GITHUB_OUTPUT || true
|
|
|
|
# This later will become obsolete
|
|
[[ $VAR == desktop && $ARC != uefi-x86 ]] && echo "runner=small" >> $GITHUB_OUTPUT || true
|
|
[[ $VAR == desktop && $ARC == uefi-x86 ]] && echo "runner=X64" >> $GITHUB_OUTPUT || true
|
|
|
|
echo "runner=$(for value in $(echo ${{ inputs.runner }} | sed "s/,/ /g"); do echo $value; done | jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT || true
|
|
|
|
image:
|
|
|
|
needs: Prepare
|
|
strategy:
|
|
max-parallel: 64
|
|
fail-fast: false
|
|
matrix:
|
|
|
|
release: ${{fromJson(needs.Prepare.outputs.release)}}
|
|
variant: ${{fromJson(needs.Prepare.outputs.variant)}}
|
|
desktop: ${{fromJson(needs.Prepare.outputs.desktop)}}
|
|
appgroup: ${{fromJson(needs.Prepare.outputs.appgroup)}}
|
|
|
|
#runs-on: ["${{ needs.Prepare.outputs.runner }}"]
|
|
runs-on: ${{ fromJSON(needs.Prepare.outputs.runner)}}
|
|
steps:
|
|
|
|
- name: Runner prepare
|
|
uses: armbian/actions/runner-prepare@main
|
|
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v3.3.0
|
|
with:
|
|
repository: armbian/build
|
|
fetch-depth: 1
|
|
ref: armbian-next
|
|
path: build
|
|
clean: true
|
|
|
|
- name: Clean rootfs cache folder
|
|
run: |
|
|
|
|
sudo mountpoint -q build/cache/rootfs && sudo fusermount -u build/cache/rootfs || true
|
|
echo "Clean folder"
|
|
sudo find build/cache/rootfs/ -maxdepth 1 -mindepth 1 -exec sudo rm -fr -- {} + || true
|
|
|
|
- name: Generate rootfs
|
|
run: |
|
|
|
|
DESKTOP_APPGROUPS=${{ matrix.appgroup }}
|
|
case "$DESKTOP_APPGROUPS" in
|
|
"01")
|
|
DESKTOP_APPGROUPS_SELECTED="3dsupport"
|
|
;;
|
|
|
|
"02")
|
|
DESKTOP_APPGROUPS_SELECTED="3dsupport browsers"
|
|
;;
|
|
"03")
|
|
DESKTOP_APPGROUPS_SELECTED="3dsupport browsers chat desktop_tools editors email internet multimedia office programming remote_desktop"
|
|
;;
|
|
"04")
|
|
DESKTOP_APPGROUPS_SELECTED="browsers chat desktop_tools editors email internet multimedia office programming remote_desktop"
|
|
;;
|
|
*)
|
|
DESKTOP_APPGROUPS_SELECTED=""
|
|
;;
|
|
esac
|
|
|
|
cd build
|
|
|
|
BUILD_DESKTOP="no"
|
|
BUILD_MINIMAL="no"
|
|
DESKTOP_ENVIRONMENT="${{ matrix.desktop }}"
|
|
[[ "${{ matrix.desktop }}" == "none" ]] && DESKTOP_ENVIRONMENT=""
|
|
[[ -n "$DESKTOP_ENVIRONMENT" ]] && BUILD_DESKTOP="yes"
|
|
[[ "${{ matrix.variant }}" == minimal ]] && BUILD_MINIMAL="yes"
|
|
|
|
./compile.sh rootfs \
|
|
BETA="yes" \
|
|
ROOT_FS_CREATE_VERSION="${{ needs.Prepare.outputs.rootfscache_version }}" \
|
|
RELEASE="${{ matrix.release }}" \
|
|
DESKTOP_ENVIRONMENT_CONFIG_NAME="config_base" \
|
|
BUILD_DESKTOP="$BUILD_DESKTOP" \
|
|
BUILD_MINIMAL="$BUILD_MINIMAL" \
|
|
DESKTOP_ENVIRONMENT="$DESKTOP_ENVIRONMENT" \
|
|
ROOTFS_COMPRESSION_RATIO="19" \
|
|
BOARD="${{ needs.Prepare.outputs.board }}" \
|
|
DESKTOP_APPGROUPS_SELECTED="$DESKTOP_APPGROUPS_SELECTED"
|
|
|
|
cd ..
|
|
sudo chown -R $USER:$USER build/cache/rootfs || true
|
|
FOLDER=$(echo ${{ inputs.variant }} | sed "s/://" )
|
|
# clean it
|
|
rm -rf $FOLDER
|
|
echo "UPLOAD=$FOLDER" >> $GITHUB_ENV
|
|
|
|
# pass value via artefacts
|
|
mkdir -p $FOLDER
|
|
echo ${{ needs.Prepare.outputs.rootfscache_version }} > $FOLDER/version
|
|
touch $FOLDER/$(ls -1 build/cache/rootfs/*.zst | awk -F/ '{print $NF}')
|
|
|
|
- name: Install SSH key for storage
|
|
uses: shimataro/ssh-key-action@v2
|
|
with:
|
|
key: ${{ secrets.SSH_KEY_TORRENTS }}
|
|
known_hosts: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
|
|
if_key_exists: ignore
|
|
|
|
- name: Upload rootfs archive and packages list
|
|
uses: ncipollo/release-action@v1
|
|
if: ${{ github.repository_owner == 'Armbian' }}
|
|
with:
|
|
artifacts: "build/cache/rootfs/*.zst,build/cache/rootfs/*.list"
|
|
artifactErrorsFailBuild: true
|
|
tag: "${{ needs.Prepare.outputs.rootfscache_version }}"
|
|
omitBody: true
|
|
omitName: true
|
|
allowUpdates: true
|
|
token: ${{ secrets.PAT1 }}
|
|
|
|
- name: Upload build list
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
path: ${{ env.UPLOAD }}
|
|
name: ${{ env.UPLOAD }}
|
|
if-no-files-found: ignore
|
|
|
|
- name: Deploy to server
|
|
if: ${{ github.repository_owner == 'Armbian' }}
|
|
run: |
|
|
#sudo apt-get -y -qq install lftp
|
|
pwd
|
|
lftp -u upload, -e "set net:timeout 4;set net:max-retries 6;mirror --Remove-source-files -R --no-empty-dirs --parallel=8 --no-perms $(pwd)/build/cache/rootfs/ rootfs/${{ needs.Prepare.outputs.rootfscache_version }} ;bye" sftp://users.armbian.com
|
|
|
|
sign-list:
|
|
|
|
needs: image
|
|
name: "Signing on powerful servers"
|
|
runs-on: [Linux]
|
|
outputs:
|
|
|
|
files: ${{steps.sign-list.outputs.files}}
|
|
|
|
steps:
|
|
|
|
- name: Read
|
|
run: |
|
|
|
|
FOLDER=$(echo ${{ inputs.variant }} | sed "s/://" )
|
|
echo "UPLOAD=$FOLDER" >> $GITHUB_ENV
|
|
rm -rf $FOLDER
|
|
|
|
- name: Runner prepare
|
|
if: ${{ github.repository_owner == 'Armbian' }}
|
|
uses: armbian/actions/runner-prepare@main
|
|
|
|
- name: Download changes
|
|
uses: actions/download-artifact@v3.0.2
|
|
with:
|
|
name: ${{ env.UPLOAD }}
|
|
path: ${{ env.UPLOAD }}
|
|
|
|
- name: Checkout repository
|
|
if: ${{ github.repository_owner == 'Armbian' }}
|
|
uses: actions/checkout@v3.3.0
|
|
with:
|
|
repository: armbian/build
|
|
fetch-depth: 1
|
|
path: build
|
|
clean: false
|
|
|
|
- name: Unmount folders
|
|
if: ${{ github.repository_owner == 'Armbian' }}
|
|
run: |
|
|
|
|
echo "Mounting SSH share"
|
|
[[ -z $(command -v sshfs) ]] && sudo apt-get -yy install sshfs
|
|
sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true
|
|
sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o nonempty,allow_other || true
|
|
sudo df
|
|
|
|
- name: Signining and torrent creation
|
|
id: sign-list
|
|
run: |
|
|
|
|
MATRIX=$(ls -1 ${{ env.UPLOAD }}/*.zst | awk -F/ '{print $NF}')
|
|
echo "files=$(for x in $(echo "${MATRIX}"); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq -c)" >> $GITHUB_OUTPUT
|
|
|
|
- name: Unmount folders
|
|
if: ${{ github.repository_owner == 'Armbian' }}
|
|
run: |
|
|
|
|
sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
|
|
|
|
sign:
|
|
|
|
needs: sign-list
|
|
#if: ${{ needs.sign-list.outputs.matrix != '[]' && needs.sign-list.outputs.matrix != '' }}
|
|
runs-on: [big,igor]
|
|
strategy:
|
|
max-parallel: 32
|
|
fail-fast: false
|
|
matrix:
|
|
|
|
node: ${{fromJson(needs.sign-list.outputs.files)}}
|
|
|
|
steps:
|
|
|
|
- name: Read
|
|
run: |
|
|
|
|
echo "UPLOAD=$(echo ${{ inputs.variant }} | sed "s/://" )" >> $GITHUB_ENV
|
|
|
|
echo {{ env.rootfscache_version }}
|
|
|
|
echo ${{ inputs.variant }}
|
|
|
|
|
|
- name: Runner prepare
|
|
uses: armbian/actions/runner-prepare@main
|
|
|
|
- name: Install SSH key for storage
|
|
uses: shimataro/ssh-key-action@v2
|
|
with:
|
|
key: ${{ secrets.SSH_KEY_TORRENTS }}
|
|
known_hosts: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
|
|
if_key_exists: ignore
|
|
|
|
- name: Download changes
|
|
uses: actions/download-artifact@v3.0.2
|
|
with:
|
|
name: ${{ env.UPLOAD }}
|
|
path: ${{ env.UPLOAD }}
|
|
|
|
- name: Checkout tracker lists
|
|
uses: actions/checkout@v3.3.0
|
|
with:
|
|
fetch-depth: 1
|
|
repository: ngosang/trackerslist
|
|
path: trackerslist
|
|
|
|
- name: Checkout repository
|
|
if: ${{ github.repository_owner == 'Armbian' }}
|
|
uses: actions/checkout@v3.3.0
|
|
with:
|
|
repository: armbian/build
|
|
fetch-depth: 1
|
|
path: build
|
|
clean: false
|
|
|
|
- name: Import GPG key
|
|
uses: crazy-max/ghaction-import-gpg@v5
|
|
with:
|
|
gpg_private_key: ${{ secrets.GPG_KEY1 }}
|
|
passphrase: ${{ secrets.GPG_PASSPHRASE1 }}
|
|
workdir: build
|
|
git_user_signingkey: true
|
|
|
|
- name: Mount folders
|
|
if: ${{ github.repository_owner == 'Armbian' }}
|
|
run: |
|
|
|
|
sudo mountpoint -q build/cache/rootfs && sudo fusermount -u build/cache/rootfs || true
|
|
sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
|
|
[[ -z $(command -v sshfs) ]] && sudo apt-get -yy install sshfs
|
|
sudo mkdir -p /root/.ssh/
|
|
sudo cp ~/.ssh/* /root/.ssh/
|
|
sudo mkdir -p build/cache/rootfs.upload/${{ needs.Prepare.outputs.rootfscache_version }} || true
|
|
sudo mkdir -p build/cache/rootfs || true
|
|
|
|
ls -l build/cache/rootfs/
|
|
echo "Clean folder"
|
|
sudo find build/cache/rootfs/ -maxdepth 1 -mindepth 1 -exec sudo rm -fr -- {} +
|
|
|
|
if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" || $(curl -s http://ifconfig.me) == "46.19.33.90" ]]; then
|
|
echo "Mounting NFS share"
|
|
sudo mount nas:/tank/armbian/users.armbian.com/upload/rootfs build/cache/rootfs.upload
|
|
else
|
|
echo "Mounting SSH share"
|
|
sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true
|
|
sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o nonempty,allow_other || true
|
|
fi
|
|
|
|
|
|
- name: Signining and torrent creation
|
|
if: ${{ github.repository_owner == 'Armbian' }}
|
|
run: |
|
|
|
|
# exit if keys were not loaded
|
|
[[ -z $(gpg -K) ]] && exit 1
|
|
|
|
#sudo apt-get -y -qq install jq buildtorrent
|
|
UPLOAD=${{ env.UPLOAD }}
|
|
ROOTFSCACHE_VERSION=$(cat $UPLOAD/version)
|
|
# cleaning
|
|
rm -rf $UPLOAD
|
|
echo "ROOTFSCACHE_VERSION=$ROOTFSCACHE_VERSION" >> $GITHUB_ENV
|
|
FILE=${{ matrix.node }}
|
|
ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | sed '/^$/d' | shuf -n 1)
|
|
TRACKERS=$(cat trackerslist/trackers_best_ip.txt | sed '/^\s*$/d' | while read line; do printf ",""${line}"; done | cut -c 2-)
|
|
WEBSEEDS="--webseeds="\
|
|
https://github.com/armbian/cache/releases/download/$ROOTFSCACHE_VERSION/$FILE,\
|
|
https://github.com/tuxwall/cache/releases/download/$ROOTFSCACHE_VERSION/$FILE,\
|
|
https://github.com/armbianworker/cache/releases/download/$ROOTFSCACHE_VERSION/$FILE,\
|
|
https://github.com/x86bian/cache/releases/download/$ROOTFSCACHE_VERSION/$FILE,\
|
|
https://github.com/riscvbian/cache/releases/download/$ROOTFSCACHE_VERSION/$FILE,\
|
|
"$(curl -s https://cache.armbian.com/mirrors | jq -r '.'default' | .[] | values' | sed -e 's/$/rootfs\/'$ROOTFSCACHE_VERSION'\/'$FILE'/' | tr '\n' , | sed 's/.$//')"
|
|
cd build/cache/rootfs.upload/$ROOTFSCACHE_VERSION
|
|
echo "Generate torrent"
|
|
sudo --preserve-env buildtorrent $WEBSEEDS --announce=$ANNOUNCE --announcelist=$TRACKERS $FILE -c "Armbian rootfs cache" ${FILE}.torrent
|
|
|
|
echo "Sign"
|
|
FOLDER=$(mktemp -d)
|
|
echo ${{ secrets.GPG_PASSPHRASE1 }} | gpg --quiet --armor --batch --yes --passphrase-fd 0 --detach-sign --pinentry-mode loopback --output ${FOLDER}/${FILE}.asc ${FILE}
|
|
sudo mv ${FOLDER}/${FILE}.asc .
|
|
|
|
echo "Verify"
|
|
zstd -t ${FILE}
|
|
echo "Fix permissions"
|
|
sudo chown 1008:1002 ${FILE}
|
|
sudo chown 1008:1002 ${FILE}.asc
|
|
sudo chown 1008:1002 ${FILE}.torrent
|
|
|
|
if [[ "$(cat ${FILE}.asc | gpg --list-packets --no-keyring | grep -oE '[0-9A-F]{40}')" == "DF00FAF1C577104B50BF1D0093D6889F9F0E78D5" ]]; then
|
|
echo "Signature is O.K."
|
|
else
|
|
exit 1
|
|
fi
|
|
|
|
- name: Upload signature and torrent
|
|
uses: ncipollo/release-action@v1
|
|
if: ${{ github.repository_owner == 'Armbian' }}
|
|
with:
|
|
artifacts: "build/cache/rootfs.upload/${{ env.ROOTFSCACHE_VERSION }}/${{ matrix.node }}.asc,build/cache/rootfs.upload/${{ env.ROOTFSCACHE_VERSION }}/${{ matrix.node }}.torrent"
|
|
artifactErrorsFailBuild: true
|
|
tag: "${{ env.ROOTFSCACHE_VERSION }}"
|
|
omitBody: true
|
|
omitName: true
|
|
allowUpdates: true
|
|
token: ${{ secrets.PAT1 }}
|
|
|
|
- name: Unmount folders
|
|
if: ${{ github.repository_owner == 'Armbian' }}
|
|
run: |
|
|
|
|
sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
|
|
|
|
- name: Runner cleanup
|
|
uses: armbian/actions/runner-prepare@main
|