Files
mirror/.github/workflows/mirror-sync.yml
2024-01-29 05:10:34 +00:00

698 lines
23 KiB
YAML

name: Sync mirror
on:
# schedule:
# - cron: '0 1 * * *'
workflow_dispatch:
jobs:
Prepare:
runs-on: ubuntu-20.04
if: ${{ github.repository_owner == 'Armbian' }}
steps:
-
run: |
wget -P temp https://www.armbian.com/userdata.csv
- name: Cache Gradle packages
uses: actions/cache@v2
env:
cache-name: cache-build
with:
path: temp
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
Build: # short name because GH will expand with the matrix values
needs: [ Prepare ]
runs-on: ubuntu-20.04
timeout-minutes: 120
strategy:
max-parallel: 4
fail-fast: false # let other jobs try to complete if one fails
matrix:
include: # build this way: cat userdata.csv | cut -d"/" -f1 | uniq | sed 's/.*/ - board: &/'
- board: aml-s9xx-box
- board: bananapi
- board: bananapicm4io
- board: bananapim1plus
- board: bananapim2plus
- board: bananapim2pro
- board: bananapim2s
- board: bananapim2ultra
- board: bananapim2zero
- board: bananapim3
- board: bananapim5
- board: bananapim64
- board: bananapipro
- board: beelinkx2
- board: bigtreetech-cb1
- board: clearfogbase
- board: clearfogpro
- board: clockworkpi-a06
- board: cubieboard
- board: cubieboard2
- board: cubieboard4
- board: cubietruck
- board: cubox-i
- board: espressobin
- board: firefly-rk3399
- board: helios4
- board: helios64
- board: hikey960
- board: inovato-quadra
- board: jethubj100
- board: jethubj80
- board: khadas-edge
- board: khadas-edge2
- board: khadas-vim1
- board: khadas-vim1s
- board: khadas-vim2
- board: khadas-vim3
- board: khadas-vim3l
- board: khadas-vim4
- board: lafrite
- board: lepotato
- board: lime-a33
- board: lime-a64
- board: lime
- board: lime2
- board: mekotronics-r58-minipc
- board: mekotronics-r58x-4g
- board: mekotronics-r58x-pro
- board: mekotronics-r58x
- board: melea1000
- board: micro
- board: mixtile-blade3
- board: nanopct6
- board: nanopi-r1
- board: nanopi-r1s-h5
- board: nanopi-r2c
- board: nanopi-r2s
- board: nanopi-r4s
- board: nanopi-r4se
- board: nanopi-r5s
- board: nanopi-r6s
- board: nanopia64
- board: nanopiair
- board: nanopiduo
- board: nanopiduo2
- board: nanopik1plus
- board: nanopik2-s905
- board: nanopim4
- board: nanopim4v2
- board: nanopineo
- board: nanopineo2
- board: nanopineo2black
- board: nanopineo3
- board: nanopineo4
- board: nanopineocore2
- board: nanopineoplus2
- board: odroidc2
- board: odroidc4
- board: odroidhc4
- board: odroidm1
- board: odroidn2
- board: odroidxu4
- board: olimex-teres-a64
- board: onecloud
- board: orangepi-r1
- board: orangepi-r1plus-lts
- board: orangepi-r1plus
- board: orangepi-rk3399
- board: orangepi2
- board: orangepi3-lts
- board: orangepi3
- board: orangepi3b
- board: orangepi4-lts
- board: orangepi4
- board: orangepi5-plus
- board: orangepi5
- board: orangepilite2
- board: orangepione
- board: orangepioneplus
- board: orangepipc
- board: orangepipc2
- board: orangepipcplus
- board: orangepiplus
- board: orangepiplus2e
- board: orangepiprime
- board: orangepiwin
- board: orangepizero
- board: orangepizero2
- board: orangepizeroplus
- board: orangepizeroplus2-h3
- board: orangepizeroplus2-h5
- board: panther-x2
- board: pcduino3
- board: pine64
- board: pine64so
- board: pinebook-a64
- board: pinebook-pro
- board: pinecube
- board: pineh64-b
- board: pineh64
- board: quartz64a
- board: quartz64b
- board: radxa-e25
- board: radxa-zero
- board: radxa-zero2
- board: recore
- board: renegade
- board: rk322x-box
- board: rk3318-box
- board: roc-rk3399-pc
- board: rock-3a
- board: rock-4se
- board: rock-5a
- board: rock-5b
- board: rock64
- board: rockpi-4a
- board: rockpi-4b
- board: rockpi-4bplus
- board: rockpi-4c
- board: rockpi-4cplus
- board: rockpi-e
- board: rockpi-n10
- board: rockpi-s
- board: rockpro64
- board: rpi4b
- board: rpi5b
- board: sk-am62b
- board: sk-am64b
- board: sk-tda4vm
- board: sunvell-r69
- board: thinkpad-x13s
- board: tinkerboard-2
- board: tinkerboard
- board: tritium-h3
- board: tritium-h5
- board: udoo
- board: uefi-arm64
- board: uefi-riscv64
- board: uefi-x86
- board: wsl2-arm64
- board: wsl2-x86
- board: xiaomi-elish
- board: xt-q8l-v10
- board: zeropi
- board: nanopi-r4s
- board: nanopiduo2
- board: nanopiduo
- board: nanopiduo2
- board: nanopiduo
- board: nanopiduo2
- board: nanopiduo
- board: nanopiduo2
- board: rockpi-4a
- board: nanopiduo
- board: nanopiduo2
- board: qemu-uboot-x86
- board: nanopiduo2
- board: nanopiduo
- board: tritium-h3
- board: nanopiduo2
- board: odroidc2
- board: nanopi-r4s
- board: rockpi-s
- board: orangepizero2
- board: orangepi-r1
- board: uefi-x86
- board: qemu-uboot-x86
- board: uefi-x86
- board: orangepi-r1
- board: jethubj100
- board: qemu-uboot-x86
- board: orangepi-r1
- board: orangepizero
- board: orangepi-r1
- board: odroidxu4
- board: onecloud
- board: tritium-h3
- board: orangepione
- board: tinkerboard
- board: odroidxu4
- board: tinkerboard
- board: odroidxu4
- board: tinkerboard
- board: odroidxu4
- board: tinkerboard
- board: orangepizero
- board: sk-am64b
- board: orangepizero
- board: khadas-vim1
- board: khadas-edge2
- board: orangepi5
- board: nanopi-r6s
- board: sk-am62b
- board: orangepione
- board: uefi-x86
- board: khadas-vim1
- board: bananapicm4io
- board: odroidc2
- board: rockpro64
- board: orangepione
- board: sk-am62b
- board: rock-5b
- board: bananapicm4io
- board: orangepizero2
- board: tinkerboard
- board: uefi-x86
- board: khadas-edge2
- board: tinkerboard
- board: qemu-uboot-arm64
- board: tritium-h3
- board: odroidm1
- board: licheepi-4a
- board: qemu-uboot-arm64
- board: odroidxu4
- board: bananapim2pro
- board: qemu-uboot-arm64
- board: odroidxu4
- board: qemu-uboot-arm64
- board: fxblox-rk1
- board: onecloud
- board: bananapim2pro
- board: orangepione
- board: pine64
- board: rockpro64
- board: orangepiprime
- board: orangepi5
- board: uefi-arm64
- board: orangepi5
- board: uefi-arm64
- board: orangepi5
- board: uefi-arm64
- board: rock-3a
- board: rpi4b
- board: mekotronics-r58-minipc
- board: orangepiprime
- board: lepotato
- board: rockpi-s
- board: lepotato
- board: radxa-e25
- board: lepotato
- board: radxa-e25
- board: rock-5b
- board: khadas-vim1s
- board: rpi4b
- board: sk-am64b
- board: fxblox-rk1
- board: pine64
- board: radxa-e25
- board: jethubj80
- board: sk-am64b
- board: khadas-vim2
- board: odroidm1
- board: licheepi-4a
- board: bananapim2s
- board: khadas-vim1
- board: mekotronics-r58x-4g
- board: odroidc2
- board: pinebook-pro
- board: orangepiprime
- board: radxa-e25
- board: bananapim2s
- board: khadas-edge2
- board: nanopi-r6s
- board: radxa-zero2
- board: khadas-vim2
- board: orangepi5-plus
- board: lafrite
- board: rpi5b
- board: orangepizero2
- board: pinebook-pro
- board: orangepizero2
- board: radxa-e25
- board: mekotronics-r58-minipc
- board: mekotronics-r58x
- board: rockpi-4a
- board: nanopi-r6s
- board: rpi5b
- board: khadas-vim3
- board: rockpi-4a
- board: mekotronics-r58x-4g
- board: odroidn2
- board: bananapim2pro
- board: rockpro64
- board: orangepi-r1plus-lts
- board: pine64
- board: orangepi4-lts
- board: radxa-zero2
- board: odroidn2
- board: bananapim2s
- board: bananapim5
- board: renegade
- board: rock-5a
- board: khadas-vim3
- board: khadas-vim1
- board: odroidc2
- board: rockpro64
- board: odroidc2
- board: rockpro64
- board: odroidc2
- board: rockpro64
- board: odroidc2
- board: rockpro64
- board: khadas-vim3l
- board: bananapicm4io
- board: olimex-teres-a64
- board: mekotronics-r58x
- board: radxa-zero2
- board: bananapim5
- board: bigtreetech-cb1
- board: sk-tda4vm
- board: orangepizero2
- board: khadas-vim3l
- board: mixtile-blade3
- board: rock-5a
- board: rpi5b
- board: bananapim2pro
- board: bigtreetech-cb1
- board: mixtile-blade3
- board: khadas-vim1s
- board: orangepi-r1plus-lts
- board: odroidm1
- board: bananapicm4io
- board: orangepi4-lts
- board: rock-5a
- board: radxa-zero
- board: tritium-h5
- board: nanopct6
- board: orangepi-r1plus-lts
- board: nanopct6
- board: rock-5a
- board: nanopct6
- board: odroidn2
- board: orangepi-r1plus-lts
- board: odroidn2
- board: radxa-zero
- board: sk-tda4vm
- board: wdk2023
- board: pinebook-pro
- board: khadas-vim2
- board: rpi4b
- board: olimex-teres-a64
- board: licheepi-4a
- board: nanopct6
- board: khadas-vim4
- board: rpi5b
- board: orangepi5-plus
- board: rock-3a
- board: lafrite
- board: khadas-vim2
- board: orangepi5-plus
- board: tritium-h5
- board: orangepizeroplus
- board: pine64
- board: odroidm1
- board: rpi4b
- board: licheepi-4a
- board: rock-3a
- board: mixtile-blade3
- board: nanopi-r5s
- board: bananapim2s
- board: jethubj100
- board: khadas-vim3
- board: orangepizeroplus
- board: jethubj80
- board: lafrite
- board: sk-tda4vm
- board: jethubj100
- board: pinebook-pro
- board: nanopi-r5s
- board: orangepizeroplus
- board: renegade
- board: orangepizeroplus
- board: renegade
- board: uefi-arm64
- board: khadas-vim3
- board: radxa-zero
- board: thinkpad-x13s
- board: radxa-zero
- board: tritium-h5
- board: orangepizero3
- board: tritium-h5
- board: orangepizero3
- board: tritium-h5
- board: orangepizero3
- board: olimex-teres-a64
- board: sk-tda4vm
- board: khadas-vim4
- board: radxa-zero
- board: lepotato
- board: uefi-arm64
- board: bananapim5
- board: uefi-arm64
- board: bananapim5
- board: orangepi5
- board: rock-5b
- board: olimex-teres-a64
- board: khadas-vim3l
- board: rock-5b
- board: nanopi-r5s
- board: rockpi-e
- board: wdk2023
- board: rockpi-e
- board: orangepizero3
- board: rockpi-4a
- board: orangepizero3
- board: bigtreetech-cb1
- board: rockpi-s
- board: lepotato
- board: renegade
- board: jethubj80
- board: khadas-vim3l
- board: rockpi-e
- board: renegade
- board: khadas-edge2
- board: bananapim5
- board: nanopi-r6s
- board: nanopi-r4s
- board: orangepiprime
- board: nanopi-r5s
- board: bigtreetech-cb1
- board: orangepi4-lts
- board: orangepi5
- board: orangepizero3
- board: lafrite
- board: tritium-h5
- board: orangepi5-plus
- board: xiaomi-elish
- board: rock-3a
- board: jethubj100
- board: xiaomi-elish
- board: qemu-uboot-arm64
- board: uefi-arm64
- board: sk-am64b
- board: radxa-zero2
steps:
- name: Install depen
run: sudo apt-get -y install parallel jq axel
- name: Cache Gradle packages
uses: actions/cache@v2
env:
cache-name: cache-build
with:
path: temp
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
#-
# uses: styfle/cancel-workflow-action@0.9.1
# with:
# access_token: ${{ github.token }}
-
name: Download from Armbian storage
id: image-build
run: |
set +e
BOARD="${{ matrix.board }}"
DEBUG="no"
FILES=$(cat temp/userdata.csv | grep ${BOARD}"/" | grep -v nightly | cut -d"|" -f2) # exclude nightly
MIRRORS=$(curl -sq http://redirect.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq)
# overrire with mirrors where file 100% exists
# MIRRORS="http://stpete-mirror.armbian.com/dl/ http://imola.armbian.com/dl/"
echo "BOARD=$BOARD" >> $GITHUB_ENV
truncate -s 0 ${BOARD}.log
for FILE in ${FILES[@]}
do
echo $FILE
PARALLEL=$(
for k in ${MIRRORS[@]}
do
echo "$k$FILE"
done | parallel --halt now,fail=1 --jobs 32 wget -q --spider --timeout=15 --tries=4 --retry-connrefused {} 2>&1 >/dev/null)
if [[ -n "${PARALLEL}" ]]; then
REMOVE=$(echo $PARALLEL | egrep -o 'https?://[^ ]+/' | sed "s/$BOARD.*//" | sort | uniq)
for l in ${REMOVE[@]}; do
[[ "${DEBUG}" == yes ]] && echo "Removing mirror: $l"
echo "Error,$l,$FILE" >> ${BOARD}.log
MIRRORS=$(echo "${MIRRORS}" | sed "s|$l ||")
done
[[ $DEBUG == yes ]] && echo "Timeout: $(echo $PARALLEL | egrep -o 'https?://[^ ]+/' | cut -d"/" -f1-4 | sort | uniq)/${BOARD}/archive/$(echo $PARALLEL | egrep -o 'https?://[^ ]+' | rev | cut -d / -f 1 | rev)"
fi
# download
DOWNLOAD="axel -c -a "
for m in ${MIRRORS[@]}; do
DOWNLOAD=${DOWNLOAD}" $m$FILE"
done
echo "https://github.com/${{ github.repository }}/releases/download/${FILE}"
if ! wget -q --spider --timeout=15 --tries=3 --retry-connrefused https://github.com/"${{ github.repository }}"/releases/download/${FILE} ; then
echo "Downloading ..."
eval "${DOWNLOAD}"
echo "UPLOAD=true" >> $GITHUB_ENV
fi
done
- name: Upload artefacts
if: ${{ env.UPLOAD == 'true' }}
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: Armbian*
tag: "${{ matrix.board }}/archive"
overwrite: true
file_glob: true
body: "<p align='center'><a href='https://www.armbian.com'><img src='https://raw.githubusercontent.com/armbian/.github/master/profile/tux-two.png' width='400' height='226' alt='Armbian Linux'></a></p>"
- name: Upload logs
uses: actions/upload-artifact@v2
with:
name: logs
path: ${{ env.BOARD }}.log
Finish:
needs: [ Build ]
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: actions/download-artifact@v2
with:
name: logs
- name: Display structure of downloaded files
run: |
function exceptions()
{
if [[ $k == *albony* ]]; then
COUNTRY=India
CITY=Nagpur
FLAG="https://cdn.ipwhois.io/flags/in.svg"
fi
if [[ $k == *aliyun* ]]; then
COUNTRY=China
CITY=Hangzhou
FLAG="https://cdn.ipwhois.io/flags/cn.svg"
fi
if [[ $k == *nju.edu* ]]; then
COUNTRY=China
CITY=Nanjing
FLAG="https://cdn.ipwhois.io/flags/cn.svg"
fi
if [[ $k == *imola.armbian* ]]; then
COUNTRY=Slovenia
CITY=Ljubljana
FLAG="https://cdn.ipwhois.io/flags/si.svg"
fi
if [[ $k == *mirrors.naho* ]]; then
COUNTRY=Korea
CITY=Seoul
FLAG="https://cdn.ipwhois.io/flags/kr.svg"
fi
}
function show_mirrors()
{
MIRRORS=$1
echo -en "\n## $2\n\n"
echo "|URL &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; |City &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; |Flag|Country|"
echo "|:--|:--|:--:|--:|"
for k in ${MIRRORS[@]}
do
HOST=$(echo "$k" | awk -F/ '{print $3}')
IP=$(dig +short $HOST | tail -1)
RES=$(
curl --max-time 5 -s http://ipwhois.app/json/${IP} | \
jq '.timezone, .city, .country, .country_flag' | \
while read -r TIMEZONE; do
read -r COUNTRY
echo "${TIMEZONE},${COUNTRY},${COUNTRYFLAG}" | tr --delete \"
done
)
TZDATA=$(echo ${RES} | cut -d"," -f1)
CITY=$(echo ${RES} | cut -d"," -f2)
COUNTRY=$(echo ${RES} | cut -d"," -f3 | xargs)
FLAG=$(echo ${RES} | cut -d"," -f4)
# automation is not perfect
exceptions
echo "|$k|$CITY|<img width=24 src=$FLAG>|$COUNTRY|"
done
}
echo '<h3 align=center><a href="#build-tools"><img src="https://raw.githubusercontent.com/armbian/build/master/.github/armbian-logo.png" alt="Armbian logo" width="144"></a><br>mirror sync</h3>' > README.md
echo -en "<p align=right>&nbsp;</p>\n" >> README.md
echo -en "\n[![Sync mirror](https://github.com/armbian/mirror/actions/workflows/mirror-sync.yml/badge.svg)](https://github.com/armbian/mirror/actions/workflows/mirror-sync.yml) [![Rootfs sync](https://github.com/armbian/mirror/actions/workflows/rootfs-sync.yml/badge.svg)](https://github.com/armbian/mirror/actions/workflows/rootfs-sync.yml)" >> README.md
echo "
## Mirroring Armbian?
We are providing rsync service for stable images (dl), repository (apt), beta (beta) and archive (archive) at:
rsync -av rsync://rsync.armbian.com/
Alternatively you can sync most valuable date from one of our mirror that provides rsync and is much faster:
rsync -av rsync://mirrors.dotsrc.org/armbian-dl
rsync -av rsync://mirrors.dotsrc.org/armbian-apt
Space needs: 500Gb (images), 100Gb (packages) and (optional) 3TB for archives
1. Setup HTTP and HTTPS hostname
2. Setup cron to sync every 2-4 hours
3. [Inform us](https://www.armbian.com/#contact)
If you want to seed our Torrent files, [check this document](https://forum.armbian.com/topic/4198-seed-our-torrents/).
" >> README.md
MIR=$(curl -sq http://redirect.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq)
show_mirrors "$MIR" "Images" >> README.md
echo "" >> README.md
MIR=$(curl -sq http://apt.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq)
show_mirrors "$MIR" "Packages" >> README.md
echo "" >> README.md
MIR=$(curl -sq http://archive.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq)
show_mirrors "$MIR" "Archive" >> README.md
echo "" >> README.md
MIR=$(curl -sq http://beta.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq)
show_mirrors "$MIR" "Beta" >> README.md
echo "" >> README.md
if [[ $(cat *.log | awk -F ',' '{print $2}' | sort | uniq -c | sort -n | wc -l) -gt 0 ]]; then
echo -en "\n## Mirrors error count\n" >> README.md
cat *.log | awk -F ',' '{print $2}' | sort | uniq -c | sort -n >> README.md
echo -en "\n\n[Detailed report](detailed.txt)" >> README.md
find *.log -not -empty -type f -print0 | xargs -0 -I % sh -c 'cat % | cut -d"," -f2-3 |tr -d \"," " ' | sort > detailed.txt
fi
- name: Add
run: |
git pull
git add README.md
git add detailed.txt
- name: Commit files
run: |
git config --local user.email "info@armbian.com"
git config --local user.name "Armbianworker"
git commit --allow-empty -m "Update github actions" -a
- name: Push changes
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.MIRROR }}
branch: ${{ github.ref }}