name: Sync mirror on: # schedule: # - cron: '0 1 * * *' workflow_dispatch: jobs: Prepare: runs-on: ubuntu-20.04 if: ${{ github.repository_owner == 'Armbian' }} steps: - run: | wget -P temp https://www.armbian.com/userdata.csv - name: Cache Gradle packages uses: actions/cache@v2 env: cache-name: cache-build with: path: temp key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }} Build: # short name because GH will expand with the matrix values needs: [ Prepare ] runs-on: ubuntu-20.04 timeout-minutes: 120 strategy: max-parallel: 4 fail-fast: false # let other jobs try to complete if one fails matrix: include: # build this way: cat userdata.csv | cut -d"/" -f1 | uniq | sed 's/.*/ - board: &/' - board: aml-s9xx-box - board: bananapi - board: bananapicm4io - board: bananapim1plus - board: bananapim2plus - board: bananapim2pro - board: bananapim2s - board: bananapim2ultra - board: bananapim2zero - board: bananapim3 - board: bananapim5 - board: bananapim64 - board: bananapipro - board: beelinkx2 - board: bigtreetech-cb1 - board: clearfogbase - board: clearfogpro - board: clockworkpi-a06 - board: cubieboard - board: cubieboard2 - board: cubieboard4 - board: cubietruck - board: cubox-i - board: espressobin - board: firefly-rk3399 - board: helios4 - board: helios64 - board: hikey960 - board: inovato-quadra - board: jethubj100 - board: jethubj80 - board: khadas-edge - board: khadas-edge2 - board: khadas-vim1 - board: khadas-vim1s - board: khadas-vim2 - board: khadas-vim3 - board: khadas-vim3l - board: khadas-vim4 - board: lafrite - board: lepotato - board: lime-a33 - board: lime-a64 - board: lime - board: lime2 - board: mekotronics-r58-minipc - board: mekotronics-r58x-4g - board: mekotronics-r58x-pro - board: mekotronics-r58x - board: melea1000 - board: micro - board: mixtile-blade3 - board: nanopct6 - board: nanopi-r1 - board: nanopi-r1s-h5 - board: nanopi-r2c - board: nanopi-r2s - board: nanopi-r4s - board: nanopi-r4se - board: nanopi-r5s - board: nanopi-r6s - board: nanopia64 - board: nanopiair - board: nanopiduo - board: nanopiduo2 - board: nanopik1plus - board: nanopik2-s905 - board: nanopim4 - board: nanopim4v2 - board: nanopineo - board: nanopineo2 - board: nanopineo2black - board: nanopineo3 - board: nanopineo4 - board: nanopineocore2 - board: nanopineoplus2 - board: odroidc2 - board: odroidc4 - board: odroidhc4 - board: odroidm1 - board: odroidn2 - board: odroidxu4 - board: olimex-teres-a64 - board: onecloud - board: orangepi-r1 - board: orangepi-r1plus-lts - board: orangepi-r1plus - board: orangepi-rk3399 - board: orangepi2 - board: orangepi3-lts - board: orangepi3 - board: orangepi3b - board: orangepi4-lts - board: orangepi4 - board: orangepi5-plus - board: orangepi5 - board: orangepilite2 - board: orangepione - board: orangepioneplus - board: orangepipc - board: orangepipc2 - board: orangepipcplus - board: orangepiplus - board: orangepiplus2e - board: orangepiprime - board: orangepiwin - board: orangepizero - board: orangepizero2 - board: orangepizeroplus - board: orangepizeroplus2-h3 - board: orangepizeroplus2-h5 - board: panther-x2 - board: pcduino3 - board: pine64 - board: pine64so - board: pinebook-a64 - board: pinebook-pro - board: pinecube - board: pineh64-b - board: pineh64 - board: quartz64a - board: quartz64b - board: radxa-e25 - board: radxa-zero - board: radxa-zero2 - board: recore - board: renegade - board: rk322x-box - board: rk3318-box - board: roc-rk3399-pc - board: rock-3a - board: rock-4se - board: rock-5a - board: rock-5b - board: rock64 - board: rockpi-4a - board: rockpi-4b - board: rockpi-4bplus - board: rockpi-4c - board: rockpi-4cplus - board: rockpi-e - board: rockpi-n10 - board: rockpi-s - board: rockpro64 - board: rpi4b - board: rpi5b - board: sk-am62b - board: sk-am64b - board: sk-tda4vm - board: sunvell-r69 - board: thinkpad-x13s - board: tinkerboard-2 - board: tinkerboard - board: tritium-h3 - board: tritium-h5 - board: udoo - board: uefi-arm64 - board: uefi-riscv64 - board: uefi-x86 - board: wsl2-arm64 - board: wsl2-x86 - board: xiaomi-elish - board: xt-q8l-v10 - board: zeropi - board: nanopi-r4s - board: nanopiduo2 - board: nanopiduo - board: nanopiduo2 - board: nanopiduo - board: nanopiduo2 - board: nanopiduo - board: nanopiduo2 - board: rockpi-4a - board: nanopiduo - board: nanopiduo2 - board: qemu-uboot-x86 - board: nanopiduo2 - board: nanopiduo - board: tritium-h3 - board: nanopiduo2 - board: odroidc2 - board: nanopi-r4s - board: rockpi-s - board: orangepizero2 - board: orangepi-r1 - board: uefi-x86 - board: qemu-uboot-x86 - board: uefi-x86 - board: orangepi-r1 - board: jethubj100 - board: qemu-uboot-x86 - board: orangepi-r1 - board: orangepizero - board: orangepi-r1 - board: odroidxu4 - board: onecloud - board: tritium-h3 - board: orangepione - board: tinkerboard - board: odroidxu4 - board: tinkerboard - board: odroidxu4 - board: tinkerboard - board: odroidxu4 - board: tinkerboard - board: orangepizero - board: sk-am64b - board: orangepizero - board: khadas-vim1 - board: khadas-edge2 - board: orangepi5 - board: nanopi-r6s - board: sk-am62b - board: orangepione - board: uefi-x86 - board: khadas-vim1 - board: bananapicm4io - board: odroidc2 - board: rockpro64 - board: orangepione - board: sk-am62b - board: rock-5b - board: bananapicm4io - board: orangepizero2 - board: tinkerboard - board: uefi-x86 - board: khadas-edge2 - board: tinkerboard - board: qemu-uboot-arm64 - board: tritium-h3 - board: odroidm1 - board: licheepi-4a - board: qemu-uboot-arm64 - board: odroidxu4 - board: bananapim2pro - board: qemu-uboot-arm64 - board: odroidxu4 - board: qemu-uboot-arm64 - board: fxblox-rk1 - board: onecloud - board: bananapim2pro - board: orangepione - board: pine64 - board: rockpro64 - board: orangepiprime - board: orangepi5 - board: uefi-arm64 - board: orangepi5 - board: uefi-arm64 - board: orangepi5 - board: uefi-arm64 - board: rock-3a - board: rpi4b - board: mekotronics-r58-minipc - board: orangepiprime - board: lepotato - board: rockpi-s - board: lepotato - board: radxa-e25 - board: lepotato - board: radxa-e25 - board: rock-5b - board: khadas-vim1s - board: rpi4b - board: sk-am64b - board: fxblox-rk1 - board: pine64 - board: radxa-e25 - board: jethubj80 - board: sk-am64b - board: khadas-vim2 - board: odroidm1 - board: licheepi-4a - board: bananapim2s - board: khadas-vim1 - board: mekotronics-r58x-4g - board: odroidc2 - board: pinebook-pro - board: orangepiprime - board: radxa-e25 - board: bananapim2s - board: khadas-edge2 - board: nanopi-r6s - board: radxa-zero2 - board: khadas-vim2 - board: orangepi5-plus - board: lafrite - board: rpi5b - board: orangepizero2 - board: pinebook-pro - board: orangepizero2 - board: radxa-e25 - board: mekotronics-r58-minipc - board: mekotronics-r58x - board: rockpi-4a - board: nanopi-r6s - board: rpi5b - board: khadas-vim3 - board: rockpi-4a - board: mekotronics-r58x-4g - board: odroidn2 - board: bananapim2pro - board: rockpro64 - board: orangepi-r1plus-lts - board: pine64 - board: orangepi4-lts - board: radxa-zero2 - board: odroidn2 - board: bananapim2s - board: bananapim5 - board: renegade - board: rock-5a - board: khadas-vim3 - board: khadas-vim1 - board: odroidc2 - board: rockpro64 - board: odroidc2 - board: rockpro64 - board: odroidc2 - board: rockpro64 - board: odroidc2 - board: rockpro64 - board: khadas-vim3l - board: bananapicm4io - board: olimex-teres-a64 - board: mekotronics-r58x - board: radxa-zero2 - board: bananapim5 - board: bigtreetech-cb1 - board: sk-tda4vm - board: orangepizero2 - board: khadas-vim3l - board: mixtile-blade3 - board: rock-5a - board: rpi5b - board: bananapim2pro - board: bigtreetech-cb1 - board: mixtile-blade3 - board: khadas-vim1s - board: orangepi-r1plus-lts - board: odroidm1 - board: bananapicm4io - board: orangepi4-lts - board: rock-5a - board: radxa-zero - board: tritium-h5 - board: nanopct6 - board: orangepi-r1plus-lts - board: nanopct6 - board: rock-5a - board: nanopct6 - board: odroidn2 - board: orangepi-r1plus-lts - board: odroidn2 - board: radxa-zero - board: sk-tda4vm - board: wdk2023 - board: pinebook-pro - board: khadas-vim2 - board: rpi4b - board: olimex-teres-a64 - board: licheepi-4a - board: nanopct6 - board: khadas-vim4 - board: rpi5b - board: orangepi5-plus - board: rock-3a - board: lafrite - board: khadas-vim2 - board: orangepi5-plus - board: tritium-h5 - board: orangepizeroplus - board: pine64 - board: odroidm1 - board: rpi4b - board: licheepi-4a - board: rock-3a - board: mixtile-blade3 - board: nanopi-r5s - board: bananapim2s - board: jethubj100 - board: khadas-vim3 - board: orangepizeroplus - board: jethubj80 - board: lafrite - board: sk-tda4vm - board: jethubj100 - board: pinebook-pro - board: nanopi-r5s - board: orangepizeroplus - board: renegade - board: orangepizeroplus - board: renegade - board: uefi-arm64 - board: khadas-vim3 - board: radxa-zero - board: thinkpad-x13s - board: radxa-zero - board: tritium-h5 - board: orangepizero3 - board: tritium-h5 - board: orangepizero3 - board: tritium-h5 - board: orangepizero3 - board: olimex-teres-a64 - board: sk-tda4vm - board: khadas-vim4 - board: radxa-zero - board: lepotato - board: uefi-arm64 - board: bananapim5 - board: uefi-arm64 - board: bananapim5 - board: orangepi5 - board: rock-5b - board: olimex-teres-a64 - board: khadas-vim3l - board: rock-5b - board: nanopi-r5s - board: rockpi-e - board: wdk2023 - board: rockpi-e - board: orangepizero3 - board: rockpi-4a - board: orangepizero3 - board: bigtreetech-cb1 - board: rockpi-s - board: lepotato - board: renegade - board: jethubj80 - board: khadas-vim3l - board: rockpi-e - board: renegade - board: khadas-edge2 - board: bananapim5 - board: nanopi-r6s - board: nanopi-r4s - board: orangepiprime - board: nanopi-r5s - board: bigtreetech-cb1 - board: orangepi4-lts - board: orangepi5 - board: orangepizero3 - board: lafrite - board: tritium-h5 - board: orangepi5-plus - board: xiaomi-elish - board: rock-3a - board: jethubj100 - board: xiaomi-elish - board: qemu-uboot-arm64 - board: uefi-arm64 - board: sk-am64b - board: radxa-zero2 steps: - name: Install depen run: sudo apt-get -y install parallel jq axel - name: Cache Gradle packages uses: actions/cache@v2 env: cache-name: cache-build with: path: temp key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }} restore-keys: | ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }} #- # uses: styfle/cancel-workflow-action@0.9.1 # with: # access_token: ${{ github.token }} - name: Download from Armbian storage id: image-build run: | set +e BOARD="${{ matrix.board }}" DEBUG="no" FILES=$(cat temp/userdata.csv | grep ${BOARD}"/" | grep -v nightly | cut -d"|" -f2) # exclude nightly MIRRORS=$(curl -sq http://redirect.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq) # overrire with mirrors where file 100% exists # MIRRORS="http://stpete-mirror.armbian.com/dl/ http://imola.armbian.com/dl/" echo "BOARD=$BOARD" >> $GITHUB_ENV truncate -s 0 ${BOARD}.log for FILE in ${FILES[@]} do echo $FILE PARALLEL=$( for k in ${MIRRORS[@]} do echo "$k$FILE" done | parallel --halt now,fail=1 --jobs 32 wget -q --spider --timeout=15 --tries=4 --retry-connrefused {} 2>&1 >/dev/null) if [[ -n "${PARALLEL}" ]]; then REMOVE=$(echo $PARALLEL | egrep -o 'https?://[^ ]+/' | sed "s/$BOARD.*//" | sort | uniq) for l in ${REMOVE[@]}; do [[ "${DEBUG}" == yes ]] && echo "Removing mirror: $l" echo "Error,$l,$FILE" >> ${BOARD}.log MIRRORS=$(echo "${MIRRORS}" | sed "s|$l ||") done [[ $DEBUG == yes ]] && echo "Timeout: $(echo $PARALLEL | egrep -o 'https?://[^ ]+/' | cut -d"/" -f1-4 | sort | uniq)/${BOARD}/archive/$(echo $PARALLEL | egrep -o 'https?://[^ ]+' | rev | cut -d / -f 1 | rev)" fi # download DOWNLOAD="axel -c -a " for m in ${MIRRORS[@]}; do DOWNLOAD=${DOWNLOAD}" $m$FILE" done echo "https://github.com/${{ github.repository }}/releases/download/${FILE}" if ! wget -q --spider --timeout=15 --tries=3 --retry-connrefused https://github.com/"${{ github.repository }}"/releases/download/${FILE} ; then echo "Downloading ..." eval "${DOWNLOAD}" echo "UPLOAD=true" >> $GITHUB_ENV fi done - name: Upload artefacts if: ${{ env.UPLOAD == 'true' }} uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} file: Armbian* tag: "${{ matrix.board }}/archive" overwrite: true file_glob: true body: "

Armbian Linux

" - name: Upload logs uses: actions/upload-artifact@v2 with: name: logs path: ${{ env.BOARD }}.log Finish: needs: [ Build ] runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: name: logs - name: Display structure of downloaded files run: | function exceptions() { if [[ $k == *albony* ]]; then COUNTRY=India CITY=Nagpur FLAG="https://cdn.ipwhois.io/flags/in.svg" fi if [[ $k == *aliyun* ]]; then COUNTRY=China CITY=Hangzhou FLAG="https://cdn.ipwhois.io/flags/cn.svg" fi if [[ $k == *nju.edu* ]]; then COUNTRY=China CITY=Nanjing FLAG="https://cdn.ipwhois.io/flags/cn.svg" fi if [[ $k == *imola.armbian* ]]; then COUNTRY=Slovenia CITY=Ljubljana FLAG="https://cdn.ipwhois.io/flags/si.svg" fi if [[ $k == *mirrors.naho* ]]; then COUNTRY=Korea CITY=Seoul FLAG="https://cdn.ipwhois.io/flags/kr.svg" fi } function show_mirrors() { MIRRORS=$1 echo -en "\n## $2\n\n" echo "|URL                                                                                           |City                         |Flag|Country|" echo "|:--|:--|:--:|--:|" for k in ${MIRRORS[@]} do HOST=$(echo "$k" | awk -F/ '{print $3}') IP=$(dig +short $HOST | tail -1) RES=$( curl --max-time 5 -s http://ipwhois.app/json/${IP} | \ jq '.timezone, .city, .country, .country_flag' | \ while read -r TIMEZONE; do read -r COUNTRY echo "${TIMEZONE},${COUNTRY},${COUNTRYFLAG}" | tr --delete \" done ) TZDATA=$(echo ${RES} | cut -d"," -f1) CITY=$(echo ${RES} | cut -d"," -f2) COUNTRY=$(echo ${RES} | cut -d"," -f3 | xargs) FLAG=$(echo ${RES} | cut -d"," -f4) # automation is not perfect exceptions echo "|$k|$CITY||$COUNTRY|" done } echo '

Armbian logo
mirror sync

' > README.md echo -en "

 

\n" >> README.md echo -en "\n[![Sync mirror](https://github.com/armbian/mirror/actions/workflows/mirror-sync.yml/badge.svg)](https://github.com/armbian/mirror/actions/workflows/mirror-sync.yml) [![Rootfs sync](https://github.com/armbian/mirror/actions/workflows/rootfs-sync.yml/badge.svg)](https://github.com/armbian/mirror/actions/workflows/rootfs-sync.yml)" >> README.md echo " ## Mirroring Armbian? We are providing rsync service for stable images (dl), repository (apt), beta (beta) and archive (archive) at: rsync -av rsync://rsync.armbian.com/ Alternatively you can sync most valuable date from one of our mirror that provides rsync and is much faster: rsync -av rsync://mirrors.dotsrc.org/armbian-dl rsync -av rsync://mirrors.dotsrc.org/armbian-apt Space needs: 500Gb (images), 100Gb (packages) and (optional) 3TB for archives 1. Setup HTTP and HTTPS hostname 2. Setup cron to sync every 2-4 hours 3. [Inform us](https://www.armbian.com/#contact) If you want to seed our Torrent files, [check this document](https://forum.armbian.com/topic/4198-seed-our-torrents/). " >> README.md MIR=$(curl -sq http://redirect.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq) show_mirrors "$MIR" "Images" >> README.md echo "" >> README.md MIR=$(curl -sq http://apt.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq) show_mirrors "$MIR" "Packages" >> README.md echo "" >> README.md MIR=$(curl -sq http://archive.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq) show_mirrors "$MIR" "Archive" >> README.md echo "" >> README.md MIR=$(curl -sq http://beta.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq) show_mirrors "$MIR" "Beta" >> README.md echo "" >> README.md if [[ $(cat *.log | awk -F ',' '{print $2}' | sort | uniq -c | sort -n | wc -l) -gt 0 ]]; then echo -en "\n## Mirrors error count\n" >> README.md cat *.log | awk -F ',' '{print $2}' | sort | uniq -c | sort -n >> README.md echo -en "\n\n[Detailed report](detailed.txt)" >> README.md find *.log -not -empty -type f -print0 | xargs -0 -I % sh -c 'cat % | cut -d"," -f2-3 |tr -d \"," " ' | sort > detailed.txt fi - name: Add run: | git pull git add README.md git add detailed.txt - name: Commit files run: | git config --local user.email "info@armbian.com" git config --local user.name "Armbianworker" git commit --allow-empty -m "Update github actions" -a - name: Push changes uses: ad-m/github-push-action@master with: github_token: ${{ secrets.MIRROR }} branch: ${{ github.ref }}