mirror of
https://github.com/armbian/mirror.git
synced 2026-01-06 10:33:45 -08:00
231 lines
9.4 KiB
Plaintext
231 lines
9.4 KiB
Plaintext
name: Sync mirror
|
|
on:
|
|
# schedule:
|
|
# - cron: '0 1 * * *'
|
|
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
|
|
Prepare:
|
|
runs-on: ubuntu-20.04
|
|
if: ${{ github.repository_owner == 'Armbian' }}
|
|
steps:
|
|
-
|
|
run: |
|
|
wget -P temp https://www.armbian.com/userdata.csv
|
|
|
|
- name: Cache Gradle packages
|
|
uses: actions/cache@v2
|
|
env:
|
|
cache-name: cache-build
|
|
with:
|
|
path: temp
|
|
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
|
|
|
|
Build: # short name because GH will expand with the matrix values
|
|
needs: [ Prepare ]
|
|
runs-on: ubuntu-20.04
|
|
timeout-minutes: 120
|
|
strategy:
|
|
max-parallel: 4
|
|
fail-fast: false # let other jobs try to complete if one fails
|
|
matrix:
|
|
include: # build this way: cat userdata.csv | cut -d"/" -f1 | uniq | sed 's/.*/ - board: &/'
|
|
steps:
|
|
- name: Install depen
|
|
run: sudo apt-get -y install parallel jq axel
|
|
- name: Cache Gradle packages
|
|
uses: actions/cache@v2
|
|
env:
|
|
cache-name: cache-build
|
|
with:
|
|
path: temp
|
|
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
|
|
restore-keys: |
|
|
${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
|
|
|
|
#-
|
|
# uses: styfle/cancel-workflow-action@0.9.1
|
|
# with:
|
|
# access_token: ${{ github.token }}
|
|
-
|
|
name: Download from Armbian storage
|
|
id: image-build
|
|
run: |
|
|
|
|
set +e
|
|
BOARD="${{ matrix.board }}"
|
|
DEBUG="no"
|
|
|
|
FILES=$(cat temp/userdata.csv | grep ${BOARD}"/" | grep -v nightly | cut -d"|" -f2) # exclude nightly
|
|
MIRRORS=$(curl -sq http://redirect.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq)
|
|
# overrire with mirrors where file 100% exists
|
|
# MIRRORS="http://stpete-mirror.armbian.com/dl/ http://imola.armbian.com/dl/"
|
|
|
|
echo "BOARD=$BOARD" >> $GITHUB_ENV
|
|
truncate -s 0 ${BOARD}.log
|
|
|
|
for FILE in ${FILES[@]}
|
|
do
|
|
echo $FILE
|
|
PARALLEL=$(
|
|
for k in ${MIRRORS[@]}
|
|
do
|
|
echo "$k$FILE"
|
|
done | parallel --halt now,fail=1 --jobs 32 wget -q --spider --timeout=15 --tries=4 --retry-connrefused {} 2>&1 >/dev/null)
|
|
if [[ -n "${PARALLEL}" ]]; then
|
|
REMOVE=$(echo $PARALLEL | egrep -o 'https?://[^ ]+/' | sed "s/$BOARD.*//" | sort | uniq)
|
|
for l in ${REMOVE[@]}; do
|
|
[[ "${DEBUG}" == yes ]] && echo "Removing mirror: $l"
|
|
echo "Error,$l,$FILE" >> ${BOARD}.log
|
|
MIRRORS=$(echo "${MIRRORS}" | sed "s|$l ||")
|
|
done
|
|
[[ $DEBUG == yes ]] && echo "Timeout: $(echo $PARALLEL | egrep -o 'https?://[^ ]+/' | cut -d"/" -f1-4 | sort | uniq)/${BOARD}/archive/$(echo $PARALLEL | egrep -o 'https?://[^ ]+' | rev | cut -d / -f 1 | rev)"
|
|
fi
|
|
# download
|
|
DOWNLOAD="axel -c -a "
|
|
for m in ${MIRRORS[@]}; do
|
|
DOWNLOAD=${DOWNLOAD}" $m$FILE"
|
|
done
|
|
echo "https://github.com/${{ github.repository }}/releases/download/${FILE}"
|
|
if ! wget -q --spider --timeout=15 --tries=3 --retry-connrefused https://github.com/"${{ github.repository }}"/releases/download/${FILE} ; then
|
|
echo "Downloading ..."
|
|
eval "${DOWNLOAD}"
|
|
echo "UPLOAD=true" >> $GITHUB_ENV
|
|
fi
|
|
done
|
|
- name: Upload artefacts
|
|
if: ${{ env.UPLOAD == 'true' }}
|
|
uses: svenstaro/upload-release-action@v2
|
|
with:
|
|
repo_token: ${{ secrets.GITHUB_TOKEN }}
|
|
file: Armbian*
|
|
tag: "${{ matrix.board }}/archive"
|
|
overwrite: true
|
|
file_glob: true
|
|
body: "<p align='center'><a href='https://www.armbian.com'><img src='https://raw.githubusercontent.com/armbian/.github/master/profile/tux-two.png' width='400' height='226' alt='Armbian Linux'></a></p>"
|
|
- name: Upload logs
|
|
uses: actions/upload-artifact@v2
|
|
with:
|
|
name: logs
|
|
path: ${{ env.BOARD }}.log
|
|
|
|
Finish:
|
|
needs: [ Build ]
|
|
runs-on: ubuntu-20.04
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
|
|
- uses: actions/download-artifact@v2
|
|
with:
|
|
name: logs
|
|
|
|
- name: Display structure of downloaded files
|
|
run: |
|
|
|
|
function exceptions()
|
|
{
|
|
if [[ $k == *aliyun* ]]; then
|
|
COUNTRY=China
|
|
CITY=Hangzhou
|
|
FLAG="https://cdn.ipwhois.io/flags/cn.svg"
|
|
fi
|
|
if [[ $k == *nju.edu* ]]; then
|
|
COUNTRY=China
|
|
CITY=Nanjing
|
|
FLAG="https://cdn.ipwhois.io/flags/cn.svg"
|
|
fi
|
|
if [[ $k == *imola.armbian* ]]; then
|
|
COUNTRY=Slovenia
|
|
CITY=Ljubljana
|
|
FLAG="https://cdn.ipwhois.io/flags/si.svg"
|
|
fi
|
|
}
|
|
|
|
function show_mirrors()
|
|
{
|
|
MIRRORS=$1
|
|
echo -en "\n## $2\n\n"
|
|
echo "|URL |City |Flag|Country |"
|
|
echo "|:--|:--|:--:|--:|"
|
|
for k in ${MIRRORS[@]}
|
|
do
|
|
HOST=$(echo "$k" | awk -F/ '{print $3}')
|
|
IP=$(dig +short $HOST | tail -1)
|
|
RES=$(
|
|
curl --max-time 5 -s http://ipwhois.app/json/${IP} | \
|
|
jq '.timezone, .city, .country, .country_flag' | \
|
|
while read -r TIMEZONE; do
|
|
read -r COUNTRY
|
|
echo "${TIMEZONE},${COUNTRY},${COUNTRYFLAG}" | tr --delete \"
|
|
done
|
|
)
|
|
TZDATA=$(echo ${RES} | cut -d"," -f1)
|
|
CITY=$(echo ${RES} | cut -d"," -f2)
|
|
COUNTRY=$(echo ${RES} | cut -d"," -f3 | xargs)
|
|
FLAG=$(echo ${RES} | cut -d"," -f4)
|
|
|
|
# automation is not perfect
|
|
exceptions
|
|
|
|
echo "|$k|$CITY|<img width=24 src=$FLAG>|$COUNTRY|"
|
|
done
|
|
}
|
|
|
|
echo '<h3 align=center><a href="#build-tools"><img src="https://raw.githubusercontent.com/armbian/build/master/.github/armbian-logo.png" alt="Armbian logo" width="144"></a><br>mirror sync</h3>' > README.md
|
|
echo -en "<p align=right> </p>\n" >> README.md
|
|
echo -en "\n[](https://github.com/armbian/mirror/actions/workflows/mirror-sync.yml) [](https://github.com/armbian/mirror/actions/workflows/rootfs-sync.yml)" >> README.md
|
|
echo "
|
|
## Mirroring Armbian?
|
|
|
|
We are providing rsync service for stable images (dl), repository (apt), beta (beta) and archive (archive) at:
|
|
|
|
rsync -av rsync://rsync.armbian.com/
|
|
|
|
Alternatively you can sync most valuable date from one of our mirror that provides rsync and is much faster:
|
|
|
|
rsync -av rsync://mirrors.dotsrc.org/armbian-dl
|
|
rsync -av rsync://mirrors.dotsrc.org/armbian-apt
|
|
|
|
Space needs: 500Gb (images), 100Gb (packages) and (optional) 3TB for archives
|
|
|
|
1. Setup HTTP and HTTPS hostname
|
|
2. Setup cron to sync every 2-4 hours
|
|
3. [Inform us](https://www.armbian.com/#contact)
|
|
" >> README.md
|
|
MIR=$(curl -sq http://redirect.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq)
|
|
show_mirrors "$MIR" "Images" >> README.md
|
|
echo "" >> README.md
|
|
MIR=$(curl -sq http://apt.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq)
|
|
show_mirrors "$MIR" "Packages" >> README.md
|
|
echo "" >> README.md
|
|
MIR=$(curl -sq http://archive.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq)
|
|
show_mirrors "$MIR" "Archive" >> README.md
|
|
echo "" >> README.md
|
|
MIR=$(curl -sq http://beta.armbian.com/mirrors | jq -Mr '.' | grep http | tr -d \"," " | sort | uniq)
|
|
show_mirrors "$MIR" "Beta" >> README.md
|
|
echo "" >> README.md
|
|
if [[ $(cat *.log | awk -F ',' '{print $2}' | sort | uniq -c | sort -n | wc -l) -gt 0 ]]; then
|
|
echo -en "\n## Mirrors error count\n" >> README.md
|
|
cat *.log | awk -F ',' '{print $2}' | sort | uniq -c | sort -n >> README.md
|
|
echo -en "\n\n[Detailed report](detailed.txt)" >> README.md
|
|
find *.log -not -empty -type f -print0 | xargs -0 -I % sh -c 'cat % | cut -d"," -f2-3 |tr -d \"," " ' | sort > detailed.txt
|
|
fi
|
|
- name: Add
|
|
run: |
|
|
git pull
|
|
git add README.md
|
|
git add detailed.txt
|
|
|
|
- name: Commit files
|
|
run: |
|
|
git config --local user.email "info@armbian.com"
|
|
git config --local user.name "Armbianworker"
|
|
git commit --allow-empty -m "Update github actions" -a
|
|
- name: Push changes
|
|
uses: ad-m/github-push-action@master
|
|
with:
|
|
github_token: ${{ secrets.MIRROR }}
|
|
branch: ${{ github.ref }}
|