armbian/armbian.github.io
0
0
Fork 0
mirror of https://github.com/armbian/armbian.github.io.git synced 2026-01-06 11:42:20 -08:00
Code Issues Packages Projects Releases Wiki Activity

Optimize workflow: cache NetBox data, simplify sync logic

Browse Source 
- Cache server configuration (name, path, port, username) in Prepare job matrix
  to avoid duplicate NetBox API calls in Sync and Index jobs
- Simplify Sync and Index jobs by removing excessive validations
  (data is already validated in Prepare job)
- Remove duplicate NetBox API fetches - reduce from 2N to 1+N where N is servers
- Simplify rsync case statements using sed for path transformation
  (debs → apt, debs-beta → beta)
- Add ENABLED input to external download workflow for conditional execution
- Reference repo_fix branch instead of main for external workflow during testing

Signed-off-by: Igor Pecovnik <igor@armbian.com>
This commit is contained in:
Igor Pecovnik
2026-01-04 13:00:48 +01:00
committed by Igor
parent 324dd240a4
commit 6a454f0824
1 changed files with 72 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

312
.github/workflows/infrastructure-repository-update.yml vendored
View File

@@ -757,23 +757,27 @@ jobs:
exit 1
fi
# Extract mirror names safely
mirror_names=$(echo "$response" | jq -r '.results[] | .name' 2>/dev/null | grep -v null || echo "")
# Extract mirror names and build enriched JSON with server configuration
# This avoids repeated NetBox API calls in downstream matrix jobs
mirror_json=$(echo "$response" | jq -r '.results[] | {
name: .name,
path: .custom_fields.path,
port: .custom_fields.port,
username: .custom_fields.username
} | select(.path != null and .port != null and .username != null)' | jq -s '.' 2>/dev/null)
if [[ -z "$mirror_names" ]]; then
if [[ -z "$mirror_json" || "$mirror_json" == "null" || "$mirror_json" == "[]" ]]; then
echo "::warning::No mirrors found in NetBox API response"
echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
echo '[]' >> $GITHUB_OUTPUT
echo 'EOF' >> $GITHUB_OUTPUT
else
# Count mirrors
mirror_count=$(echo "$mirror_names" | wc -l)
mirror_count=$(echo "$mirror_json" | jq 'length')
echo "Found $mirror_count active mirrors" | tee -a "$GITHUB_STEP_SUMMARY"
# Format as JSON array, filtering empty lines
mirror_json=$(echo "$mirror_names" | jq -cnR '[inputs | select(length>0)]' 2>/dev/null)
if [[ -z "$mirror_json" || "$mirror_json" == "null" ]]; then
# Validate JSON structure
if ! echo "$mirror_json" | jq empty 2>/dev/null; then
echo "::error::Failed to format mirror list as JSON"
exit 1
fi
@@ -786,7 +790,7 @@ jobs:
# List mirrors in summary
echo "" | tee -a "$GITHUB_STEP_SUMMARY"
echo "Mirror servers:" | tee -a "$GITHUB_STEP_SUMMARY"
echo "$mirror_names" | sed 's/^/ - /' | tee -a "$GITHUB_STEP_SUMMARY"
echo "$mirror_json" | jq -r '.[].name' | sed 's/^/ - /' | tee -a "$GITHUB_STEP_SUMMARY"
fi
Sync:
@@ -819,105 +823,29 @@ jobs:
set -e
set -o pipefail
# Validate secrets
if [[ -z "${{ secrets.NETBOX_API }}" || "${{ secrets.NETBOX_API }}" == "" ]]; then
echo "::error::NETBOX_API secret is not set or is empty"
exit 1
fi
# Use server configuration from matrix (fetched and validated in Prepare job)
HOSTNAME="${{ matrix.node.name }}"
SERVER_PATH="${{ matrix.node.path }}"
SERVER_PORT="${{ matrix.node.port }}"
SERVER_USERNAME="${{ matrix.node.username }}"
if [[ -z "${{ secrets.NETBOX_TOKEN }}" || "${{ secrets.NETBOX_TOKEN }}" == "" ]]; then
echo "::error::NETBOX_TOKEN secret is not set or is empty"
exit 1
fi
# Validate API URL format
NETBOX_API="${{ secrets.NETBOX_API }}"
if [[ ! "$NETBOX_API" =~ ^https?:// ]]; then
echo "::error::NETBOX_API must start with http:// or https://"
exit 1
fi
# Validate hostname format
HOSTNAME="${{ matrix.node }}"
if [[ ! "$HOSTNAME" =~ ^[a-zA-Z0-9.-]+$ ]]; then
echo "::error::Invalid hostname format: $HOSTNAME"
exit 1
fi
echo "### Fetching server configuration for $HOSTNAME" | tee -a "$GITHUB_STEP_SUMMARY"
echo "### Server: $HOSTNAME" | tee -a "$GITHUB_STEP_SUMMARY"
echo " Path: $SERVER_PATH" | tee -a "$GITHUB_STEP_SUMMARY"
echo " Port: $SERVER_PORT" | tee -a "$GITHUB_STEP_SUMMARY"
echo " Username: $SERVER_USERNAME" | tee -a "$GITHUB_STEP_SUMMARY"
echo ""
# Fetch server configuration with timeout
API_URL="${NETBOX_API}/virtualization/virtual-machines/?limit=500&name__empty=false&name=${HOSTNAME}"
# Fetch targets from NetBox API (need tags for determining sync targets)
API_URL="${{ secrets.NETBOX_API }}/virtualization/virtual-machines/?limit=500&name__empty=false&name=${HOSTNAME}"
response=$(curl -fsSL \
--max-time 30 \
--connect-timeout 10 \
-H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" \
-H "Accept: application/json" \
"$API_URL" 2>&1)
"$API_URL" 2>&1) || exit 1
curl_exit_code=$?
if [[ $curl_exit_code -ne 0 ]]; then
echo "::error::Failed to fetch server config (curl exit code: $curl_exit_code)"
echo "::error::Response: $response"
exit 1
fi
# Validate JSON response
if ! echo "$response" | jq empty 2>/dev/null; then
echo "::error::Invalid JSON response from NetBox API"
echo "::error::Response: $response"
exit 1
fi
# Extract and validate custom fields
SERVER_PATH=$(echo "$response" | jq -r '.results[] | .custom_fields["path"]' 2>/dev/null)
SERVER_PORT=$(echo "$response" | jq -r '.results[] | .custom_fields["port"]' 2>/dev/null)
SERVER_USERNAME=$(echo "$response" | jq -r '.results[] | .custom_fields["username"]' 2>/dev/null)
# Validate required fields
if [[ -z "$SERVER_PATH" || "$SERVER_PATH" == "null" ]]; then
echo "::error::Server path not found in NetBox for $HOSTNAME"
exit 1
fi
if [[ -z "$SERVER_PORT" || "$SERVER_PORT" == "null" ]]; then
echo "::error::Server port not found in NetBox for $HOSTNAME"
exit 1
fi
# Validate port is numeric and in valid range
if ! [[ "$SERVER_PORT" =~ ^[0-9]+$ ]] || [ "$SERVER_PORT" -lt 1 ] || [ "$SERVER_PORT" -gt 65535 ]; then
echo "::error::Invalid server port: $SERVER_PORT"
exit 1
fi
if [[ -z "$SERVER_USERNAME" || "$SERVER_USERNAME" == "null" ]]; then
echo "::error::Server username not found in NetBox for $HOSTNAME"
exit 1
fi
# Validate username format (alphanumeric, underscore, hyphen, dot)
if [[ ! "$SERVER_USERNAME" =~ ^[a-zA-Z0-9._-]+$ ]]; then
echo "::error::Invalid username format: $SERVER_USERNAME"
exit 1
fi
# Validate server path format (prevent path traversal)
if [[ "$SERVER_PATH" =~ \.\. ]]; then
echo "::error::Server path contains directory traversal: $SERVER_PATH"
exit 1
fi
echo "Server configuration:" | tee -a "$GITHUB_STEP_SUMMARY"
echo " Path: $SERVER_PATH" | tee -a "$GITHUB_STEP_SUMMARY"
echo " Port: $SERVER_PORT" | tee -a "$GITHUB_STEP_SUMMARY"
echo " Username: $SERVER_USERNAME" | tee -a "$GITHUB_STEP_SUMMARY"
echo ""
# Extract targets
# Extract targets from tags
TARGETS=($(echo "$response" | jq -r '.results[] | .tags[] | .name' 2>/dev/null | grep -v "Push" || echo ""))
if [[ ${#TARGETS[@]} -eq 0 ]]; then
@@ -975,39 +903,22 @@ jobs:
# Remove old host key
ssh-keygen -f "${HOME}/.ssh/known_hosts" -R "$HOSTNAME" 2>/dev/null || true
case "$target" in
debs)
REPO_PATH="${PUBLISHING_PATH}-debs"
if [[ ! -d "$REPO_PATH/public" ]]; then
echo "::error::Source repository path does not exist: $REPO_PATH/public"
exit 1
fi
RSYNC_CMD="rsync $RSYNC_OPTIONS -e \"ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30\" --exclude \"dists\" --exclude \"control\" \"$REPO_PATH/public/\" ${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/apt"
echo "Command: \`$RSYNC_CMD\`" | tee -a "$GITHUB_STEP_SUMMARY"
echo "" | tee -a "$GITHUB_STEP_SUMMARY"
rsync $RSYNC_OPTIONS -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \
--exclude "dists" --exclude "control" \
"$REPO_PATH/public/" \
${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/apt
;;
debs-beta)
REPO_PATH="${PUBLISHING_PATH}-debs-beta"
if [[ ! -d "$REPO_PATH/public" ]]; then
echo "::warning::Beta repository path does not exist: $REPO_PATH/public, skipping"
continue
fi
RSYNC_CMD="rsync $RSYNC_OPTIONS -e \"ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30\" --exclude \"dists\" --exclude \"control\" \"$REPO_PATH/public/\" ${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/beta"
echo "Command: \`$RSYNC_CMD\`" | tee -a "$GITHUB_STEP_SUMMARY"
echo "" | tee -a "$GITHUB_STEP_SUMMARY"
rsync $RSYNC_OPTIONS -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \
--exclude "dists" --exclude "control" \
"$REPO_PATH/public/" \
${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/beta
;;
*)
echo "::warning::Unknown target: $target"
;;
esac
REPO_PATH="${PUBLISHING_PATH}-${target}"
if [[ ! -d "$REPO_PATH/public" ]]; then
if [[ "$target" == "debs" ]]; then
echo "::error::Source repository path does not exist: $REPO_PATH/public"
exit 1
else
echo "::warning::Repository path does not exist: $REPO_PATH/public, skipping"
continue
fi
fi
DEST_PATH="${SERVER_PATH}/$(echo "$target" | sed 's/debs-beta$/beta/;s/^debs$/apt/')"
rsync $RSYNC_OPTIONS -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \
--exclude "dists" --exclude "control" \
"$REPO_PATH/public/" \
${SERVER_USERNAME}@${HOSTNAME}:"${DEST_PATH}"
done
echo "" | tee -a "$GITHUB_STEP_SUMMARY"
@@ -1041,99 +952,29 @@ jobs:
set -e
set -o pipefail
# Validate secrets
if [[ -z "${{ secrets.NETBOX_API }}" || "${{ secrets.NETBOX_API }}" == "" ]]; then
echo "::error::NETBOX_API secret is not set or is empty"
exit 1
fi
# Use server configuration from matrix (fetched and validated in Prepare job)
HOSTNAME="${{ matrix.node.name }}"
SERVER_PATH="${{ matrix.node.path }}"
SERVER_PORT="${{ matrix.node.port }}"
SERVER_USERNAME="${{ matrix.node.username }}"
if [[ -z "${{ secrets.NETBOX_TOKEN }}" || "${{ secrets.NETBOX_TOKEN }}" == "" ]]; then
echo "::error::NETBOX_TOKEN secret is not set or is empty"
exit 1
fi
# Validate API URL format
NETBOX_API="${{ secrets.NETBOX_API }}"
if [[ ! "$NETBOX_API" =~ ^https?:// ]]; then
echo "::error::NETBOX_API must start with http:// or https://"
exit 1
fi
# Validate hostname format
HOSTNAME="${{ matrix.node }}"
if [[ ! "$HOSTNAME" =~ ^[a-zA-Z0-9.-]+$ ]]; then
echo "::error::Invalid hostname format: $HOSTNAME"
exit 1
fi
echo "### Finalizing sync for $HOSTNAME" | tee -a "$GITHUB_STEP_SUMMARY"
echo "### Server: $HOSTNAME" | tee -a "$GITHUB_STEP_SUMMARY"
echo " Path: $SERVER_PATH" | tee -a "$GITHUB_STEP_SUMMARY"
echo " Port: $SERVER_PORT" | tee -a "$GITHUB_STEP_SUMMARY"
echo " Username: $SERVER_USERNAME" | tee -a "$GITHUB_STEP_SUMMARY"
echo ""
# Fetch server configuration with timeout
API_URL="${NETBOX_API}/virtualization/virtual-machines/?limit=500&name__empty=false&name=${HOSTNAME}"
# Fetch targets from NetBox API (need tags for determining sync targets)
API_URL="${{ secrets.NETBOX_API }}/virtualization/virtual-machines/?limit=500&name__empty=false&name=${HOSTNAME}"
response=$(curl -fsSL \
--max-time 30 \
--connect-timeout 10 \
-H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" \
-H "Accept: application/json" \
"$API_URL" 2>&1)
"$API_URL" 2>&1) || exit 1
curl_exit_code=$?
if [[ $curl_exit_code -ne 0 ]]; then
echo "::error::Failed to fetch server config (curl exit code: $curl_exit_code)"
echo "::error::Response: $response"
exit 1
fi
# Validate JSON response
if ! echo "$response" | jq empty 2>/dev/null; then
echo "::error::Invalid JSON response from NetBox API"
echo "::error::Response: $response"
exit 1
fi
# Extract and validate custom fields
SERVER_PATH=$(echo "$response" | jq -r '.results[] | .custom_fields["path"]' 2>/dev/null)
SERVER_PORT=$(echo "$response" | jq -r '.results[] | .custom_fields["port"]' 2>/dev/null)
SERVER_USERNAME=$(echo "$response" | jq -r '.results[] | .custom_fields["username"]' 2>/dev/null)
# Validate required fields
if [[ -z "$SERVER_PATH" || "$SERVER_PATH" == "null" ]]; then
echo "::error::Server path not found in NetBox for $HOSTNAME"
exit 1
fi
if [[ -z "$SERVER_PORT" || "$SERVER_PORT" == "null" ]]; then
echo "::error::Server port not found in NetBox for $HOSTNAME"
exit 1
fi
# Validate port is numeric and in valid range
if ! [[ "$SERVER_PORT" =~ ^[0-9]+$ ]] || [ "$SERVER_PORT" -lt 1 ] || [ "$SERVER_PORT" -gt 65535 ]; then
echo "::error::Invalid server port: $SERVER_PORT"
exit 1
fi
if [[ -z "$SERVER_USERNAME" || "$SERVER_USERNAME" == "null" ]]; then
echo "::error::Server username not found in NetBox for $HOSTNAME"
exit 1
fi
# Validate username format
if [[ ! "$SERVER_USERNAME" =~ ^[a-zA-Z0-9._-]+$ ]]; then
echo "::error::Invalid username format: $SERVER_USERNAME"
exit 1
fi
# Validate server path format (prevent path traversal)
if [[ "$SERVER_PATH" =~ \.\. ]]; then
echo "::error::Server path contains directory traversal: $SERVER_PATH"
exit 1
fi
# Extract targets
# Extract targets from tags
TARGETS=($(echo "$response" | jq -r '.results[] | .tags[] | .name' 2>/dev/null | grep -v "Push" || echo ""))
if [[ ${#TARGETS[@]} -eq 0 ]]; then
@@ -1186,32 +1027,23 @@ jobs:
for target in "${TARGETS[@]}"; do
echo "→ Finalizing $target" | tee -a "$GITHUB_STEP_SUMMARY"
case "$target" in
debs-beta)
REPO_PATH="${PUBLISHING_PATH}-debs-beta"
if [[ ! -d "$REPO_PATH/public" ]]; then
echo "::warning::Beta repository path does not exist: $REPO_PATH/public, skipping"
continue
fi
# Final sync without excludes
RSYNC_CMD="rsync $RSYNC_OPTIONS -e \"ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30\" \"$REPO_PATH/public/\" ${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/beta"
echo "Command (final sync): \`$RSYNC_CMD\`" | tee -a "$GITHUB_STEP_SUMMARY"
echo "" | tee -a "$GITHUB_STEP_SUMMARY"
rsync $RSYNC_OPTIONS -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \
"$REPO_PATH/public/" \
${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/beta
# Cleanup sync with --delete
RSYNC_CMD="rsync $RSYNC_OPTIONS --delete -e \"ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30\" \"$REPO_PATH/public/\" ${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/beta"
echo "Command (cleanup sync): \`$RSYNC_CMD\`" | tee -a "$GITHUB_STEP_SUMMARY"
echo "" | tee -a "$GITHUB_STEP_SUMMARY"
rsync $RSYNC_OPTIONS --delete -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \
"$REPO_PATH/public/" \
${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/beta
;;
*)
echo "::warning::Unknown target: $target"
;;
esac
REPO_PATH="${PUBLISHING_PATH}-${target}"
if [[ ! -d "$REPO_PATH/public" ]]; then
echo "::warning::Repository path does not exist: $REPO_PATH/public, skipping"
continue
fi
DEST_PATH="${SERVER_PATH}/$(echo "$target" | sed 's/debs-beta$/beta/')"
# Final sync without excludes
rsync $RSYNC_OPTIONS -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \
"$REPO_PATH/public/" \
${SERVER_USERNAME}@${HOSTNAME}:"${DEST_PATH}"
# Cleanup sync with --delete
rsync $RSYNC_OPTIONS --delete -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \
"$REPO_PATH/public/" \
${SERVER_USERNAME}@${HOSTNAME}:"${DEST_PATH}"
done
echo "" | tee -a "$GITHUB_STEP_SUMMARY"