diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index bb54a386..78994e02 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -757,23 +757,27 @@ jobs: exit 1 fi - # Extract mirror names safely - mirror_names=$(echo "$response" | jq -r '.results[] | .name' 2>/dev/null | grep -v null || echo "") + # Extract mirror names and build enriched JSON with server configuration + # This avoids repeated NetBox API calls in downstream matrix jobs + mirror_json=$(echo "$response" | jq -r '.results[] | { + name: .name, + path: .custom_fields.path, + port: .custom_fields.port, + username: .custom_fields.username + } | select(.path != null and .port != null and .username != null)' | jq -s '.' 2>/dev/null) - if [[ -z "$mirror_names" ]]; then + if [[ -z "$mirror_json" || "$mirror_json" == "null" || "$mirror_json" == "[]" ]]; then echo "::warning::No mirrors found in NetBox API response" echo 'JSON_CONTENT<> $GITHUB_OUTPUT echo '[]' >> $GITHUB_OUTPUT echo 'EOF' >> $GITHUB_OUTPUT else # Count mirrors - mirror_count=$(echo "$mirror_names" | wc -l) + mirror_count=$(echo "$mirror_json" | jq 'length') echo "Found $mirror_count active mirrors" | tee -a "$GITHUB_STEP_SUMMARY" - # Format as JSON array, filtering empty lines - mirror_json=$(echo "$mirror_names" | jq -cnR '[inputs | select(length>0)]' 2>/dev/null) - - if [[ -z "$mirror_json" || "$mirror_json" == "null" ]]; then + # Validate JSON structure + if ! echo "$mirror_json" | jq empty 2>/dev/null; then echo "::error::Failed to format mirror list as JSON" exit 1 fi @@ -786,7 +790,7 @@ jobs: # List mirrors in summary echo "" | tee -a "$GITHUB_STEP_SUMMARY" echo "Mirror servers:" | tee -a "$GITHUB_STEP_SUMMARY" - echo "$mirror_names" | sed 's/^/ - /' | tee -a "$GITHUB_STEP_SUMMARY" + echo "$mirror_json" | jq -r '.[].name' | sed 's/^/ - /' | tee -a "$GITHUB_STEP_SUMMARY" fi Sync: @@ -819,105 +823,29 @@ jobs: set -e set -o pipefail - # Validate secrets - if [[ -z "${{ secrets.NETBOX_API }}" || "${{ secrets.NETBOX_API }}" == "" ]]; then - echo "::error::NETBOX_API secret is not set or is empty" - exit 1 - fi + # Use server configuration from matrix (fetched and validated in Prepare job) + HOSTNAME="${{ matrix.node.name }}" + SERVER_PATH="${{ matrix.node.path }}" + SERVER_PORT="${{ matrix.node.port }}" + SERVER_USERNAME="${{ matrix.node.username }}" - if [[ -z "${{ secrets.NETBOX_TOKEN }}" || "${{ secrets.NETBOX_TOKEN }}" == "" ]]; then - echo "::error::NETBOX_TOKEN secret is not set or is empty" - exit 1 - fi - - # Validate API URL format - NETBOX_API="${{ secrets.NETBOX_API }}" - if [[ ! "$NETBOX_API" =~ ^https?:// ]]; then - echo "::error::NETBOX_API must start with http:// or https://" - exit 1 - fi - - # Validate hostname format - HOSTNAME="${{ matrix.node }}" - if [[ ! "$HOSTNAME" =~ ^[a-zA-Z0-9.-]+$ ]]; then - echo "::error::Invalid hostname format: $HOSTNAME" - exit 1 - fi - - echo "### Fetching server configuration for $HOSTNAME" | tee -a "$GITHUB_STEP_SUMMARY" + echo "### Server: $HOSTNAME" | tee -a "$GITHUB_STEP_SUMMARY" + echo " Path: $SERVER_PATH" | tee -a "$GITHUB_STEP_SUMMARY" + echo " Port: $SERVER_PORT" | tee -a "$GITHUB_STEP_SUMMARY" + echo " Username: $SERVER_USERNAME" | tee -a "$GITHUB_STEP_SUMMARY" echo "" - # Fetch server configuration with timeout - API_URL="${NETBOX_API}/virtualization/virtual-machines/?limit=500&name__empty=false&name=${HOSTNAME}" + # Fetch targets from NetBox API (need tags for determining sync targets) + API_URL="${{ secrets.NETBOX_API }}/virtualization/virtual-machines/?limit=500&name__empty=false&name=${HOSTNAME}" response=$(curl -fsSL \ --max-time 30 \ --connect-timeout 10 \ -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" \ -H "Accept: application/json" \ - "$API_URL" 2>&1) + "$API_URL" 2>&1) || exit 1 - curl_exit_code=$? - - if [[ $curl_exit_code -ne 0 ]]; then - echo "::error::Failed to fetch server config (curl exit code: $curl_exit_code)" - echo "::error::Response: $response" - exit 1 - fi - - # Validate JSON response - if ! echo "$response" | jq empty 2>/dev/null; then - echo "::error::Invalid JSON response from NetBox API" - echo "::error::Response: $response" - exit 1 - fi - - # Extract and validate custom fields - SERVER_PATH=$(echo "$response" | jq -r '.results[] | .custom_fields["path"]' 2>/dev/null) - SERVER_PORT=$(echo "$response" | jq -r '.results[] | .custom_fields["port"]' 2>/dev/null) - SERVER_USERNAME=$(echo "$response" | jq -r '.results[] | .custom_fields["username"]' 2>/dev/null) - - # Validate required fields - if [[ -z "$SERVER_PATH" || "$SERVER_PATH" == "null" ]]; then - echo "::error::Server path not found in NetBox for $HOSTNAME" - exit 1 - fi - - if [[ -z "$SERVER_PORT" || "$SERVER_PORT" == "null" ]]; then - echo "::error::Server port not found in NetBox for $HOSTNAME" - exit 1 - fi - - # Validate port is numeric and in valid range - if ! [[ "$SERVER_PORT" =~ ^[0-9]+$ ]] || [ "$SERVER_PORT" -lt 1 ] || [ "$SERVER_PORT" -gt 65535 ]; then - echo "::error::Invalid server port: $SERVER_PORT" - exit 1 - fi - - if [[ -z "$SERVER_USERNAME" || "$SERVER_USERNAME" == "null" ]]; then - echo "::error::Server username not found in NetBox for $HOSTNAME" - exit 1 - fi - - # Validate username format (alphanumeric, underscore, hyphen, dot) - if [[ ! "$SERVER_USERNAME" =~ ^[a-zA-Z0-9._-]+$ ]]; then - echo "::error::Invalid username format: $SERVER_USERNAME" - exit 1 - fi - - # Validate server path format (prevent path traversal) - if [[ "$SERVER_PATH" =~ \.\. ]]; then - echo "::error::Server path contains directory traversal: $SERVER_PATH" - exit 1 - fi - - echo "Server configuration:" | tee -a "$GITHUB_STEP_SUMMARY" - echo " Path: $SERVER_PATH" | tee -a "$GITHUB_STEP_SUMMARY" - echo " Port: $SERVER_PORT" | tee -a "$GITHUB_STEP_SUMMARY" - echo " Username: $SERVER_USERNAME" | tee -a "$GITHUB_STEP_SUMMARY" - echo "" - - # Extract targets + # Extract targets from tags TARGETS=($(echo "$response" | jq -r '.results[] | .tags[] | .name' 2>/dev/null | grep -v "Push" || echo "")) if [[ ${#TARGETS[@]} -eq 0 ]]; then @@ -975,39 +903,22 @@ jobs: # Remove old host key ssh-keygen -f "${HOME}/.ssh/known_hosts" -R "$HOSTNAME" 2>/dev/null || true - case "$target" in - debs) - REPO_PATH="${PUBLISHING_PATH}-debs" - if [[ ! -d "$REPO_PATH/public" ]]; then - echo "::error::Source repository path does not exist: $REPO_PATH/public" - exit 1 - fi - RSYNC_CMD="rsync $RSYNC_OPTIONS -e \"ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30\" --exclude \"dists\" --exclude \"control\" \"$REPO_PATH/public/\" ${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/apt" - echo "Command: \`$RSYNC_CMD\`" | tee -a "$GITHUB_STEP_SUMMARY" - echo "" | tee -a "$GITHUB_STEP_SUMMARY" - rsync $RSYNC_OPTIONS -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \ - --exclude "dists" --exclude "control" \ - "$REPO_PATH/public/" \ - ${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/apt - ;; - debs-beta) - REPO_PATH="${PUBLISHING_PATH}-debs-beta" - if [[ ! -d "$REPO_PATH/public" ]]; then - echo "::warning::Beta repository path does not exist: $REPO_PATH/public, skipping" - continue - fi - RSYNC_CMD="rsync $RSYNC_OPTIONS -e \"ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30\" --exclude \"dists\" --exclude \"control\" \"$REPO_PATH/public/\" ${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/beta" - echo "Command: \`$RSYNC_CMD\`" | tee -a "$GITHUB_STEP_SUMMARY" - echo "" | tee -a "$GITHUB_STEP_SUMMARY" - rsync $RSYNC_OPTIONS -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \ - --exclude "dists" --exclude "control" \ - "$REPO_PATH/public/" \ - ${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/beta - ;; - *) - echo "::warning::Unknown target: $target" - ;; - esac + REPO_PATH="${PUBLISHING_PATH}-${target}" + if [[ ! -d "$REPO_PATH/public" ]]; then + if [[ "$target" == "debs" ]]; then + echo "::error::Source repository path does not exist: $REPO_PATH/public" + exit 1 + else + echo "::warning::Repository path does not exist: $REPO_PATH/public, skipping" + continue + fi + fi + + DEST_PATH="${SERVER_PATH}/$(echo "$target" | sed 's/debs-beta$/beta/;s/^debs$/apt/')" + rsync $RSYNC_OPTIONS -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \ + --exclude "dists" --exclude "control" \ + "$REPO_PATH/public/" \ + ${SERVER_USERNAME}@${HOSTNAME}:"${DEST_PATH}" done echo "" | tee -a "$GITHUB_STEP_SUMMARY" @@ -1041,99 +952,29 @@ jobs: set -e set -o pipefail - # Validate secrets - if [[ -z "${{ secrets.NETBOX_API }}" || "${{ secrets.NETBOX_API }}" == "" ]]; then - echo "::error::NETBOX_API secret is not set or is empty" - exit 1 - fi + # Use server configuration from matrix (fetched and validated in Prepare job) + HOSTNAME="${{ matrix.node.name }}" + SERVER_PATH="${{ matrix.node.path }}" + SERVER_PORT="${{ matrix.node.port }}" + SERVER_USERNAME="${{ matrix.node.username }}" - if [[ -z "${{ secrets.NETBOX_TOKEN }}" || "${{ secrets.NETBOX_TOKEN }}" == "" ]]; then - echo "::error::NETBOX_TOKEN secret is not set or is empty" - exit 1 - fi - - # Validate API URL format - NETBOX_API="${{ secrets.NETBOX_API }}" - if [[ ! "$NETBOX_API" =~ ^https?:// ]]; then - echo "::error::NETBOX_API must start with http:// or https://" - exit 1 - fi - - # Validate hostname format - HOSTNAME="${{ matrix.node }}" - if [[ ! "$HOSTNAME" =~ ^[a-zA-Z0-9.-]+$ ]]; then - echo "::error::Invalid hostname format: $HOSTNAME" - exit 1 - fi - - echo "### Finalizing sync for $HOSTNAME" | tee -a "$GITHUB_STEP_SUMMARY" + echo "### Server: $HOSTNAME" | tee -a "$GITHUB_STEP_SUMMARY" + echo " Path: $SERVER_PATH" | tee -a "$GITHUB_STEP_SUMMARY" + echo " Port: $SERVER_PORT" | tee -a "$GITHUB_STEP_SUMMARY" + echo " Username: $SERVER_USERNAME" | tee -a "$GITHUB_STEP_SUMMARY" echo "" - # Fetch server configuration with timeout - API_URL="${NETBOX_API}/virtualization/virtual-machines/?limit=500&name__empty=false&name=${HOSTNAME}" + # Fetch targets from NetBox API (need tags for determining sync targets) + API_URL="${{ secrets.NETBOX_API }}/virtualization/virtual-machines/?limit=500&name__empty=false&name=${HOSTNAME}" response=$(curl -fsSL \ --max-time 30 \ --connect-timeout 10 \ -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" \ -H "Accept: application/json" \ - "$API_URL" 2>&1) + "$API_URL" 2>&1) || exit 1 - curl_exit_code=$? - - if [[ $curl_exit_code -ne 0 ]]; then - echo "::error::Failed to fetch server config (curl exit code: $curl_exit_code)" - echo "::error::Response: $response" - exit 1 - fi - - # Validate JSON response - if ! echo "$response" | jq empty 2>/dev/null; then - echo "::error::Invalid JSON response from NetBox API" - echo "::error::Response: $response" - exit 1 - fi - - # Extract and validate custom fields - SERVER_PATH=$(echo "$response" | jq -r '.results[] | .custom_fields["path"]' 2>/dev/null) - SERVER_PORT=$(echo "$response" | jq -r '.results[] | .custom_fields["port"]' 2>/dev/null) - SERVER_USERNAME=$(echo "$response" | jq -r '.results[] | .custom_fields["username"]' 2>/dev/null) - - # Validate required fields - if [[ -z "$SERVER_PATH" || "$SERVER_PATH" == "null" ]]; then - echo "::error::Server path not found in NetBox for $HOSTNAME" - exit 1 - fi - - if [[ -z "$SERVER_PORT" || "$SERVER_PORT" == "null" ]]; then - echo "::error::Server port not found in NetBox for $HOSTNAME" - exit 1 - fi - - # Validate port is numeric and in valid range - if ! [[ "$SERVER_PORT" =~ ^[0-9]+$ ]] || [ "$SERVER_PORT" -lt 1 ] || [ "$SERVER_PORT" -gt 65535 ]; then - echo "::error::Invalid server port: $SERVER_PORT" - exit 1 - fi - - if [[ -z "$SERVER_USERNAME" || "$SERVER_USERNAME" == "null" ]]; then - echo "::error::Server username not found in NetBox for $HOSTNAME" - exit 1 - fi - - # Validate username format - if [[ ! "$SERVER_USERNAME" =~ ^[a-zA-Z0-9._-]+$ ]]; then - echo "::error::Invalid username format: $SERVER_USERNAME" - exit 1 - fi - - # Validate server path format (prevent path traversal) - if [[ "$SERVER_PATH" =~ \.\. ]]; then - echo "::error::Server path contains directory traversal: $SERVER_PATH" - exit 1 - fi - - # Extract targets + # Extract targets from tags TARGETS=($(echo "$response" | jq -r '.results[] | .tags[] | .name' 2>/dev/null | grep -v "Push" || echo "")) if [[ ${#TARGETS[@]} -eq 0 ]]; then @@ -1186,32 +1027,23 @@ jobs: for target in "${TARGETS[@]}"; do echo "→ Finalizing $target" | tee -a "$GITHUB_STEP_SUMMARY" - case "$target" in - debs-beta) - REPO_PATH="${PUBLISHING_PATH}-debs-beta" - if [[ ! -d "$REPO_PATH/public" ]]; then - echo "::warning::Beta repository path does not exist: $REPO_PATH/public, skipping" - continue - fi - # Final sync without excludes - RSYNC_CMD="rsync $RSYNC_OPTIONS -e \"ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30\" \"$REPO_PATH/public/\" ${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/beta" - echo "Command (final sync): \`$RSYNC_CMD\`" | tee -a "$GITHUB_STEP_SUMMARY" - echo "" | tee -a "$GITHUB_STEP_SUMMARY" - rsync $RSYNC_OPTIONS -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \ - "$REPO_PATH/public/" \ - ${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/beta - # Cleanup sync with --delete - RSYNC_CMD="rsync $RSYNC_OPTIONS --delete -e \"ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30\" \"$REPO_PATH/public/\" ${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/beta" - echo "Command (cleanup sync): \`$RSYNC_CMD\`" | tee -a "$GITHUB_STEP_SUMMARY" - echo "" | tee -a "$GITHUB_STEP_SUMMARY" - rsync $RSYNC_OPTIONS --delete -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \ - "$REPO_PATH/public/" \ - ${SERVER_USERNAME}@${HOSTNAME}:${SERVER_PATH}/beta - ;; - *) - echo "::warning::Unknown target: $target" - ;; - esac + REPO_PATH="${PUBLISHING_PATH}-${target}" + if [[ ! -d "$REPO_PATH/public" ]]; then + echo "::warning::Repository path does not exist: $REPO_PATH/public, skipping" + continue + fi + + DEST_PATH="${SERVER_PATH}/$(echo "$target" | sed 's/debs-beta$/beta/')" + + # Final sync without excludes + rsync $RSYNC_OPTIONS -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \ + "$REPO_PATH/public/" \ + ${SERVER_USERNAME}@${HOSTNAME}:"${DEST_PATH}" + + # Cleanup sync with --delete + rsync $RSYNC_OPTIONS --delete -e "ssh -p ${SERVER_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30" \ + "$REPO_PATH/public/" \ + ${SERVER_USERNAME}@${HOSTNAME}:"${DEST_PATH}" done echo "" | tee -a "$GITHUB_STEP_SUMMARY"