mirror of
https://github.com/armbian/gitutils.git
synced 2026-01-06 10:36:09 -08:00
first try
This commit is contained in:
Ricardo Pardini
51
.github/workflows/main-latest.yml
vendored
Normal file
51
.github/workflows/main-latest.yml
vendored
Normal file
@@ -0,0 +1,51 @@
|
||||
name: main-latest
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
|
||||
jobs:
|
||||
|
||||
prepare:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Prepare release ID
|
||||
id: prep
|
||||
run: |
|
||||
echo ::set-output name=created::$(date -u +'%Y%m%d-%H%M')
|
||||
outputs:
|
||||
created: ${{ steps.prep.outputs.created }} # refer to as ${{needs.prepare.outputs.created}}
|
||||
|
||||
|
||||
build:
|
||||
needs: [ prepare ]
|
||||
runs-on: ubuntu-latest # soon to support hosted runners for arm64
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Cache git worktree
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: /tmp/workdir/kernel/worktree
|
||||
key: ${{ runner.os }}-kernel-worktree-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-kernel-worktree
|
||||
|
||||
- name: Prepare git trees, bulk of work
|
||||
run: |
|
||||
BASE_WORK_DIR="/tmp/workdir" bash shallow_kernel_tree.sh
|
||||
|
||||
|
||||
- name: "GH latest release"
|
||||
uses: "marvinpinto/action-automatic-releases@latest"
|
||||
with:
|
||||
repo_token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
automatic_release_tag: "latest"
|
||||
prerelease: false
|
||||
title: "Latest: ${{needs.prepare.outputs.created}}"
|
||||
files: |
|
||||
/tmp/workdir/output/*.gitshallow
|
||||
/tmp/workdir/output/*.gitbundle
|
||||
|
||||
31
lib_git.sh
Normal file
31
lib_git.sh
Normal file
@@ -0,0 +1,31 @@
|
||||
function display_alert() {
|
||||
echo "--> $*"
|
||||
}
|
||||
|
||||
function run_host_command_logged() {
|
||||
echo "==> $*"
|
||||
"$@"
|
||||
}
|
||||
|
||||
|
||||
function git_fetch_from_bundle_file() {
|
||||
local bundle_file="${1}" remote_name="${2}" shallow_file="${3}"
|
||||
git bundle verify "${bundle_file}" # Make sure bundle is valid.
|
||||
git remote add "${remote_name}" "${bundle_file}" # Add the remote pointing to the cold bundle file
|
||||
if [[ -f "${shallow_file}" ]]; then
|
||||
display_alert "Bundle is shallow" "${shallow_file}" "git"
|
||||
cp -p "${shallow_file}" ".git/shallow"
|
||||
fi
|
||||
git fetch --progress --verbose --tags "${remote_name}" # Fetch it! (including tags!)
|
||||
display_alert "Bundle fetch '${remote_name}' completed"
|
||||
}
|
||||
|
||||
function download_git_bundle_from_http() {
|
||||
local bundle_file="${1}" bundle_url="${2}"
|
||||
if [[ ! -f "${git_cold_bundle_cache_file}" ]]; then # Download the bundle file if it does not exist.
|
||||
display_alert "Downloading Git cold bundle via HTTP" "${bundle_url}" "info" # This gonna take a while. And waste bandwidth
|
||||
run_host_command_logged wget --continue --progress=dot:giga --output-document="${bundle_file}" "${bundle_url}"
|
||||
else
|
||||
display_alert "Cold bundle file exists, using it" "${bundle_file}" "git"
|
||||
fi
|
||||
}
|
||||
171
shallow_kernel_tree.sh
Executable file
171
shallow_kernel_tree.sh
Executable file
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
# source the lib
|
||||
. lib_git.sh
|
||||
|
||||
ONLINE="yes"
|
||||
|
||||
BASE_WORK_DIR="${BASE_WORK_DIR:-"/Volumes/LinuxDev/shallow_git_tree_work"}"
|
||||
mkdir -p "${BASE_WORK_DIR}"
|
||||
|
||||
WORKDIR="${BASE_WORK_DIR}/kernel"
|
||||
mkdir -p "${WORKDIR}"
|
||||
|
||||
SHALLOWED_TREES_DIR="${WORKDIR}/shallow_trees"
|
||||
mkdir -p "${SHALLOWED_TREES_DIR}"
|
||||
|
||||
OUTPUT_DIR="${WORKDIR}/output"
|
||||
mkdir -p "${OUTPUT_DIR}"
|
||||
|
||||
KERNEL_GIT_TREE="${WORKDIR}/worktree"
|
||||
mkdir -p "${KERNEL_GIT_TREE}"
|
||||
|
||||
KERNEL_TORVALDS_BUNDLE_DIR="${WORKDIR}/bundle-torvalds"
|
||||
mkdir -p "${KERNEL_TORVALDS_BUNDLE_DIR}"
|
||||
|
||||
GIT_TORVALDS_BUNDLE_URL="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/clone.bundle"
|
||||
GIT_TORVALDS_BUNDLE_ID="$(echo -n "${GIT_TORVALDS_BUNDLE_URL}" | md5sum | awk '{print $1}')" # md5 of the URL.
|
||||
GIT_TORVALDS_BUNDLE_FILE="${KERNEL_TORVALDS_BUNDLE_DIR}/${GIT_TORVALDS_BUNDLE_ID}.gitbundle" # final filename of bundle
|
||||
GIT_TORVALDS_BUNDLE_REMOTE_NAME="torvalds-gitbundle" # name of the remote that will point to bundle
|
||||
|
||||
# TORVALDS LIVE info
|
||||
GIT_TORVALDS_LIVE_GIT_URL="git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git"
|
||||
GIT_TORVALDS_LIVE_REMOTE_NAME="torvalds-live"
|
||||
|
||||
GIT_STABLE_LIVE_GIT_URL="git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git"
|
||||
GIT_STABLE_LIVE_REMOTE_NAME="stable-live"
|
||||
|
||||
# Versions we're interested in, array
|
||||
declare -ag WANTED_KERNEL_VERSIONS=("5.18" "5.17" "5.15" "5.12" "5.10")
|
||||
|
||||
# Minimal version
|
||||
|
||||
# Kernel versions: 5.18, 5.17, 5.15, 5.10, 4.19, 4.9, 4.4
|
||||
|
||||
# 1st stage Global:
|
||||
|
||||
# Init an empty git repo
|
||||
# Fetch from Torvalds bundle (very slow) into 'torvalds-gitbundle' branch
|
||||
# include tags?
|
||||
if [[ ! -d "${KERNEL_GIT_TREE}/.git" ]]; then
|
||||
display_alert "Initting git tree"
|
||||
git init --initial-branch="armbian_unused_initial_branch" "${KERNEL_GIT_TREE}"
|
||||
else
|
||||
display_alert "Git tree already initted"
|
||||
fi
|
||||
|
||||
## From now on, everything is done inside the git worktree...
|
||||
cd "${KERNEL_GIT_TREE}" || exit 2
|
||||
|
||||
if ! git config "remote.${GIT_TORVALDS_BUNDLE_REMOTE_NAME}.url"; then
|
||||
# Grab torvald's gitbundle via http from kernel.org
|
||||
download_git_bundle_from_http "${GIT_TORVALDS_BUNDLE_FILE}" "${GIT_TORVALDS_BUNDLE_URL}"
|
||||
display_alert "Fetching from cold git bundle, wait" "${GIT_TORVALDS_BUNDLE_ID}"
|
||||
git_fetch_from_bundle_file "${GIT_TORVALDS_BUNDLE_FILE}" "${GIT_TORVALDS_BUNDLE_REMOTE_NAME}"
|
||||
else
|
||||
display_alert "Torvalds bundle already fetched..."
|
||||
fi
|
||||
|
||||
# At this stage, we've all blobs, but no tags!
|
||||
|
||||
# 2nd stage Global:
|
||||
# Add torvalds live git as remote 'torvalds-live'
|
||||
if ! git config "remote.${GIT_TORVALDS_LIVE_REMOTE_NAME}.url"; then
|
||||
display_alert "Adding torvalds live remote" "${GIT_TORVALDS_LIVE_REMOTE_NAME}"
|
||||
git remote add "${GIT_TORVALDS_LIVE_REMOTE_NAME}" "${GIT_TORVALDS_LIVE_GIT_URL}"
|
||||
else
|
||||
display_alert "Torvalds live remote already exists..."
|
||||
fi
|
||||
|
||||
# Fetch from it (to update), also bring in the tags. Around a 60mb download, quite fast.
|
||||
[[ "${ONLINE}" == "yes" ]] && git fetch --progress --verbose --tags "${GIT_TORVALDS_LIVE_REMOTE_NAME}" # Fetch it! (including tags!)
|
||||
|
||||
# Now, add the stable remote. Do NOT fetch from it, it's huge and has a lot more than we need.
|
||||
if ! git config "remote.${GIT_STABLE_LIVE_REMOTE_NAME}.url"; then
|
||||
display_alert "Adding stable live remote" "${GIT_STABLE_LIVE_REMOTE_NAME}"
|
||||
git remote add "${GIT_STABLE_LIVE_REMOTE_NAME}" "${GIT_STABLE_LIVE_GIT_URL}"
|
||||
else
|
||||
display_alert "Stable live remote already exists..."
|
||||
fi
|
||||
|
||||
# 3rd stage: For each version, eg: 5.17
|
||||
# - Fetch from stable git source (not bundle) into `stable-5.17` branch
|
||||
# - include tags?
|
||||
# - if this fails (eg: an unreleased kernel at that moment, tolerate and go ahead, torvalds should have -rc1)
|
||||
WANTED_KERNEL_VERSIONS_COUNT=${#WANTED_KERNEL_VERSIONS[@]}
|
||||
display_alert "Wanted kernel versions: ${WANTED_KERNEL_VERSIONS_COUNT}"
|
||||
|
||||
for KERNEL_VERSION in "${WANTED_KERNEL_VERSIONS[@]}"; do
|
||||
display_alert "Fetching stable kernel version: ${KERNEL_VERSION}"
|
||||
|
||||
KERNEL_VERSION_LOCAL_BRANCH_NAME="linux-${KERNEL_VERSION}.y"
|
||||
KERNEL_VERSION_REMOTE_BRANCH_NAME="linux-${KERNEL_VERSION}.y"
|
||||
|
||||
# Fetch the branch from the stable live into the local branch. Since I don't specify "--tags", it will only fetch the tags for the branch. Those DON'T include the -rc tags which came from torvalds live
|
||||
[[ "${ONLINE}" == "yes" ]] && git fetch --progress --verbose "${GIT_STABLE_LIVE_REMOTE_NAME}" "${KERNEL_VERSION_REMOTE_BRANCH_NAME}:${KERNEL_VERSION_LOCAL_BRANCH_NAME}"
|
||||
done
|
||||
|
||||
# 4th stage: For each version, eg 5.17
|
||||
# - Find the earliest tag with 5.17 in it, or 5.17-rc1 if all else fails;
|
||||
# - find the _date_ for such a tag
|
||||
# - Export a shallow bundle via the date for that version;
|
||||
# - include the shallow marker file (eg: .git/shallow)
|
||||
# - lightweight tarball of those things, many -0 zstd
|
||||
# - publish the tarball as GH releases, always in "latest" release
|
||||
for KERNEL_VERSION in "${WANTED_KERNEL_VERSIONS[@]}"; do
|
||||
cd "${KERNEL_GIT_TREE}" || exit 2
|
||||
|
||||
KERNEL_VERSION_LOCAL_BRANCH_NAME="linux-${KERNEL_VERSION}.y"
|
||||
display_alert "Finding shallow point for version: ${KERNEL_VERSION}" "on local branch" "${KERNEL_VERSION_LOCAL_BRANCH_NAME}"
|
||||
|
||||
# shit happens upstream too, so filter out "-dontuse" tags.
|
||||
KERNEL_VERSION_FIRST_RC_TAG_NAME="$(git tag -l | grep "^v$(echo -n "${KERNEL_VERSION}" | sed -e 's/\./\\\./')-rc" | grep -v "\-dontuse" | sort -n | head -1)"
|
||||
display_alert "Found first RC for version:" "${KERNEL_VERSION}" "${KERNEL_VERSION_FIRST_RC_TAG_NAME}"
|
||||
|
||||
# Now translate that tag into a date, which what we're gonna use to shallow the bundle.
|
||||
# Attention: date has timezone part.
|
||||
KERNEL_VERSION_SHALLOW_AT_DATE="$(git tag --list --format="%(creatordate)" "${KERNEL_VERSION_FIRST_RC_TAG_NAME}")"
|
||||
display_alert "Date for first RC tag:" "${KERNEL_VERSION}" "${KERNEL_VERSION_FIRST_RC_TAG_NAME}" "'${KERNEL_VERSION_SHALLOW_AT_DATE}'"
|
||||
|
||||
# Clone from the worktree into a new directory, shallowing in the process. This is the only way to make it consistently shallow without jumping through hoops.
|
||||
KERNEL_VERSION_SHALLOWED_WORKDIR="${SHALLOWED_TREES_DIR}/shallow-${KERNEL_VERSION}-${KERNEL_VERSION_FIRST_RC_TAG_NAME}"
|
||||
#rm -rf "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
|
||||
|
||||
if [[ ! -d "${KERNEL_VERSION_SHALLOWED_WORKDIR}" ]]; then
|
||||
display_alert "Making shallow tree" "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
|
||||
git clone --no-checkout --progress --verbose \
|
||||
--single-branch --branch="${KERNEL_VERSION_LOCAL_BRANCH_NAME}" \
|
||||
--tags --shallow-since="${KERNEL_VERSION_SHALLOW_AT_DATE}" \
|
||||
"file://${KERNEL_GIT_TREE}" "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
|
||||
else
|
||||
display_alert "Shallow tree already exists" "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
|
||||
fi
|
||||
|
||||
OUTPUT_BUNDLE_DIR="${OUTPUT_DIR}" # /${KERNEL_VERSION}"
|
||||
mkdir -p "${OUTPUT_BUNDLE_DIR}"
|
||||
|
||||
OUTPUT_BUNDLE_FILE_NAME_BUNDLE="${OUTPUT_BUNDLE_DIR}/linux-${KERNEL_VERSION}.gitbundle"
|
||||
OUTPUT_BUNDLE_FILE_NAME_SHALLOW="${OUTPUT_BUNDLE_DIR}/linux-${KERNEL_VERSION}.gitshallow"
|
||||
|
||||
cd "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
|
||||
|
||||
# Remove the origin remote, otherwise it would be exported due to "--all" below.
|
||||
if git config "remote.origin.url"; then
|
||||
git remote rm origin
|
||||
fi
|
||||
|
||||
# Now, export a bundle from the shallow tree. This is gonna be a shallow bundle, of course!
|
||||
git bundle create "${OUTPUT_BUNDLE_FILE_NAME_BUNDLE}" --all
|
||||
|
||||
# export the shallow file, so it can be actually used.
|
||||
cp -pv ".git/shallow" "${OUTPUT_BUNDLE_FILE_NAME_SHALLOW}"
|
||||
|
||||
# sanity check
|
||||
git bundle list-heads "${OUTPUT_BUNDLE_FILE_NAME_BUNDLE}"
|
||||
done
|
||||
|
||||
# In GHA, cache the full git work tree, if it fits in GHA cache;
|
||||
# If cache hit, skip 1st stage, but _always_ execute stages 2-4 (to update the cache)
|
||||
Reference in New Issue
Block a user