mirror of
https://github.com/armbian/gitutils.git
synced 2026-01-06 10:36:09 -08:00
182 lines
8.7 KiB
Bash
Executable File
182 lines
8.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -e
|
|
|
|
function display_alert() {
|
|
echo "--> $*"
|
|
}
|
|
|
|
echo "::group::Prepare basics"
|
|
|
|
# Versions we're interested in, array - @TODO this needs smarts. maybe parse kernel.org's Atom Release feed?
|
|
declare -ag WANTED_KERNEL_VERSIONS=("5.18" "5.17" "5.15" "5.10" "4.19" "4.9" "4.4")
|
|
ONLINE="yes"
|
|
|
|
BASE_WORK_DIR="${BASE_WORK_DIR:-"/Volumes/LinuxDev/shallow_git_tree_work"}"
|
|
WORKDIR="${BASE_WORK_DIR}/kernel"
|
|
SHALLOWED_TREES_DIR="${WORKDIR}/shallow_trees"
|
|
OUTPUT_DIR="${WORKDIR}/output"
|
|
KERNEL_GIT_TREE="${WORKDIR}/worktree"
|
|
KERNEL_TORVALDS_BUNDLE_DIR="${WORKDIR}/bundle-torvalds"
|
|
mkdir -p "${BASE_WORK_DIR}" "${WORKDIR}" "${SHALLOWED_TREES_DIR}" "${OUTPUT_DIR}" "${KERNEL_GIT_TREE}" "${KERNEL_TORVALDS_BUNDLE_DIR}"
|
|
|
|
GIT_TORVALDS_BUNDLE_URL="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/clone.bundle" # Thanks, kernel.org!
|
|
GIT_TORVALDS_BUNDLE_ID="$(echo -n "${GIT_TORVALDS_BUNDLE_URL}" | md5sum | awk '{print $1}')" # md5 of the URL.
|
|
GIT_TORVALDS_BUNDLE_FILE="${KERNEL_TORVALDS_BUNDLE_DIR}/${GIT_TORVALDS_BUNDLE_ID}.gitbundle" # final filename of bundle
|
|
GIT_TORVALDS_BUNDLE_REMOTE_NAME="torvalds-gitbundle" # name of the remote that will point to bundle
|
|
GIT_TORVALDS_LIVE_GIT_URL="git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" # Torvalds live tree git:// URL
|
|
GIT_TORVALDS_LIVE_REMOTE_NAME="torvalds-live" # name of the remote that will point to live Torvalds tree
|
|
GIT_STABLE_LIVE_GIT_URL="git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git" # Stable live tree git:// URL
|
|
GIT_STABLE_LIVE_REMOTE_NAME="stable-live" # name of the remote that will point to live stable tree
|
|
|
|
# 1st stage Global:
|
|
# Init an empty git repo
|
|
if [[ ! -d "${KERNEL_GIT_TREE}/.git" ]]; then
|
|
display_alert "Initting git tree"
|
|
git init --initial-branch="armbian_unused_initial_branch" "${KERNEL_GIT_TREE}"
|
|
else
|
|
display_alert "Git tree already initted"
|
|
fi
|
|
|
|
echo "::endgroup::"
|
|
echo "::group::Fetching Torvalds bundle"
|
|
|
|
## From now on, everything is done inside the git worktree...
|
|
cd "${KERNEL_GIT_TREE}" || exit 2
|
|
|
|
if ! git config "remote.${GIT_TORVALDS_BUNDLE_REMOTE_NAME}.url"; then
|
|
# Grab torvald's gitbundle via http from kernel.org
|
|
if [[ ! -f "${GIT_TORVALDS_BUNDLE_FILE}" ]]; then # Download the bundle file if it does not exist.
|
|
display_alert "Downloading Git cold bundle via HTTP" "${GIT_TORVALDS_BUNDLE_URL}"
|
|
wget --continue --progress=dot:giga --output-document="${GIT_TORVALDS_BUNDLE_FILE}" "${GIT_TORVALDS_BUNDLE_URL}"
|
|
else
|
|
display_alert "Cold bundle file exists, using it" "${GIT_TORVALDS_BUNDLE_FILE}" "git"
|
|
fi
|
|
|
|
# Fetch from Torvalds bundle (very slow) into 'torvalds-gitbundle' branch
|
|
display_alert "Fetching from cold git bundle, wait" "${GIT_TORVALDS_BUNDLE_ID}"
|
|
git bundle verify "${GIT_TORVALDS_BUNDLE_FILE}" # Make sure bundle is valid.
|
|
git remote add "${GIT_TORVALDS_BUNDLE_REMOTE_NAME}" "${GIT_TORVALDS_BUNDLE_FILE}" # Add the remote pointing to the cold bundle file
|
|
git fetch --progress --verbose "${GIT_TORVALDS_BUNDLE_REMOTE_NAME}" # Fetch it!
|
|
else
|
|
display_alert "Torvalds bundle already fetched..."
|
|
fi
|
|
|
|
# At this stage, we've all blobs, but no tags!
|
|
|
|
echo "::endgroup::"
|
|
echo "::group::Fetching Torvalds live"
|
|
|
|
# 2nd stage Global:
|
|
# Add torvalds live git as remote 'torvalds-live'
|
|
if ! git config "remote.${GIT_TORVALDS_LIVE_REMOTE_NAME}.url"; then
|
|
display_alert "Adding torvalds live remote" "${GIT_TORVALDS_LIVE_REMOTE_NAME}"
|
|
git remote add "${GIT_TORVALDS_LIVE_REMOTE_NAME}" "${GIT_TORVALDS_LIVE_GIT_URL}"
|
|
else
|
|
display_alert "Torvalds live remote already exists..."
|
|
fi
|
|
|
|
# Fetch from it (to update), also bring in the tags. Around a 60mb download, quite fast.
|
|
[[ "${ONLINE}" == "yes" ]] && git fetch --progress --verbose --tags "${GIT_TORVALDS_LIVE_REMOTE_NAME}" # Fetch it! (including tags!)
|
|
|
|
echo "::endgroup::"
|
|
echo "::group::Adding stable remote"
|
|
|
|
# Now, add the stable remote. Do NOT fetch from it, it's huge and has a lot more than we need.
|
|
if ! git config "remote.${GIT_STABLE_LIVE_REMOTE_NAME}.url"; then
|
|
display_alert "Adding stable live remote" "${GIT_STABLE_LIVE_REMOTE_NAME}"
|
|
git remote add "${GIT_STABLE_LIVE_REMOTE_NAME}" "${GIT_STABLE_LIVE_GIT_URL}"
|
|
else
|
|
display_alert "Stable live remote already exists..."
|
|
fi
|
|
|
|
# 3rd stage: For each version, eg: 5.17
|
|
# - Fetch from stable git source (not bundle) into `stable-5.17` branch
|
|
# - include tags
|
|
# - if this fails (eg: an unreleased kernel at that moment, tolerate and go ahead, torvalds should have -rc1)
|
|
WANTED_KERNEL_VERSIONS_COUNT=${#WANTED_KERNEL_VERSIONS[@]}
|
|
display_alert "Wanted kernel versions: ${WANTED_KERNEL_VERSIONS_COUNT}"
|
|
|
|
echo "::endgroup::"
|
|
|
|
for KERNEL_VERSION in "${WANTED_KERNEL_VERSIONS[@]}"; do
|
|
echo "::group::Fetching stable remote for ${KERNEL_VERSION}"
|
|
display_alert "Fetching stable kernel version: ${KERNEL_VERSION}"
|
|
|
|
KERNEL_VERSION_LOCAL_BRANCH_NAME="linux-${KERNEL_VERSION}.y"
|
|
KERNEL_VERSION_REMOTE_BRANCH_NAME="linux-${KERNEL_VERSION}.y"
|
|
|
|
# Fetch the branch from the stable live into the local branch. Since I don't specify "--tags", it will only fetch the tags for the branch. Those DON'T include the -rc tags which came from torvalds live
|
|
[[ "${ONLINE}" == "yes" ]] && git fetch --progress --verbose "${GIT_STABLE_LIVE_REMOTE_NAME}" "${KERNEL_VERSION_REMOTE_BRANCH_NAME}:${KERNEL_VERSION_LOCAL_BRANCH_NAME}"
|
|
echo "::endgroup::"
|
|
|
|
done
|
|
|
|
# 4th stage: For each version, eg 5.17
|
|
# - Find the earliest tag with 5.17 in it
|
|
# - find the _date_ for such a tag
|
|
# - Export a shallow bundle via the date for that version;
|
|
# - include the shallow marker file (.git/shallow)
|
|
# - ready to publish as GH releases, always in "latest" release
|
|
for KERNEL_VERSION in "${WANTED_KERNEL_VERSIONS[@]}"; do
|
|
echo "::group::Processing shallow for ${KERNEL_VERSION}"
|
|
|
|
cd "${KERNEL_GIT_TREE}" || exit 2
|
|
|
|
KERNEL_VERSION_LOCAL_BRANCH_NAME="linux-${KERNEL_VERSION}.y"
|
|
display_alert "Finding shallow point for version: ${KERNEL_VERSION}" "on local branch" "${KERNEL_VERSION_LOCAL_BRANCH_NAME}"
|
|
|
|
# shit happens upstream too, so filter out "-dontuse" tags.
|
|
KERNEL_VERSION_FIRST_RC_TAG_NAME="$(git tag -l | grep "^v$(echo -n "${KERNEL_VERSION}" | sed -e 's/\./\\\./')-rc" | grep -v "\-dontuse" | sort -n | head -1)"
|
|
display_alert "Found first RC for version:" "${KERNEL_VERSION}" "${KERNEL_VERSION_FIRST_RC_TAG_NAME}"
|
|
|
|
# Now translate that tag into a date, which what we're gonna use to shallow the bundle.
|
|
# Attention: date has timezone part.
|
|
KERNEL_VERSION_SHALLOW_AT_DATE="$(git tag --list --format="%(creatordate)" "${KERNEL_VERSION_FIRST_RC_TAG_NAME}")"
|
|
display_alert "Date for first RC tag:" "${KERNEL_VERSION}" "${KERNEL_VERSION_FIRST_RC_TAG_NAME}" "'${KERNEL_VERSION_SHALLOW_AT_DATE}'"
|
|
|
|
# Clone from the worktree into a new directory, shallowing in the process. This is the only way to make it consistently shallow without jumping through hoops.
|
|
KERNEL_VERSION_SHALLOWED_WORKDIR="${SHALLOWED_TREES_DIR}/shallow-${KERNEL_VERSION}-${KERNEL_VERSION_FIRST_RC_TAG_NAME}"
|
|
|
|
if [[ ! -d "${KERNEL_VERSION_SHALLOWED_WORKDIR}" ]]; then
|
|
display_alert "Making shallow tree" "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
|
|
git clone --no-checkout --progress --verbose \
|
|
--single-branch --branch="${KERNEL_VERSION_LOCAL_BRANCH_NAME}" \
|
|
--tags --shallow-since="${KERNEL_VERSION_SHALLOW_AT_DATE}" \
|
|
"file://${KERNEL_GIT_TREE}" "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
|
|
else
|
|
display_alert "Shallow tree already exists" "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
|
|
fi
|
|
|
|
OUTPUT_BUNDLE_FILE_NAME_BUNDLE="${OUTPUT_DIR}/linux-${KERNEL_VERSION}.gitbundle"
|
|
OUTPUT_BUNDLE_FILE_NAME_SHALLOW="${OUTPUT_DIR}/linux-${KERNEL_VERSION}.gitshallow"
|
|
OUTPUT_BUNDLE_FILE_NAME_TARBALL="${OUTPUT_DIR}/linux-${KERNEL_VERSION}.git.tar"
|
|
|
|
cd "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
|
|
|
|
# Remove the origin remote, otherwise it would be exported due to "--all" below.
|
|
if git config "remote.origin.url"; then
|
|
git remote rm origin
|
|
fi
|
|
|
|
# Now, export a bundle from the shallow tree.
|
|
git bundle create "${OUTPUT_BUNDLE_FILE_NAME_BUNDLE}" --all
|
|
|
|
# export the shallow file, so it can be actually used.
|
|
cp -pv ".git/shallow" "${OUTPUT_BUNDLE_FILE_NAME_SHALLOW}"
|
|
|
|
# sanity check
|
|
git bundle list-heads "${OUTPUT_BUNDLE_FILE_NAME_BUNDLE}"
|
|
|
|
# Also export a .tar of .git as well, for convenience; ppl might wanna use them directly to save a fetch.
|
|
tar cf "${OUTPUT_BUNDLE_FILE_NAME_TARBALL}" .git
|
|
|
|
# List the outputs with sizes
|
|
ls -laht "${OUTPUT_DIR}/linux-${KERNEL_VERSION}".*
|
|
|
|
echo "::endgroup::"
|
|
done
|
|
|
|
# In GHA, cache the full git work tree, if it fits in GHA cache;
|
|
# If cache hit, skip 1st stage, but _always_ execute stages 2-4 (to update the cache)
|