From afe3358b39fefbed9af3807bd853a85773fdc702 Mon Sep 17 00:00:00 2001 From: Ricardo Pardini Date: Mon, 23 May 2022 23:31:49 +0200 Subject: [PATCH] first try --- .github/workflows/main-latest.yml | 51 +++++++++ lib_git.sh | 31 ++++++ shallow_kernel_tree.sh | 171 ++++++++++++++++++++++++++++++ 3 files changed, 253 insertions(+) create mode 100644 .github/workflows/main-latest.yml create mode 100644 lib_git.sh create mode 100755 shallow_kernel_tree.sh diff --git a/.github/workflows/main-latest.yml b/.github/workflows/main-latest.yml new file mode 100644 index 0000000..9dbf88b --- /dev/null +++ b/.github/workflows/main-latest.yml @@ -0,0 +1,51 @@ +name: main-latest + +on: + push: + branches: [ main ] + +jobs: + + prepare: + runs-on: ubuntu-latest + steps: + - name: Prepare release ID + id: prep + run: | + echo ::set-output name=created::$(date -u +'%Y%m%d-%H%M') + outputs: + created: ${{ steps.prep.outputs.created }} # refer to as ${{needs.prepare.outputs.created}} + + + build: + needs: [ prepare ] + runs-on: ubuntu-latest # soon to support hosted runners for arm64 + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Cache git worktree + uses: actions/cache@v2 + with: + path: /tmp/workdir/kernel/worktree + key: ${{ runner.os }}-kernel-worktree-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-kernel-worktree + + - name: Prepare git trees, bulk of work + run: | + BASE_WORK_DIR="/tmp/workdir" bash shallow_kernel_tree.sh + + + - name: "GH latest release" + uses: "marvinpinto/action-automatic-releases@latest" + with: + repo_token: "${{ secrets.GITHUB_TOKEN }}" + automatic_release_tag: "latest" + prerelease: false + title: "Latest: ${{needs.prepare.outputs.created}}" + files: | + /tmp/workdir/output/*.gitshallow + /tmp/workdir/output/*.gitbundle + diff --git a/lib_git.sh b/lib_git.sh new file mode 100644 index 0000000..d5a0fa4 --- /dev/null +++ b/lib_git.sh @@ -0,0 +1,31 @@ +function display_alert() { + echo "--> $*" +} + +function run_host_command_logged() { + echo "==> $*" + "$@" +} + + +function git_fetch_from_bundle_file() { + local bundle_file="${1}" remote_name="${2}" shallow_file="${3}" + git bundle verify "${bundle_file}" # Make sure bundle is valid. + git remote add "${remote_name}" "${bundle_file}" # Add the remote pointing to the cold bundle file + if [[ -f "${shallow_file}" ]]; then + display_alert "Bundle is shallow" "${shallow_file}" "git" + cp -p "${shallow_file}" ".git/shallow" + fi + git fetch --progress --verbose --tags "${remote_name}" # Fetch it! (including tags!) + display_alert "Bundle fetch '${remote_name}' completed" +} + +function download_git_bundle_from_http() { + local bundle_file="${1}" bundle_url="${2}" + if [[ ! -f "${git_cold_bundle_cache_file}" ]]; then # Download the bundle file if it does not exist. + display_alert "Downloading Git cold bundle via HTTP" "${bundle_url}" "info" # This gonna take a while. And waste bandwidth + run_host_command_logged wget --continue --progress=dot:giga --output-document="${bundle_file}" "${bundle_url}" + else + display_alert "Cold bundle file exists, using it" "${bundle_file}" "git" + fi +} diff --git a/shallow_kernel_tree.sh b/shallow_kernel_tree.sh new file mode 100755 index 0000000..01200ef --- /dev/null +++ b/shallow_kernel_tree.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash + +set -e +set -x + +# source the lib +. lib_git.sh + +ONLINE="yes" + +BASE_WORK_DIR="${BASE_WORK_DIR:-"/Volumes/LinuxDev/shallow_git_tree_work"}" +mkdir -p "${BASE_WORK_DIR}" + +WORKDIR="${BASE_WORK_DIR}/kernel" +mkdir -p "${WORKDIR}" + +SHALLOWED_TREES_DIR="${WORKDIR}/shallow_trees" +mkdir -p "${SHALLOWED_TREES_DIR}" + +OUTPUT_DIR="${WORKDIR}/output" +mkdir -p "${OUTPUT_DIR}" + +KERNEL_GIT_TREE="${WORKDIR}/worktree" +mkdir -p "${KERNEL_GIT_TREE}" + +KERNEL_TORVALDS_BUNDLE_DIR="${WORKDIR}/bundle-torvalds" +mkdir -p "${KERNEL_TORVALDS_BUNDLE_DIR}" + +GIT_TORVALDS_BUNDLE_URL="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/clone.bundle" +GIT_TORVALDS_BUNDLE_ID="$(echo -n "${GIT_TORVALDS_BUNDLE_URL}" | md5sum | awk '{print $1}')" # md5 of the URL. +GIT_TORVALDS_BUNDLE_FILE="${KERNEL_TORVALDS_BUNDLE_DIR}/${GIT_TORVALDS_BUNDLE_ID}.gitbundle" # final filename of bundle +GIT_TORVALDS_BUNDLE_REMOTE_NAME="torvalds-gitbundle" # name of the remote that will point to bundle + +# TORVALDS LIVE info +GIT_TORVALDS_LIVE_GIT_URL="git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" +GIT_TORVALDS_LIVE_REMOTE_NAME="torvalds-live" + +GIT_STABLE_LIVE_GIT_URL="git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git" +GIT_STABLE_LIVE_REMOTE_NAME="stable-live" + +# Versions we're interested in, array +declare -ag WANTED_KERNEL_VERSIONS=("5.18" "5.17" "5.15" "5.12" "5.10") + +# Minimal version + +# Kernel versions: 5.18, 5.17, 5.15, 5.10, 4.19, 4.9, 4.4 + +# 1st stage Global: + +# Init an empty git repo +# Fetch from Torvalds bundle (very slow) into 'torvalds-gitbundle' branch +# include tags? +if [[ ! -d "${KERNEL_GIT_TREE}/.git" ]]; then + display_alert "Initting git tree" + git init --initial-branch="armbian_unused_initial_branch" "${KERNEL_GIT_TREE}" +else + display_alert "Git tree already initted" +fi + +## From now on, everything is done inside the git worktree... +cd "${KERNEL_GIT_TREE}" || exit 2 + +if ! git config "remote.${GIT_TORVALDS_BUNDLE_REMOTE_NAME}.url"; then + # Grab torvald's gitbundle via http from kernel.org + download_git_bundle_from_http "${GIT_TORVALDS_BUNDLE_FILE}" "${GIT_TORVALDS_BUNDLE_URL}" + display_alert "Fetching from cold git bundle, wait" "${GIT_TORVALDS_BUNDLE_ID}" + git_fetch_from_bundle_file "${GIT_TORVALDS_BUNDLE_FILE}" "${GIT_TORVALDS_BUNDLE_REMOTE_NAME}" +else + display_alert "Torvalds bundle already fetched..." +fi + +# At this stage, we've all blobs, but no tags! + +# 2nd stage Global: +# Add torvalds live git as remote 'torvalds-live' +if ! git config "remote.${GIT_TORVALDS_LIVE_REMOTE_NAME}.url"; then + display_alert "Adding torvalds live remote" "${GIT_TORVALDS_LIVE_REMOTE_NAME}" + git remote add "${GIT_TORVALDS_LIVE_REMOTE_NAME}" "${GIT_TORVALDS_LIVE_GIT_URL}" +else + display_alert "Torvalds live remote already exists..." +fi + +# Fetch from it (to update), also bring in the tags. Around a 60mb download, quite fast. +[[ "${ONLINE}" == "yes" ]] && git fetch --progress --verbose --tags "${GIT_TORVALDS_LIVE_REMOTE_NAME}" # Fetch it! (including tags!) + +# Now, add the stable remote. Do NOT fetch from it, it's huge and has a lot more than we need. +if ! git config "remote.${GIT_STABLE_LIVE_REMOTE_NAME}.url"; then + display_alert "Adding stable live remote" "${GIT_STABLE_LIVE_REMOTE_NAME}" + git remote add "${GIT_STABLE_LIVE_REMOTE_NAME}" "${GIT_STABLE_LIVE_GIT_URL}" +else + display_alert "Stable live remote already exists..." +fi + +# 3rd stage: For each version, eg: 5.17 +# - Fetch from stable git source (not bundle) into `stable-5.17` branch +# - include tags? +# - if this fails (eg: an unreleased kernel at that moment, tolerate and go ahead, torvalds should have -rc1) +WANTED_KERNEL_VERSIONS_COUNT=${#WANTED_KERNEL_VERSIONS[@]} +display_alert "Wanted kernel versions: ${WANTED_KERNEL_VERSIONS_COUNT}" + +for KERNEL_VERSION in "${WANTED_KERNEL_VERSIONS[@]}"; do + display_alert "Fetching stable kernel version: ${KERNEL_VERSION}" + + KERNEL_VERSION_LOCAL_BRANCH_NAME="linux-${KERNEL_VERSION}.y" + KERNEL_VERSION_REMOTE_BRANCH_NAME="linux-${KERNEL_VERSION}.y" + + # Fetch the branch from the stable live into the local branch. Since I don't specify "--tags", it will only fetch the tags for the branch. Those DON'T include the -rc tags which came from torvalds live + [[ "${ONLINE}" == "yes" ]] && git fetch --progress --verbose "${GIT_STABLE_LIVE_REMOTE_NAME}" "${KERNEL_VERSION_REMOTE_BRANCH_NAME}:${KERNEL_VERSION_LOCAL_BRANCH_NAME}" +done + +# 4th stage: For each version, eg 5.17 +# - Find the earliest tag with 5.17 in it, or 5.17-rc1 if all else fails; +# - find the _date_ for such a tag +# - Export a shallow bundle via the date for that version; +# - include the shallow marker file (eg: .git/shallow) +# - lightweight tarball of those things, many -0 zstd +# - publish the tarball as GH releases, always in "latest" release +for KERNEL_VERSION in "${WANTED_KERNEL_VERSIONS[@]}"; do + cd "${KERNEL_GIT_TREE}" || exit 2 + + KERNEL_VERSION_LOCAL_BRANCH_NAME="linux-${KERNEL_VERSION}.y" + display_alert "Finding shallow point for version: ${KERNEL_VERSION}" "on local branch" "${KERNEL_VERSION_LOCAL_BRANCH_NAME}" + + # shit happens upstream too, so filter out "-dontuse" tags. + KERNEL_VERSION_FIRST_RC_TAG_NAME="$(git tag -l | grep "^v$(echo -n "${KERNEL_VERSION}" | sed -e 's/\./\\\./')-rc" | grep -v "\-dontuse" | sort -n | head -1)" + display_alert "Found first RC for version:" "${KERNEL_VERSION}" "${KERNEL_VERSION_FIRST_RC_TAG_NAME}" + + # Now translate that tag into a date, which what we're gonna use to shallow the bundle. + # Attention: date has timezone part. + KERNEL_VERSION_SHALLOW_AT_DATE="$(git tag --list --format="%(creatordate)" "${KERNEL_VERSION_FIRST_RC_TAG_NAME}")" + display_alert "Date for first RC tag:" "${KERNEL_VERSION}" "${KERNEL_VERSION_FIRST_RC_TAG_NAME}" "'${KERNEL_VERSION_SHALLOW_AT_DATE}'" + + # Clone from the worktree into a new directory, shallowing in the process. This is the only way to make it consistently shallow without jumping through hoops. + KERNEL_VERSION_SHALLOWED_WORKDIR="${SHALLOWED_TREES_DIR}/shallow-${KERNEL_VERSION}-${KERNEL_VERSION_FIRST_RC_TAG_NAME}" + #rm -rf "${KERNEL_VERSION_SHALLOWED_WORKDIR}" + + if [[ ! -d "${KERNEL_VERSION_SHALLOWED_WORKDIR}" ]]; then + display_alert "Making shallow tree" "${KERNEL_VERSION_SHALLOWED_WORKDIR}" + git clone --no-checkout --progress --verbose \ + --single-branch --branch="${KERNEL_VERSION_LOCAL_BRANCH_NAME}" \ + --tags --shallow-since="${KERNEL_VERSION_SHALLOW_AT_DATE}" \ + "file://${KERNEL_GIT_TREE}" "${KERNEL_VERSION_SHALLOWED_WORKDIR}" + else + display_alert "Shallow tree already exists" "${KERNEL_VERSION_SHALLOWED_WORKDIR}" + fi + + OUTPUT_BUNDLE_DIR="${OUTPUT_DIR}" # /${KERNEL_VERSION}" + mkdir -p "${OUTPUT_BUNDLE_DIR}" + + OUTPUT_BUNDLE_FILE_NAME_BUNDLE="${OUTPUT_BUNDLE_DIR}/linux-${KERNEL_VERSION}.gitbundle" + OUTPUT_BUNDLE_FILE_NAME_SHALLOW="${OUTPUT_BUNDLE_DIR}/linux-${KERNEL_VERSION}.gitshallow" + + cd "${KERNEL_VERSION_SHALLOWED_WORKDIR}" + + # Remove the origin remote, otherwise it would be exported due to "--all" below. + if git config "remote.origin.url"; then + git remote rm origin + fi + + # Now, export a bundle from the shallow tree. This is gonna be a shallow bundle, of course! + git bundle create "${OUTPUT_BUNDLE_FILE_NAME_BUNDLE}" --all + + # export the shallow file, so it can be actually used. + cp -pv ".git/shallow" "${OUTPUT_BUNDLE_FILE_NAME_SHALLOW}" + + # sanity check + git bundle list-heads "${OUTPUT_BUNDLE_FILE_NAME_BUNDLE}" +done + +# In GHA, cache the full git work tree, if it fits in GHA cache; +# If cache hit, skip 1st stage, but _always_ execute stages 2-4 (to update the cache)