diff --git a/extensions/kernel-localmodconfig.sh b/extensions/kernel-localmodconfig.sh index 5ff0aa32f..4700a646a 100644 --- a/extensions/kernel-localmodconfig.sh +++ b/extensions/kernel-localmodconfig.sh @@ -13,10 +13,9 @@ function extension_prepare_config__prepare_localmodconfig() { # This needs much more love than this. can be used to make "light" versions of kernels, that compile 3x-5x faster or more function custom_kernel_config_post_defconfig__apply_localmodconfig() { - display_alert "localmodconfig with lsmod" "${KERNEL_CONFIG_FROM_LSMOD}" "warn" if [[ "a${KERNEL_CONFIG_FROM_LSMOD}a" != "aa" ]]; then + display_alert "localmodconfig with lsmod" "${KERNEL_CONFIG_FROM_LSMOD}" "warn" local lsmod_file="${SRC}/userpatches/lsmod/${KERNEL_CONFIG_FROM_LSMOD}.lsmod" - run_kernel_make "LSMOD=${lsmod_file}" localmodconfig - kernel_config_mtime=$(get_file_modification_time ".config") # capture the mtime of the config file after the localmodconfig + run_kernel_make "LSMOD=${lsmod_file}" localmodconfig "> /dev/null" # hide output, it's way too long. stderr still shows fi } diff --git a/lib/functions/compilation/kernel-config.sh b/lib/functions/compilation/kernel-config.sh index 9e3f7bb1f..f5ad209a2 100644 --- a/lib/functions/compilation/kernel-config.sh +++ b/lib/functions/compilation/kernel-config.sh @@ -1,98 +1,85 @@ -function kernel_config_maybe_interactive() { - # Check if we're gonna do some interactive configuration; if so, don't run kernel_config under logging manager. - if [[ $KERNEL_CONFIGURE != yes ]]; then - LOG_SECTION="kernel_config" do_with_logging do_with_hooks kernel_config - else - LOG_SECTION="kernel_config_interactive" do_with_hooks kernel_config +function kernel_config() { + # check $kernel_work_dir is set and exists, or bail + [[ -z "${kernel_work_dir}" ]] && exit_with_error "kernel_work_dir is not set" + [[ ! -d "${kernel_work_dir}" ]] && exit_with_error "kernel_work_dir does not exist: ${kernel_work_dir}" + declare previous_config_filename=".config.armbian.previous" + declare kernel_config_source_filename="" # which actual .config was used? + + LOG_SECTION="kernel_config_initialize" do_with_logging do_with_hooks kernel_config_initialize + + if [[ "${KERNEL_CONFIGURE}" == "yes" ]]; then + # This piece is interactive, no logging + display_alert "Starting (interactive) kernel ${KERNEL_MENUCONFIG:-menuconfig}" "${LINUXCONFIG}" "debug" + run_kernel_make_dialog "${KERNEL_MENUCONFIG:-menuconfig}" + + # Export, but log about it too. + LOG_SECTION="kernel_config_export" do_with_logging do_with_hooks kernel_config_export fi + + LOG_SECTION="kernel_config_finalize" do_with_logging do_with_hooks kernel_config_finalize } -function kernel_config() { - # re-read kernel version after patching - version=$(grab_version "$kernel_work_dir") +function kernel_config_initialize() { + display_alert "Configuring kernel" "${LINUXCONFIG}" "info" - display_alert "Compiling $BRANCH kernel" "$version" "info" - - # compare with the architecture of the current Debian node - # if it matches we use the system compiler - if dpkg-architecture -e "${ARCH}"; then - display_alert "Native compilation" "target ${ARCH} on host $(dpkg --print-architecture)" - else - display_alert "Cross compilation" "target ${ARCH} on host $(dpkg --print-architecture)" - toolchain=$(find_toolchain "$KERNEL_COMPILER" "$KERNEL_USE_GCC") - [[ -z $toolchain ]] && exit_with_error "Could not find required toolchain" "${KERNEL_COMPILER}gcc $KERNEL_USE_GCC" + # If a `.config` already exists (from previous build), store it, preserving date. + # We will compare the result of the new configuration to it, and if the contents are the same, we'll restore the original date. + # This way we avoid unnecessary recompilation of the kernel; even if the .config contents + # have not changed, the date will be different, and Kbuild will at least re-link everything. + if [[ -f "${kernel_work_dir}/.config" ]]; then + display_alert "Preserving previous kernel configuration" "${previous_config_filename}" "debug" + run_host_command_logged cp -pv "${kernel_work_dir}/.config" "${kernel_work_dir}/${previous_config_filename}" fi - kernel_compiler_version="$(eval env PATH="${toolchain}:${PATH}" "${KERNEL_COMPILER}gcc" -dumpfullversion -dumpversion)" - display_alert "Compiler version" "${KERNEL_COMPILER}gcc ${kernel_compiler_version}" "info" - - # copy kernel config - local COPY_CONFIG_BACK_TO="" - - if [[ $KERNEL_KEEP_CONFIG == yes && -f "${DEST}"/config/$LINUXCONFIG.config ]]; then - display_alert "Using previous kernel config" "${DEST}/config/$LINUXCONFIG.config" "info" + # copy kernel config from configuration, userpatches + if [[ "${KERNEL_KEEP_CONFIG}" == yes && -f "${DEST}"/config/$LINUXCONFIG.config ]]; then + display_alert "Using previously-exported kernel config" "${DEST}/config/$LINUXCONFIG.config" "info" run_host_command_logged cp -pv "${DEST}/config/${LINUXCONFIG}.config" .config else - # @TODO: rpardini: this is too contrived, make obvious, use an array and a loop and stop repeating itself + # @TODO: rpardini: this is too contrived if [[ -f $USERPATCHES_PATH/$LINUXCONFIG.config ]]; then display_alert "Using kernel config provided by user" "userpatches/$LINUXCONFIG.config" "info" run_host_command_logged cp -pv "${USERPATCHES_PATH}/${LINUXCONFIG}.config" .config - COPY_CONFIG_BACK_TO="${USERPATCHES_PATH}/${LINUXCONFIG}.config" + kernel_config_source_filename="${USERPATCHES_PATH}/${LINUXCONFIG}.config" elif [[ -f "${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" ]]; then display_alert "Using kernel config provided by user in config/kernel folder" "config/kernel/${LINUXCONFIG}.config" "info" run_host_command_logged cp -pv "${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" .config - COPY_CONFIG_BACK_TO="${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" + kernel_config_source_filename="${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" else display_alert "Using kernel config file" "config/kernel/$LINUXCONFIG.config" "info" run_host_command_logged cp -pv "${SRC}/config/kernel/${LINUXCONFIG}.config" .config - COPY_CONFIG_BACK_TO="${SRC}/config/kernel/${LINUXCONFIG}.config" + kernel_config_source_filename="${SRC}/config/kernel/${LINUXCONFIG}.config" fi fi - # Store the .config modification date at this time, for restoring later. Otherwise rebuilds. - local kernel_config_mtime - kernel_config_mtime=$(get_file_modification_time ".config") + # Start by running olddefconfig -- always. + # It "updates" the config, using defaults from Kbuild files in the source tree. + # It is worthy noting that on the first run, it builds the tools, so the host-side compiler has to be working, + # regardless of the cross-build toolchain. + run_kernel_make olddefconfig + # Run the core-armbian config modifications here, built-in extensions: + # 1) Enable the options for the extrawifi/drivers; so we don't need to worry about them when updating configs + # 2) Disable: debug, version shenanigans, signing, and compression of modules, to ensure sanity + call_extension_method "armbian_kernel_config" <<- 'ARMBIAN_KERNEL_CONFIG' + *Armbian-core default hook point for pre-olddefconfig Kernel config modifications* + NOT for user consumption. Do NOT use this hook, this is internal to Armbian. + Instead, use `custom_kernel_config` which runs later and can undo anything done by this step. + ARMBIAN_KERNEL_CONFIG + + # Custom hooks receive a clean / updated config; depending on their modifications, they may need to run olddefconfig again. call_extension_method "custom_kernel_config" <<- 'CUSTOM_KERNEL_CONFIG' *Kernel .config is in place, still clean from git version* Called after ${LINUXCONFIG}.config is put in place (.config). - Before any olddefconfig any Kconfig make is called. A good place to customize the .config directly. + Armbian default Kconfig modifications have already been applied and can be overriden. CUSTOM_KERNEL_CONFIG display_alert "Kernel configuration" "${LINUXCONFIG}" "info" - if [[ $KERNEL_CONFIGURE != yes ]]; then - run_kernel_make olddefconfig # @TODO: what is this? does it fuck up dates? - else - display_alert "Starting (non-interactive) kernel olddefconfig" "${LINUXCONFIG}" "debug" - - run_kernel_make olddefconfig - - # No logging for this. this is UI piece - display_alert "Starting (interactive) kernel ${KERNEL_MENUCONFIG:-menuconfig}" "${LINUXCONFIG}" "debug" - run_kernel_make_dialog "${KERNEL_MENUCONFIG:-menuconfig}" - - # Capture new date. Otherwise changes not detected by make. - kernel_config_mtime=$(get_file_modification_time ".config") - - # store kernel config in easily reachable place - mkdir -p "${DEST}"/config - display_alert "Exporting new kernel config" "$DEST/config/$LINUXCONFIG.config" "info" - run_host_command_logged cp -pv .config "${DEST}/config/${LINUXCONFIG}.config" - - # store back into original LINUXCONFIG too, if it came from there, so it's pending commits when done. - if [[ "${COPY_CONFIG_BACK_TO}" != "" ]]; then - display_alert "Exporting new kernel config - git commit pending" "${COPY_CONFIG_BACK_TO}" "info" - run_host_command_logged cp -pv .config "${COPY_CONFIG_BACK_TO}" - - # export defconfig - run_kernel_make savedefconfig - run_host_command_logged cp -pv defconfig "${DEST}/config/${LINUXCONFIG}.defconfig" - run_host_command_logged cp -pv defconfig "${COPY_CONFIG_BACK_TO}.defconfig" - fi - fi +} +function kernel_config_finalize() { call_extension_method "custom_kernel_config_post_defconfig" <<- 'CUSTOM_KERNEL_CONFIG_POST_DEFCONFIG' *Kernel .config is in place, already processed by Armbian* Called after ${LINUXCONFIG}.config is put in place (.config). @@ -100,6 +87,38 @@ function kernel_config() { A good place to customize the .config last-minute. CUSTOM_KERNEL_CONFIG_POST_DEFCONFIG - # Restore the date of .config. Above delta is a pure function, theoretically. - set_files_modification_time "${kernel_config_mtime}" ".config" + # Now, compare the .config with the previous one, and if they are the same, restore the original date. + # This way we avoid unnecessary recompilation of the kernel; even if the .config contents + # have not changed, the date will be different, and Kbuild will at least re-link everything. + if [[ -f "${kernel_work_dir}/${previous_config_filename}" ]]; then + # from "man cmp": Exit status is 0 if inputs are the same, 1 if different, 2 if trouble. + if cmp "${kernel_work_dir}/.config" "${kernel_work_dir}/${previous_config_filename}"; then + display_alert "Kernel configuration unchanged from previous run" "optimizing for fast rebuilds" "cachehit" + run_host_command_logged cp -pv "${kernel_work_dir}/${previous_config_filename}" "${kernel_work_dir}/.config" + else + display_alert "Kernel configuration changed from previous build" "optimizing for correctness" "info" + # ad: added lines; de: deleted lines; hd: header lines; ln: line numbers + # run_host_command_logged diff -u --color=always "--palette='rs=0:hd=1:ad=33:de=37:ln=36'" "${kernel_work_dir}/${previous_config_filename}" "${kernel_work_dir}/.config" "|| true" # no errors please + fi + # either way, remove the previous file + run_host_command_logged rm -f "${kernel_work_dir}/${previous_config_filename}" + fi +} + +function kernel_config_export() { + # store kernel config in easily reachable place + mkdir -p "${DEST}"/config + display_alert "Exporting new kernel config" "$DEST/config/$LINUXCONFIG.config" "info" + run_host_command_logged cp -pv .config "${DEST}/config/${LINUXCONFIG}.config" + + # store back into original LINUXCONFIG too, if it came from there, so it's pending commits when done. + if [[ "${kernel_config_source_filename}" != "" ]]; then + display_alert "Exporting new kernel config - git commit pending" "${kernel_config_source_filename}" "info" + run_host_command_logged cp -pv .config "${kernel_config_source_filename}" + + # export defconfig + run_kernel_make savedefconfig + run_host_command_logged cp -pv defconfig "${DEST}/config/${LINUXCONFIG}.defconfig" + run_host_command_logged cp -pv defconfig "${kernel_config_source_filename}.defconfig" + fi } diff --git a/lib/functions/compilation/kernel-make.sh b/lib/functions/compilation/kernel-make.sh index f026d7753..4f48c0651 100644 --- a/lib/functions/compilation/kernel-make.sh +++ b/lib/functions/compilation/kernel-make.sh @@ -65,3 +65,17 @@ function run_kernel_make_long_running() { display_alert "Kernel Make '$*' took" "$((SECONDS - seconds_start)) seconds" "debug" } +function kernel_determine_toolchain() { + # compare with the architecture of the current Debian node + # if it matches we use the system compiler + if dpkg-architecture -e "${ARCH}"; then + display_alert "Native compilation" "target ${ARCH} on host $(dpkg --print-architecture)" + else + display_alert "Cross compilation" "target ${ARCH} on host $(dpkg --print-architecture)" + toolchain=$(find_toolchain "$KERNEL_COMPILER" "$KERNEL_USE_GCC") + [[ -z $toolchain ]] && exit_with_error "Could not find required toolchain" "${KERNEL_COMPILER}gcc $KERNEL_USE_GCC" + fi + + kernel_compiler_version="$(eval env PATH="${toolchain}:${PATH}" "${KERNEL_COMPILER}gcc" -dumpfullversion -dumpversion)" + display_alert "Compiler version" "${KERNEL_COMPILER}gcc ${kernel_compiler_version}" "info" +} diff --git a/lib/functions/compilation/kernel.sh b/lib/functions/compilation/kernel.sh index f14657777..3af45ad09 100644 --- a/lib/functions/compilation/kernel.sh +++ b/lib/functions/compilation/kernel.sh @@ -26,8 +26,8 @@ function compile_kernel() { LOG_SECTION="kernel_maybe_clean" do_with_logging do_with_hooks kernel_maybe_clean # Patching. - local version hash pre_patch_version - kernel_main_patching + declare hash pre_patch_version + kernel_main_patching # has it's own logging sections inside # Stop after patching; if [[ "${PATCH_ONLY}" == yes ]]; then @@ -35,23 +35,30 @@ function compile_kernel() { return 0 fi - local toolchain - kernel_config_maybe_interactive + # re-read kernel version after patching + declare version + version=$(grab_version "$kernel_work_dir") + display_alert "Compiling $BRANCH kernel" "$version" "info" + + # determine the toolchain + declare toolchain + LOG_SECTION="kernel_determine_toolchain" do_with_logging do_with_hooks kernel_determine_toolchain + + kernel_config # has it's own logging sections inside # package the kernel-source .deb LOG_SECTION="kernel_package_source" do_with_logging do_with_hooks kernel_package_source # build via make and package .debs; they're separate sub-steps - LOG_SECTION="kernel_build_and_package" do_with_logging do_with_hooks kernel_build_and_package + kernel_prepare_build_and_package # has it's own logging sections inside display_alert "Done with" "kernel compile" "debug" - cd "${kernel_work_dir}/.." || exit - rm -f linux-firmware-image-*.deb # remove firmware image packages here - easier than patching ~40 packaging scripts at once - run_host_command_logged rsync --remove-source-files -r ./*.deb "${DEB_STORAGE}/" + LOG_SECTION="kernel_deploy_pkg" do_with_logging do_with_hooks kernel_deploy_pkg # kernel build worked; let's clean up the git-bundle cache, since the git-bare cache is proven working. - kernel_cleanup_bundle_artifacts + # @TODO: armbian-oleg: clean this earlier, so we save some 2gb on disk _during_ kernel compile, not after + LOG_SECTION="kernel_cleanup_bundle_artifacts" do_with_logging do_with_hooks kernel_cleanup_bundle_artifacts return 0 } @@ -115,14 +122,14 @@ function kernel_package_source() { display_alert "$(basename "${sources_pkg_dir}.deb" ".deb") packaged" "$((SECONDS - ts)) seconds, ${tarball_size} tarball, ${package_size} .deb" "info" } -function kernel_build_and_package() { - local ts=${SECONDS} - - cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}" - - local -a build_targets=("all") # "All" builds the vmlinux/Image/Image.gz default for the ${ARCH} +function kernel_prepare_build_and_package() { + declare -a build_targets declare kernel_dest_install_dir - kernel_dest_install_dir=$(mktemp -d "${WORKDIR}/kernel.temp.install.target.XXXXXXXXX") # subject to TMPDIR/WORKDIR, so is protected by single/common error trapmanager to clean-up. + declare -a install_make_params_quoted + declare -A kernel_install_dirs + + build_targets=("all") # "All" builds the vmlinux/Image/Image.gz default for the ${ARCH} + kernel_dest_install_dir=$(mktemp -d "${WORKDIR}/kernel.XXXXXXXXX") # subject to TMPDIR/WORKDIR, so is protected by single/common error trapmanager to clean-up. # define dict with vars passed and target directories declare -A kernel_install_dirs=( @@ -139,7 +146,6 @@ function kernel_build_and_package() { fi # loop over the keys above, get the value, create param value in array; also mkdir the dir - declare -a install_make_params_quoted local dir_key for dir_key in "${!kernel_install_dirs[@]}"; do local dir="${kernel_install_dirs["${dir_key}"]}" @@ -148,14 +154,38 @@ function kernel_build_and_package() { install_make_params_quoted+=("${value}") done + # Fire off the build & package + LOG_SECTION="kernel_build" do_with_logging do_with_hooks kernel_build + + LOG_SECTION="kernel_package" do_with_logging do_with_hooks kernel_package +} + +function kernel_build() { + local ts=${SECONDS} + cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}" + display_alert "Building kernel" "${LINUXFAMILY} ${LINUXCONFIG} ${build_targets[*]}" "info" make_filter="| grep --line-buffered -v -e 'LD' -e 'AR' -e 'INSTALL' -e 'SIGN' -e 'XZ' " \ do_with_ccache_statistics \ run_kernel_make_long_running "${install_make_params_quoted[@]@Q}" "${build_targets[@]}" + display_alert "Kernel built in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info" +} + +function kernel_package() { + local ts=${SECONDS} cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}" display_alert "Packaging kernel" "${LINUXFAMILY} ${LINUXCONFIG}" "info" prepare_kernel_packaging_debs "${kernel_work_dir}" "${kernel_dest_install_dir}" "${version}" kernel_install_dirs - - display_alert "Kernel built and packaged in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info" + display_alert "Kernel packaged in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info" +} + +function kernel_deploy_pkg() { + cd "${kernel_work_dir}/.." || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}" + + # @TODO: rpardini: this is kept for historical reasons... wth? + # remove firmware image packages here - easier than patching ~40 packaging scripts at once + run_host_command_logged rm -fv "linux-firmware-image-*.deb" + + run_host_command_logged rsync -v --remove-source-files -r ./*.deb "${DEB_STORAGE}/" } diff --git a/lib/functions/general/file-mtime.sh b/lib/functions/general/file-mtime.sh deleted file mode 100644 index 6fa46065d..000000000 --- a/lib/functions/general/file-mtime.sh +++ /dev/null @@ -1,38 +0,0 @@ - -function get_file_modification_time() { # @TODO: This is almost always called from a subshell. No use throwing errors? - local -i file_date - if [[ ! -f "${1}" ]]; then - exit_with_error "Can't get modification time of nonexisting file" "${1}" - return 1 - fi - # YYYYMMDDhhmm.ss - it is NOT a valid integer, but is what 'touch' wants for its "-t" parameter - # YYYYMMDDhhmmss - IS a valid integer and we can do math to it. 'touch' code will format it later - file_date=$(date +%Y%m%d%H%M%S -r "${1}") - display_alert "Read modification date for file" "${1} - ${file_date}" "timestamp" - echo -n "${file_date}" - return 0 -} - -function get_dir_modification_time() { - local -i file_date - if [[ ! -d "${1}" ]]; then - exit_with_error "Can't get modification time of nonexisting dir" "${1}" - return 1 - fi - # YYYYMMDDhhmm.ss - it is NOT a valid integer, but is what 'touch' wants for its "-t" parameter - # YYYYMMDDhhmmss - IS a valid integer and we can do math to it. 'touch' code will format it later - file_date=$(date +%Y%m%d%H%M%S -r "${1}") - display_alert "Read modification date for DIRECTORY" "${1} - ${file_date}" "timestamp" - echo -n "${file_date}" - return 0 -} - -# This is for simple "set without thinking" usage, date preservation is done directly by process_patch_file -function set_files_modification_time() { - local -i mtime="${1}" - local formatted_mtime - shift - display_alert "Setting date ${mtime}" "${*} (no newer check)" "timestamp" - formatted_mtime="${mtime:0:12}.${mtime:12}" - touch --no-create -m -t "${formatted_mtime}" "${@}" -} diff --git a/lib/library-functions.sh b/lib/library-functions.sh index 015b800fc..574c20918 100644 --- a/lib/library-functions.sh +++ b/lib/library-functions.sh @@ -442,15 +442,6 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true # shellcheck source=lib/functions/general/downloads.sh source "${SRC}"/lib/functions/general/downloads.sh -# no errors tolerated. invoked before each sourced file to make sure. -#set -o pipefail # trace ERR through pipes - will be enabled "soon" -#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled -set -o errtrace # trace ERR through - enabled -set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled -### lib/functions/general/file-mtime.sh -# shellcheck source=lib/functions/general/file-mtime.sh -source "${SRC}"/lib/functions/general/file-mtime.sh - # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled