diff --git a/lib/functions/cli/cli-jsoninfo.sh b/lib/functions/cli/cli-jsoninfo.sh index 07a6f5427..22ee241e6 100644 --- a/lib/functions/cli/cli-jsoninfo.sh +++ b/lib/functions/cli/cli-jsoninfo.sh @@ -111,6 +111,7 @@ function cli_json_info_run() { declare TARGETS_OUTPUT_FILE="${BASE_INFO_OUTPUT_DIR}/all-targets.json" declare IMAGE_INFO_FILE="${BASE_INFO_OUTPUT_DIR}/image-info.json" declare IMAGE_INFO_CSV_FILE="${BASE_INFO_OUTPUT_DIR}/image-info.csv" + declare INVENTORY_BOARDS_CSV_FILE="${BASE_INFO_OUTPUT_DIR}/boards-inventory.csv" declare REDUCED_ARTIFACTS_FILE="${BASE_INFO_OUTPUT_DIR}/artifacts-reduced.json" declare ARTIFACTS_INFO_FILE="${BASE_INFO_OUTPUT_DIR}/artifacts-info.json" declare ARTIFACTS_INFO_UPTODATE_FILE="${BASE_INFO_OUTPUT_DIR}/artifacts-info-uptodate.json" @@ -172,6 +173,13 @@ function cli_json_info_run() { run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${INFO_TOOLS_DIR}"/json2csv.py "<" "${IMAGE_INFO_FILE}" ">" ${IMAGE_INFO_CSV_FILE} fi + if [[ "${ARMBIAN_COMMAND}" == "inventory-boards" ]]; then + display_alert "Preparing" "inventory-boards" "info" + run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${INFO_TOOLS_DIR}"/inventory-boards-csv.py "<" "${IMAGE_INFO_FILE}" ">" ${INVENTORY_BOARDS_CSV_FILE} + display_alert "Done with" "inventory-boards: ${INVENTORY_BOARDS_CSV_FILE}" "info" + exit 0 + fi + if [[ "${ARMBIAN_COMMAND}" == "targets-dashboard" ]]; then display_alert "To load the OpenSearch dashboards:" " pip3 install opensearch-py # install needed lib to talk to OpenSearch diff --git a/lib/functions/cli/commands.sh b/lib/functions/cli/commands.sh index 8de1123be..d0995068c 100644 --- a/lib/functions/cli/commands.sh +++ b/lib/functions/cli/commands.sh @@ -28,6 +28,7 @@ function armbian_register_commands() { ["inventory"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run ["targets"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run ["targets-dashboard"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run + ["inventory-boards"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run ["targets-composed"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run ["debs-to-repo-json"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run ["gha-matrix"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run @@ -103,6 +104,9 @@ function armbian_register_commands() { ["debs-to-repo-download"]="CLEAN_MATRIX='no' CLEAN_INFO='no'" ["debs-to-repo-reprepro"]="CLEAN_MATRIX='no' CLEAN_INFO='no'" + # inventory + ["inventory-boards"]="TARGETS_FILE='something_that_does_not_exist_so_defaults_are_used'" + # patching ["rewrite-kernel-patches"]="REWRITE_PATCHES=yes" # rewrite the patches after round-tripping to git: "rebase patches" diff --git a/lib/tools/info/inventory-boards-csv.py b/lib/tools/info/inventory-boards-csv.py new file mode 100755 index 000000000..2059d818c --- /dev/null +++ b/lib/tools/info/inventory-boards-csv.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 + +# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2023,2024 Ricardo Pardini +# This file is a part of the Armbian Build Framework https://github.com/armbian/build/ +# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹ +import collections.abc +import json +import logging +import os + +import sys + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from common import armbian_utils + +# Prepare logging +armbian_utils.setup_logging() +log: logging.Logger = logging.getLogger("info-gatherer-image") + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def flatten(d, parent_key='', sep='_'): + items = [] + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, collections.abc.MutableMapping): + items.extend(flatten(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + +# read json from filename + +# if ran with an argument, use it: +filename = None +if len(sys.argv) > 1: + filename = sys.argv[1] + with open(filename) as f: + json_object = json.load(f) + log.info("Loaded {} objects from file...".format(len(json_object))) +else: # load from stdin + log.info("Reading from stdin...") + json_object = json.load(sys.stdin) + log.info("Loaded {} objects from stdin...".format(len(json_object))) + +flat = [] +for obj in json_object: + flat.append(flatten(obj, '', '.')) + +columns_map = {} +for obj in flat: + # get the string keys + for key in obj.keys(): + value = obj[key] + if type(value) == str: + columns_map[key] = True + if type(value) == bool: + columns_map[key] = True + +columns = columns_map.keys() + +log.info("size of columns: {}".format(len(columns))) +log.info("columns: {}".format((columns))) + +# Now, find the columns of which all values are the same +# and remove them +columns_to_remove = [] +for column in columns: + values = [] + for obj in flat: + value = obj.get(column) + values.append(value) + if len(set(values)) == 1: + columns_to_remove.append(column) + +eprint("columns with all-identical values: {}: '{}'".format(len(columns_to_remove), columns_to_remove)) + +# Now actually filter columns, removing columns_to_remove +columns = [column for column in columns if column not in columns_to_remove] + +# Actually remove the columns from the objects as well. +for obj in flat: + for column in columns_to_remove: + del obj[column] + +# Now, group by 'in.inventory.BOARD' field +flat_grouped_by_board: dict[str, list[dict[str, str]]] = {} +for obj in flat: + # if 'out.BOOT_SOC' not in obj: + # continue + board = obj['in.inventory.BOARD'] + if board not in flat_grouped_by_board: + flat_grouped_by_board[board] = [] + flat_grouped_by_board[board].append(obj) + +log.info(f"Grouped by board: {flat_grouped_by_board.keys()}") + +unique_boards: list[str] = flat_grouped_by_board.keys() + +board_info: dict[str, dict[str, str]] = {} + +# Now, for each unique board, find the columns of which all values are the same in that group. +for board in unique_boards: + flats_for_board = flat_grouped_by_board[board] + columns_to_keep = [] + forced_values = {} + for column in columns: + # Skip if column begins with 'in.inventory.BOARD_TOP_LEVEL_VARS' + if column.startswith('in.inventory.BOARD_TOP_LEVEL_VARS'): + continue + values = [] + for obj in flats_for_board: + value = obj.get(column) + if value is None or value == "": + continue + values.append(str(value)) + set_of_values = set(values) + num_diff_values = len(set_of_values) + if num_diff_values == 1: + columns_to_keep.append(column) + elif num_diff_values > 0: + forced_values[column] = "<> " + " /// ".join(set_of_values) + else: + forced_values[column] = "<>" + # log.info(f"Board {board} has {len(columns_to_keep)} columns with all-identical values: {columns_to_keep}") + # Now actually filter columns + # make a copy of only the common columns + obj = flats_for_board[0] # use the first value, they're all the same anyway + obj_common = {} + for column in columns_to_keep: + if column in obj: + obj_common[column] = obj[column] + else: + log.warning(f"Column {column} not found in first of board {board}!") + for column in forced_values.keys(): + obj_common[column] = forced_values[column] + # add to board_info + board_info[board] = obj_common + +common_columns: set[str] = set() +for board in board_info.keys(): + for key in board_info[board].keys(): + common_columns.add(key) + +log.info(f"Common columns: {common_columns}") + +# remove non-common columns from board_info +for board in board_info.keys(): + obj = board_info[board] + for key in obj.keys(): + if key not in common_columns: + del obj[key] + +import csv + +sorted_common_columns = sorted(common_columns) + +handpicked_columns = [ + 'in.inventory.BOARD', + 'in.inventory.BOARD_FILE_HARDWARE_DESC', + 'in.inventory.BOARD_SUPPORT_LEVEL', + 'out.BOARDFAMILY', + 'out.KERNEL_TARGET', + 'out.LINUXFAMILY', + 'out.BOARD_NAME', + 'out.BOARD_MAINTAINER', + 'out.ARCH', + 'out.BOOT_FDT_FILE', + 'out.BOOTCONFIG', + 'out.BOOT_SOC', + 'out.ATF_COMPILE', + 'out.KERNEL_MAJOR_MINOR', + 'out.BOOTBRANCH', + 'out.BOOT_SCENARIO', + 'out.OVERLAY_PREFIX' +] + + +def write_csv(writer, board_info): + global board, obj + writer.writeheader() + for board in board_info.keys(): + obj = board_info[board] + writer.writerow(obj) + + +if filename is not None: + output_filename = filename + ".inventory.csv" + log.info(f"Writing to {output_filename} ...") + with open(output_filename, 'w') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=handpicked_columns, extrasaction='ignore') + write_csv(writer, board_info) + log.info(f"Wrote to file {output_filename} ...") +else: + writer = csv.DictWriter(sys.stdout, fieldnames=handpicked_columns, extrasaction='ignore') + write_csv(writer, board_info) + log.info("Wrote to stdout...")