mirror of
https://github.com/m5stack/build.git
synced 2026-05-20 11:48:25 -07:00
Ricardo Pardini
f8ddf7f9e2
- pipeline: add `pipeline` context object to targets; use it to filter artifacts and images to build; warn about oci-name with multiple oci-tags - pipeline: better error messages when info's fail; show some (out-of-order) progress messages during parallel info gathering - pipeline: targets-compositor: add `not-eos` inventory - TARGETS_FILENAME, log all OCI lookups - SKIP_IMAGES - IMAGES_ONLY_OUTDATED_ARTIFACTS - no dash in chunk id in JSON - pipeline: very initial chunking, using the same outputs - pipeline: template targets, `items-from-inventory:` inventory expansion, CHECK_OCI=yes, CLEAN_MATRIX=yes, CLEAN_INFO=yes, many fixes - cli: `inventory` / `targets` / `matrix` / `workflow` - pipeline: workflow beginnings - pipeline: general log cleanup + OCI stats / better miss handling - pipeline: fixes/reorg - pipeline: catch & log JSON parsing errors - pipeline: gha matrix: use IMAGE_FILE_ID as job description - pipeline (delusion): gha workflow output, based on old matrix code - pipeline: better parsing and reporting of stderr log lines (under `ANSI_COLOR=none`) - pipeline: mapper-oci-uptodate: use separate positive/negative cache dirs (GHA will only cache positives); cache negs for 5 minutes locally - pipeline: output-gha-matrix artifacts + images - pipeline: output-gha-matrix artifacts + images: "really" and fake 1-item matrix if empty - pipeline: move files into subdir; update copyright & cleanup - pipeline: refactor bash jsoninfo driver a bit - pipeline: outdated-artifact-image-reducer - pipeline: introduce `target_id` at the compositor, aggregate it at the reducer, carry it over in the artifact info mapper - pipeline: mapper-oci-uptodate - pipeline: info-gatherer-artifact, with PRE_PREPARED_HOST - pipeline: refactor/rename info-gatherer-image.py - pipeline: beginnings
84 lines
2.7 KiB
Python
Executable File
84 lines
2.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹
|
|
# SPDX-License-Identifier: GPL-2.0
|
|
# Copyright (c) 2023 Ricardo Pardini <ricardo@pardini.net>
|
|
# This file is a part of the Armbian Build Framework https://github.com/armbian/build/
|
|
# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹
|
|
import collections.abc
|
|
import json
|
|
import logging
|
|
import os
|
|
|
|
import sys
|
|
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
from common import armbian_utils
|
|
|
|
# Prepare logging
|
|
armbian_utils.setup_logging()
|
|
log: logging.Logger = logging.getLogger("info-gatherer-image")
|
|
|
|
|
|
def eprint(*args, **kwargs):
|
|
print(*args, file=sys.stderr, **kwargs)
|
|
|
|
|
|
def flatten(d, parent_key='', sep='_'):
|
|
items = []
|
|
for k, v in d.items():
|
|
new_key = parent_key + sep + k if parent_key else k
|
|
if isinstance(v, collections.abc.MutableMapping):
|
|
items.extend(flatten(v, new_key, sep=sep).items())
|
|
else:
|
|
items.append((new_key, v))
|
|
return dict(items)
|
|
|
|
|
|
json_object = json.load(sys.stdin)
|
|
eprint("Loaded {} objects from stdin...".format(len(json_object)))
|
|
|
|
flat = []
|
|
for obj in json_object:
|
|
flat.append(flatten(obj, '', '.'))
|
|
|
|
columns_map = {}
|
|
for obj in flat:
|
|
# get the string keys
|
|
for key in obj.keys():
|
|
value = obj[key]
|
|
if type(value) == str:
|
|
columns_map[key] = True
|
|
if type(value) == bool:
|
|
columns_map[key] = True
|
|
|
|
columns = columns_map.keys()
|
|
|
|
eprint("columns: {}".format(len(columns)))
|
|
|
|
# Now, find the columns of which all values are the same
|
|
# and remove them
|
|
columns_to_remove = []
|
|
for column in columns:
|
|
values = []
|
|
for obj in flat:
|
|
value = obj.get(column)
|
|
values.append(value)
|
|
if len(set(values)) == 1:
|
|
columns_to_remove.append(column)
|
|
|
|
# eprint("columns with all-identical values: {}: '{}'".format(len(columns_to_remove), columns_to_remove))
|
|
|
|
# Now actually filter columns, removing columns_to_remove
|
|
columns = [column for column in columns if column not in columns_to_remove]
|
|
|
|
import csv
|
|
|
|
writer = csv.DictWriter(sys.stdout, fieldnames=columns, extrasaction='ignore')
|
|
|
|
writer.writeheader()
|
|
for obj in flat:
|
|
writer.writerow(obj)
|
|
|
|
eprint("Done writing CSV to stdout.")
|