libloot/docs/scripts/licenses.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import datetime
import json
import os
import re
import sys
import subprocess

from urllib.request import urlopen

SKIPPABLE_NOTICE_LICENSES = [
    # CC0 and Unlicense licenses don't require license redistribution.
    'CC0-1.0',
    'Unlicense',
    # Zlib and MPL licenses don't need to be included in binary distributions.
    'Zlib',
    'MPL-2.0'
]
SKIPPABLE_LICENSE_FILES = SKIPPABLE_NOTICE_LICENSES + [
    # Skip GPL v3 because it's already present as libloot's own license.
    'GPL-3.0',
    'GPL-3.0-only',
    'GPL-3.0-or-later'
]
# Don't include notices for libloot's own crates.
SKIPPABLE_CRATES = [
        'libloot',
        'libloot-ffi-errors',
]
# Notice extraction is not flawless and may fail to find notices, so I've
# verified that these have no notices.
NO_NOTICES_WHITELIST = [
    ('allocator-api2',  '0.2.21'),
    ('cxx',             '1.0.192'),
    ('cxxbridge-macro', '1.0.192'),
    ('delegate',        '0.13.4'),
    ('link-cplusplus',  '1.0.12'),
    ('once_cell',       '1.21.4'),
    ('option-ext',      '0.2.0'),
    ('pelite-macros',   '0.1.1'),
    ('pest',            '2.8.0'),
    ('proc-macro2',     '1.0.106'),
    ('quote',           '1.0.45'),
    ('rustc-hash',      '2.1.2'),
    ('rustversion',     '1.0.20'),
    ('serde',           '1.0.228'),
    ('serde_core',      '1.0.228'),
    ('serde_derive',    '1.0.228'),
    ('syn',             '2.0.117'),
    ('thiserror',       '2.0.12'),
    ('thiserror-impl',  '2.0.12'),
]

YEAR_REGEX = re.compile(r'\d{4}')

def recurse_dependencies(packages, current_package, dependency_names):
    for dependency in current_package['dependencies']:
        name = dependency['name']
        if name in dependency_names or dependency['kind'] in ['dev', 'build'] or dependency['target'] == 'cfg(target_os = \"redox\")':
            continue

        dependency_names.add(name)
        for package in packages:
            if package['name'] == name:
                recurse_dependencies(packages, package, dependency_names)

def get_package_dependencies(cargo_toml_path, package_name):
    result = subprocess.run(
        [
            'cargo',
            'metadata',
            '--manifest-path',
            cargo_toml_path,
            '--filter-platform',
            'x86_64-pc-windows-msvc',
            '--filter-platform',
            'x86_64-unknown-linux-gnu',
            '--format-version',
            '1'
        ],
        capture_output=True,
        text=True,
        check=True
    )

    json_output = json.loads(result.stdout[:-1])
    packages = json_output['packages']

    dependency_names = set()
    for package in packages:
        if package['name'] == package_name:
            recurse_dependencies(packages, package, dependency_names)

    dependencies = []
    for package in packages:
        if package['name'] in dependency_names:
            dependencies.append({
                'name': package['name'],
                'version': package['version'],
                'license': package['license'],
                'manifest_path': package['manifest_path'],
            })

    return dependencies

def get_url(dependency):
    return f'https://crates.io/crates/{dependency['name']}/{dependency['version']}'

def parse_licenses(dependency):
    # This is not robust, but it's good enough with a couple of specific workarounds.

    if 'license' not in dependency:
        print(f'Found dependency with no license: {dependency['name']}', file=sys.stderr)
        exit(1)

    # Fix non-SPDX expression
    if dependency['license'] == 'MIT/Apache-2.0':
        expression = 'MIT OR Apache-2.0'

    # Preselect a license to simplify a couple of expressions.
    elif dependency['license'] == '(Apache-2.0 OR MIT) AND BSD-3-Clause':
        expression = 'MIT AND BSD-3-Clause'

    elif dependency['license'] == '(MIT OR Apache-2.0) AND Unicode-3.0':
        expression = 'MIT AND Unicode-3.0'

    else:
        expression = dependency['license']

    licenses_any = list(map(str.strip, expression.split('OR')))

    # If it's possible to pick a license that doesn't need a notice, pick it.
    for license in licenses_any:
        licenses_all = list(map(str.strip, license.split('AND')))
        if all(license in SKIPPABLE_NOTICE_LICENSES for license in licenses_all):
            return licenses_all

    # MIT OR Apache-2.0 is a common combination, MIT also appears on its own.
    if 'MIT' in licenses_any:
        return ['MIT']

    return list(map(str.strip, licenses_any[0].split('AND')))

def download_licenses(dependencies, output_directory):
    unique_licenses = set()
    for dependency in dependencies:
        licenses = parse_licenses(dependency)
        unique_licenses.update(licenses)

    # These three all have the same license text.
    if 'GPL-3.0-or-later' in unique_licenses:
        if 'GPL-3.0' in unique_licenses:
            unique_licenses.remove('GPL-3.0')

        if 'GPL-3.0-only' in unique_licenses:
            unique_licenses.remove('GPL-3.0-only')

    for license in unique_licenses:
        if license in SKIPPABLE_LICENSE_FILES:
            continue

        license_url = f'https://raw.githubusercontent.com/spdx/license-list-data/refs/heads/main/text/{license}.txt'

        print(f'Downloading {license} from {license_url}...')

        response = urlopen(license_url)

        if response.status != 200:
            print(f'Error fetching license: {response.status} {response.reason}, {response.read()}', file=sys.stderr)
            exit(1)

        with open(os.path.join(output_directory, f'{license}'), 'wb') as outfile:
            outfile.write(response.read())

def read_notices_from_file(file_path):
    notices = []
    with open(file_path, 'r', encoding='utf-8') as file:
        for line in file:
            lowercase_line = line.lower()
            if 'copyright' in lowercase_line and ('(c)' in lowercase_line or '©' in line or re.search(YEAR_REGEX, line)):
                notices.append(line.strip())

    return notices

def read_notices(dependency):
    # First hardcode a couple of notices that get mangled by the code below.
    if dependency['name'] == 'hashlink':
        return ['Copyright (c) 2015 The Rust Project Developers']
    elif dependency['name'] == 'libloadorder':
        return ['Copyright (C) 2017 Oliver Hamlet']
    elif dependency['name'] == 'esplugin':
        return ['Copyright (C) 2017 Oliver Hamlet']

    crate_path = os.path.dirname(dependency['manifest_path'])

    notices = []

    for entry in os.scandir(crate_path):
        lowercase_filename = entry.name.lower()
        if 'license' in lowercase_filename or 'copying' in lowercase_filename:
            if entry.is_file():
                notices.extend(read_notices_from_file(entry.path))
            elif entry.is_dir():
                for entry in os.scandir(entry.path):
                    if entry.is_file():
                        notices.extend(read_notices_from_file(entry.path))

    notices = list(set(notices))
    notices.sort()

    return notices

if __name__ == "__main__":
    target_package_name = 'libloot-cpp'
    cargo_toml_path = os.path.join('..', 'cpp', 'Cargo.toml')
    output_dir = os.path.join('api', 'copyright')

    package_dependencies = get_package_dependencies(cargo_toml_path, target_package_name)

    download_licenses(package_dependencies, os.path.join(output_dir, 'licenses'))

    notices_rst = f'.. This file was generated by scripts/licenses.py at {datetime.datetime.now().isoformat()}.\n\n'

    heading = 'Dependency Copyright Notices'
    notices_rst += heading
    notices_rst += '\n'
    notices_rst += '=' * len(heading)
    notices_rst += '\n\n'

    for dependency in package_dependencies:
        if dependency['name'] in SKIPPABLE_CRATES:
            continue

        if all(license in SKIPPABLE_NOTICE_LICENSES for license in parse_licenses(dependency)):
            print(f'Skipping {dependency['name']} because its license allows it: {dependency['license']}')
            continue

        notices = read_notices(dependency)

        if not notices:
            if (dependency['name'], dependency['version']) not in NO_NOTICES_WHITELIST:
                print(f'Found dependency with no notices: {dependency['name']} v{dependency['version']}', file=sys.stderr)
                exit(1)
            else:
                print(f'Skipping dependency with no notices: {dependency['name']}')
                continue

        link_rst = f'`{dependency['name']} v{dependency['version']} <{get_url(dependency)}>`_'
        underline_rst = '-' * len(link_rst)
        dep_notices_rst = '\n    '.join(notices)

        section_rst = f'{link_rst}\n{underline_rst}\n\n::\n\n    {dep_notices_rst}\n\n'

        notices_rst += section_rst

    notices_rst_path = os.path.join(output_dir, 'dependency-notices.rst')
    with open(notices_rst_path, 'w', encoding="utf-8") as outfile:
        outfile.write(notices_rst)