archipelago
3a27b6095b
Inventory all four pinned Python runtime/build roots and 20 resolved artifacts with hashes and environment evidence. Reject stale reports; retain custom license names and exact per-file obligations. Trace all eleven prior missing declarations to exact artifacts, registry metadata or parent POMs. Record the newly exposed high ecdsa advisory without suppression and refresh integrated input/BOM hashes while retaining prior receipts. Four pure inventory regressions pass; no host install, release build or live node operation.
222 lines
12 KiB
Python
Executable File
222 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Inventory exact lockfile components/licenses; emit a source CycloneDX BOM.
|
|
|
|
No builds, installs or network calls. Supply cargo metadata from the same lockfile
|
|
and optional completed Gradle dependency output. Unknown license declarations and
|
|
unlocked Python constraints stay explicit; this is not an installed-image SBOM.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import tomllib
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
import yaml
|
|
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--repo', type=Path, default=Path(__file__).resolve().parents[1])
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--cargo-metadata', type=Path)
|
|
parser.add_argument('--android-metadata', type=Path)
|
|
parser.add_argument('--gradle-report', type=Path)
|
|
parser.add_argument('--npm-metadata', type=Path, help='Saved public registry name@version metadata, never executes code')
|
|
parser.add_argument('--maven-metadata', type=Path, help='Saved license declarations from resolved POM files')
|
|
parser.add_argument('--python-report', type=Path, help='Disposable pip install --dry-run --ignore-installed --report output')
|
|
parser.add_argument('--python-metadata', type=Path, help='Exact Python license declarations and artifact provenance')
|
|
args = parser.parse_args()
|
|
root = args.repo.resolve()
|
|
components = {}
|
|
inputs = []
|
|
unresolved = []
|
|
cargo_licenses = {}
|
|
selected_cargo = set()
|
|
npm_metadata = json.loads(args.npm_metadata.read_text()) if args.npm_metadata else {}
|
|
maven_metadata = json.loads(args.maven_metadata.read_text()) if args.maven_metadata else {}
|
|
python_metadata = json.loads(args.python_metadata.read_text()) if args.python_metadata else {}
|
|
python_report = json.loads(args.python_report.read_text()) if args.python_report else None
|
|
for file in [args.cargo_metadata, args.android_metadata]:
|
|
if file:
|
|
metadata = json.loads(file.read_text())
|
|
for package in metadata['packages']:
|
|
selected_cargo.add((package['name'], package['version']))
|
|
cargo_licenses[(package['name'], package['version'])] = package.get('license')
|
|
|
|
|
|
def add(ecosystem, name, version, path, license=None, checksum=None):
|
|
if ecosystem == 'pypi':
|
|
name = re.sub(r'[-_.]+', '-', name).lower()
|
|
if not license and ecosystem == 'npm':
|
|
license = npm_metadata.get(name + '@' + version, {}).get('license')
|
|
if isinstance(license, dict):
|
|
license = license.get('type')
|
|
purl = f'pkg:{ecosystem}/{quote(name, safe="/")}@{quote(version, safe="")}'
|
|
item = components.setdefault(purl, {'type': 'library', 'bom-ref': purl,
|
|
'name': name, 'version': version, 'purl': purl, 'properties': []})
|
|
prop = {'name': 'archipelago:lockfile', 'value': str(path)}
|
|
if prop not in item['properties']:
|
|
item['properties'].append(prop)
|
|
if ecosystem == 'cargo':
|
|
item['properties'].append({'name': 'archipelago:selected-cargo-metadata', 'value': str((name, version) in selected_cargo).lower()})
|
|
if not license:
|
|
for cached in Path.home().glob('.cargo/registry/src/*/' + name + '-' + version + '/Cargo.toml'):
|
|
try:
|
|
license = tomllib.loads(cached.read_text()).get('package', {}).get('license')
|
|
except (OSError, tomllib.TOMLDecodeError):
|
|
pass
|
|
if license:
|
|
break
|
|
if isinstance(license, str) and license:
|
|
item['licenses'] = [{'expression': license}]
|
|
if ecosystem == 'maven':
|
|
declarations = maven_metadata.get(name + '@' + version, {}).get('licenses', [])
|
|
if declarations:
|
|
item['licenses'] = [{'license': declaration} for declaration in declarations]
|
|
if ecosystem == 'pypi':
|
|
metadata = python_metadata.get(name + '@' + version, {})
|
|
# Custom license names and per-file terms must not become invented SPDX
|
|
# expressions. The evidence explicitly distinguishes those two formats.
|
|
if metadata.get('licenses'):
|
|
item['licenses'] = metadata['licenses']
|
|
if metadata.get('provenance'):
|
|
prop = {'name': 'archipelago:license-provenance',
|
|
'value': json.dumps(metadata['provenance'], sort_keys=True)}
|
|
if prop not in item['properties']:
|
|
item['properties'].append(prop)
|
|
if checksum:
|
|
item['hashes'] = [{'alg': 'SHA-256', 'content': checksum}]
|
|
return item
|
|
|
|
|
|
def read(path):
|
|
data = (root / path).read_bytes()
|
|
inputs.append({'path': str(path), 'sha256': hashlib.sha256(data).hexdigest()})
|
|
return data
|
|
|
|
|
|
for path in [Path('core/Cargo.lock'), Path('Android/rust/archy-fips-core/Cargo.lock')]:
|
|
lock = tomllib.loads(read(path).decode())
|
|
for package in lock['package']:
|
|
add('cargo', package['name'], package['version'], path,
|
|
cargo_licenses.get((package['name'], package['version'])), package.get('checksum'))
|
|
|
|
for path in [Path('neode-ui/package-lock.json'), Path('apps/router/package-lock.json'),
|
|
Path('examples/archipelago-app-starter/package-lock.json')]:
|
|
lock = json.loads(read(path))
|
|
for location, package in lock.get('packages', {}).items():
|
|
if not location or package.get('link'):
|
|
continue
|
|
name = package.get('name') or location.rsplit('node_modules/', 1)[-1]
|
|
if not package.get('version'):
|
|
unresolved.append({'input': str(path), 'package': name, 'reason': 'missing version'})
|
|
continue
|
|
add('npm', name, package['version'], path, package.get('license'))
|
|
|
|
for path in [Path('aiui/pnpm-lock.yaml'), Path('docker/gashboard/pnpm-lock.yaml')]:
|
|
lock = yaml.safe_load(read(path))
|
|
for key in lock.get('packages', {}):
|
|
coordinate = key.split('(', 1)[0].lstrip('/')
|
|
name, separator, version = coordinate.rpartition('@')
|
|
if not separator or not name or not re.match(r'^\d', version):
|
|
unresolved.append({'input': str(path), 'package': key, 'reason': 'non-registry coordinate'})
|
|
continue
|
|
add('npm', name, version, path)
|
|
|
|
if args.gradle_report:
|
|
text = args.gradle_report.read_text()
|
|
if 'BUILD SUCCESSFUL' not in text:
|
|
unresolved.append({'input': str(args.gradle_report), 'reason': 'Gradle resolution failed; no complete Maven graph'})
|
|
else:
|
|
for match in re.finditer(r'[-+]--- ([\w.-]+):([\w.-]+):([\w.+-]+)(?: -> ([\w.+-]+))?', text):
|
|
group, name, version, selected = match.groups()
|
|
add('maven', group + '/' + name, selected or version, 'Android releaseRuntimeClasspath')
|
|
else:
|
|
unresolved.append({'input': 'Android Gradle', 'reason': 'resolved Maven graph not supplied'})
|
|
|
|
python_roots = []
|
|
for path in sorted((root / 'reticulum-daemon').glob('requirements*.txt')):
|
|
relative = path.relative_to(root)
|
|
for line in read(relative).decode().splitlines():
|
|
line = line.split('#', 1)[0].strip()
|
|
if line and not line.startswith('-'):
|
|
pinned = re.fullmatch(r'([A-Za-z0-9_.-]+)==([A-Za-z0-9_.+-]+)', line)
|
|
if pinned:
|
|
add('pypi', pinned[1], pinned[2], relative)
|
|
python_roots.append((re.sub(r'[-_.]+', '-', pinned[1]).lower(), pinned[2], str(relative)))
|
|
else:
|
|
unresolved.append({'input': str(relative), 'constraint': line,
|
|
'reason': 'Python requirement is not an exact pin'})
|
|
|
|
# esptool is currently pinned in the two build entry points rather than the
|
|
# requirements files. Include both sources, and fail a stale report if they drift.
|
|
for path in [Path('reticulum-daemon/build.sh'), Path('reticulum-daemon/build-esptool.sh')]:
|
|
for version in re.findall(r'esptool==([A-Za-z0-9_.+-]+)', read(path).decode()):
|
|
add('pypi', 'esptool', version, path)
|
|
python_roots.append(('esptool', version, str(path)))
|
|
|
|
if python_report:
|
|
if python_report.get('version') != '1' or not python_report.get('environment'):
|
|
parser.error('Python report must use pip report version 1 and include its environment')
|
|
resolved = {}
|
|
for package in python_report.get('install', []):
|
|
metadata = package['metadata']
|
|
name = re.sub(r'[-_.]+', '-', metadata['name']).lower()
|
|
version = metadata['version']
|
|
if name in resolved:
|
|
parser.error('Duplicate Python report package: ' + name)
|
|
resolved[name] = version
|
|
download = package['download_info']
|
|
checksum = download.get('archive_info', {}).get('hashes', {}).get('sha256')
|
|
if not checksum or not re.fullmatch(r'[0-9a-f]{64}', checksum):
|
|
parser.error('Python report lacks exact SHA-256 artifact evidence: ' + name)
|
|
item = add('pypi', name, version, args.python_report, checksum=checksum)
|
|
if not item.get('licenses'):
|
|
if metadata.get('license_expression'):
|
|
item['licenses'] = [{'expression': metadata['license_expression']}]
|
|
elif metadata.get('license'):
|
|
item['licenses'] = [{'license': {'name': metadata['license']}}]
|
|
item['externalReferences'] = [{'type': 'distribution', 'url': download['url']}]
|
|
item['properties'].extend([
|
|
{'name': 'archipelago:python-requested', 'value': str(package.get('requested', False)).lower()},
|
|
{'name': 'archipelago:python-requires-dist', 'value': json.dumps(metadata.get('requires_dist', []))},
|
|
{'name': 'archipelago:python-resolution-environment',
|
|
'value': json.dumps(python_report['environment'], sort_keys=True)},
|
|
])
|
|
for name, version, path in python_roots:
|
|
if resolved.get(name) != version:
|
|
parser.error(f'Python report does not match {path}: {name}=={version}')
|
|
unresolved.append({'input': str(args.python_report),
|
|
'reason': 'point-in-time resolution for the recorded environment; not a checked-in transitive lock, other-platform graph, embedded-vendor inventory or installed artifact'})
|
|
else:
|
|
for name, version, path in python_roots:
|
|
unresolved.append({'input': path, 'constraint': name + '==' + version,
|
|
'reason': 'requirements constraint is not an exact resolved lock graph'})
|
|
|
|
for evidence in [args.cargo_metadata, args.android_metadata, args.gradle_report, args.npm_metadata,
|
|
args.maven_metadata, args.python_report, args.python_metadata]:
|
|
if evidence:
|
|
inputs.append({'evidence': str(evidence), 'sha256': hashlib.sha256(evidence.read_bytes()).hexdigest()})
|
|
|
|
args.output.mkdir(parents=True, exist_ok=True)
|
|
bom = {'bomFormat': 'CycloneDX', 'specVersion': '1.5', 'version': 1,
|
|
'metadata': {'component': {'type': 'application', 'name': 'Archipelago source dependency inventory'},
|
|
'properties': [{'name': 'archipelago:scope', 'value': 'source lockfiles; not an installed-artifact SBOM'}]},
|
|
'components': sorted(components.values(), key=lambda item: item['bom-ref'])}
|
|
(args.output / 'source-bom.cdx.json').write_text(json.dumps(bom, indent=2) + '\n')
|
|
license_rows = [{'purl': item['purl'], 'licenses': item.get('licenses', []),
|
|
'inputs': [p['value'] for p in item['properties'] if p['name'] == 'archipelago:lockfile']}
|
|
for item in bom['components']]
|
|
report = {'inputs': inputs, 'components': len(license_rows),
|
|
'missing_license_declarations': sum(not item['licenses'] for item in license_rows),
|
|
'cargo_lock_only_components': sum(c['purl'].startswith('pkg:cargo/') and any(p['name'] == 'archipelago:selected-cargo-metadata' and p['value'] == 'false' for p in c['properties']) for c in bom['components']),
|
|
'licenses': license_rows, 'unresolved': unresolved}
|
|
if python_report:
|
|
report['python_resolution'] = {'environment': python_report['environment'],
|
|
'components': len(python_report['install']),
|
|
'roots': sorted(set(name + '==' + version for name, version, _ in python_roots))}
|
|
(args.output / 'source-inventory.json').write_text(json.dumps(report, indent=2) + '\n')
|
|
print(json.dumps({'components': len(license_rows), 'inputs': len(inputs),
|
|
'missing_license_declarations': report['missing_license_declarations'],
|
|
'unresolved': len(unresolved)}))
|