Created
July 13, 2026 16:47
-
-
Save RobertKielty/bf372f3807679905f79c13055b6a0af8 to your computer and use it in GitHub Desktop.
fossa_license_triage.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| """Generate a FOSSA licensing-issue triage document for a CNCF project repo. | |
| Run from the top level of any CNCF project's git checkout. Pulls the current | |
| FOSSA policy-flag (licensing) issues for HEAD's revision via `fossa test`, | |
| cross-checks each flagged package's own registry-declared license against the | |
| CNCF allowed-third-party-license policy, locates where each dependency enters | |
| the repo, and writes a markdown triage table. | |
| Requires the `fossa` CLI (https://github.com/fossas/fossa-cli) on PATH. | |
| Usage: | |
| python3 scripts/fossa_license_triage.py [FOSSA_API_TOKEN] [--revision SHA] | |
| [--project URL] [--output PATH] | |
| If FOSSA_API_TOKEN is not given as a positional argument, it is read from the | |
| FOSSA_API_KEY environment variable. | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import re | |
| import subprocess | |
| import sys | |
| import urllib.error | |
| import urllib.request | |
| # Source: https://github.com/cncf/foundation/blob/main/policies-guidance/allowed-third-party-license-policy.md | |
| # Verified 2026-07-13. Re-check this list periodically -- CNCF governance can | |
| # extend the allowlist and this snapshot will not reflect that automatically. | |
| CNCF_ALLOWLIST = { | |
| "0BSD", "BSD-2-Clause", "BSD-2-Clause-FreeBSD", "BSD-3-Clause", "MIT", | |
| "MIT-0", "ISC", "OpenSSL", "OpenSSL-standalone", "PSF-2.0", "Python-2.0", | |
| "Python-2.0.1", "PostgreSQL", "SSLeay-standalone", "UPL-1.0", "X11", | |
| "Zlib", "Apache-2.0", | |
| } | |
| CNCF_ALLOWLIST_URL = "https://raw.githubusercontent.com/cncf/foundation/main/policies-guidance/allowed-third-party-license-policy.md" | |
| REGISTRY_TIMEOUT = 15 | |
| def run(cmd): | |
| return subprocess.run(cmd, capture_output=True, text=True) | |
| def git_remote_url(): | |
| result = run(["git", "config", "--get", "remote.origin.url"]) | |
| url = result.stdout.strip() | |
| if not url: | |
| raise SystemExit("Could not determine git remote 'origin' -- pass --project explicitly.") | |
| # Normalize git@github.com:org/repo.git and https://...git to a plain https URL. | |
| m = re.search(r"github\.com[:/]([^/]+)/([^/.]+)", url) | |
| if not m: | |
| raise SystemExit(f"Unrecognized remote URL format: {url!r} -- pass --project explicitly.") | |
| org, repo = m.groups() | |
| return f"https://github.com/{org}/{repo}" | |
| def git_head_revision(): | |
| result = run(["git", "rev-parse", "HEAD"]) | |
| sha = result.stdout.strip() | |
| if not sha: | |
| raise SystemExit("Could not determine HEAD revision -- pass --revision explicitly.") | |
| return sha | |
| def fetch_cncf_allowlist(): | |
| """Best-effort refresh of the allowlist from the live CNCF policy doc. | |
| Falls back to the hardcoded CNCF_ALLOWLIST snapshot on any failure | |
| (offline, doc restructured, etc.) rather than producing a partial list. | |
| """ | |
| try: | |
| req = urllib.request.Request(CNCF_ALLOWLIST_URL, headers={"User-Agent": "fossa-license-triage"}) | |
| with urllib.request.urlopen(req, timeout=REGISTRY_TIMEOUT) as resp: | |
| text = resp.read().decode("utf-8") | |
| # The policy doc lists allowed licenses as backtick-quoted SPDX ids. | |
| found = set(re.findall(r"`([A-Za-z0-9.\-]+)`", text)) | |
| allowed = {tok for tok in found if tok in CNCF_ALLOWLIST or re.match(r"^[A-Z0-9][A-Za-z0-9.\-]*$", tok)} | |
| # Only trust the scrape if it's a plausible superset of what we already know; | |
| # otherwise the doc's structure changed and our regex is no longer valid. | |
| if CNCF_ALLOWLIST.issubset(allowed): | |
| return allowed | |
| except (urllib.error.URLError, TimeoutError, OSError): | |
| pass | |
| return set(CNCF_ALLOWLIST) | |
| def parse_locator(locator): | |
| """Split a FOSSA locator like 'cargo+zstd-sys$2.0.16+zstd.1.5.7' into parts.""" | |
| fetcher, _, rest = locator.partition("+") | |
| name, _, version = rest.partition("$") | |
| return fetcher, name, version | |
| def run_fossa_test(token, project, revision): | |
| cmd = [ | |
| "fossa", "test", | |
| "--fossa-api-key", token, | |
| "--project", project, | |
| "--revision", revision, | |
| "--format", "json", | |
| ] | |
| result = run(cmd) | |
| # `fossa test` exits 1 when issues are found -- that is expected, not a failure. | |
| try: | |
| return json.loads(result.stdout) | |
| except json.JSONDecodeError: | |
| sys.stderr.write(result.stderr or result.stdout) | |
| raise SystemExit("`fossa test` did not return parseable JSON -- see output above.") | |
| def registry_declared_license(fetcher, name, version): | |
| """Best-effort lookup of a package's own declared license from its registry. | |
| Returns (license_string_or_None, note). A None license means the lookup | |
| failed or the registry didn't expose a usable license field -- callers | |
| must treat that as "needs manual review", not as "no license". | |
| """ | |
| try: | |
| if fetcher == "npm": | |
| url = f"https://registry.npmjs.org/{name}/{version}" | |
| with urllib.request.urlopen(url, timeout=REGISTRY_TIMEOUT) as resp: | |
| data = json.load(resp) | |
| lic = data.get("license") | |
| if isinstance(lic, dict): | |
| lic = lic.get("type") | |
| return lic, "npm registry" | |
| if fetcher == "pip": | |
| url = f"https://pypi.org/pypi/{name}/{version}/json" | |
| with urllib.request.urlopen(url, timeout=REGISTRY_TIMEOUT) as resp: | |
| data = json.load(resp) | |
| info = data.get("info", {}) | |
| lic = info.get("license_expression") or info.get("license") | |
| if not lic: | |
| for classifier in info.get("classifiers", []): | |
| if "License ::" in classifier: | |
| lic = classifier.split("::")[-1].strip() | |
| break | |
| return lic, "PyPI registry" | |
| if fetcher == "cargo": | |
| url = f"https://crates.io/api/v1/crates/{name}/{version}" | |
| req = urllib.request.Request(url, headers={"User-Agent": "fossa-license-triage (contact via CNCF project maintainers)"}) | |
| with urllib.request.urlopen(req, timeout=REGISTRY_TIMEOUT) as resp: | |
| data = json.load(resp) | |
| return data.get("version", {}).get("license"), "crates.io" | |
| if fetcher == "gem": | |
| url = f"https://rubygems.org/api/v1/gems/{name}.json" | |
| with urllib.request.urlopen(url, timeout=REGISTRY_TIMEOUT) as resp: | |
| data = json.load(resp) | |
| licenses = data.get("licenses") or [] | |
| return (" OR ".join(licenses) if licenses else None), "RubyGems (latest version metadata; may differ from flagged version)" | |
| if fetcher == "go": | |
| # Go modules have no central license registry; only a reliable | |
| # answer for github.com-hosted modules via the GitHub license API. | |
| m = re.match(r"^github\.com/([^/]+)/([^/]+)", name) | |
| if m: | |
| owner, repo = m.groups() | |
| url = f"https://api.github.com/repos/{owner}/{repo}/license" | |
| req = urllib.request.Request(url, headers={"User-Agent": "fossa-license-triage"}) | |
| with urllib.request.urlopen(req, timeout=REGISTRY_TIMEOUT) as resp: | |
| data = json.load(resp) | |
| return data.get("license", {}).get("spdx_id"), "GitHub repo license API" | |
| return None, "no registry available for non-GitHub Go modules -- check manually" | |
| if fetcher == "nuget": | |
| url = f"https://api.nuget.org/v3/registration5-semver1/{name.lower()}/{version}.json" | |
| with urllib.request.urlopen(url, timeout=REGISTRY_TIMEOUT) as resp: | |
| data = json.load(resp) | |
| return data.get("catalogEntry", {}).get("licenseExpression"), "NuGet registry" | |
| if fetcher == "composer": | |
| vendor_pkg = name | |
| url = f"https://repo.packagist.org/p2/{vendor_pkg}.json" | |
| with urllib.request.urlopen(url, timeout=REGISTRY_TIMEOUT) as resp: | |
| data = json.load(resp) | |
| versions = next(iter(data.get("packages", {}).values()), []) | |
| for v in versions: | |
| if v.get("version") == version: | |
| licenses = v.get("license") or [] | |
| return (" OR ".join(licenses) if licenses else None), "Packagist" | |
| return None, "version not found on Packagist" | |
| except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError, json.JSONDecodeError, KeyError): | |
| return None, "registry lookup failed -- check manually" | |
| return None, f"no registry lookup implemented for fetcher '{fetcher}' -- check manually" | |
| # Manifest filenames to search for each fetcher, in rough priority order: | |
| # first entry is treated as the "direct dependency declaration" file, the | |
| # rest as lockfiles that only prove presence (direct-vs-transitive unclear). | |
| MANIFEST_FILES = { | |
| "npm": (["package.json"], ["package-lock.json"]), | |
| "pip": (["requirements.txt", "pyproject.toml"], []), | |
| "cargo": (["Cargo.toml"], ["Cargo.lock"]), | |
| "gem": (["Gemfile"], ["Gemfile.lock"]), | |
| "go": (["go.mod"], ["go.sum"]), | |
| "nuget": (["*.csproj"], []), | |
| "composer": (["composer.json"], ["composer.lock"]), | |
| "mix": (["mix.exs"], ["mix.lock"]), | |
| } | |
| def find_manifests(filenames): | |
| matches = [] | |
| for root, dirs, files in os.walk("."): | |
| dirs[:] = [d for d in dirs if d not in (".git", "node_modules", "vendor", ".venv")] | |
| for pattern in filenames: | |
| if "*" in pattern: | |
| regex = re.compile(re.escape(pattern).replace(r"\*", ".*") + "$") | |
| matches += [os.path.join(root, f) for f in files if regex.match(f)] | |
| elif pattern in files: | |
| matches.append(os.path.join(root, pattern)) | |
| return matches | |
| def locate_dependency(fetcher, name): | |
| """Best-effort: which manifest(s) mention this dependency, and does it look direct? | |
| This is a text search, not a real dependency-graph walk -- it tells you | |
| where to look, not a guaranteed direct/transitive verdict. Treat the | |
| result as a starting point for manual confirmation, same as the rest of | |
| this script's "root cause" fields. | |
| """ | |
| direct_files, lock_files = MANIFEST_FILES.get(fetcher, ([], [])) | |
| short_name = name.rsplit("/", 1)[-1] # for e.g. go module paths, gem/npm scoped names | |
| found_direct = [] | |
| for path in find_manifests(direct_files): | |
| try: | |
| content = open(path, encoding="utf-8", errors="ignore").read() | |
| except OSError: | |
| continue | |
| if name in content or short_name in content: | |
| found_direct.append(path) | |
| if found_direct: | |
| return found_direct, "direct (declared in a primary manifest -- verify manually)" | |
| found_lock = [] | |
| for path in find_manifests(lock_files): | |
| try: | |
| content = open(path, encoding="utf-8", errors="ignore").read() | |
| except OSError: | |
| continue | |
| if name in content or short_name in content: | |
| found_lock.append(path) | |
| if found_lock: | |
| return found_lock, "transitive (only found in a lockfile, not a primary manifest)" | |
| return [], "not found by text search -- check manually" | |
| def classify(flagged_license, declared_license, allowlist): | |
| if declared_license is None: | |
| return "UNKNOWN -- needs manual/FOSSA-UI review", None | |
| # Dual/multi-license expressions show up in several conventions across | |
| # registries: SPDX "A OR B", crates.io's legacy "A/B", or npm's "A|B". | |
| stripped = declared_license.strip().strip("()") | |
| branches = [b.strip() for b in re.split(r"\s+OR\s+|\s*/\s*|\s*\|\s*", stripped)] | |
| allowed_branches = [b for b in branches if b in allowlist] | |
| if len(branches) > 1 and allowed_branches: | |
| return f"Dual/multi-licensed -- permissive branch available: {allowed_branches[0]}", allowed_branches[0] | |
| if declared_license in allowlist: | |
| return "Declared license is already CNCF-allowlisted -- flagged license likely a file-scan mismatch", declared_license | |
| if declared_license.replace(" ", "") == (flagged_license or "").replace(" ", ""): | |
| return "Declared license matches the flagged license -- likely a genuine non-allowlist dependency", None | |
| return "Declared license differs from flagged license and is not allowlisted -- needs manual review", None | |
| def build_table(issues, allowlist): | |
| rows = [] | |
| for issue in issues: | |
| if issue.get("type") != "policy_flag" or not issue.get("license"): | |
| continue | |
| fetcher, name, version = parse_locator(issue["revisionId"]) | |
| declared, source_note = registry_declared_license(fetcher, name, version) | |
| manifests, origin_note = locate_dependency(fetcher, name) | |
| verdict, recommend_license = classify(issue["license"], declared, allowlist) | |
| rows.append({ | |
| "id": issue["id"], | |
| "url": issue.get("issueDashURL", ""), | |
| "locator": issue["revisionId"], | |
| "flagged_license": issue["license"], | |
| "declared_license": declared or "(lookup failed)", | |
| "declared_source": source_note, | |
| "manifests": manifests, | |
| "origin_note": origin_note, | |
| "verdict": verdict, | |
| }) | |
| return rows | |
| def render_markdown(rows, project, revision, allowlist): | |
| lines = [ | |
| "# FOSSA Licensing Triage", | |
| "", | |
| f"Project: `{project}` ", | |
| f"Revision: `{revision}` ", | |
| f"CNCF allowlist checked against: {', '.join(sorted(allowlist))}", | |
| "", | |
| f"{len(rows)} unresolved licensing (`policy_flag`) issue(s) found.", | |
| "", | |
| "**Every field below except Issue/License/Locator is a best-effort automated lookup.**", | |
| "Declared license comes from the package's own registry metadata (npm/PyPI/crates.io/RubyGems/" | |
| "NuGet/Packagist/GitHub). Manifest origin comes from a plain text search of the repo, not a real " | |
| "dependency graph walk. Treat the Verdict column as a starting hypothesis to confirm in the FOSSA " | |
| "UI's evidence panel before resolving any issue, especially anything marked UNKNOWN or 'needs manual review'.", | |
| "", | |
| "| Issue | Locator | Flagged license | Declared license (source) | Found in | Verdict |", | |
| "|---|---|---|---|---|---|", | |
| ] | |
| for row in rows: | |
| manifests = "<br>".join(row["manifests"]) if row["manifests"] else "(none found)" | |
| lines.append( | |
| f"| [{row['id']}]({row['url']}) | `{row['locator']}` | {row['flagged_license']} | " | |
| f"{row['declared_license']} ({row['declared_source']}) | {manifests}<br>{row['origin_note']} | " | |
| f"{row['verdict']} |" | |
| ) | |
| lines += [ | |
| "", | |
| "## How this was generated", | |
| "", | |
| "```bash", | |
| f"python3 scripts/fossa_license_triage.py [FOSSA_API_TOKEN] --revision {revision}", | |
| "```", | |
| "", | |
| "Re-run any time; it never uploads or modifies anything on FOSSA (`fossa test` is read-only).", | |
| ] | |
| return "\n".join(lines) | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument("token", nargs="?", help="FOSSA API token (falls back to $FOSSA_API_KEY)") | |
| parser.add_argument("--project", help="Project URL FOSSA knows this repo by (default: derived from git remote 'origin')") | |
| parser.add_argument("--revision", help="Revision/SHA to check (default: current HEAD)") | |
| parser.add_argument("--output", default="licensing-triage.md", help="Output markdown path (default: licensing-triage.md)") | |
| args = parser.parse_args() | |
| token = args.token or os.environ.get("FOSSA_API_KEY") | |
| if not token: | |
| raise SystemExit("No FOSSA API token given: pass it as an argument or set FOSSA_API_KEY.") | |
| project = args.project or git_remote_url() | |
| revision = args.revision or git_head_revision() | |
| print(f"Fetching FOSSA issues for {project} @ {revision} ...", file=sys.stderr) | |
| result = run_fossa_test(token, project, revision) | |
| issues = result.get("issues", []) | |
| allowlist = fetch_cncf_allowlist() | |
| rows = build_table(issues, allowlist) | |
| print(f"{len(rows)} licensing issue(s); resolving declared licenses ...", file=sys.stderr) | |
| markdown = render_markdown(rows, project, revision, allowlist) | |
| with open(args.output, "w", encoding="utf-8") as f: | |
| f.write(markdown + "\n") | |
| print(f"Wrote {args.output}", file=sys.stderr) | |
| if __name__ == "__main__": | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment