|
| 1 | +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +"""Check same-repository branch URLs against a Git tree, without HTTP requests. |
| 16 | +
|
| 17 | +Input is Lychee's unfiltered HTTP(S) ``--dump`` output, one URL per line. |
| 18 | +Explicit tags, SHAs, other branches, and external destinations are written to |
| 19 | +a separate file for Lychee's normal online check. Relative Fern routes and |
| 20 | +fragments are outside this file/directory existence check. |
| 21 | +""" |
| 22 | + |
| 23 | +from __future__ import annotations |
| 24 | + |
| 25 | +import argparse |
| 26 | +import logging |
| 27 | +import subprocess |
| 28 | +from pathlib import Path |
| 29 | +from urllib.parse import unquote, urlsplit |
| 30 | + |
| 31 | + |
| 32 | +LOGGER = logging.getLogger(__name__) |
| 33 | + |
| 34 | + |
| 35 | +def repository_target(url: str, *, repository: str, branch: str) -> tuple[str, str] | None: |
| 36 | + """Return the expected Git object type and path for a candidate-tree URL. |
| 37 | +
|
| 38 | + Args: |
| 39 | + url: Absolute URL extracted by Lychee. |
| 40 | + repository: Upstream owner/repository, independent of a fork's origin. |
| 41 | + branch: Branch whose URLs should resolve against the checked tree. |
| 42 | +
|
| 43 | + Returns: |
| 44 | + A Git type and decoded path, or None for an online-check destination. |
| 45 | + """ |
| 46 | + parsed = urlsplit(url) |
| 47 | + if parsed.scheme not in {"http", "https"}: |
| 48 | + raise ValueError(f"Expected an HTTP(S) URL from Lychee, got {url!r}") |
| 49 | + # A different authority is an online destination even when hostname matches. |
| 50 | + default_port = 443 if parsed.scheme == "https" else 80 |
| 51 | + if parsed.username is not None or parsed.password is not None or parsed.port not in {None, default_port}: |
| 52 | + return None |
| 53 | + parts = parsed.path.lstrip("/").split("/", 2) |
| 54 | + if len(parts) != 3 or "/".join(parts[:2]).casefold() != repository.casefold(): |
| 55 | + return None |
| 56 | + remainder = parts[2] |
| 57 | + if parsed.hostname == "github.com": |
| 58 | + route, separator, remainder = remainder.partition("/") |
| 59 | + if not separator or route not in {"blob", "tree"}: |
| 60 | + return None |
| 61 | + # GitHub redirects tree/file and blob/directory to their canonical route. |
| 62 | + kind = "object" |
| 63 | + elif parsed.hostname == "raw.githubusercontent.com": |
| 64 | + kind = "blob" |
| 65 | + else: |
| 66 | + return None |
| 67 | + |
| 68 | + # Match only at literal URL segment boundaries. Decoding the entire string |
| 69 | + # first would mistake an explicit main%2Fother ref for branch main + a path. |
| 70 | + segments = remainder.split("/") |
| 71 | + for ref in (f"refs/heads/{branch}", branch): |
| 72 | + for end in range(1, len(segments) + 1): |
| 73 | + candidate = unquote("/".join(segments[:end])) |
| 74 | + if candidate == ref: |
| 75 | + path = unquote("/".join(segments[end:])) |
| 76 | + # GitHub HTML redirects trailing slashes; raw content returns 404. |
| 77 | + return kind, path.rstrip("/") if kind == "object" else path |
| 78 | + if len(candidate) >= len(ref): |
| 79 | + break |
| 80 | + return None |
| 81 | + |
| 82 | + |
| 83 | +def git_tree(repo_root: Path, revision: str) -> dict[str, str]: |
| 84 | + """Read files and directories from an immutable tree, including shallow clones.""" |
| 85 | + tree_id = subprocess.check_output( |
| 86 | + ["git", "rev-parse", "--verify", f"{revision}^{{tree}}"], cwd=repo_root, text=True |
| 87 | + ).strip() |
| 88 | + LOGGER.info("Checking repository links against %s (tree %s)", revision, tree_id) |
| 89 | + entries = subprocess.check_output(["git", "ls-tree", "-r", "-t", "-z", tree_id], cwd=repo_root) |
| 90 | + result = {"": "tree"} |
| 91 | + for entry in entries.split(b"\0"): |
| 92 | + if entry: |
| 93 | + metadata, path = entry.split(b"\t", 1) |
| 94 | + result[path.decode("utf-8")] = metadata.split()[1].decode("ascii") |
| 95 | + return result |
| 96 | + |
| 97 | + |
| 98 | +def check_links( |
| 99 | + links: list[str], *, tree: dict[str, str], repository: str, branch: str |
| 100 | +) -> tuple[list[str], list[str]]: |
| 101 | + """Partition extracted URLs into online destinations and local errors.""" |
| 102 | + external = [] |
| 103 | + errors = [] |
| 104 | + checked = 0 |
| 105 | + for url in sorted(set(links)): |
| 106 | + target = repository_target(url, repository=repository, branch=branch) |
| 107 | + if target is None: |
| 108 | + external.append(url) |
| 109 | + continue |
| 110 | + checked += 1 |
| 111 | + expected_type, path = target |
| 112 | + actual_type = tree.get(path) |
| 113 | + if actual_type is None or (expected_type != "object" and actual_type != expected_type): |
| 114 | + errors.append(f"{url}: expected {expected_type} at {path!r}, found {actual_type or 'missing'}") |
| 115 | + LOGGER.info("Checked %d candidate-tree URLs; %d URLs remain for online checking", checked, len(external)) |
| 116 | + return external, errors |
| 117 | + |
| 118 | + |
| 119 | +def main() -> int: |
| 120 | + """Validate the candidate tree and write only online destinations for Lychee.""" |
| 121 | + parser = argparse.ArgumentParser(description=__doc__) |
| 122 | + parser.add_argument("--repo-root", type=Path, default=Path.cwd()) |
| 123 | + parser.add_argument("--revision", default="HEAD") |
| 124 | + parser.add_argument("--repository", required=True) |
| 125 | + parser.add_argument("--branch", required=True) |
| 126 | + parser.add_argument("--links-file", type=Path, required=True) |
| 127 | + parser.add_argument("--external-links-file", type=Path, required=True) |
| 128 | + args = parser.parse_args() |
| 129 | + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") |
| 130 | + links = [line.strip() for line in args.links_file.read_text().splitlines() if line.strip()] |
| 131 | + if not links: |
| 132 | + LOGGER.error("No URLs extracted; refusing to pass an empty documentation scan") |
| 133 | + return 1 |
| 134 | + external, errors = check_links( |
| 135 | + links, tree=git_tree(args.repo_root, args.revision), repository=args.repository, branch=args.branch |
| 136 | + ) |
| 137 | + for error in errors: |
| 138 | + LOGGER.error("%s", error) |
| 139 | + if errors: |
| 140 | + return 1 |
| 141 | + args.external_links_file.write_text("".join(url + "\n" for url in external)) |
| 142 | + return 0 |
| 143 | + |
| 144 | + |
| 145 | +if __name__ == "__main__": |
| 146 | + raise SystemExit(main()) |
0 commit comments