|
| 1 | +#!/usr/bin/env python3 |
| 2 | + |
| 3 | +import concurrent.futures |
| 4 | +import json |
| 5 | +import os |
| 6 | +import re |
| 7 | +import subprocess |
| 8 | +import sys |
| 9 | +from pathlib import Path |
| 10 | +from urllib.error import HTTPError, URLError |
| 11 | +from urllib.parse import quote, urlencode |
| 12 | +from urllib.request import Request, urlopen |
| 13 | + |
| 14 | + |
| 15 | +SCRIPT_DIR = Path(__file__).resolve().parent |
| 16 | +sys.path.append(str(SCRIPT_DIR.parent)) |
| 17 | +from utils import defaultConfig |
| 18 | + |
| 19 | + |
| 20 | +REPOSITORY_ROOT = SCRIPT_DIR.parents[1] |
| 21 | +CONFIG_PATH = SCRIPT_DIR / "update_config.json" |
| 22 | +INDEX_PATH = REPOSITORY_ROOT / "docs" / "libraries" / "index.md" |
| 23 | +BEGIN_MARKER = "<!-- BEGIN GENERATED STM32duino library list -->" |
| 24 | +END_MARKER = "<!-- END GENERATED STM32duino library list -->" |
| 25 | +ORG_NAME = "stm32duino" |
| 26 | +GITHUB_API = "https://api.github.com" |
| 27 | +RAW_GITHUB = "https://raw.githubusercontent.com" |
| 28 | +PER_PAGE = 100 |
| 29 | +MAX_WORKERS = 8 |
| 30 | + |
| 31 | + |
| 32 | +def check_config(): |
| 33 | + if CONFIG_PATH.is_file(): |
| 34 | + try: |
| 35 | + config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) |
| 36 | + except (OSError, json.JSONDecodeError) as error: |
| 37 | + raise RuntimeError(f"cannot read {CONFIG_PATH}: {error}") from error |
| 38 | + if not isinstance(config, dict): |
| 39 | + raise RuntimeError(f"{CONFIG_PATH} must contain a JSON object") |
| 40 | + else: |
| 41 | + config = {} |
| 42 | + |
| 43 | + if "REPO_STM32DUINO" not in config: |
| 44 | + config["REPO_STM32DUINO"] = str(REPOSITORY_ROOT) |
| 45 | + defaultConfig(CONFIG_PATH, config) |
| 46 | + |
| 47 | + configured_path = config["REPO_STM32DUINO"] |
| 48 | + if not isinstance(configured_path, str) or not configured_path: |
| 49 | + raise RuntimeError("REPO_STM32DUINO must be a non-empty path") |
| 50 | + |
| 51 | + repository_path = Path(configured_path).expanduser() |
| 52 | + if not repository_path.is_absolute(): |
| 53 | + repository_path = REPOSITORY_ROOT / repository_path |
| 54 | + return repository_path.resolve() |
| 55 | + |
| 56 | + |
| 57 | +def request_headers(accept): |
| 58 | + headers = { |
| 59 | + "Accept": accept, |
| 60 | + "User-Agent": "stm32duino-library-list-updater", |
| 61 | + } |
| 62 | + token = os.environ.get("GITHUB_TOKEN") |
| 63 | + if token: |
| 64 | + headers["Authorization"] = f"Bearer {token}" |
| 65 | + return headers |
| 66 | + |
| 67 | + |
| 68 | +def fetch_json(url): |
| 69 | + request = Request(url, headers=request_headers("application/vnd.github+json")) |
| 70 | + try: |
| 71 | + with urlopen(request, timeout=30) as response: |
| 72 | + return json.loads(response.read().decode("utf-8")) |
| 73 | + except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as error: |
| 74 | + raise RuntimeError( |
| 75 | + f"failed to fetch GitHub API response from {url}: {error}" |
| 76 | + ) from error |
| 77 | + |
| 78 | + |
| 79 | +def fetch_organization_repositories(): |
| 80 | + repositories = [] |
| 81 | + page = 1 |
| 82 | + while True: |
| 83 | + query = urlencode( |
| 84 | + { |
| 85 | + "type": "public", |
| 86 | + "per_page": PER_PAGE, |
| 87 | + "page": page, |
| 88 | + } |
| 89 | + ) |
| 90 | + page_repositories = fetch_json(f"{GITHUB_API}/orgs/{ORG_NAME}/repos?{query}") |
| 91 | + if not isinstance(page_repositories, list): |
| 92 | + raise RuntimeError( |
| 93 | + "GitHub returned an invalid organization repository list" |
| 94 | + ) |
| 95 | + repositories.extend(page_repositories) |
| 96 | + if len(page_repositories) < PER_PAGE: |
| 97 | + return repositories |
| 98 | + page += 1 |
| 99 | + |
| 100 | + |
| 101 | +def fetch_manifest(repository): |
| 102 | + branch = repository.get("default_branch") |
| 103 | + if not branch: |
| 104 | + return None |
| 105 | + |
| 106 | + full_name = repository.get("full_name") |
| 107 | + if not full_name: |
| 108 | + raise RuntimeError("GitHub returned a repository without a full_name") |
| 109 | + |
| 110 | + branch_path = quote(branch, safe="/") |
| 111 | + url = f"{RAW_GITHUB}/{full_name}/{branch_path}/library.properties" |
| 112 | + request = Request(url, headers=request_headers("text/plain")) |
| 113 | + try: |
| 114 | + with urlopen(request, timeout=30) as response: |
| 115 | + return response.read().decode("utf-8") |
| 116 | + except HTTPError as error: |
| 117 | + if error.code == 404: |
| 118 | + return None |
| 119 | + raise RuntimeError(f"failed to fetch {url}: {error}") from error |
| 120 | + except (URLError, TimeoutError, UnicodeDecodeError) as error: |
| 121 | + raise RuntimeError(f"failed to fetch {url}: {error}") from error |
| 122 | + |
| 123 | + |
| 124 | +def parse_manifest(text): |
| 125 | + properties = {} |
| 126 | + for line in text.splitlines(): |
| 127 | + line = line.strip() |
| 128 | + if not line or line.startswith("#") or "=" not in line: |
| 129 | + continue |
| 130 | + key, value = line.split("=", 1) |
| 131 | + properties[key.strip()] = value.strip() |
| 132 | + return properties |
| 133 | + |
| 134 | + |
| 135 | +def normalize_remote(url): |
| 136 | + if url.startswith("git@github.com:"): |
| 137 | + url = "https://github.com/" + url.removeprefix("git@github.com:") |
| 138 | + return url.removesuffix(".git").rstrip("/") |
| 139 | + |
| 140 | + |
| 141 | +def run_git(repository_path, *arguments): |
| 142 | + result = subprocess.run( |
| 143 | + ["git", "-C", str(repository_path), *arguments], |
| 144 | + check=False, |
| 145 | + capture_output=True, |
| 146 | + text=True, |
| 147 | + ) |
| 148 | + if result.returncode: |
| 149 | + message = result.stderr.strip() or result.stdout.strip() |
| 150 | + raise RuntimeError( |
| 151 | + f"git {' '.join(arguments)} failed in {repository_path}: {message}" |
| 152 | + ) |
| 153 | + return result.stdout.strip() |
| 154 | + |
| 155 | + |
| 156 | +def fetch_repository(repository, destination_root): |
| 157 | + full_name = repository["full_name"] |
| 158 | + name = repository["name"] |
| 159 | + branch = repository["default_branch"] |
| 160 | + clone_url = repository["clone_url"] |
| 161 | + destination = destination_root / name |
| 162 | + |
| 163 | + if destination.exists(): |
| 164 | + if not (destination / ".git").exists(): |
| 165 | + raise RuntimeError( |
| 166 | + f"{destination} already exists but is not a Git repository" |
| 167 | + ) |
| 168 | + origin = run_git(destination, "remote", "get-url", "origin") |
| 169 | + if normalize_remote(origin) != normalize_remote(clone_url): |
| 170 | + raise RuntimeError( |
| 171 | + f"{destination} has unexpected origin {origin!r}; expected {clone_url!r}" |
| 172 | + ) |
| 173 | + run_git(destination, "fetch", "--depth=1", "origin", branch) |
| 174 | + revision = "FETCH_HEAD" |
| 175 | + else: |
| 176 | + destination_root.mkdir(parents=True, exist_ok=True) |
| 177 | + result = subprocess.run( |
| 178 | + [ |
| 179 | + "git", |
| 180 | + "clone", |
| 181 | + "--depth=1", |
| 182 | + "--single-branch", |
| 183 | + "--branch", |
| 184 | + branch, |
| 185 | + clone_url, |
| 186 | + str(destination), |
| 187 | + ], |
| 188 | + check=False, |
| 189 | + capture_output=True, |
| 190 | + text=True, |
| 191 | + ) |
| 192 | + if result.returncode: |
| 193 | + message = result.stderr.strip() or result.stdout.strip() |
| 194 | + raise RuntimeError(f"failed to clone {full_name}: {message}") |
| 195 | + revision = "HEAD" |
| 196 | + manifest = run_git(destination, "show", f"{revision}:library.properties") |
| 197 | + manifest = run_git(destination, "show", f"{revision}:library.properties") |
| 198 | + return parse_manifest(manifest) |
| 199 | + |
| 200 | + |
| 201 | +def is_expansion_board(repository, properties): |
| 202 | + names = (properties.get("name", ""), repository["name"]) |
| 203 | + return any(name.upper().startswith(("X-NUCLEO-", "I-NUCLEO-")) for name in names) |
| 204 | + |
| 205 | + |
| 206 | +def library_description(repository, properties): |
| 207 | + description = ( |
| 208 | + properties.get("sentence") |
| 209 | + or properties.get("paragraph") |
| 210 | + or repository.get("description") |
| 211 | + or "" |
| 212 | + ) |
| 213 | + return " ".join(description.split()) |
| 214 | + |
| 215 | + |
| 216 | +def library_dependencies(properties): |
| 217 | + return [ |
| 218 | + dependency.strip() |
| 219 | + for dependency in properties.get("depends", "").split(",") |
| 220 | + if dependency.strip() |
| 221 | + ] |
| 222 | + |
| 223 | + |
| 224 | +def normalize_dependency_name(name): |
| 225 | + name = re.sub(r"\s*\([^)]*\)\s*$", "", name.strip()) |
| 226 | + name = re.split(r"\s*@\s*", name, maxsplit=1)[0].strip() |
| 227 | + return name.casefold() |
| 228 | + |
| 229 | + |
| 230 | +def dependency_links(libraries): |
| 231 | + candidates = {} |
| 232 | + for library in libraries: |
| 233 | + repository = library["repository"] |
| 234 | + properties = library["properties"] |
| 235 | + repository_url = f"https://github.com/{repository['full_name']}" |
| 236 | + names = { |
| 237 | + properties.get("name", ""), |
| 238 | + repository["name"], |
| 239 | + } |
| 240 | + for name in names: |
| 241 | + normalized = normalize_dependency_name(name) |
| 242 | + if normalized: |
| 243 | + candidates.setdefault(normalized, set()).add(repository_url) |
| 244 | + |
| 245 | + aliases = {} |
| 246 | + for name, urls in candidates.items(): |
| 247 | + if name.startswith("stm32duino "): |
| 248 | + aliases.setdefault(name.removeprefix("stm32duino "), set()).update(urls) |
| 249 | + for name, urls in aliases.items(): |
| 250 | + candidates.setdefault(name, set()).update(urls) |
| 251 | + |
| 252 | + return { |
| 253 | + name: next(iter(urls)) for name, urls in candidates.items() if len(urls) == 1 |
| 254 | + } |
| 255 | + |
| 256 | + |
| 257 | +def render_library_entry(library, links): |
| 258 | + repository = library["repository"] |
| 259 | + properties = library["properties"] |
| 260 | + name = properties.get("name", repository["name"]) |
| 261 | + description = library_description(repository, properties) |
| 262 | + entry = f"* [{name}](https://github.com/{repository['full_name']})" |
| 263 | + if description: |
| 264 | + entry += f": {description}" |
| 265 | + |
| 266 | + dependencies = [] |
| 267 | + for dependency in library_dependencies(properties): |
| 268 | + url = links.get(normalize_dependency_name(dependency)) |
| 269 | + if url: |
| 270 | + dependencies.append(f"[{dependency}]({url})") |
| 271 | + else: |
| 272 | + dependencies.append(f"`{dependency}`") |
| 273 | + if dependencies: |
| 274 | + entry += "\n * Depends on: " + ", ".join(dependencies) |
| 275 | + return entry |
| 276 | + |
| 277 | + |
| 278 | +def render_libraries(libraries): |
| 279 | + categories = {} |
| 280 | + expansion_boards = [] |
| 281 | + for library in libraries: |
| 282 | + if is_expansion_board(library["repository"], library["properties"]): |
| 283 | + expansion_boards.append(library) |
| 284 | + else: |
| 285 | + category = library["properties"].get("category", "").strip() or "Other" |
| 286 | + categories.setdefault(category, []).append(library) |
| 287 | + |
| 288 | + links = dependency_links(libraries) |
| 289 | + |
| 290 | + lines = [ |
| 291 | + BEGIN_MARKER, |
| 292 | + "", |
| 293 | + "This list is generated from public STM32duino repositories that contain " |
| 294 | + "a `library.properties` file.", |
| 295 | + "", |
| 296 | + ] |
| 297 | + for category in sorted(categories, key=str.casefold): |
| 298 | + lines.extend((f"### {category}", "")) |
| 299 | + entries = sorted( |
| 300 | + categories[category], |
| 301 | + key=lambda library: library["properties"] |
| 302 | + .get("name", library["repository"]["name"]) |
| 303 | + .casefold(), |
| 304 | + ) |
| 305 | + for library in entries: |
| 306 | + lines.append(render_library_entry(library, links)) |
| 307 | + lines.append("") |
| 308 | + if expansion_boards: |
| 309 | + lines.extend(("## Expansion boards", "")) |
| 310 | + for library in sorted( |
| 311 | + expansion_boards, |
| 312 | + key=lambda item: item["properties"] |
| 313 | + .get("name", item["repository"]["name"]) |
| 314 | + .casefold(), |
| 315 | + ): |
| 316 | + lines.append(render_library_entry(library, links)) |
| 317 | + lines.append("") |
| 318 | + lines.append(END_MARKER) |
| 319 | + return "\n".join(lines) |
| 320 | + |
| 321 | + |
| 322 | +def replace_generated_section(document, generated_section): |
| 323 | + start = document.find(BEGIN_MARKER) |
| 324 | + end = document.find(END_MARKER) |
| 325 | + if start < 0 or end < 0 or end < start: |
| 326 | + raise RuntimeError( |
| 327 | + f"{INDEX_PATH} must contain {BEGIN_MARKER!r} before {END_MARKER!r}" |
| 328 | + ) |
| 329 | + end += len(END_MARKER) |
| 330 | + return document[:start] + generated_section + document[end:] |
| 331 | + |
| 332 | + |
| 333 | +def update_index(libraries): |
| 334 | + if not libraries: |
| 335 | + raise RuntimeError("no STM32duino libraries with library.properties were found") |
| 336 | + original = INDEX_PATH.read_text(encoding="utf-8") |
| 337 | + updated = replace_generated_section(original, render_libraries(libraries)) |
| 338 | + INDEX_PATH.write_text(updated, encoding="utf-8") |
| 339 | + |
| 340 | + |
| 341 | +def main(): |
| 342 | + destination_root = check_config() |
| 343 | + repositories = fetch_organization_repositories() |
| 344 | + candidates = [] |
| 345 | + |
| 346 | + with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: |
| 347 | + futures = { |
| 348 | + executor.submit(fetch_manifest, repository): repository |
| 349 | + for repository in repositories |
| 350 | + if repository.get("default_branch") and repository.get("clone_url") |
| 351 | + } |
| 352 | + for future in concurrent.futures.as_completed(futures): |
| 353 | + repository = futures[future] |
| 354 | + if future.result() is not None: |
| 355 | + candidates.append(repository) |
| 356 | + |
| 357 | + libraries = [] |
| 358 | + for repository in sorted(candidates, key=lambda item: item["name"].casefold()): |
| 359 | + print(f"Fetching {repository['full_name']}...") |
| 360 | + properties = fetch_repository(repository, destination_root) |
| 361 | + libraries.append({"repository": repository, "properties": properties}) |
| 362 | + |
| 363 | + update_index(libraries) |
| 364 | + print(f"Updated {INDEX_PATH} with {len(libraries)} libraries.") |
| 365 | + |
| 366 | + |
| 367 | +if __name__ == "__main__": |
| 368 | + try: |
| 369 | + main() |
| 370 | + except (OSError, RuntimeError) as error: |
| 371 | + print(f"Error: {error}", file=sys.stderr) |
| 372 | + sys.exit(1) |
0 commit comments