Skip to content

Commit 45ca575

Browse files
committed
docs: add local STM32duino library list generator
Fetch STM32duino library repositories and regenerate the categorized, alphabetized library index locally. Include expansion-board libraries and configure the repository fetch path. Signed-off-by: Frederic Pillon <frederic.pillon@st.com> Assisted-by: GitHub Copilot: GPT-6 Luna
1 parent 05fb284 commit 45ca575

1 file changed

Lines changed: 372 additions & 0 deletions

File tree

‎CI/update/stm32libraries.py‎

Lines changed: 372 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,372 @@
1+
#!/usr/bin/env python3
2+
3+
import concurrent.futures
4+
import json
5+
import os
6+
import re
7+
import subprocess
8+
import sys
9+
from pathlib import Path
10+
from urllib.error import HTTPError, URLError
11+
from urllib.parse import quote, urlencode
12+
from urllib.request import Request, urlopen
13+
14+
15+
SCRIPT_DIR = Path(__file__).resolve().parent
16+
sys.path.append(str(SCRIPT_DIR.parent))
17+
from utils import defaultConfig
18+
19+
20+
REPOSITORY_ROOT = SCRIPT_DIR.parents[1]
21+
CONFIG_PATH = SCRIPT_DIR / "update_config.json"
22+
INDEX_PATH = REPOSITORY_ROOT / "docs" / "libraries" / "index.md"
23+
BEGIN_MARKER = "<!-- BEGIN GENERATED STM32duino library list -->"
24+
END_MARKER = "<!-- END GENERATED STM32duino library list -->"
25+
ORG_NAME = "stm32duino"
26+
GITHUB_API = "https://api.github.com"
27+
RAW_GITHUB = "https://raw.githubusercontent.com"
28+
PER_PAGE = 100
29+
MAX_WORKERS = 8
30+
31+
32+
def check_config():
33+
if CONFIG_PATH.is_file():
34+
try:
35+
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
36+
except (OSError, json.JSONDecodeError) as error:
37+
raise RuntimeError(f"cannot read {CONFIG_PATH}: {error}") from error
38+
if not isinstance(config, dict):
39+
raise RuntimeError(f"{CONFIG_PATH} must contain a JSON object")
40+
else:
41+
config = {}
42+
43+
if "REPO_STM32DUINO" not in config:
44+
config["REPO_STM32DUINO"] = str(REPOSITORY_ROOT)
45+
defaultConfig(CONFIG_PATH, config)
46+
47+
configured_path = config["REPO_STM32DUINO"]
48+
if not isinstance(configured_path, str) or not configured_path:
49+
raise RuntimeError("REPO_STM32DUINO must be a non-empty path")
50+
51+
repository_path = Path(configured_path).expanduser()
52+
if not repository_path.is_absolute():
53+
repository_path = REPOSITORY_ROOT / repository_path
54+
return repository_path.resolve()
55+
56+
57+
def request_headers(accept):
58+
headers = {
59+
"Accept": accept,
60+
"User-Agent": "stm32duino-library-list-updater",
61+
}
62+
token = os.environ.get("GITHUB_TOKEN")
63+
if token:
64+
headers["Authorization"] = f"Bearer {token}"
65+
return headers
66+
67+
68+
def fetch_json(url):
69+
request = Request(url, headers=request_headers("application/vnd.github+json"))
70+
try:
71+
with urlopen(request, timeout=30) as response:
72+
return json.loads(response.read().decode("utf-8"))
73+
except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as error:
74+
raise RuntimeError(
75+
f"failed to fetch GitHub API response from {url}: {error}"
76+
) from error
77+
78+
79+
def fetch_organization_repositories():
80+
repositories = []
81+
page = 1
82+
while True:
83+
query = urlencode(
84+
{
85+
"type": "public",
86+
"per_page": PER_PAGE,
87+
"page": page,
88+
}
89+
)
90+
page_repositories = fetch_json(f"{GITHUB_API}/orgs/{ORG_NAME}/repos?{query}")
91+
if not isinstance(page_repositories, list):
92+
raise RuntimeError(
93+
"GitHub returned an invalid organization repository list"
94+
)
95+
repositories.extend(page_repositories)
96+
if len(page_repositories) < PER_PAGE:
97+
return repositories
98+
page += 1
99+
100+
101+
def fetch_manifest(repository):
102+
branch = repository.get("default_branch")
103+
if not branch:
104+
return None
105+
106+
full_name = repository.get("full_name")
107+
if not full_name:
108+
raise RuntimeError("GitHub returned a repository without a full_name")
109+
110+
branch_path = quote(branch, safe="/")
111+
url = f"{RAW_GITHUB}/{full_name}/{branch_path}/library.properties"
112+
request = Request(url, headers=request_headers("text/plain"))
113+
try:
114+
with urlopen(request, timeout=30) as response:
115+
return response.read().decode("utf-8")
116+
except HTTPError as error:
117+
if error.code == 404:
118+
return None
119+
raise RuntimeError(f"failed to fetch {url}: {error}") from error
120+
except (URLError, TimeoutError, UnicodeDecodeError) as error:
121+
raise RuntimeError(f"failed to fetch {url}: {error}") from error
122+
123+
124+
def parse_manifest(text):
125+
properties = {}
126+
for line in text.splitlines():
127+
line = line.strip()
128+
if not line or line.startswith("#") or "=" not in line:
129+
continue
130+
key, value = line.split("=", 1)
131+
properties[key.strip()] = value.strip()
132+
return properties
133+
134+
135+
def normalize_remote(url):
136+
if url.startswith("git@github.com:"):
137+
url = "https://github.com/" + url.removeprefix("git@github.com:")
138+
return url.removesuffix(".git").rstrip("/")
139+
140+
141+
def run_git(repository_path, *arguments):
142+
result = subprocess.run(
143+
["git", "-C", str(repository_path), *arguments],
144+
check=False,
145+
capture_output=True,
146+
text=True,
147+
)
148+
if result.returncode:
149+
message = result.stderr.strip() or result.stdout.strip()
150+
raise RuntimeError(
151+
f"git {' '.join(arguments)} failed in {repository_path}: {message}"
152+
)
153+
return result.stdout.strip()
154+
155+
156+
def fetch_repository(repository, destination_root):
157+
full_name = repository["full_name"]
158+
name = repository["name"]
159+
branch = repository["default_branch"]
160+
clone_url = repository["clone_url"]
161+
destination = destination_root / name
162+
163+
if destination.exists():
164+
if not (destination / ".git").exists():
165+
raise RuntimeError(
166+
f"{destination} already exists but is not a Git repository"
167+
)
168+
origin = run_git(destination, "remote", "get-url", "origin")
169+
if normalize_remote(origin) != normalize_remote(clone_url):
170+
raise RuntimeError(
171+
f"{destination} has unexpected origin {origin!r}; expected {clone_url!r}"
172+
)
173+
run_git(destination, "fetch", "--depth=1", "origin", branch)
174+
revision = "FETCH_HEAD"
175+
else:
176+
destination_root.mkdir(parents=True, exist_ok=True)
177+
result = subprocess.run(
178+
[
179+
"git",
180+
"clone",
181+
"--depth=1",
182+
"--single-branch",
183+
"--branch",
184+
branch,
185+
clone_url,
186+
str(destination),
187+
],
188+
check=False,
189+
capture_output=True,
190+
text=True,
191+
)
192+
if result.returncode:
193+
message = result.stderr.strip() or result.stdout.strip()
194+
raise RuntimeError(f"failed to clone {full_name}: {message}")
195+
revision = "HEAD"
196+
manifest = run_git(destination, "show", f"{revision}:library.properties")
197+
manifest = run_git(destination, "show", f"{revision}:library.properties")
198+
return parse_manifest(manifest)
199+
200+
201+
def is_expansion_board(repository, properties):
202+
names = (properties.get("name", ""), repository["name"])
203+
return any(name.upper().startswith(("X-NUCLEO-", "I-NUCLEO-")) for name in names)
204+
205+
206+
def library_description(repository, properties):
207+
description = (
208+
properties.get("sentence")
209+
or properties.get("paragraph")
210+
or repository.get("description")
211+
or ""
212+
)
213+
return " ".join(description.split())
214+
215+
216+
def library_dependencies(properties):
217+
return [
218+
dependency.strip()
219+
for dependency in properties.get("depends", "").split(",")
220+
if dependency.strip()
221+
]
222+
223+
224+
def normalize_dependency_name(name):
225+
name = re.sub(r"\s*\([^)]*\)\s*$", "", name.strip())
226+
name = re.split(r"\s*@\s*", name, maxsplit=1)[0].strip()
227+
return name.casefold()
228+
229+
230+
def dependency_links(libraries):
231+
candidates = {}
232+
for library in libraries:
233+
repository = library["repository"]
234+
properties = library["properties"]
235+
repository_url = f"https://github.com/{repository['full_name']}"
236+
names = {
237+
properties.get("name", ""),
238+
repository["name"],
239+
}
240+
for name in names:
241+
normalized = normalize_dependency_name(name)
242+
if normalized:
243+
candidates.setdefault(normalized, set()).add(repository_url)
244+
245+
aliases = {}
246+
for name, urls in candidates.items():
247+
if name.startswith("stm32duino "):
248+
aliases.setdefault(name.removeprefix("stm32duino "), set()).update(urls)
249+
for name, urls in aliases.items():
250+
candidates.setdefault(name, set()).update(urls)
251+
252+
return {
253+
name: next(iter(urls)) for name, urls in candidates.items() if len(urls) == 1
254+
}
255+
256+
257+
def render_library_entry(library, links):
258+
repository = library["repository"]
259+
properties = library["properties"]
260+
name = properties.get("name", repository["name"])
261+
description = library_description(repository, properties)
262+
entry = f"* [{name}](https://github.com/{repository['full_name']})"
263+
if description:
264+
entry += f": {description}"
265+
266+
dependencies = []
267+
for dependency in library_dependencies(properties):
268+
url = links.get(normalize_dependency_name(dependency))
269+
if url:
270+
dependencies.append(f"[{dependency}]({url})")
271+
else:
272+
dependencies.append(f"`{dependency}`")
273+
if dependencies:
274+
entry += "\n * Depends on: " + ", ".join(dependencies)
275+
return entry
276+
277+
278+
def render_libraries(libraries):
279+
categories = {}
280+
expansion_boards = []
281+
for library in libraries:
282+
if is_expansion_board(library["repository"], library["properties"]):
283+
expansion_boards.append(library)
284+
else:
285+
category = library["properties"].get("category", "").strip() or "Other"
286+
categories.setdefault(category, []).append(library)
287+
288+
links = dependency_links(libraries)
289+
290+
lines = [
291+
BEGIN_MARKER,
292+
"",
293+
"This list is generated from public STM32duino repositories that contain "
294+
"a `library.properties` file.",
295+
"",
296+
]
297+
for category in sorted(categories, key=str.casefold):
298+
lines.extend((f"### {category}", ""))
299+
entries = sorted(
300+
categories[category],
301+
key=lambda library: library["properties"]
302+
.get("name", library["repository"]["name"])
303+
.casefold(),
304+
)
305+
for library in entries:
306+
lines.append(render_library_entry(library, links))
307+
lines.append("")
308+
if expansion_boards:
309+
lines.extend(("## Expansion boards", ""))
310+
for library in sorted(
311+
expansion_boards,
312+
key=lambda item: item["properties"]
313+
.get("name", item["repository"]["name"])
314+
.casefold(),
315+
):
316+
lines.append(render_library_entry(library, links))
317+
lines.append("")
318+
lines.append(END_MARKER)
319+
return "\n".join(lines)
320+
321+
322+
def replace_generated_section(document, generated_section):
323+
start = document.find(BEGIN_MARKER)
324+
end = document.find(END_MARKER)
325+
if start < 0 or end < 0 or end < start:
326+
raise RuntimeError(
327+
f"{INDEX_PATH} must contain {BEGIN_MARKER!r} before {END_MARKER!r}"
328+
)
329+
end += len(END_MARKER)
330+
return document[:start] + generated_section + document[end:]
331+
332+
333+
def update_index(libraries):
334+
if not libraries:
335+
raise RuntimeError("no STM32duino libraries with library.properties were found")
336+
original = INDEX_PATH.read_text(encoding="utf-8")
337+
updated = replace_generated_section(original, render_libraries(libraries))
338+
INDEX_PATH.write_text(updated, encoding="utf-8")
339+
340+
341+
def main():
342+
destination_root = check_config()
343+
repositories = fetch_organization_repositories()
344+
candidates = []
345+
346+
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
347+
futures = {
348+
executor.submit(fetch_manifest, repository): repository
349+
for repository in repositories
350+
if repository.get("default_branch") and repository.get("clone_url")
351+
}
352+
for future in concurrent.futures.as_completed(futures):
353+
repository = futures[future]
354+
if future.result() is not None:
355+
candidates.append(repository)
356+
357+
libraries = []
358+
for repository in sorted(candidates, key=lambda item: item["name"].casefold()):
359+
print(f"Fetching {repository['full_name']}...")
360+
properties = fetch_repository(repository, destination_root)
361+
libraries.append({"repository": repository, "properties": properties})
362+
363+
update_index(libraries)
364+
print(f"Updated {INDEX_PATH} with {len(libraries)} libraries.")
365+
366+
367+
if __name__ == "__main__":
368+
try:
369+
main()
370+
except (OSError, RuntimeError) as error:
371+
print(f"Error: {error}", file=sys.stderr)
372+
sys.exit(1)

0 commit comments

Comments
 (0)