diff --git a/scripts/changelog/generate_changelog_json.py b/scripts/changelog/generate_changelog_json.py new file mode 100644 index 00000000000..2c83e454128 --- /dev/null +++ b/scripts/changelog/generate_changelog_json.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# Converts Thunderbird/K-9 Mail release notes into JSON changelog resources. +# +# The script: +# - Loads release notes from https://github.com/thunderbird/thunderbird-notes. +# - Extracts changelog data for a specific release version. +# - Generates version-specific JSON changelog files. +# - Updates the changelog index. +# - Validates all generated files against the project's JSON schemas. +# +# Supported application IDs: +# - com.fsck.k9 +# - net.thunderbird.android +# - net.thunderbird.android.beta +# +# Usage: +# python scripts/changelog/migrate_changelog_to_json.py +# +# Example: +# python scripts/changelog/migrate_changelog_to_json.py net.thunderbird.android 11.0 + +import argparse +import json +import re +from pathlib import Path + +import requests +import yaml +from jsonschema import Draft202012Validator + +APP_CONFIG = { + "com.fsck.k9": {"app_dir": "app-k9mail", "build_type": "release"}, + "net.thunderbird.android": {"app_dir": "app-thunderbird", "build_type": "release"}, + "net.thunderbird.android.beta": {"app_dir": "app-thunderbird", "build_type": "beta"}, +} + +def to_resource_name(version: str) -> str: + normalized = re.sub(r"[^a-z0-9]+", "_", version.lower()).strip("_") + return f"changelog_release_{normalized}" + +def load_schema(schema_path: Path) -> dict: + with schema_path.open("r", encoding="utf-8") as file: + return json.load(file) + +def validate_json( + data: dict, + schema: dict, + description: str, +) -> None: + validator = Draft202012Validator( + schema + ) + + errors = sorted( + validator.iter_errors(data), + key=lambda error: list( + error.absolute_path + ), + ) + + if not errors: + return + + messages = [] + + for error in errors: + location = ( + "/".join( + str(part) + for part in error.absolute_path + ) + or "" + ) + + messages.append( + f"{location}: " + f"{error.message}" + ) + + raise ValueError( + f"{description} failed schema " + f"validation:\n" + + "\n".join(messages) + ) + +def load_release_notes(version: str, notesrepo: str, notesbranch: str) -> dict: + filename = f"{version}.yml" + directory = "android_release" + if "0b" in version: + filename = f"{version[:-1]}eta.yml" + directory = "android_beta" + + repo_path = Path(notesrepo).expanduser() + if repo_path.is_dir(): + yaml_file = repo_path / directory / filename + with yaml_file.open("r", encoding="utf-8") as f: + return yaml.safe_load(f) + + url = ( + f"https://api.github.com/repos/{notesrepo}/contents/" + f"{directory}/{filename}?ref={notesbranch}" + ) + response = requests.get( + url, + headers={"Accept": "application/vnd.github.v3.raw"}, + timeout=30, + ) + response.raise_for_status() + return yaml.safe_load(response.text) + +def extract_release(version: str, application: str, yaml_content: dict) -> dict: + release_info = next( + (r for r in yaml_content["release"]["releases"] if r["version"] == version), + None, + ) + if release_info is None: + raise ValueError(f"Version '{version}' not found") + + beta_group = None + match = re.search(r"0b(\d+)$", version) + if match: + beta_group = int(match.group(1)) + + notes = [] + for note in yaml_content["notes"]: + if beta_group is not None and note.get("group") != beta_group: + continue + if note.get("thunderbird_only", False) and application == "k9mail": + continue + if note.get("k9mail_only", False) and application == "thunderbird": + continue + + text = note.get("note", "").strip() + if not text: + continue + + change_type = determine_change_type(note.get("tag", "changed").lower()) + entry = { + "type": change_type, + "text": text, + "source": "thunderbird-notes", + } + if "issues" in note: + entry["issues"] = note["issues"] + notes.append(entry) + + return { + "schemaVersion": 1, + "version": version, + "date": release_info["release_date"], + "notes": notes, + } + +def determine_change_type( + text: str, +) -> str: + normalized = text.lower() + + if normalized.startswith( + "fixed" + ): + return "fixed" + + if ( + normalized.startswith( + "added" + ) + or normalized.startswith( + "new" + ) + or normalized.startswith( + "support" + ) + ): + return "new" + + return "changed" + +def write_release_file(release_data: dict, output_dir: Path) -> str: + resource_name = to_resource_name(release_data["version"]) + output_file = output_dir / f"{resource_name}.json" + with output_file.open("w", encoding="utf-8") as f: + json.dump(release_data, f, indent=2, ensure_ascii=False) + f.write("\n") + return resource_name + +def create_index_entry(release_data: dict, resource_name: str) -> dict: + return { + "version": release_data["version"], + "date": release_data["date"], + "resourceName": resource_name, + } + +def load_existing_index(output_dir: Path) -> dict: + index_file = output_dir / "changelog_index.json" + if not index_file.exists(): + return {"schemaVersion": 1, "releases": []} + with index_file.open("r", encoding="utf-8") as f: + return json.load(f) + +def update_index(index: dict, release_data: dict, resource_name: str) -> dict: + releases = [ + r for r in index["releases"] + if r["version"] != release_data["version"] + ] + releases.append(create_index_entry(release_data, resource_name)) + releases.sort(key=lambda r: r["date"], reverse=True) + index["releases"] = releases + return index + +def write_index_file(index: dict, output_dir: Path) -> None: + with (output_dir / "changelog_index.json").open("w", encoding="utf-8") as f: + json.dump(index, f, indent=2, ensure_ascii=False) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "applicationid", + choices=[ + "net.thunderbird.android", + "net.thunderbird.android.beta", + "com.fsck.k9", + ], + ) + parser.add_argument("version") + parser.add_argument("versioncode", nargs="?", default="0") + parser.add_argument( + "--repository", "-r", + default="thunderbird/thunderbird-notes", + ) + parser.add_argument("--branch", "-b", default="master") + args = parser.parse_args() + + config = APP_CONFIG[args.applicationid] + application = "k9mail" if args.applicationid == "com.fsck.k9" else "thunderbird" + + repo_root = Path(__file__).resolve().parents[2] + + output_dir = ( + repo_root / config["app_dir"] / "src" / config["build_type"] + / "res" / "raw" + ) + + schema_dir = repo_root / "schemas" + release_schema = load_schema(schema_dir / "changelog-release.schema.json") + index_schema = load_schema(schema_dir / "changelog-index.schema.json") + + output_dir.mkdir(parents=True, exist_ok=True) + + yaml_content = load_release_notes( + args.version, + args.repository, + args.branch, + ) + + release_data = extract_release( + args.version, + application, + yaml_content, + ) + print(release_data) + + validate_json( + release_data, + release_schema, + f"Release {args.version}", + ) + + resource_name = write_release_file(release_data, output_dir) + + index = load_existing_index(output_dir) + index = update_index(index, release_data, resource_name) + + validate_json(index, index_schema, "Changelog index") + + write_index_file(index, output_dir) + + print( + f"Generated {resource_name}.json and updated changelog_index.json" + ) + +if __name__ == "__main__": + main() diff --git a/scripts/changelog/migrate_changelog_to_json.py b/scripts/changelog/migrate_changelog_to_json.py index 56b48ca4100..aa61f19f20f 100755 --- a/scripts/changelog/migrate_changelog_to_json.py +++ b/scripts/changelog/migrate_changelog_to_json.py @@ -371,8 +371,7 @@ def main(): "src" / build_type / "res" / - "raw" / - "changelog" + "raw" ) schema_dir = ( diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 3fa64eb1bd1..829474e0d10 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -18,3 +18,7 @@ Jinja2==3.1.6 # Required by: scripts/ci/setup_release_automation, scripts/ci/render-notes.py # HTTP library for GitHub API interactions requests==2.34.2 + +# Required by: scripts/changelog/generate_changelog_json.py +# JSON Schema validator (used with Draft202012Validator) +jsonschema==4.26.0