Skip to content

1-update-github-repositories-details #1114

1-update-github-repositories-details

1-update-github-repositories-details #1114

name: 1-update-github-repositories-details
# Update the details of all the GitHub repositories corresponding to the list of considered topics
# Workflow triggered every day at 06:00 AM EST - but can also be triggered manually.
on:
schedule:
- cron: "0 11 * * *" # Every day at 06:00 AM EST
workflow_dispatch:
inputs:
pipeline:
description: 'Generation pipeline to run. Use powershell only as a manual rollback fallback; both paths use a 15% count guard.'
required: true
default: typescript
type: choice
options:
- typescript
- powershell
allow_suspicious_repository_delta:
description: 'Allow committing a suspiciously large repository-count delta for an intentional scope change or rollback run'
required: false
default: false
type: boolean
permissions:
contents: write
# Concurrency configuration for the current workflow - Keep only the latest workflow queued for the considered group
concurrency:
group: update-github-repositories-details
cancel-in-progress: true
jobs:
update-github-repositories-details:
runs-on: ubuntu-latest
concurrency:
group: main-repository-writers
cancel-in-progress: false
env:
RUNNER_DEBUG: 1
PIPELINE: ${{ inputs.pipeline || 'typescript' }}
ALLOW_SUSPICIOUS_REPOSITORY_DELTA: ${{ inputs.allow_suspicious_repository_delta || false }}
CONFIGURATION_FILE_PATH: ${{ vars.CONFIGURATION_FILE_PATH }}
DATA_FILE_PATH: ${{ vars.DATA_FILE_PATH }}
DATA_GENERATED_DIR: Data/GeneratedRepositories
CURATED_OVERLAY_DIR: Data/CuratedRepositories
TAXONOMY_DIR: Configuration/Taxonomy
SCHEMA_FILE_PATH: Configuration/Schemas/GitHubRepositoriesDetails.schema.json
GENERATED_SCHEMA_FILE_PATH: Configuration/Schemas/GitHubRepositoryGenerated.schema.json
TYPESCRIPT_OUTPUT_DIR: Pipeline/Output/production
steps:
# Action used to checkout the main branch in the current repository
# Community action: https://github.com/actions/checkout
- name: Checkout
uses: actions/checkout@v7
with:
ref: main
fetch-depth: 0
- name: Validate production pipeline inputs
run: |
set -euo pipefail
normalize_repo_path() {
local label="$1"
local raw="${2:-}"
local normalized="${raw//\\//}"
normalized="${normalized#./}"
if [[ -z "$normalized" || "$normalized" == /* || "$normalized" == *".."* ]]; then
echo "::error::${label} must be a repository-relative path."
exit 1
fi
printf '%s' "$normalized"
}
case "$PIPELINE" in
typescript|powershell) ;;
*)
echo "::error::Unsupported pipeline '$PIPELINE'."
exit 1
;;
esac
config_path="$(normalize_repo_path CONFIGURATION_FILE_PATH "$CONFIGURATION_FILE_PATH")"
data_path="$(normalize_repo_path DATA_FILE_PATH "$DATA_FILE_PATH")"
generated_dir="$(normalize_repo_path DATA_GENERATED_DIR "$DATA_GENERATED_DIR")"
overlay_dir="$(normalize_repo_path CURATED_OVERLAY_DIR "$CURATED_OVERLAY_DIR")"
taxonomy_dir="$(normalize_repo_path TAXONOMY_DIR "$TAXONOMY_DIR")"
schema_path="$(normalize_repo_path SCHEMA_FILE_PATH "$SCHEMA_FILE_PATH")"
generated_schema_path="$(normalize_repo_path GENERATED_SCHEMA_FILE_PATH "$GENERATED_SCHEMA_FILE_PATH")"
{
echo "CONFIGURATION_FILE_PATH=$config_path"
echo "DATA_FILE_PATH=$data_path"
echo "DATA_GENERATED_DIR=$generated_dir"
echo "CURATED_OVERLAY_DIR=$overlay_dir"
echo "TAXONOMY_DIR=$taxonomy_dir"
echo "SCHEMA_FILE_PATH=$schema_path"
echo "GENERATED_SCHEMA_FILE_PATH=$generated_schema_path"
echo "TYPESCRIPT_BASELINE_PATH=$TYPESCRIPT_OUTPUT_DIR/baseline-before-typescript.json"
echo "TYPESCRIPT_GENERATED_ARRAY_PATH=$TYPESCRIPT_OUTPUT_DIR/generated-layer.json"
echo "TYPESCRIPT_METRICS_PATH=$TYPESCRIPT_OUTPUT_DIR/typescript-metrics.json"
echo "TYPESCRIPT_PARITY_REPORT_PATH=$TYPESCRIPT_OUTPUT_DIR/typescript-parity-report.json"
echo "TYPESCRIPT_PARITY_SUMMARY_PATH=$TYPESCRIPT_OUTPUT_DIR/typescript-parity-summary.md"
echo "TYPESCRIPT_MERGE_SUMMARY_PATH=$TYPESCRIPT_OUTPUT_DIR/typescript-merge-summary.json"
} >> "$GITHUB_ENV"
{
echo "### Repository details production generation"
echo ""
echo "| Setting | Value |"
echo "| --- | --- |"
echo "| Pipeline | \`$PIPELINE\` |"
echo "| Configuration | \`$config_path\` |"
echo "| Data output | \`$data_path\` |"
echo "| Generated directory | \`$generated_dir\` |"
echo "| Curated overlays | \`$overlay_dir\` |"
echo "| Schema | \`$schema_path\` |"
echo "| Suspicious delta override | \`$ALLOW_SUSPICIOUS_REPOSITORY_DELTA\` |"
echo ""
} >> "$GITHUB_STEP_SUMMARY"
shell: bash
# Set a current date environment variable in the following format: YYYYMMDD
- name: Set current date as env variable
id: current_date
run: echo "NOW=$(date +'%Y%m%d')" >> $env:GITHUB_OUTPUT
shell: pwsh
- name: Set up Node.js
if: ${{ env.PIPELINE == 'typescript' }}
uses: actions/setup-node@v7
with:
node-version: 20
cache: npm
cache-dependency-path: Pipeline/package-lock.json
- name: Install TypeScript pipeline dependencies
if: ${{ env.PIPELINE == 'typescript' }}
working-directory: Pipeline
run: npm ci
- name: Build TypeScript pipeline CLI
if: ${{ env.PIPELINE == 'typescript' }}
working-directory: Pipeline
run: npm run build
- name: Capture existing repository details baseline
if: ${{ env.PIPELINE == 'typescript' }}
run: |
set -euo pipefail
mkdir -p "$TYPESCRIPT_OUTPUT_DIR"
if [[ -d "$DATA_GENERATED_DIR" ]]; then
BASELINE_PATH="$TYPESCRIPT_BASELINE_PATH" GENERATED_DIR="$DATA_GENERATED_DIR" node --input-type=module <<'NODE'
import fs from 'node:fs';
import path from 'node:path';
const generatedDir = process.env.GENERATED_DIR;
const baselinePath = process.env.BASELINE_PATH;
if (!generatedDir || !baselinePath) {
throw new Error('Generated baseline paths were not provided.');
}
const files = [];
const walk = (directory) => {
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
const entryPath = path.join(directory, entry.name);
if (entry.isDirectory()) {
walk(entryPath);
} else if (entry.isFile() && entry.name.endsWith('.json')) {
files.push(entryPath);
}
}
};
walk(generatedDir);
const records = files.sort().map((filePath) => JSON.parse(fs.readFileSync(filePath, 'utf8')));
fs.writeFileSync(baselinePath, `${JSON.stringify(records, null, 2)}\n`);
NODE
echo "Captured generated-layer baseline from '$DATA_GENERATED_DIR'."
echo "TYPESCRIPT_BASELINE_PRESENT=true" >> "$GITHUB_ENV"
echo "TYPESCRIPT_BASELINE_SOURCE=generated-dir" >> "$GITHUB_ENV"
elif [[ -f "$DATA_FILE_PATH" ]]; then
cp "$DATA_FILE_PATH" "$TYPESCRIPT_BASELINE_PATH"
echo "::warning::No generated-layer baseline found at '$DATA_GENERATED_DIR'; falling back to merged artifact '$DATA_FILE_PATH' for this run."
echo "TYPESCRIPT_BASELINE_PRESENT=true" >> "$GITHUB_ENV"
echo "TYPESCRIPT_BASELINE_SOURCE=merged-artifact-fallback" >> "$GITHUB_ENV"
else
echo "::warning::No existing repository details data found at '$DATA_FILE_PATH'; using an empty baseline so candidate duplicate, sentinel, and count guards still run."
printf '[]\n' > "$TYPESCRIPT_BASELINE_PATH"
echo "TYPESCRIPT_BASELINE_PRESENT=false" >> "$GITHUB_ENV"
echo "TYPESCRIPT_BASELINE_SOURCE=empty" >> "$GITHUB_ENV"
fi
shell: bash
- name: Generate repository details with TypeScript pipeline
if: ${{ env.PIPELINE == 'typescript' }}
run: |
set -euo pipefail
mkdir -p "$(dirname "$DATA_FILE_PATH")" "$TYPESCRIPT_OUTPUT_DIR" "$DATA_GENERATED_DIR"
node Pipeline/dist/cli.js generate \
--live \
--config "$CONFIGURATION_FILE_PATH" \
--schema "$SCHEMA_FILE_PATH" \
--output "$TYPESCRIPT_GENERATED_ARRAY_PATH" \
--generated-dir "$DATA_GENERATED_DIR" \
--generated-schema "$GENERATED_SCHEMA_FILE_PATH" \
--metrics "$TYPESCRIPT_METRICS_PATH"
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.PAT_ACCESS_PUBLIC_GITHUB_REPOSITORIES }}
- name: Guard TypeScript repository-count delta
if: ${{ env.PIPELINE == 'typescript' }}
run: |
set -euo pipefail
threshold="0.15"
if [[ "$ALLOW_SUSPICIOUS_REPOSITORY_DELTA" == "true" ]]; then
threshold="999"
echo "::warning::Repository-count delta guard is explicitly overridden for this run; schema, input, duplicate, and sentinel failures still block commits."
fi
set +e
node Pipeline/dist/cli.js compare \
--baseline "$TYPESCRIPT_BASELINE_PATH" \
--candidate "$TYPESCRIPT_GENERATED_ARRAY_PATH" \
--sentinels Configuration/SentinelRepositories.json \
--report "$TYPESCRIPT_PARITY_REPORT_PATH" \
--count-delta-threshold "$threshold" > "$TYPESCRIPT_PARITY_SUMMARY_PATH" 2>&1
comparison_exit_code=$?
set -e
{
echo "### TypeScript output parity guard"
echo ""
echo "| Setting | Value |"
echo "| --- | --- |"
echo "| Baseline | \`$TYPESCRIPT_BASELINE_PATH\` |"
echo "| Existing baseline present | \`${TYPESCRIPT_BASELINE_PRESENT:-unknown}\` |"
echo "| Baseline source | \`${TYPESCRIPT_BASELINE_SOURCE:-unknown}\` |"
echo "| Candidate generated layer | \`$TYPESCRIPT_GENERATED_ARRAY_PATH\` |"
echo "| Count delta threshold | \`$threshold\` |"
echo "| Report | \`$TYPESCRIPT_PARITY_REPORT_PATH\` |"
echo "| Exit code | \`$comparison_exit_code\` |"
echo "| Production blocking scope | repository-count delta, invalid inputs, candidate duplicates, candidate missing sentinels |"
echo ""
cat "$TYPESCRIPT_PARITY_SUMMARY_PATH"
echo ""
} >> "$GITHUB_STEP_SUMMARY"
node --input-type=module <<'NODE'
import fs from 'node:fs';
const reportPath = process.env.TYPESCRIPT_PARITY_REPORT_PATH;
const metricsPath = process.env.TYPESCRIPT_METRICS_PATH;
const allowSuspiciousDelta = process.env.ALLOW_SUSPICIOUS_REPOSITORY_DELTA === 'true';
const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
const blockingFailures = [];
// Block on any repository detail hydration failure: silent skips in live mode must not reach production.
if (metricsPath && fs.existsSync(metricsPath)) {
const metrics = JSON.parse(fs.readFileSync(metricsPath, 'utf8'));
const failures = metrics.detailFailures ?? 0;
const missingRepoSkips = metrics.missingRepoSkips ?? 0;
if (missingRepoSkips > 0) {
const missingRepoSkipNames = metrics.missingRepoSkipNames ?? [];
const displayedNames = missingRepoSkipNames.slice(0, 5).join(', ');
const moreCount = Math.max(0, missingRepoSkipNames.length - 5);
const affectedRepos =
displayedNames.length > 0
? `${displayedNames}${moreCount > 0 ? `, and ${moreCount} more` : ''}`
: 'names not recorded';
console.log(`::warning::TypeScript pipeline skipped ${missingRepoSkips} repo(s) whose GraphQL alias was absent (likely deleted/private). Affected: ${affectedRepos}`);
}
if (failures > 0) {
blockingFailures.push(`TypeScript pipeline had ${failures} repository detail hydration failure(s); all must succeed in production live mode`);
}
const patPolicyFailures = metrics.patPolicyFailures ?? 0;
if (patPolicyFailures > 0) {
const names = (metrics.patPolicyFailureNames ?? []).join(', ');
console.log(`::warning::TypeScript pipeline skipped ${patPolicyFailures} repo(s) due to PAT policy restrictions (fine-grained PAT lifetime exceeded enterprise limit). Affected repos: ${names}. Consider switching to a classic PAT with public_repo scope.`);
}
}
if ((report.failures.invalidInputs ?? []).length > 0) {
blockingFailures.push('comparison inputs are invalid');
}
const candidateDuplicates = (report.failures.duplicateFullNames ?? []).filter((failure) => failure.input === 'candidate');
if (candidateDuplicates.length > 0) {
blockingFailures.push('candidate output contains duplicate fullName values');
}
if (!allowSuspiciousDelta && report.failures.repositoryCountDelta !== undefined) {
const delta = report.failures.repositoryCountDelta;
blockingFailures.push(
`repository-count delta ${(delta.percentDelta * 100).toFixed(2)}% exceeds ${(delta.threshold * 100).toFixed(2)}%`
);
}
const missingCandidateSentinels = report.failures.missingSentinels?.candidate ?? [];
if (missingCandidateSentinels.length > 0) {
blockingFailures.push(`candidate output is missing sentinel repositories: ${missingCandidateSentinels.join(', ')}`);
}
if (blockingFailures.length > 0) {
for (const failure of blockingFailures) {
console.error(`::error::${failure}`);
}
process.exit(1);
}
NODE
shell: bash
- name: Merge TypeScript generated layer with curated overlays
if: ${{ env.PIPELINE == 'typescript' }}
run: |
set -euo pipefail
node Pipeline/dist/cli.js merge \
--generated-dir "$DATA_GENERATED_DIR" \
--overlay-dir "$CURATED_OVERLAY_DIR" \
--schema "$SCHEMA_FILE_PATH" \
--output "$DATA_FILE_PATH" \
--taxonomy-dir "$TAXONOMY_DIR" \
--sentinels Configuration/SentinelRepositories.json \
--generated-schema "$GENERATED_SCHEMA_FILE_PATH" > "$TYPESCRIPT_MERGE_SUMMARY_PATH"
{
echo "### TypeScript generated + curated merge"
echo ""
echo "| Setting | Value |"
echo "| --- | --- |"
echo "| Generated directory | \`$DATA_GENERATED_DIR\` |"
echo "| Curated overlays | \`$CURATED_OVERLAY_DIR\` |"
echo "| Merged artifact | \`$DATA_FILE_PATH\` |"
echo "| Merge summary | \`$TYPESCRIPT_MERGE_SUMMARY_PATH\` |"
echo ""
echo '```json'
cat "$TYPESCRIPT_MERGE_SUMMARY_PATH"
echo '```'
echo ""
} >> "$GITHUB_STEP_SUMMARY"
shell: bash
- name: Upload TypeScript production guard artifacts
if: ${{ always() && env.PIPELINE == 'typescript' }}
uses: actions/upload-artifact@v7
with:
name: repository-details-typescript-production-${{ github.run_id }}
path: |
Pipeline/Output/production/
Data/GeneratedRepositories/
Data/GitHubRepositoriesDetails.json
if-no-files-found: warn
# Update the details of all the GitHub repositories corresponding to the list of considered topics
- name: Update GitHub repositories details with PowerShell fallback
if: ${{ env.PIPELINE == 'powershell' }}
run: |
Write-Host "/********************************************************************************/"
Write-Host "Install required modules"
Import-Module .\Scripts\Search-GitHubRepositories.ps1 -Force
Import-Module .\Scripts\Get-GitHubRepositoryDetails.ps1 -Force
Import-Module .\Scripts\Export-GitHubRepositoriesDetails.ps1 -Force
Write-Host "/********************************************************************************/"
Write-Host "Update GitHub repositories details"
$exportParameters = @{
ConfigurationFilePath = $env:CONFIGURATION_FILE_PATH
OutputFilePath = $env:DATA_FILE_PATH
MaximumRepositoryCountDeltaPercentage = 15
}
if ($env:ALLOW_SUSPICIOUS_REPOSITORY_DELTA -eq "true") {
$exportParameters.AllowSuspiciousRepositoryCountDelta = $true
Write-Warning "Suspicious repository-count delta guard is explicitly overridden for this run."
}
$results = Export-GitHubRepositoriesDetails @exportParameters -Verbose
Write-Host "Number of GitHub repositories referenced: $($results.Count)"
shell: pwsh
env:
GITHUB_TOKEN: ${{ secrets.PAT_ACCESS_PUBLIC_GITHUB_REPOSITORIES }}
- name: Validate merged repository details data
run: |
$schemaFilePath = $env:SCHEMA_FILE_PATH
$dataFilePath = $env:DATA_FILE_PATH
if (-not (Test-Path -Path $schemaFilePath)) {
throw "No schema file found at the path '$schemaFilePath'."
}
if (-not (Test-Path -Path $dataFilePath)) {
throw "No repository details data file found at the path '$dataFilePath'."
}
$dataAsJson = Get-Content -Path $dataFilePath -Raw
if (-not ($dataAsJson | Test-Json -SchemaFile $schemaFilePath)) {
throw "The merged repository details data does not match the schema file '$schemaFilePath'."
}
$repositories = @($dataAsJson | ConvertFrom-Json)
$schemaVersions = ($repositories | ForEach-Object { $_._schemaVersion } | Sort-Object -Unique) -join ", "
if ([string]::IsNullOrWhiteSpace($schemaVersions)) {
$schemaVersions = "none"
}
Write-Host "Repository details count: $($repositories.Count)"
Write-Host "Repository details schema versions: $schemaVersions"
$sentinelStatus = "not checked"
$sentinelsFilePath = "Configuration/SentinelRepositories.json"
if (Test-Path -Path $sentinelsFilePath) {
$sentinelConfiguration = Get-Content -Path $sentinelsFilePath -Raw | ConvertFrom-Json
$sentinelFullNames = @($sentinelConfiguration.repositories | ForEach-Object { $_.fullName })
$repositoryFullNames = @($repositories | ForEach-Object { $_.fullName })
$missingSentinels = @($sentinelFullNames | Where-Object { $repositoryFullNames -notcontains $_ })
if ($missingSentinels.Count -gt 0) {
throw "Merged data is missing $($missingSentinels.Count) sentinel repository/repositories: $($missingSentinels -join ', ')"
}
Write-Host "All $($sentinelFullNames.Count) sentinel repositories are present in merged data."
$sentinelStatus = "all $($sentinelFullNames.Count) present"
}
if ($env:GITHUB_STEP_SUMMARY) {
@(
"### Merged repository details"
""
"| Metric | Value |"
"| --- | ---: |"
"| Repository count | $($repositories.Count) |"
"| Schema versions | $schemaVersions |"
"| Sentinel check | $sentinelStatus |"
) | Add-Content -Path $env:GITHUB_STEP_SUMMARY
}
shell: pwsh
# Push the changes in the current repository
- name: Push changes
id: push_changes
run: |
git config --global user.name 'action@github.com'
git config --global user.email 'GitHub Action'
git add -- "$env:DATA_FILE_PATH"
if ($env:PIPELINE -eq "typescript") {
git add -- "$env:DATA_GENERATED_DIR"
}
git diff --cached --quiet -- "$env:DATA_FILE_PATH"
$mergedArtifactDiffExitCode = $LASTEXITCODE
if ($mergedArtifactDiffExitCode -eq 0) {
$hasMergedArtifactChanges = $false
} elseif ($mergedArtifactDiffExitCode -eq 1) {
$hasMergedArtifactChanges = $true
} else {
throw "Unable to check staged merged repository details changes. git diff exited with code $mergedArtifactDiffExitCode."
}
git diff --cached --quiet
$gitDiffExitCode = $LASTEXITCODE
if ($gitDiffExitCode -eq 0) {
Write-Host "No GitHub repositories details changes to commit."
"has_changes=false" >> $env:GITHUB_OUTPUT
"has_merged_artifact_changes=false" >> $env:GITHUB_OUTPUT
if ($env:GITHUB_STEP_SUMMARY) {
@(
"### Push changes"
""
"- No repository details changes were detected, so commit and push were skipped."
) | Add-Content -Path $env:GITHUB_STEP_SUMMARY
}
} elseif ($gitDiffExitCode -eq 1) {
git commit -m "GitHub repositories details updated - ${{ steps.current_date.outputs.NOW }}.${{ github.run_number }}"
$pushSucceeded = $false
$usedRetry = $false
for ($attempt = 1; $attempt -le 2; $attempt++) {
$pushOutput = & git -c http.extraheader="AUTHORIZATION: Bearer ${{ secrets.GITHUB_TOKEN }}" push origin HEAD:main 2>&1
$pushExitCode = $LASTEXITCODE
$pushOutput | ForEach-Object { Write-Host $_ }
if ($pushExitCode -eq 0) {
$pushSucceeded = $true
break
}
$isFastForwardConflict = ($pushOutput | Out-String) -match 'fetch first|non-fast-forward|failed to push some refs'
if ($attempt -eq 2 -or -not $isFastForwardConflict) {
throw "Unable to push repository details changes. git push exited with code $pushExitCode."
}
$usedRetry = $true
Write-Warning "Remote main moved during push attempt $attempt. Fetching origin/main, rebasing, and retrying once."
if ($env:GITHUB_STEP_SUMMARY) {
@(
"### Push retry"
""
"- Push attempt $attempt was rejected because `main` moved."
"- Fetching `origin/main`, rebasing the generated commit, and retrying once."
) | Add-Content -Path $env:GITHUB_STEP_SUMMARY
}
& git fetch origin main 2>&1 | ForEach-Object { Write-Host $_ }
if ($LASTEXITCODE -ne 0) {
throw "Unable to fetch origin/main before retrying the repository details push."
}
& git rebase origin/main 2>&1 | ForEach-Object { Write-Host $_ }
if ($LASTEXITCODE -ne 0) {
& git rebase --abort 2>&1 | ForEach-Object { Write-Host $_ }
throw "Unable to rebase repository details changes onto origin/main before retrying the push."
}
}
if (-not $pushSucceeded) {
throw "Unable to push repository details changes after retrying."
}
if ($usedRetry -and $env:GITHUB_STEP_SUMMARY) {
@(
"- The retry path completed successfully and the rebased repository details commit was pushed to `main`."
) | Add-Content -Path $env:GITHUB_STEP_SUMMARY
}
"has_changes=true" >> $env:GITHUB_OUTPUT
"has_merged_artifact_changes=$($hasMergedArtifactChanges.ToString().ToLowerInvariant())" >> $env:GITHUB_OUTPUT
} else {
throw "Unable to check staged GitHub repositories details changes. git diff exited with code $gitDiffExitCode."
}
shell: pwsh
# Triggere the build-push-website-to-gh-pages workflow in the current repository using GitHub CLI
- name: Trigger build-push-website-to-gh-pages workflow
if: steps.push_changes.outputs.has_merged_artifact_changes == 'true'
run: |
gh workflow run build-push-website-to-gh-pages.yml --ref main
shell: pwsh
env:
GITHUB_TOKEN: ${{ secrets.PAT_WORKFLOW_UPDATE }}