Create builde-repos-db.yml

Author: Pro-Bandey

.github/workflows/builde-repos-db.yml

@@ -0,0 +1,237 @@
+name: Build Repositories Database
+
+on:
+  schedule:
+    - cron: '0 0 * * 0' # Runs every Sunday at midnight UTC
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  build-json:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout main repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Fetch data and build db.json
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+
+            // Utility to handle API rate limits via delay
+            const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+            async function generateDatabase() {
+              const username = 'Pro-bandey';
+
+              // Repositories to exclude
+              const excludedRepos = [
+                'pro-bandey',
+                'pro-bandey.github.io',
+                '.github'
+              ].map(repo => repo.toLowerCase().trim());
+
+              console.log(`Fetching repositories for ${username}...`);
+
+              // Fetch all public repositories
+              const reposResponse = await github.rest.repos.listForUser({
+                username: username,
+                type: 'public',
+                per_page: 100
+              });
+
+              // Filter repositories BEFORE processing
+              const repositories = reposResponse.data.filter(repo => {
+                const repoName = repo.name.toLowerCase().trim();
+
+                // Skip private repositories
+                if (repo.private) {
+                  console.log(`Skipping private repository: ${repo.name}`);
+                  return false;
+                }
+
+                // Skip excluded repositories
+                if (excludedRepos.includes(repoName)) {
+                  console.log(`Skipping excluded repository: ${repo.name}`);
+                  return false;
+                }
+
+                return true;
+              });
+
+              console.log(
+                'Repositories after filtering:',
+                repositories.map(r => r.name)
+              );
+
+              const db = [];
+              const repoDetails = {};
+
+              try {
+                console.log('Fetching repository GraphQL details (batched)...');
+                
+                // Batch GraphQL query to avoid secondary rate limits
+                const batchSize = 10;
+                for (let i = 0; i < repositories.length; i += batchSize) {
+                  const batch = repositories.slice(i, i + batchSize);
+                  
+                  // Construct dynamic aliases for the batched query
+                  let queryFragments = batch.map((repo, idx) => `
+                    repo_${idx}: repository(owner: "${username}", name: "${repo.name}") {
+                      name
+                      openGraphImageUrl
+                      defaultBranchRef {
+                        target {
+                          ... on Commit {
+                            history {
+                              totalCount
+                            }
+                          }
+                        }
+                      }
+                    }
+                  `).join('\n');
+                  
+                  const query = `query { ${queryFragments} }`;
+                  
+                  const result = await github.graphql(query);
+                  
+                  // Extract results back to our map
+                  Object.values(result).forEach(node => {
+                    if (node && node.name) {
+                      repoDetails[node.name] = {
+                        ogImage: node.openGraphImageUrl,
+                        commits: node.defaultBranchRef?.target?.history?.totalCount || 0
+                      };
+                    }
+                  });
+                  
+                  // Sleep between batches to prevent secondary rate limiting
+                  console.log(`Batch fetched. Sleeping for 1000ms...`);
+                  await sleep(1000);
+                }
+              } catch (err) {
+                console.warn(
+                  'Could not fetch GraphQL details:',
+                  err.message
+                );
+              }
+
+              // Process repositories sequentially with deliberate pacing
+              for (const repo of repositories) {
+                console.log(`Processing: ${repo.name}`);
+
+                // Fetch languages
+                let languagesData = {};
+                let languageBytes = {};
+
+                try {
+                  const langResponse = await github.rest.repos.listLanguages({
+                    owner: username,
+                    repo: repo.name
+                  });
+
+                  const languages = langResponse.data;
+
+                  const totalBytes = Object.values(languages).reduce(
+                    (a, b) => a + b,
+                    0
+                  );
+
+                  if (totalBytes > 0) {
+                    for (const [lang, bytes] of Object.entries(languages)) {
+                      languagesData[lang] = parseFloat(
+                        ((bytes / totalBytes) * 100).toFixed(1)
+                      );
+                      languageBytes[lang] = bytes;
+                    }
+                  }
+                } catch (err) {
+                  console.warn(
+                    `Could not fetch languages for ${repo.name}:`,
+                    err.message
+                  );
+                }
+
+                // Check README existence
+                let readmeExists = false;
+
+                try {
+                  await github.rest.repos.getReadme({
+                    owner: username,
+                    repo: repo.name
+                  });
+
+                  readmeExists = true;
+                } catch (err) {
+                  readmeExists = false;
+                }
+
+                const details = repoDetails[repo.name] || {};
+
+                // Add repository to database
+                db.push({
+                  Repo: repo.name,
+                  Date: repo.created_at,
+                  UpdatedAt: repo.pushed_at || repo.updated_at,
+                  Status: 'Pub',
+                  Desc: repo.description || 'No description provided.',
+                  Langs: languagesData,
+                  LangBytes: languageBytes,
+                  PreviewUrl: repo.homepage || '',
+                  ReadMeIs: readmeExists,
+                  Banner: details.ogImage || '',
+                  Commits: details.commits || 0,
+                  Stars: repo.stargazers_count || 0,
+                  Forks: repo.forks_count || 0
+                });
+
+                // Crucial: Sleep 1.5s between REST API requests to respect GitHub Secondary Rate Limits
+                await sleep(1500);
+              }
+
+              // Save db.json
+              fs.writeFileSync('db.json', JSON.stringify(db, null, 2));
+
+              console.log(
+                `Successfully created db.json with ${db.length} records.`
+              );
+            }
+
+            await generateDatabase();
+
+      - name: Deploy db.json to db branch
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Save generated file temporarily
+          mv db.json /tmp/db.json
+
+          # Fetch db branch if it exists
+          git fetch origin db || true
+
+          # Checkout db branch or create it
+          git checkout db || git checkout --orphan db
+
+          # Remove old files
+          find . -mindepth 1 -maxdepth 1 ! -name '.git' -exec rm -rf {} +
+
+          # Restore db.json
+          mv /tmp/db.json db.json
+
+          # Commit changes
+          git add db.json
+          git commit -m "Automated update of repository database" || echo "No changes to commit"
+
+          # Force push db branch
+          git push origin db --force