Skip to content

Commit 6fd3f68

Browse files
Priyanshu-byte-coderPriyanshu-byte-coder
authored
fix: stop GitHub rate-limit lockouts and unblock Vercel deploys (#3528)
* fix: stop GitHub rate-limit lockouts and unblock Vercel deploys Dashboard loads issued ~95 Search API calls against a 30/minute quota, and paging a year of commit search tripped GitHub's secondary (anti-scraping) limit, which then froze every API for the user. - Build the commit timeline from GraphQL (contributionsCollection repo discovery + per-repo default-branch history filtered by author id). No Search API quota is spent and totals now match GitHub's own contribution graph instead of double-counting mirrored commits. - Derive every commit window (streak, consistency, contributions, productive hours, commit times, hourly, weekly summary, repos, languages, repo explorer, repo health, CI) from that one shared, memoized timeline. - Repo health: one GraphQL query per repo replaces four searches. - Request layer: bounded per-resource concurrency instead of strict serialization, per-resource freshness, a short revalidation floor so a forced refresh revalidates shared queries once, and a 429 with remaining=0 is treated as a primary limit for that resource only. - Move the two hourly crons to a GitHub Actions schedule; Vercel Hobby rejects sub-daily crons, which has failed every deploy since Sept 12. Forced refresh of all 24 dashboard endpoints: ~95 -> 15 searches, 0 429s. * fix(metrics): derive repo/org-scoped commit views from the shared timeline Repo and org filters on contributions and top repos now slice the shared commit timeline in memory instead of issuing their own commit searches. This removes the last commit-search fan-out and the user-controlled branch between two fetch paths that CodeQL flagged.
1 parent 49f81cf commit 6fd3f68

26 files changed

Lines changed: 2005 additions & 990 deletions

File tree

‎.github/workflows/hourly-jobs.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: Hourly jobs
2+
3+
# Vercel Hobby only runs crons once a day, so hourly jobs are triggered from here.
4+
# Requires repository secrets APP_URL (e.g. https://devtrack.example.com) and CRON_SECRET.
5+
on:
6+
schedule:
7+
- cron: "5 * * * *"
8+
workflow_dispatch:
9+
10+
permissions: {}
11+
12+
concurrency:
13+
group: hourly-jobs
14+
cancel-in-progress: false
15+
16+
jobs:
17+
trigger:
18+
if: github.repository == 'Priyanshu-byte-coder/devtrack'
19+
runs-on: ubuntu-latest
20+
timeout-minutes: 15
21+
strategy:
22+
fail-fast: false
23+
matrix:
24+
path:
25+
- /api/cron/goals-sync
26+
- /api/notifications/weekly
27+
steps:
28+
- name: Call ${{ matrix.path }}
29+
env:
30+
APP_URL: ${{ secrets.APP_URL }}
31+
CRON_SECRET: ${{ secrets.CRON_SECRET }}
32+
run: |
33+
if [ -z "$APP_URL" ] || [ -z "$CRON_SECRET" ]; then
34+
echo "::notice::APP_URL or CRON_SECRET is not configured; skipping."
35+
exit 0
36+
fi
37+
status=$(curl -sS -o response.json -w '%{http_code}' --max-time 310 \
38+
-H "Authorization: Bearer $CRON_SECRET" "${APP_URL%/}${{ matrix.path }}")
39+
echo "HTTP $status"
40+
head -c 500 response.json; echo
41+
test "$status" -lt 400

‎src/app/api/metrics/coding-activity-insights/route.ts‎

Lines changed: 29 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
11
import { githubRateLimitResponse } from "@/lib/github-rate-limit";
2-
import { githubRequest } from "@/lib/github-request";
2+
import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search";
33
import { getSessionWithToken } from "@/lib/get-session-token";
44
import { NextRequest } from "next/server";
5-
import {
6-
getAccountToken,
7-
getAllAccounts,
8-
} from "@/lib/github-accounts";
9-
import { GITHUB_API } from "@/lib/github";
10-
import { GitHubAuthError, githubAuthErrorResponse } from "@/lib/github-fetch";
5+
import { getAccountToken, getAllAccounts } from "@/lib/github-accounts";
6+
import { githubAuthErrorResponse } from "@/lib/github-fetch";
117
import {
128
isMetricsCacheBypassed,
139
METRICS_CACHE_TTL_SECONDS,
@@ -68,58 +64,19 @@ async function fetchCommitTimestampsForAccount(
6864
since.setDate(since.getDate() - SEARCH_WINDOW_DAYS);
6965
const sinceStr = since.toISOString().slice(0, 10);
7066

71-
const timestamps: string[] = [];
72-
let totalCount = 0;
73-
let page = 1;
74-
75-
while (page <= 10) {
76-
const response = await githubRequest(
77-
`${GITHUB_API}/search/commits?q=author:${githubLogin}+author-date:>=${sinceStr}&per_page=100&page=${page}&sort=author-date&order=desc`,
67+
const { items } = requireUnthrottled(
68+
await collectAuthorCommits<CommitSearchItem>(
69+
githubLogin,
70+
sinceStr,
71+
token,
7872
{
79-
headers: {
80-
Authorization: `Bearer ${token}`,
81-
Accept: "application/vnd.github+json",
82-
},
83-
cache: "no-store",
84-
}
85-
);
86-
87-
if (!response.ok) {
88-
if (response.status === 401) throw new GitHubAuthError();
89-
if ((response.status === 403 || response.status === 429) && timestamps.length > 0) {
90-
break;
73+
fresh: cacheContext.bypass,
9174
}
92-
93-
throw new Error("GitHub API error");
94-
}
95-
96-
const payload = (await response.json()) as {
97-
total_count: number;
98-
items: CommitSearchItem[];
99-
};
100-
101-
if (page === 1) {
102-
totalCount = payload.total_count;
103-
}
104-
105-
timestamps.push(
106-
...payload.items
107-
.map((item) => item.commit?.author?.date)
108-
.filter((date): date is string => Boolean(date))
109-
);
110-
111-
if (payload.items.length < 100) {
112-
break;
113-
}
114-
115-
if (timestamps.length >= 1000 || timestamps.length >= totalCount) {
116-
break;
117-
}
118-
119-
page += 1;
120-
}
121-
122-
return timestamps;
75+
)
76+
);
77+
return items
78+
.map((item) => item.commit?.author?.date)
79+
.filter((date): date is string => Boolean(date));
12380
}
12481
);
12582
}
@@ -142,7 +99,11 @@ async function buildInsightsForAccount(
14299
export async function GET(req: NextRequest) {
143100
const sessionData = await getSessionWithToken();
144101

145-
if (!sessionData || !sessionData.session.githubLogin || !sessionData.session.githubId) {
102+
if (
103+
!sessionData ||
104+
!sessionData.session.githubLogin ||
105+
!sessionData.session.githubId
106+
) {
146107
return Response.json({ error: "Unauthorized" }, { status: 401 });
147108
}
148109

@@ -244,10 +205,15 @@ export async function GET(req: NextRequest) {
244205
return Response.json({ error: "Account not found" }, { status: 404 });
245206
}
246207

247-
const data = await buildInsightsForAccount(token, accountRow.github_login, timeZone, {
248-
bypass,
249-
userId: accountId,
250-
});
208+
const data = await buildInsightsForAccount(
209+
token,
210+
accountRow.github_login,
211+
timeZone,
212+
{
213+
bypass,
214+
userId: accountId,
215+
}
216+
);
251217

252218
return Response.json(data);
253219
} catch (e) {
@@ -256,4 +222,4 @@ export async function GET(req: NextRequest) {
256222

257223
return Response.json({ error: "GitHub API error" }, { status: 502 });
258224
}
259-
}
225+
}

‎src/app/api/metrics/commit-times/route.ts‎

Lines changed: 15 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
import { githubRateLimitResponse } from "@/lib/github-rate-limit";
2-
import { githubRequest } from "@/lib/github-request";
2+
import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search";
33
import { getServerSession } from "@/lib/server-session";
44
import { NextRequest } from "next/server";
55
import { authOptions } from "@/lib/auth";
6-
import { GITHUB_API } from "@/lib/github";
76
import {
87
isMetricsCacheBypassed,
98
METRICS_CACHE_TTL_SECONDS,
@@ -39,28 +38,19 @@ export async function GET(req: NextRequest) {
3938
new Array(24).fill(0)
4039
);
4140

42-
let page = 1;
43-
while (true) {
44-
const res = await githubRequest(
45-
`${GITHUB_API}/search/commits?q=author:${session.githubLogin}+author-date:>=${sinceStr}&per_page=100&page=${page}&sort=author-date&order=desc`,
41+
const { items } = requireUnthrottled(
42+
await collectAuthorCommits(
43+
session.githubLogin!,
44+
sinceStr,
45+
session.accessToken,
4646
{
47-
headers: {
48-
Authorization: `Bearer ${session.accessToken}`,
49-
Accept: "application/vnd.github+json",
50-
},
51-
cache: "no-store",
47+
fresh: bypass,
5248
}
53-
);
54-
if (!res.ok) throw new Error("GitHub API error");
55-
const data = (await res.json()) as {
56-
items: Array<{ commit: { author: { date: string } } }>;
57-
};
58-
for (const item of data.items) {
59-
const d = new Date(item.commit.author.date);
60-
matrix[d.getDay()][d.getHours()]++;
61-
}
62-
if (data.items.length < 100 || page >= 10) break;
63-
page++;
49+
)
50+
);
51+
for (const item of items) {
52+
const d = new Date(item.commit.author.date);
53+
matrix[d.getDay()][d.getHours()]++;
6454
}
6555

6656
// Find peak slot
@@ -83,8 +73,8 @@ export async function GET(req: NextRequest) {
8373

8474
return Response.json(result);
8575
} catch (error) {
86-
const githubLimit = githubRateLimitResponse(error);
87-
if (githubLimit) return githubLimit;
76+
const githubLimit = githubRateLimitResponse(error);
77+
if (githubLimit) return githubLimit;
8878
return Response.json({ error: "GitHub API error" }, { status: 502 });
8979
}
90-
}
80+
}

‎src/app/api/metrics/consistency-score/route.ts‎

Lines changed: 21 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
import { githubRateLimitResponse } from "@/lib/github-rate-limit";
2-
import { githubRequest } from "@/lib/github-request";
2+
import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search";
33
import { getServerSession } from "@/lib/server-session";
44
import { NextRequest } from "next/server";
55
import { authOptions } from "@/lib/auth";
66
import { getAccountToken, getAllAccounts } from "@/lib/github-accounts";
7-
import { GITHUB_API } from "@/lib/github";
87
import {
98
isMetricsCacheBypassed,
109
METRICS_CACHE_TTL_SECONDS,
@@ -44,46 +43,26 @@ async function fetchActiveDates(
4443
const sinceStr = since.toISOString().slice(0, 10);
4544

4645
const activeDates = new Set<string>();
47-
let page = 1;
48-
49-
while (true) {
50-
const searchRes = await githubRequest(
51-
`${GITHUB_API}/search/commits?q=author:${githubLogin}+author-date:>=${sinceStr}&per_page=100&page=${page}&sort=author-date&order=desc`,
52-
{
53-
headers: {
54-
Authorization: `Bearer ${token}`,
55-
Accept: "application/vnd.github+json",
56-
},
57-
cache: "no-store",
58-
}
59-
);
60-
61-
if (!searchRes.ok) {
62-
throw new Error("GitHub API error");
63-
}
46+
const { items } = requireUnthrottled(
47+
await collectAuthorCommits(githubLogin, sinceStr, token, {
48+
fresh: cacheContext.bypass,
49+
})
50+
);
6451

65-
const data = (await searchRes.json()) as {
66-
items: Array<{ commit: { author: { date: string } } }>;
67-
};
68-
69-
for (const item of data.items) {
70-
const commitDate = new Date(item.commit.author.date);
71-
const parts = new Intl.DateTimeFormat("en", {
72-
timeZone,
73-
year: "numeric",
74-
month: "2-digit",
75-
day: "2-digit",
76-
}).formatToParts(commitDate);
77-
const year = parts.find((p) => p.type === "year")?.value;
78-
const month = parts.find((p) => p.type === "month")?.value;
79-
const day = parts.find((p) => p.type === "day")?.value;
80-
if (year && month && day) {
81-
activeDates.add(`${year}-${month}-${day}`);
82-
}
52+
for (const item of items) {
53+
const commitDate = new Date(item.commit.author.date);
54+
const parts = new Intl.DateTimeFormat("en", {
55+
timeZone,
56+
year: "numeric",
57+
month: "2-digit",
58+
day: "2-digit",
59+
}).formatToParts(commitDate);
60+
const year = parts.find((p) => p.type === "year")?.value;
61+
const month = parts.find((p) => p.type === "month")?.value;
62+
const day = parts.find((p) => p.type === "day")?.value;
63+
if (year && month && day) {
64+
activeDates.add(`${year}-${month}-${day}`);
8365
}
84-
85-
if (data.items.length < 100 || page >= 10) break;
86-
page += 1;
8766
}
8867

8968
return Array.from(activeDates);
@@ -240,8 +219,8 @@ export async function GET(req: NextRequest) {
240219
});
241220
return Response.json(result);
242221
} catch (error) {
243-
const githubLimit = githubRateLimitResponse(error);
244-
if (githubLimit) return githubLimit;
222+
const githubLimit = githubRateLimitResponse(error);
223+
if (githubLimit) return githubLimit;
245224
return Response.json({ error: "GitHub API error" }, { status: 502 });
246225
}
247226
}

‎src/app/api/metrics/contributions/hourly/route.ts‎

Lines changed: 13 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
import { githubRateLimitResponse } from "@/lib/github-rate-limit";
2-
import { githubRequest } from "@/lib/github-request";
2+
import { collectAuthorCommits, requireUnthrottled } from "@/lib/github-search";
33
import { getServerSession } from "@/lib/server-session";
44
import { NextRequest } from "next/server";
55
import { authOptions } from "@/lib/auth";
6-
import { GITHUB_API } from "@/lib/github";
76
import {
87
isMetricsCacheBypassed,
98
METRICS_CACHE_TTL_SECONDS,
@@ -21,7 +20,9 @@ export async function GET(req: NextRequest) {
2120

2221
const daysParam = req.nextUrl.searchParams.get("days");
2322
const parsedDays = daysParam ? parseInt(daysParam, 10) : NaN;
24-
const days = Number.isNaN(parsedDays) ? 30 : Math.max(1, Math.min(365, parsedDays));
23+
const days = Number.isNaN(parsedDays)
24+
? 30
25+
: Math.max(1, Math.min(365, parsedDays));
2526
const bypass = isMetricsCacheBypassed(req);
2627
const key = metricsCacheKey(
2728
session.githubId ?? session.githubLogin,
@@ -37,32 +38,20 @@ export async function GET(req: NextRequest) {
3738
ttlSeconds: METRICS_CACHE_TTL_SECONDS.contributions,
3839
},
3940
async () => {
40-
const since = new Date();
41+
const since = new Date();
4142
since.setDate(since.getDate() - days);
4243
const sinceStr = since.toISOString().slice(0, 10);
4344

44-
const allItems: Array<{ commit: { author: { date: string } } }> = [];
45-
let page = 1;
46-
47-
while (true) {
48-
const searchRes = await githubRequest(
49-
`${GITHUB_API}/search/commits?q=author:${session.githubLogin}+author-date:>=${sinceStr}&per_page=100&page=${page}&sort=author-date&order=desc`,
45+
const { items: allItems } = requireUnthrottled(
46+
await collectAuthorCommits(
47+
session.githubLogin!,
48+
sinceStr,
49+
session.accessToken,
5050
{
51-
headers: {
52-
Authorization: `Bearer ${session.accessToken}`,
53-
Accept: "application/vnd.github+json",
54-
},
55-
cache: "no-store",
51+
fresh: bypass,
5652
}
57-
);
58-
if (!searchRes.ok) throw new Error("GitHub API error");
59-
const data = (await searchRes.json()) as {
60-
items: Array<{ commit: { author: { date: string } } }>;
61-
};
62-
allItems.push(...data.items);
63-
if (data.items.length < 100 || page >= 10) break;
64-
page++;
65-
}
53+
)
54+
);
6655

6756
// Initialize all 24 hours to 0
6857
const hourMap: Record<number, number> = {};
@@ -74,7 +63,6 @@ export async function GET(req: NextRequest) {
7463
hourMap[hour]++;
7564
}
7665

77-
7866
const hours = Array.from({ length: 24 }, (_, i) => ({
7967
hour: i,
8068
commits: hourMap[i],

0 commit comments

Comments
 (0)