Skip to content
This repository was archived by the owner on Jun 3, 2026. It is now read-only.

Commit ca9884a

Browse files
ishaanxguptaishaanxgupta
committed
add scanner endpoints
1 parent 9370ae1 commit ca9884a

3 files changed

Lines changed: 296 additions & 1 deletion

File tree

‎admin/app.js‎

Lines changed: 98 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,14 +49,15 @@ function navigate(page) {
4949
if (nav) nav.classList.add('active');
5050

5151
// Update page title
52-
const titles = { overview:'Overview', logs:'Live Terminal', llm:'LLM & Costs', github:'GitHub Traffic', users:'Users' };
52+
const titles = { overview:'Overview', logs:'Live Terminal', llm:'LLM & Costs', github:'GitHub Traffic', users:'Users', scanner:'Scanner Analytics' };
5353
const titleEl = document.getElementById('page-title');
5454
if (titleEl) titleEl.textContent = titles[page] || page;
5555

5656
if (page === 'overview') loadOverview();
5757
if (page === 'llm') loadLLM();
5858
if (page === 'github') loadGitHub();
5959
if (page === 'users') loadUsers();
60+
if (page === 'scanner') loadScanner();
6061
if (page === 'logs' && !_sseSource) connectSystemLogs();
6162
}
6263

@@ -75,11 +76,13 @@ async function loadAll() {
7576
loadOverview();
7677
loadLLM();
7778
loadGitHub();
79+
loadScanner();
7880
// Auto-refresh every 30s
7981
setInterval(() => {
8082
if (currentPage === 'overview') loadOverview();
8183
if (currentPage === 'llm') loadLLM();
8284
if (currentPage === 'users') loadUsers();
85+
if (currentPage === 'scanner') loadScanner();
8386
}, 30000);
8487
}
8588

@@ -362,6 +365,100 @@ async function loadGitHub() {
362365
document.getElementById('gh-paths-body').innerHTML = pathRows || '<tr><td colspan="3" style="color:var(--text2)">No data</td></tr>';
363366
}
364367

368+
// ── Scanner page ──────────────────────────────────────────────────
369+
async function loadScanner() {
370+
const data = await apiFetch('/scanner/analytics');
371+
if (!data) return;
372+
373+
if (data.error) {
374+
console.error('Scanner analytics error:', data.error);
375+
return;
376+
}
377+
378+
// Update summary cards
379+
const jobs = data.scanner_jobs || {};
380+
const repos = data.user_repos || {};
381+
const runs = data.scan_runs || {};
382+
const stars = data.community_stars || {};
383+
const visibility = data.index_visibility || {};
384+
385+
document.getElementById('scanner-jobs-total').textContent = formatNum(jobs.total || 0);
386+
document.getElementById('scanner-repos-total').textContent = formatNum(repos.total || 0);
387+
document.getElementById('scanner-runs-total').textContent = formatNum(runs.total || 0);
388+
document.getElementById('scanner-stars-total').textContent = formatNum(stars.total_stars || 0);
389+
document.getElementById('scanner-visibility-total').textContent = formatNum(visibility.total || 0);
390+
document.getElementById('scanner-stargazers').textContent = formatNum(stars.unique_users || 0);
391+
392+
// Recent scanner jobs table
393+
const jobsRows = (jobs.recent || []).map(j => {
394+
const statusBadge = (status) => {
395+
const color = status === 'complete' ? 'var(--green)' : status === 'failed' ? 'var(--red)' : status === 'running' ? 'var(--amber)' : 'var(--text2)';
396+
return `<span style="color:${color};font-weight:600">${status || 'N/A'}</span>`;
397+
};
398+
const updated = j.updated_at ? formatDateShort(j.updated_at) : '—';
399+
return `<tr>
400+
<td style="font-family:'JetBrains Mono',monospace;font-size:11px">${escHtml(j.job_id || 'N/A')}</td>
401+
<td>${escHtml(j.username || 'N/A')}</td>
402+
<td>${escHtml(j.org || '')}/${escHtml(j.repo || '')}</td>
403+
<td>${statusBadge(j.phase1_status)}</td>
404+
<td>${statusBadge(j.phase2_status)}</td>
405+
<td>${updated}</td>
406+
</tr>`;
407+
}).join('');
408+
document.getElementById('scanner-jobs-body').innerHTML = jobsRows || '<tr><td colspan="6" style="color:var(--text2)">No scanner jobs found</td></tr>';
409+
410+
// Recent scan runs table
411+
const runsRows = (runs.latest || []).map(r => {
412+
const status = r.status || 'unknown';
413+
const statusColor = status === 'success' ? 'var(--green)' : status === 'failed' ? 'var(--red)' : 'var(--text2)';
414+
const scanned = r.last_scanned_at ? formatDateShort(r.last_scanned_at) : '—';
415+
return `<tr>
416+
<td>${escHtml(r.org_id || 'N/A')}</td>
417+
<td>${escHtml(r.repo || 'N/A')}</td>
418+
<td style="color:${statusColor};font-weight:600">${status}</td>
419+
<td>${scanned}</td>
420+
</tr>`;
421+
}).join('');
422+
document.getElementById('scanner-runs-body').innerHTML = runsRows || '<tr><td colspan="4" style="color:var(--text2)">No scan runs found</td></tr>';
423+
424+
// Top starred repos table
425+
const starsRows = (stars.top_repos || []).map(s => {
426+
const org = (s._id && s._id.org) || 'N/A';
427+
const repo = (s._id && s._id.repo) || 'N/A';
428+
return `<tr>
429+
<td>${escHtml(org)}</td>
430+
<td>${escHtml(repo)}</td>
431+
<td style="color:#fbbf24;font-weight:600">${s.star_count || 0}</td>
432+
</tr>`;
433+
}).join('');
434+
document.getElementById('scanner-stars-body').innerHTML = starsRows || '<tr><td colspan="3" style="color:var(--text2)">No stars data found</td></tr>';
435+
436+
// Repos per user table
437+
const reposPerUserRows = (repos.per_user || []).map(u =>
438+
`<tr><td>${escHtml(u._id || 'N/A')}</td><td>${u.repo_count || 0}</td></tr>`
439+
).join('');
440+
document.getElementById('scanner-repos-per-user-body').innerHTML = reposPerUserRows || '<tr><td colspan="2" style="color:var(--text2)">No user repos found</td></tr>';
441+
442+
// Recent user repos table
443+
const recentReposRows = (repos.recent || []).map(r =>
444+
`<tr>
445+
<td>${escHtml(r.username || 'N/A')}</td>
446+
<td>${escHtml(r.github_org || 'N/A')}</td>
447+
<td>${escHtml(r.repo || 'N/A')}</td>
448+
<td>${escHtml(r.branch || 'main')}</td>
449+
</tr>`
450+
).join('');
451+
document.getElementById('scanner-recent-repos-body').innerHTML = recentReposRows || '<tr><td colspan="4" style="color:var(--text2)">No recent repos found</td></tr>';
452+
453+
// Job status breakdown table
454+
const statusRows = (jobs.by_status || []).map(s => {
455+
const p1 = (s._id && s._id.phase1) || 'N/A';
456+
const p2 = (s._id && s._id.phase2) || 'N/A';
457+
return `<tr><td>${escHtml(p1)}</td><td>${escHtml(p2)}</td><td>${s.count || 0}</td></tr>`;
458+
}).join('');
459+
document.getElementById('scanner-job-status-body').innerHTML = statusRows || '<tr><td colspan="3" style="color:var(--text2)">No status data found</td></tr>';
460+
}
461+
365462

366463
// ═════════════════════════════════════════════════════════════════
367464
// Live Logs — SSE (Server-Sent Events) + journalctl subprocess

‎admin/index.html‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,10 @@ <h1>XMem</h1>
7272
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5"><path d="M17 21v-2a4 4 0 00-4-4H5a4 4 0 00-4 4v2"/><circle cx="9" cy="7" r="4"/><path d="M23 21v-2a4 4 0 00-3-3.87M16 3.13a4 4 0 010 7.75"/></svg>
7373
<span>Users</span>
7474
</div>
75+
<div class="nav-item" data-page="scanner" onclick="navigate('scanner')">
76+
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5"><path d="M4 4h16c1.1 0 2 .9 2 2v12c0 1.1-.9 2-2 2H4c-1.1 0-2-.9-2-2V6c0-1.1.9-2 2-2z"/><polyline points="22,6 12,13 2,6"/></svg>
77+
<span>Scanner</span>
78+
</div>
7579

7680
<div class="sidebar-footer">
7781
<div class="server-status" id="sidebar-status">
@@ -366,6 +370,103 @@ <h3 id="trail-modal-title">User Activity Trail</h3>
366370
</div>
367371
</div>
368372

373+
<!-- Scanner Analytics Page -->
374+
<div id="page-scanner" class="page">
375+
<div class="cards">
376+
<div class="card card-glow-blue">
377+
<div class="card-icon"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M12 2L2 7l10 5 10-5-10-5zM2 17l10 5 10-5M2 12l10 5 10-5"/></svg></div>
378+
<div class="card-content"><div class="label">Total Jobs</div><div class="value" id="scanner-jobs-total">—</div></div>
379+
</div>
380+
<div class="card card-glow-green">
381+
<div class="card-icon"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M21 15a2 2 0 01-2 2H7l-4 4V5a2 2 0 012-2h14a2 2 0 012 2z"/></svg></div>
382+
<div class="card-content"><div class="label">User Repos</div><div class="value" id="scanner-repos-total">—</div></div>
383+
</div>
384+
<div class="card card-glow-purple">
385+
<div class="card-icon"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><circle cx="12" cy="12" r="10"/><path d="M12 2v20M2 12h20"/></svg></div>
386+
<div class="card-content"><div class="label">Scan Runs</div><div class="value" id="scanner-runs-total">—</div></div>
387+
</div>
388+
<div class="card card-glow-amber">
389+
<div class="card-icon"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M12 2l3.09 6.26L22 9.27l-5 4.87 1.18 6.88L12 17.77l-6.18 3.25L7 14.14 2 9.27l6.91-1.01L12 2z"/></svg></div>
390+
<div class="card-content"><div class="label">Community Stars</div><div class="value" id="scanner-stars-total">—</div></div>
391+
</div>
392+
</div>
393+
394+
<!-- Second row of cards -->
395+
<div class="cards">
396+
<div class="card card-glow-blue">
397+
<div class="card-icon"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M1 12s4-8 11-8 11 8 11 8-4 8-11 8-11-8-11-8z"/><circle cx="12" cy="12" r="3"/></svg></div>
398+
<div class="card-content"><div class="label">Index Visibility</div><div class="value" id="scanner-visibility-total">—</div></div>
399+
</div>
400+
<div class="card card-glow-green">
401+
<div class="card-icon"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M17 21v-2a4 4 0 00-4-4H5a4 4 0 00-4 4v2"/><circle cx="9" cy="7" r="4"/><path d="M23 21v-2a4 4 0 00-3-3.87M16 3.13a4 4 0 010 7.75"/></svg></div>
402+
<div class="card-content"><div class="label">Unique Stargazers</div><div class="value" id="scanner-stargazers">—</div></div>
403+
</div>
404+
</div>
405+
406+
<div class="row">
407+
<div class="panel">
408+
<div class="panel-header"><h3>Recent Scanner Jobs</h3></div>
409+
<div class="table-wrap">
410+
<table class="tbl">
411+
<thead><tr><th>Job ID</th><th>User</th><th>Org/Repo</th><th>Phase 1</th><th>Phase 2</th><th>Updated</th></tr></thead>
412+
<tbody id="scanner-jobs-body"><tr><td colspan="6" class="empty-state">Loading...</td></tr></tbody>
413+
</table>
414+
</div>
415+
</div>
416+
<div class="panel">
417+
<div class="panel-header"><h3>Recent Scan Runs</h3></div>
418+
<div class="table-wrap">
419+
<table class="tbl">
420+
<thead><tr><th>Org</th><th>Repo</th><th>Status</th><th>Last Scanned</th></tr></thead>
421+
<tbody id="scanner-runs-body"><tr><td colspan="4" class="empty-state">Loading...</td></tr></tbody>
422+
</table>
423+
</div>
424+
</div>
425+
</div>
426+
427+
<div class="row">
428+
<div class="panel">
429+
<div class="panel-header"><h3>Top Starred Repositories</h3></div>
430+
<div class="table-wrap">
431+
<table class="tbl">
432+
<thead><tr><th>Organization</th><th>Repository</th><th>Stars</th></tr></thead>
433+
<tbody id="scanner-stars-body"><tr><td colspan="3" class="empty-state">Loading...</td></tr></tbody>
434+
</table>
435+
</div>
436+
</div>
437+
<div class="panel">
438+
<div class="panel-header"><h3>Repos Per User (Top 10)</h3></div>
439+
<div class="table-wrap">
440+
<table class="tbl">
441+
<thead><tr><th>User</th><th>Repo Count</th></tr></thead>
442+
<tbody id="scanner-repos-per-user-body"><tr><td colspan="2" class="empty-state">Loading...</td></tr></tbody>
443+
</table>
444+
</div>
445+
</div>
446+
</div>
447+
448+
<div class="row">
449+
<div class="panel">
450+
<div class="panel-header"><h3>Recent User Repos</h3></div>
451+
<div class="table-wrap">
452+
<table class="tbl">
453+
<thead><tr><th>User</th><th>Organization</th><th>Repository</th><th>Branch</th></tr></thead>
454+
<tbody id="scanner-recent-repos-body"><tr><td colspan="4" class="empty-state">Loading...</td></tr></tbody>
455+
</table>
456+
</div>
457+
</div>
458+
<div class="panel">
459+
<div class="panel-header"><h3>Job Status Breakdown</h3></div>
460+
<div class="table-wrap">
461+
<table class="tbl">
462+
<thead><tr><th>Phase 1 Status</th><th>Phase 2 Status</th><th>Count</th></tr></thead>
463+
<tbody id="scanner-job-status-body"><tr><td colspan="3" class="empty-state">Loading...</td></tr></tbody>
464+
</table>
465+
</div>
466+
</div>
467+
</div>
468+
</div>
469+
369470
</div>
370471
</div>
371472

‎src/api/routes/admin.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -672,6 +672,103 @@ async def test_llm_track(request: Request, user: dict = Depends(_verify_admin_to
672672
})
673673

674674

675+
# ═══════════════════════════════════════════════════════════════════════════
676+
# Scanner analytics endpoints
677+
# ═══════════════════════════════════════════════════════════════════════════
678+
679+
@router.get("/api/scanner/analytics")
680+
async def scanner_analytics(request: Request, user: dict = Depends(_verify_admin_token)):
681+
"""Return scanner analytics from all scanner collections."""
682+
try:
683+
from pymongo import MongoClient
684+
client = MongoClient(settings.mongodb_uri, serverSelectionTimeoutMS=3000)
685+
db = client[settings.mongodb_database]
686+
687+
# Collection references
688+
scan_runs = db["scan_runs"]
689+
scanner_jobs = db["scanner_jobs"]
690+
scanner_user_repos = db["scanner_user_repos"]
691+
scanner_index_visibility = db["scanner_index_visibility"]
692+
scanner_community_stars = db["scanner_community_stars"]
693+
694+
# 1. Scan Runs stats
695+
scan_runs_count = scan_runs.count_documents({})
696+
scan_runs_latest = list(scan_runs.find({}, {"_id": 0, "org_id": 1, "repo": 1, "last_sha": 1, "last_scanned_at": 1, "status": 1})
697+
.sort("last_scanned_at", -1)
698+
.limit(10))
699+
700+
# 2. Scanner Jobs stats
701+
scanner_jobs_count = scanner_jobs.count_documents({})
702+
jobs_by_status = list(scanner_jobs.aggregate([
703+
{"$group": {"_id": {"phase1": "$phase1_status", "phase2": "$phase2_status"}, "count": {"$sum": 1}}}
704+
]))
705+
706+
recent_jobs = list(scanner_jobs.find(
707+
{},
708+
{"_id": 0, "job_id": 1, "username": 1, "org": 1, "repo": 1, "branch": 1,
709+
"phase1_status": 1, "phase2_status": 1, "updated_at": 1, "error": 1}
710+
).sort("updated_at", -1).limit(20))
711+
712+
# 3. User Repos stats
713+
user_repos_count = scanner_user_repos.count_documents({})
714+
repos_per_user = list(scanner_user_repos.aggregate([
715+
{"$group": {"_id": "$username", "repo_count": {"$sum": 1}}},
716+
{"$sort": {"repo_count": -1}},
717+
{"$limit": 10}
718+
]))
719+
720+
recent_repos = list(scanner_user_repos.find(
721+
{},
722+
{"_id": 0, "username": 1, "github_org": 1, "repo": 1, "branch": 1, "last_seen_commit": 1}
723+
).sort("_id", -1).limit(20))
724+
725+
# 4. Index Visibility stats
726+
visibility_count = scanner_index_visibility.count_documents({})
727+
visibility_breakdown = list(scanner_index_visibility.aggregate([
728+
{"$group": {"_id": "$is_visible", "count": {"$sum": 1}}}
729+
]))
730+
731+
# 5. Community Stars stats
732+
stars_count = scanner_community_stars.count_documents({})
733+
top_starred_repos = list(scanner_community_stars.aggregate([
734+
{"$group": {"_id": {"org": "$org_id", "repo": "$repo"}, "star_count": {"$sum": 1}}},
735+
{"$sort": {"star_count": -1}},
736+
{"$limit": 10}
737+
]))
738+
739+
# Unique users who starred
740+
unique_stargazers = scanner_community_stars.distinct("username")
741+
742+
return JSONResponse({
743+
"scan_runs": {
744+
"total": scan_runs_count,
745+
"latest": _bson_safe(scan_runs_latest),
746+
},
747+
"scanner_jobs": {
748+
"total": scanner_jobs_count,
749+
"by_status": _bson_safe(jobs_by_status),
750+
"recent": _bson_safe(recent_jobs),
751+
},
752+
"user_repos": {
753+
"total": user_repos_count,
754+
"per_user": _bson_safe(repos_per_user),
755+
"recent": _bson_safe(recent_repos),
756+
},
757+
"index_visibility": {
758+
"total": visibility_count,
759+
"breakdown": _bson_safe(visibility_breakdown),
760+
},
761+
"community_stars": {
762+
"total_stars": stars_count,
763+
"unique_users": len(unique_stargazers),
764+
"top_repos": _bson_safe(top_starred_repos),
765+
},
766+
})
767+
except Exception as exc:
768+
logger.exception("Scanner analytics failed")
769+
return JSONResponse({"error": str(exc)}, status_code=500)
770+
771+
675772
# ═══════════════════════════════════════════════════════════════════════════
676773
# Users management endpoints
677774
# ═══════════════════════════════════════════════════════════════════════════

0 commit comments

Comments
 (0)