Pipeline update: collect activity for all non-bot org members and expand bot filtering (#2)

Author: Piinks

.github/workflows/sync_activity.yml

@@ -8,8 +8,14 @@ on:
     inputs:
       days:
         description: 'Number of past days of activity to sync'
+        type: string
         required: false
         default: '7'
+      week:
+        description: 'Optional specific ISO week to sync (e.g. 2026-W32)'
+        type: string
+        required: false
+        default: ''
 
 permissions:
   contents: write
@@ -39,7 +45,13 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DAYS: ${{ inputs.days || '7' }}
-        run: dart run tool/sync_activity.dart --days "$DAYS"
+          WEEK: ${{ inputs.week || '' }}
+        run: |
+          if [ -n "$WEEK" ]; then
+            dart run tool/sync_activity.dart --week "$WEEK"
+          else
+            dart run tool/sync_activity.dart --days "$DAYS"
+          fi
 
       - name: Commit and push changes
         run: |
@@ -50,6 +62,6 @@ jobs:
             echo "No activity changes to commit."
           else
             git commit -m "chore: sync contributor activity data [skip ci]"
-            git pull --rebase origin main || true
+            git pull --rebase || true
             git push
           fi

docs/DATA_PIPELINE.md

@@ -15,8 +15,7 @@ The pipeline records 3 qualifying contribution types:
 
 ### Exclusions and filtering
 
-- Accounts in the `@flutter/robots` GitHub team and known automation usernames defined in `tool/bots.dart` are excluded.
-- Members of `@flutter/googlers` and `@flutter/partners` are excluded to conserve GitHub API rate limits.
+Accounts in the `@flutter/robots` and `@flutter/bots` GitHub teams and known automation usernames defined in `tool/bots.dart` (along with the deleted-account placeholder `ghost`) are excluded, while all human contributors and organization members (including Maintainers) are recorded so that 12-month contribution history is available for every organization member.
 
 ## Dataset structure
 
@@ -33,8 +32,7 @@ data/
 └── activity_summary.json
 ```
 
-- `data/activity/` holds 52 discrete weekly JSON snapshots (`YYYY-Wxx.json`). Each file records the contributions for that ISO week.
-- `data/activity_summary.json` holds the rolling 12-month totals per contributor, calculated directly as the sum of all active weekly files.
+The `data/activity/` directory holds 52 discrete weekly JSON snapshots (`YYYY-Wxx.json`), where each file records the contributions for that ISO week, and `data/activity_summary.json` holds the rolling 12-month totals per contributor calculated directly as the sum of all active weekly files.
 
 ## Daily execution and rolling purge
 
@@ -47,7 +45,4 @@ graph LR
     Sum --> Commit["Commit updates<br/>[skip ci]"]
 ```
 
-- Each night at 02:17 UTC, the GitHub Actions workflow (`.github/workflows/sync_activity.yml`) queries the current ISO week of activity using GraphQL with cursor pagination.
-- The week's snapshot in `data/activity/` is updated idempotently, avoiding double-counting across runs. Reviews are deduplicated per merged pull request.
-- Snapshot files older than 52 weeks are pruned based on their ISO week filenames.
-- `data/activity_summary.json` is recalculated by summing the active weekly files with deterministically sorted keys, rolling off activity older than 12 months without requiring historical re-scans.
+Each night at 02:17 UTC, the GitHub Actions workflow (`.github/workflows/sync_activity.yml`) queries the current ISO week of activity using GraphQL with cursor pagination and updates the week's snapshot in `data/activity/` idempotently, avoiding double-counting across runs and deduplicating reviews per merged pull request. Snapshot files older than 52 weeks are pruned based on their ISO week filenames, and `data/activity_summary.json` is recalculated by summing the active weekly files with deterministically sorted keys, rolling off activity older than 12 months without requiring historical re-scans.

test/activity_sync_test.dart

@@ -1,6 +1,9 @@
 import 'dart:convert';
 import 'dart:io';
+import 'package:http/http.dart' as http;
+import 'package:http/testing.dart';
 import 'package:test/test.dart';
+import '../tool/github.dart';
 import '../tool/sync_activity.dart';
 
 void main() {
@@ -98,5 +101,157 @@ void main() {
       final remaining = activityDir.listSync().whereType<File>().length;
       expect(remaining, equals(52));
     });
+
+    test(
+        'pruneOldActivityFiles skips invalid filenames and handles missing dir',
+        () {
+      final missingDir = Directory('${tempDir.path}/non_existent');
+      expect(pruneOldActivityFiles(missingDir), equals(0));
+
+      final activityDir = Directory('${tempDir.path}/activity')..createSync();
+      File('${activityDir.path}/not-a-week.json').writeAsStringSync('{}');
+      File('${activityDir.path}/README.txt').writeAsStringSync('ignore');
+      expect(
+          pruneOldActivityFiles(activityDir,
+              referenceDate: DateTime.utc(2026, 8, 3)),
+          equals(0));
+    });
+
+    test(
+        'fetchActivityForRange aggregates PRs, deduplicated reviews, and issues while excluding bots and ghost',
+        () async {
+      final mockClient = MockClient((request) async {
+        final body = jsonDecode(request.body) as Map<String, dynamic>;
+        final variables = body['variables'] as Map<String, dynamic>;
+        final query = variables['query'] as String;
+
+        if (query.contains('is:pr')) {
+          return http.Response(
+            jsonEncode({
+              'data': {
+                'search': {
+                  'pageInfo': {'hasNextPage': false, 'endCursor': null},
+                  'nodes': [
+                    {
+                      'author': {'login': 'alice'},
+                      'reviews': {
+                        'nodes': [
+                          // Self-review by author should be ignored
+                          {
+                            'author': {'login': 'alice'}
+                          },
+                          // Duplicate review by bob on the same PR should count once
+                          {
+                            'author': {'login': 'bob'}
+                          },
+                          {
+                            'author': {'login': 'bob'}
+                          },
+                          // Maintainer review should be recorded
+                          {
+                            'author': {'login': 'Piinks'}
+                          },
+                          // Bot review should be skipped
+                          {
+                            'author': {'login': 'auto-submit'}
+                          },
+                        ]
+                      }
+                    },
+                    // PR by maintainer and community review on it should both be recorded
+                    {
+                      'author': {'login': 'Piinks'},
+                      'reviews': {
+                        'nodes': [
+                          {
+                            'author': {'login': 'alice'}
+                          }
+                        ]
+                      }
+                    },
+                    // PR by bot should be skipped
+                    {
+                      'author': {'login': 'engine-flutter-autoroll'},
+                      'reviews': {'nodes': <Map<String, dynamic>>[]}
+                    },
+                  ]
+                }
+              }
+            }),
+            200,
+          );
+        } else {
+          // Issue search
+          return http.Response(
+            jsonEncode({
+              'data': {
+                'search': {
+                  'pageInfo': {'hasNextPage': false, 'endCursor': null},
+                  'nodes': [
+                    {
+                      'author': {'login': 'alice'}
+                    },
+                    {
+                      'author': {'login': 'charlie'}
+                    },
+                    // Ghost issue should be skipped
+                    {
+                      'author': {'login': 'ghost'}
+                    },
+                    {
+                      'author': {'login': 'Piinks'}
+                    },
+                  ]
+                }
+              }
+            }),
+            200,
+          );
+        }
+      });
+
+      final client = GitHubClient(token: 'test', httpClient: mockClient);
+      final stats = await fetchActivityForRange(
+        client,
+        since: DateTime.utc(2026, 8, 1),
+        until: DateTime.utc(2026, 8, 8),
+      );
+
+      expect(
+          stats['alice'],
+          equals({
+            'merged_prs': 1,
+            'issues': 1,
+            'reviews': 1,
+            'total': 3,
+          }));
+      expect(
+          stats['bob'],
+          equals({
+            'merged_prs': 0,
+            'issues': 0,
+            'reviews': 1,
+            'total': 1,
+          }));
+      expect(
+          stats['charlie'],
+          equals({
+            'merged_prs': 0,
+            'issues': 1,
+            'reviews': 0,
+            'total': 1,
+          }));
+      expect(
+          stats['Piinks'],
+          equals({
+            'merged_prs': 1,
+            'issues': 1,
+            'reviews': 1,
+            'total': 3,
+          }));
+      expect(stats.containsKey('auto-submit'), isFalse);
+      expect(stats.containsKey('engine-flutter-autoroll'), isFalse);
+      expect(stats.containsKey('ghost'), isFalse);
+    });
   });
 }

test/bots_test.dart

@@ -1,8 +1,14 @@
+import 'dart:convert';
+import 'package:http/http.dart' as http;
+import 'package:http/testing.dart';
 import 'package:test/test.dart';
 import '../tool/bots.dart';
+import '../tool/github.dart';
 
 void main() {
   group('Bot Detection (isBot)', () {
+    tearDown(resetRobotTeamCache);
+
     test('identifies known bots in knownBots set', () {
       expect(isBot('dependabot'), isTrue);
       expect(isBot('dependabot[bot]'), isTrue);
@@ -13,7 +19,9 @@ void main() {
       expect(isBot('github-actions'), isTrue);
       expect(isBot('github-actions[bot]'), isTrue);
       expect(isBot('engine-flutter-autoroll'), isTrue);
+      expect(isBot('auto-submit'), isTrue);
       expect(isBot('auto-submit[bot]'), isTrue);
+      expect(isBot('google-cla'), isTrue);
       expect(isBot('google-cla[bot]'), isTrue);
       expect(isBot('renovate[bot]'), isTrue);
       expect(isBot('DartDevtoolWorkflowBot'), isTrue);
@@ -69,5 +77,28 @@ void main() {
       expect(isBot('FlutterGithubBot'), isTrue);
       expect(isBot('  piinks  '), isFalse);
     });
+
+    test(
+        'loadRobotTeams dynamically adds members of @flutter/robots and @flutter/bots to bots',
+        () async {
+      final mockClient = MockClient((request) async {
+        expect(
+          request.url.path,
+          anyOf(
+            equals('/orgs/flutter/teams/robots/members'),
+            equals('/orgs/flutter/teams/bots/members'),
+          ),
+        );
+        return http.Response(
+          jsonEncode([
+            {'login': 'dynamic-team-robot-account'}
+          ]),
+          200,
+        );
+      });
+      final gh = GitHubClient(token: 'test', httpClient: mockClient);
+      await loadRobotTeams(gh);
+      expect(isBot('dynamic-team-robot-account'), isTrue);
+    });
   });
 }