Skip to content

Commit 66727cb

Browse files
committed
fix(guard): 按中心周期判定连续超限
1 parent f3b09ca commit 66727cb

7 files changed

Lines changed: 166 additions & 37 deletions

File tree

docs/account-max-ips-anti-sharing-plan.md

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -426,15 +426,15 @@ Map<accountNo, Map<nodeIp, NodeStat>>
426426
NodeStat = { connections, ipSet, reportedAtEpochSeconds }
427427
```
428428

429-
处理一次 `guard-sync` 请求的步骤
429+
`guard-sync` 请求只负责更新快照并返回已经确认的黑名单;中心每 10 秒统一采样一次
430430

431431
1. **更新**:用请求体覆盖 `聚合表[各accountNo][nodeIp]` 这一格,刷新 `reportedAt`
432-
2. **求和**(只针对本请求涉及的账户,避免全表扫描):对每个账户,累加所有节点
432+
2. **中心采样求和**:对每个账户,累加所有节点
433433
`connections`、合并所有节点的 `ipSet` 去重,得到全局 `totalConnections` /
434434
`totalIps`
435-
3. **判定**:与 `Account.maxConnections`(连接数)及 IP 上限比较,超限则计入本次
436-
响应的 `blockedAccounts`
437-
4. **返回**把结论写入响应体回给该节点 agent。
435+
3. **连续判定**:与 `Account.maxConnections`(连接数)及 IP 上限比较;连续超限
436+
达到配置的采样次数后加入中心黑名单,任一采样周期恢复到限制内时立即移除
437+
4. **返回**后续 `guard-sync` 请求把已经确认的黑名单结论回给对应节点 agent。
438438

439439
**每格必须带 TTL**——这是合并方案不出错的关键:
440440

@@ -446,14 +446,14 @@ Map<accountNo, Map<nodeIp, NodeStat>>
446446
> 内存表不落库(秒级高频,落库无必要);中心重启后由 agent 在数个周期内重新
447447
> 上报重建,无需持久化。
448448
449-
### 4.3 无需独立的聚合定时任务
449+
### 4.3 中心统一采样任务
450450

451-
由于聚合与判定都在**收到 `guard-sync` 请求时同步完成**(4.2),中心**不需要**
452-
再跑一个独立的秒级 `@Scheduled` 聚合任务——每个节点自己的周期上报就是聚合的
453-
驱动时钟。这比「上报入表 + 独立任务定时算配额 + agent 再来拉」少了一整条链路
451+
中心使用每 10 秒执行一次的 `@Scheduled` 任务统一计算连续超限次数。这样同一周期
452+
内无论收到多少个服务器 agent 上报,每个账户都只累计一次,避免服务器数量改变
453+
连续超限阈值的实际含义。任务使用 `ConcurrentExecution.SKIP` 防止重叠执行
454454

455-
> 唯一可选的后台任务是低频(如每 30 秒)清理超 TTL 的僵尸节点格,防止长期不上报
456-
> 的节点在表中堆积;此任务不在热路径,用普通 `@Scheduled` 即可
455+
另有低频(如每 30 秒)任务清理超 TTL 的僵尸节点格,防止长期不上报的节点在表中
456+
堆积;清理任务不参与连续超限次数累计
457457

458458
### 4.4 单节点兜底字段下发(低频,随部署)
459459

@@ -577,9 +577,9 @@ Map<accountNo, Map<nodeIp, NodeStat>>
577577
4. **中心开发**(AirOpsCat):
578578
- **数据模型先行**`account``max_ips` 列 Flyway 迁移 → `Account` 实体、
579579
`AccountDto``AccountRequest``NodeClient``maxIps`(§5.2)。
580-
- `guard-sync` 端点(含鉴权)更新该节点在内存聚合表中的格 → 实时求和
581-
(剔除超 TTL 的僵尸节点格)`maxConnections` / `maxIps` 双维度判定并在
582-
同一响应回结论 → 低频清理任务 → 单节点兜底字段下发(`NodeClient` +
580+
- `guard-sync` 端点(含鉴权)更新该节点在内存聚合表中的格;中心定时任务统一
581+
求和(剔除超 TTL 的僵尸节点格)`maxConnections` / `maxIps` 双维度执行
582+
连续超限判定;后续同步响应返回已确认黑名单 → 低频清理任务 → 单节点兜底字段下发(`NodeClient` +
583583
`SingBoxConfigBuilder`)→ 反射注册 → 账户页新增「最大 IP / 设备数」输入项。
584584
5. **联调验证**
585585
- 同账户在**多个节点**合计超过 `maxConnections` → 新连接被内核静默拒绝;

src/main/java/com/fun90/airopscat/controller/OpenController.java

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import com.fun90.airopscat.model.entity.Account;
66
import com.fun90.airopscat.repository.AccountRepository;
77
import com.fun90.airopscat.service.AccountOnlineIpService;
8-
import com.fun90.airopscat.service.AccountOnlineLimitAlertService;
98
import com.fun90.airopscat.service.SubscriptionService;
109
import com.fun90.airopscat.service.SystemConfigService;
1110
import com.fun90.airopscat.service.guard.AccountGuardAggregator;
@@ -48,9 +47,6 @@ public class OpenController {
4847
@Inject
4948
AccountGuardAggregator accountGuardAggregator;
5049

51-
@Inject
52-
AccountOnlineLimitAlertService accountOnlineLimitAlertService;
53-
5450
@Inject
5551
AccountRepository accountRepository;
5652

@@ -122,11 +118,6 @@ public Response guardSync(GuardSyncRequest request,
122118
log.warn("guard-sync 处理失败: nodeIp={}, error={}", request.getNodeIp(), e.getMessage(), e);
123119
evaluation = new AccountGuardEvaluation(Map.of(), Map.of());
124120
}
125-
try {
126-
accountOnlineLimitAlertService.checkAndNotifyFromGuard(evaluation.getStatsByAccountNo());
127-
} catch (Exception e) {
128-
log.warn("guard-sync 实时告警检查失败: nodeIp={}, error={}", request.getNodeIp(), e.getMessage(), e);
129-
}
130121
try {
131122
if (request.getOnlineAccountIps() != null) {
132123
int refreshed = accountOnlineIpService.refreshFromGuardAccountIps(request.getNodeIp(), request.getOnlineAccountIps());
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package com.fun90.airopscat.scheduler;
2+
3+
import com.fun90.airopscat.service.AccountOnlineLimitAlertService;
4+
import com.fun90.airopscat.service.guard.AccountGuardAggregator;
5+
import com.fun90.airopscat.service.guard.AccountGuardEvaluation;
6+
import io.quarkus.scheduler.Scheduled;
7+
import jakarta.enterprise.context.ApplicationScoped;
8+
import jakarta.inject.Inject;
9+
import jakarta.transaction.Transactional;
10+
import lombok.extern.slf4j.Slf4j;
11+
12+
/**
13+
* 账户 Guard 中心采样任务。同一周期无论收到多少个服务器 agent 上报,
14+
* 黑名单与告警的连续超限次数都只累计一次。
15+
*/
16+
@Slf4j
17+
@ApplicationScoped
18+
public class AccountGuardEvaluationTask {
19+
20+
@Inject
21+
AccountGuardAggregator accountGuardAggregator;
22+
23+
@Inject
24+
AccountOnlineLimitAlertService accountOnlineLimitAlertService;
25+
26+
@Scheduled(every = "10s", concurrentExecution = Scheduled.ConcurrentExecution.SKIP)
27+
@Transactional
28+
public void evaluateAccounts() {
29+
AccountGuardEvaluation evaluation;
30+
try {
31+
evaluation = accountGuardAggregator.evaluateTrackedAccounts();
32+
} catch (Exception e) {
33+
log.warn("Guard 中心采样评估失败: {}", e.getMessage(), e);
34+
return;
35+
}
36+
37+
try {
38+
accountOnlineLimitAlertService.checkAndNotifyFromGuard(evaluation.getStatsByAccountNo());
39+
} catch (Exception e) {
40+
log.warn("Guard 中心采样告警检查失败: {}", e.getMessage(), e);
41+
}
42+
}
43+
}

src/main/java/com/fun90/airopscat/service/SystemConfigService.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,7 @@ private Map<String, ConfigGroupDefinition> buildGroupDefinitions() {
291291
item("airopscat.account.online.history.retention-hours", "在线记录保留小时数", "定期清理超过此小时数的历史在线连接记录,任务执行间隔与此值相同。", INPUT_NUMBER, true, false, false, true, "8", "8"),
292292
item("airopscat.node.online-account.stats.sample-minutes", "节点在线趋势采样间隔分钟数", "采样当前在线账户数并沉淀为节点每日在线账户趋势的间隔。", INPUT_NUMBER, true, false, false, true, "5", "5"),
293293
item("airopscat.account.connection-limit.alert.enabled", "连接数超限告警", "账户当前在线连接数超过最大连接数时发送告警。", INPUT_CHECKBOX, false, false, false, true, "", "true"),
294-
item("airopscat.account.connection-limit.alert.consecutive-times", "连接数告警连续触发次数", "连续多少次检测到连接数超限才触发告警,避免瞬时抖动误报。", INPUT_NUMBER, true, false, false, true, "2", "2"),
294+
item("airopscat.account.connection-limit.alert.consecutive-times", "Guard 连续超限次数", "中心连续多少个采样周期检测到连接数或 IP 数超限后加入黑名单;连接数告警复用同一采样周期。", INPUT_NUMBER, true, false, false, true, "2", "2"),
295295
item("airopscat.account.connection-limit.alert.min-interval-minutes", "连接数告警间隔分钟数", "同一账户连接数持续超限时,两次通知之间的最小间隔。", INPUT_NUMBER, true, false, false, true, "60", "60"),
296296
item("airopscat.account.connection-limit.alert.recovery-notify-enabled", "连接数恢复通知", "账户连接数恢复到限制以内时是否发送恢复通知。", INPUT_CHECKBOX, false, false, false, true, "", "false"),
297297
item("airopscat.sing-box.clash-api.host", "Clash API 远端地址", "sing-box Clash API 监听地址。", INPUT_TEXT, true, false, false, true, "127.0.0.1", "127.0.0.1"),

src/main/java/com/fun90/airopscat/service/guard/AccountGuardAggregator.java

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
public class AccountGuardAggregator {
3030

3131
private static final int DEFAULT_TTL_SECONDS = 30;
32+
private static final int DEFAULT_CONSECUTIVE_TIMES = 2;
3233

3334
private static final class NodeStat {
3435
final int connections;
@@ -43,6 +44,8 @@ private static final class NodeStat {
4344
}
4445

4546
private final Map<String, Map<String, NodeStat>> table = new ConcurrentHashMap<>();
47+
private final Map<String, Integer> consecutiveExceedCounts = new ConcurrentHashMap<>();
48+
private final Map<String, GuardBlockedEntry> blockedAccounts = new ConcurrentHashMap<>();
4649

4750
private final AccountRepository accountRepository;
4851
private final SystemConfigService systemConfigService;
@@ -111,7 +114,7 @@ public AccountGuardEvaluation reportAndEvaluateWithStats(GuardSyncRequest reques
111114
if (!reportedAccountNos.contains(accountNo)) {
112115
continue;
113116
}
114-
GuardBlockedEntry entry = evaluateAccount(account, stats);
117+
GuardBlockedEntry entry = blockedAccounts.get(accountNo);
115118
if (entry != null) {
116119
blocked.put(accountNo, entry);
117120
}
@@ -120,6 +123,53 @@ public AccountGuardEvaluation reportAndEvaluateWithStats(GuardSyncRequest reques
120123
return new AccountGuardEvaluation(blocked, statsByAccountNo);
121124
}
122125

126+
/**
127+
* 按中心统一采样周期评估所有已跟踪账户。只有连续超限达到配置次数后,
128+
* 才将账户加入黑名单;任一采样周期恢复到限制内时立即清零并移出黑名单。
129+
*/
130+
public AccountGuardEvaluation evaluateTrackedAccounts() {
131+
Set<String> accountNos = new HashSet<>(table.keySet());
132+
accountNos.addAll(consecutiveExceedCounts.keySet());
133+
accountNos.addAll(blockedAccounts.keySet());
134+
if (accountNos.isEmpty()) {
135+
return new AccountGuardEvaluation(Map.of(), Map.of());
136+
}
137+
138+
long now = nowEpochSeconds();
139+
int ttl = getTtlSeconds();
140+
int requiredTimes = getConsecutiveTimesThreshold();
141+
Map<String, Account> accountMap = loadAccounts(accountNos);
142+
Map<String, AccountGuardStats> statsByAccountNo = new HashMap<>();
143+
144+
for (String accountNo : accountNos) {
145+
Account account = accountMap.get(accountNo);
146+
if (account == null) {
147+
consecutiveExceedCounts.remove(accountNo);
148+
blockedAccounts.remove(accountNo);
149+
continue;
150+
}
151+
152+
AccountGuardStats stats = calculateStats(accountNo, now, ttl);
153+
statsByAccountNo.put(accountNo, stats);
154+
GuardBlockedEntry exceeded = evaluateAccount(account, stats);
155+
if (exceeded == null) {
156+
consecutiveExceedCounts.remove(accountNo);
157+
blockedAccounts.remove(accountNo);
158+
continue;
159+
}
160+
161+
int consecutiveTimes = consecutiveExceedCounts.merge(
162+
accountNo,
163+
1,
164+
(current, increment) -> Math.min(requiredTimes, current + increment));
165+
if (consecutiveTimes >= requiredTimes) {
166+
blockedAccounts.put(accountNo, exceeded);
167+
}
168+
}
169+
170+
return new AccountGuardEvaluation(new HashMap<>(blockedAccounts), statsByAccountNo);
171+
}
172+
123173
private AccountGuardStats calculateStats(String accountNo, long now, int ttl) {
124174
Map<String, NodeStat> byNode = table.get(accountNo);
125175
if (byNode == null || byNode.isEmpty()) {
@@ -205,6 +255,11 @@ public int getTtlSeconds() {
205255
"airopscat.account.guard.ttl-seconds", DEFAULT_TTL_SECONDS));
206256
}
207257

258+
private int getConsecutiveTimesThreshold() {
259+
return Math.max(1, systemConfigService.getIntValue(
260+
"airopscat.account.connection-limit.alert.consecutive-times", DEFAULT_CONSECUTIVE_TIMES));
261+
}
262+
208263
private long nowEpochSeconds() {
209264
return java.time.Instant.now().getEpochSecond();
210265
}

src/test/java/com/fun90/airopscat/controller/OpenControllerGuardSyncTest.java

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import com.fun90.airopscat.model.dto.guard.GuardSyncAccountReport;
66
import com.fun90.airopscat.model.dto.guard.GuardSyncRequest;
77
import com.fun90.airopscat.model.dto.guard.GuardSyncResponse;
8-
import com.fun90.airopscat.service.AccountOnlineLimitAlertService;
98
import com.fun90.airopscat.service.AccountOnlineIpService;
109
import com.fun90.airopscat.service.SystemConfigService;
1110
import com.fun90.airopscat.service.guard.AccountGuardAggregator;
@@ -55,7 +54,6 @@ private static OpenController controller(AccountOnlineIpService onlineIpService)
5554
OpenController controller = new OpenController();
5655
controller.accountGuardAggregator = new FakeAggregator();
5756
controller.accountOnlineIpService = onlineIpService;
58-
controller.accountOnlineLimitAlertService = new FakeAlertService();
5957
controller.systemConfigService = new FakeSystemConfigService();
6058
return controller;
6159
}
@@ -111,16 +109,6 @@ public int refreshFromGuardAccountIps(String nodeIp, java.util.List<GuardOnlineA
111109
}
112110
}
113111

114-
static class FakeAlertService extends AccountOnlineLimitAlertService {
115-
FakeAlertService() {
116-
super(null, null, null, null);
117-
}
118-
119-
@Override
120-
public void checkAndNotifyFromGuard(Map<String, com.fun90.airopscat.service.guard.AccountGuardStats> statsByAccountNo) {
121-
}
122-
}
123-
124112
static class FakeSystemConfigService extends SystemConfigService {
125113
FakeSystemConfigService() {
126114
super(null, null, null);

src/test/java/com/fun90/airopscat/service/guard/AccountGuardAggregatorTest.java

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
import java.util.Map;
1212

1313
import static org.junit.jupiter.api.Assertions.assertEquals;
14+
import static org.junit.jupiter.api.Assertions.assertFalse;
15+
import static org.junit.jupiter.api.Assertions.assertTrue;
1416

1517
class AccountGuardAggregatorTest {
1618

@@ -36,11 +38,61 @@ void shouldSnapshotActualConnectionCountFromGuardReports() {
3638
assertEquals(1, stats.get("acct-001").getTotalIps());
3739
}
3840

41+
@Test
42+
void shouldBlockOnlyAfterConsecutiveCentralEvaluations() {
43+
FakeAccountRepository repository = new FakeAccountRepository();
44+
repository.maxConnections = 5;
45+
AccountGuardAggregator aggregator = new AccountGuardAggregator(
46+
repository,
47+
new FakeSystemConfigService());
48+
49+
AccountGuardEvaluation firstNodeReport = aggregator.reportAndEvaluateWithStats(
50+
request("192.0.2.10", "acct-001", 4));
51+
AccountGuardEvaluation secondNodeReport = aggregator.reportAndEvaluateWithStats(
52+
request("192.0.2.11", "acct-001", 3));
53+
54+
assertTrue(firstNodeReport.getBlockedAccounts().isEmpty());
55+
assertTrue(secondNodeReport.getBlockedAccounts().isEmpty());
56+
57+
AccountGuardEvaluation firstEvaluation = aggregator.evaluateTrackedAccounts();
58+
assertEquals(7, firstEvaluation.getStatsByAccountNo().get("acct-001").getTotalConnections());
59+
assertTrue(firstEvaluation.getBlockedAccounts().isEmpty());
60+
61+
AccountGuardEvaluation reportInSamePeriod = aggregator.reportAndEvaluateWithStats(
62+
request("192.0.2.10", "acct-001", 4));
63+
assertTrue(reportInSamePeriod.getBlockedAccounts().isEmpty());
64+
65+
AccountGuardEvaluation secondEvaluation = aggregator.evaluateTrackedAccounts();
66+
assertTrue(secondEvaluation.getBlockedAccounts().containsKey("acct-001"));
67+
68+
AccountGuardEvaluation blockedResponse = aggregator.reportAndEvaluateWithStats(
69+
request("192.0.2.11", "acct-001", 3));
70+
assertTrue(blockedResponse.getBlockedAccounts().containsKey("acct-001"));
71+
72+
aggregator.reportAndEvaluateWithStats(request("192.0.2.11", "acct-001", 1));
73+
AccountGuardEvaluation recoveredEvaluation = aggregator.evaluateTrackedAccounts();
74+
assertFalse(recoveredEvaluation.getBlockedAccounts().containsKey("acct-001"));
75+
}
76+
77+
private static GuardSyncRequest request(String nodeIp, String accountNo, int connections) {
78+
GuardSyncRequest request = new GuardSyncRequest();
79+
request.setNodeIp(nodeIp);
80+
GuardSyncAccountReport report = new GuardSyncAccountReport();
81+
report.setAccountNo(accountNo);
82+
report.setConnections(connections);
83+
report.setIps(List.of("203.0.113." + connections));
84+
request.setAccounts(List.of(report));
85+
return request;
86+
}
87+
3988
static class FakeAccountRepository extends AccountRepository {
89+
int maxConnections;
90+
4091
@Override
4192
public List<Account> list(String query, Object... params) {
4293
Account account = new Account();
4394
account.setAccountNo("acct-001");
95+
account.setMaxConnections(maxConnections);
4496
return List.of(account);
4597
}
4698
}

0 commit comments

Comments
 (0)