|
1 | | -import { describe, it, expect, vi } from 'vitest'; |
| 1 | +import { afterEach, describe, it, expect, vi } from 'vitest'; |
| 2 | +// Paid at collection, not inside a case's clocked window: the dispatcher's |
| 3 | +// reach loads `@objectstack/metadata` lazily (`check:test-source-alias`). |
| 4 | +import '@objectstack/metadata'; |
| 5 | +import { offerClusterDriverConnection } from '@objectstack/service-cluster'; |
2 | 6 | import { HttpDispatcher, type HttpDispatcherResult } from './http-dispatcher.js'; |
3 | 7 |
|
4 | 8 | function kernel(state: string, dataService?: unknown): any { |
@@ -313,6 +317,188 @@ describe('HttpDispatcher — GET /ready primary-vs-secondary drain (#13408)', () |
313 | 317 | }); |
314 | 318 | }); |
315 | 319 |
|
| 320 | +// #31601 (child of cloud epic #3814) — a declared multi-node deployment on the |
| 321 | +// redis cluster driver reports the cluster store's reachability on `/ready`. |
| 322 | +// |
| 323 | +// R3 「同意:每张卡都只在"声明多副本并接上 Redis 驱动"时才生效」 |
| 324 | +// PM decision D4: an unreachable store answers 200 `status: degraded`, not |
| 325 | +// 503 — every replica loses the store together, so a 503 would turn a |
| 326 | +// degraded cluster into a total outage while the database can still serve. |
| 327 | +describe('HttpDispatcher — GET /ready cluster driver health (#31601)', () => { |
| 328 | + /** A `cluster` service on the redis driver, offering a client whose ping is `ping`. */ |
| 329 | + function redisCluster(ping: () => Promise<unknown>) { |
| 330 | + const cluster = { driver: 'redis', nodeId: 'web-1' }; |
| 331 | + offerClusterDriverConnection(cluster, { driver: 'redis', client: { ping }, keyPrefix: 'os:' }); |
| 332 | + return cluster; |
| 333 | + } |
| 334 | + const redisDown = () => vi.fn(async () => { throw new Error('connect ECONNREFUSED 10.0.0.7:6379'); }); |
| 335 | + const redisUp = () => vi.fn(async () => 'PONG'); |
| 336 | + |
| 337 | + /** A host kernel serving `data` and `cluster`. */ |
| 338 | + function clusterKernel(state: string, cluster: unknown, dataService?: unknown): any { |
| 339 | + return { |
| 340 | + getState: () => state, |
| 341 | + getService: (name: string) => (name === 'data' ? dataService : name === 'cluster' ? cluster : undefined), |
| 342 | + getServiceAsync: async () => undefined, |
| 343 | + }; |
| 344 | + } |
| 345 | + const ready = (k: any) => new HttpDispatcher(k).dispatch('GET', '/ready', undefined, undefined, ctx); |
| 346 | + |
| 347 | + afterEach(() => { vi.unstubAllEnvs(); }); |
| 348 | + |
| 349 | + describe('declared multi-node on the redis driver', () => { |
| 350 | + it('Redis down: 200 — not 503 — with status degraded and a cluster entry naming the driver unreachable', async () => { |
| 351 | + vi.stubEnv('OS_CLUSTER_REPLICAS', '3'); |
| 352 | + const ping = redisDown(); |
| 353 | + const res = await ready(clusterKernel('running', redisCluster(ping))); |
| 354 | + |
| 355 | + expect(responseOf(res).status).toBe(200); |
| 356 | + expect(responseOf(res).body.data).toEqual({ |
| 357 | + status: 'degraded', |
| 358 | + state: 'running', |
| 359 | + cluster: { driver: 'redis', reachable: false, reason: 'ping-failed' }, |
| 360 | + }); |
| 361 | + // The reading is served anonymously: the store's address stays out. |
| 362 | + expect(JSON.stringify(responseOf(res).body)).not.toContain('10.0.0.7'); |
| 363 | + expect(ping).toHaveBeenCalledTimes(1); |
| 364 | + }); |
| 365 | + |
| 366 | + it('Redis up: 200 with status ready and a reachable cluster entry', async () => { |
| 367 | + vi.stubEnv('OS_EXPECT_MULTI_NODE', 'true'); |
| 368 | + const res = await ready(clusterKernel('running', redisCluster(redisUp()))); |
| 369 | + |
| 370 | + expect(responseOf(res).status).toBe(200); |
| 371 | + expect(responseOf(res).body.data).toEqual({ |
| 372 | + status: 'ready', |
| 373 | + state: 'running', |
| 374 | + cluster: { driver: 'redis', reachable: true }, |
| 375 | + }); |
| 376 | + }); |
| 377 | + |
| 378 | + it('Redis down beside a degraded SECONDARY datasource: both are reported, still 200', async () => { |
| 379 | + vi.stubEnv('OS_CLUSTER_REPLICAS', '2'); |
| 380 | + const res = await ready(clusterKernel( |
| 381 | + 'running', |
| 382 | + redisCluster(redisDown()), |
| 383 | + engineWithPrimary([{ driverName: 'pg', healthy: true }, { driverName: 'tenant_mongo', healthy: false }], primary('pg')), |
| 384 | + )); |
| 385 | + |
| 386 | + expect(responseOf(res).status).toBe(200); |
| 387 | + expect(responseOf(res).body.data).toEqual({ |
| 388 | + status: 'degraded', |
| 389 | + state: 'running', |
| 390 | + degraded: { drivers: ['tenant_mongo'], primaryDatasource: 'pg' }, |
| 391 | + cluster: { driver: 'redis', reachable: false, reason: 'ping-failed' }, |
| 392 | + }); |
| 393 | + }); |
| 394 | + |
| 395 | + it('the primary datasource going down still answers the byte-identical 503, and never probes Redis', async () => { |
| 396 | + vi.stubEnv('OS_CLUSTER_REPLICAS', '3'); |
| 397 | + const ping = redisDown(); |
| 398 | + const res = await ready(clusterKernel( |
| 399 | + 'running', |
| 400 | + redisCluster(ping), |
| 401 | + engineWithPrimary([{ driverName: 'pg', healthy: false }], primary('pg')), |
| 402 | + )); |
| 403 | + |
| 404 | + expect(responseOf(res).status).toBe(503); |
| 405 | + expect(responseOf(res).body.error.message).toBe('Data driver unavailable'); |
| 406 | + expect(responseOf(res).body.error.details).toEqual({ state: 'running', drivers: ['pg'] }); |
| 407 | + expect(ping).not.toHaveBeenCalled(); |
| 408 | + }); |
| 409 | + |
| 410 | + it('the cluster probe is memoized with the same TTL — k8s polls must not each round-trip to Redis', async () => { |
| 411 | + vi.stubEnv('OS_CLUSTER_REPLICAS', '3'); |
| 412 | + const ping = redisDown(); |
| 413 | + const dispatcher = new HttpDispatcher(clusterKernel('running', redisCluster(ping))); |
| 414 | + |
| 415 | + await dispatcher.dispatch('GET', '/ready', undefined, undefined, ctx); |
| 416 | + await dispatcher.dispatch('GET', '/ready', undefined, undefined, ctx); |
| 417 | + await dispatcher.dispatch('GET', '/ready', undefined, undefined, ctx); |
| 418 | + |
| 419 | + expect(ping).toHaveBeenCalledTimes(1); |
| 420 | + }); |
| 421 | + }); |
| 422 | + |
| 423 | + // ⛔ The stopping (SIGTERM) / booting 503 — the graceful-drain path — is |
| 424 | + // reached before any probe, so a declared deployment with Redis down answers |
| 425 | + // it exactly as a single node does. |
| 426 | + it.each(['idle', 'initializing', 'stopping', 'stopped'])( |
| 427 | + 'kernel %s: the 503 envelope is identical to the single-node one, and Redis is never probed', |
| 428 | + async (state) => { |
| 429 | + const single = responseOf(await ready(kernel(state))); |
| 430 | + vi.stubEnv('OS_CLUSTER_REPLICAS', '3'); |
| 431 | + const ping = redisDown(); |
| 432 | + const declared = responseOf(await ready(clusterKernel(state, redisCluster(ping)))); |
| 433 | + |
| 434 | + expect(declared.status).toBe(503); |
| 435 | + expect(declared.body.error.message).toBe('Service not ready'); |
| 436 | + expect(declared.body).toEqual(single.body); |
| 437 | + expect(declared.body.error.details).toEqual({ state }); |
| 438 | + expect(ping).not.toHaveBeenCalled(); |
| 439 | + }, |
| 440 | + ); |
| 441 | + |
| 442 | + // ⛔ R3: nothing changes unless BOTH halves hold — the declaration AND the |
| 443 | + // redis driver. Each case below is missing one half; its body must be the |
| 444 | + // pre-#31601 body to the byte, and its cluster store is never pinged. |
| 445 | + describe('single node, or no redis driver: body and status unchanged', () => { |
| 446 | + it('single node (no declaration) with a redis cluster whose store is DOWN: the old body, no ping', async () => { |
| 447 | + const ping = redisDown(); |
| 448 | + const res = await ready(clusterKernel('running', redisCluster(ping))); |
| 449 | + |
| 450 | + expect(responseOf(res).status).toBe(200); |
| 451 | + expect(JSON.stringify(responseOf(res).body.data)).toBe(JSON.stringify({ status: 'ready', state: 'running' })); |
| 452 | + expect(ping).not.toHaveBeenCalled(); |
| 453 | + }); |
| 454 | + |
| 455 | + it('single node: OS_CLUSTER_REPLICAS=1 is not a declaration', async () => { |
| 456 | + vi.stubEnv('OS_CLUSTER_REPLICAS', '1'); |
| 457 | + const ping = redisDown(); |
| 458 | + const res = await ready(clusterKernel('running', redisCluster(ping))); |
| 459 | + |
| 460 | + expect(responseOf(res).body.data).toEqual({ status: 'ready', state: 'running' }); |
| 461 | + expect(ping).not.toHaveBeenCalled(); |
| 462 | + }); |
| 463 | + |
| 464 | + it('declared, but the cluster runs the in-process memory driver (guard escape hatch): the old body', async () => { |
| 465 | + vi.stubEnv('OS_CLUSTER_REPLICAS', '3'); |
| 466 | + const res = await ready(clusterKernel('running', { driver: 'memory', nodeId: 'web-1' })); |
| 467 | + |
| 468 | + expect(responseOf(res).status).toBe(200); |
| 469 | + expect(responseOf(res).body.data).toEqual({ status: 'ready', state: 'running' }); |
| 470 | + }); |
| 471 | + |
| 472 | + it('declared, but no cluster service at all — or a lookup that throws: the old body', async () => { |
| 473 | + vi.stubEnv('OS_CLUSTER_REPLICAS', '3'); |
| 474 | + const none = await ready(clusterKernel('running', undefined)); |
| 475 | + expect(responseOf(none).body.data).toEqual({ status: 'ready', state: 'running' }); |
| 476 | + |
| 477 | + const throwing = await ready({ |
| 478 | + getState: () => 'running', |
| 479 | + getService: (name: string) => { if (name === 'cluster') throw new Error('not registered'); return undefined; }, |
| 480 | + getServiceAsync: async () => undefined, |
| 481 | + }); |
| 482 | + expect(responseOf(throwing).status).toBe(200); |
| 483 | + expect(responseOf(throwing).body.data).toEqual({ status: 'ready', state: 'running' }); |
| 484 | + }); |
| 485 | + |
| 486 | + it('single node with a degraded secondary: the #13408 body, no cluster key', async () => { |
| 487 | + const res = await ready(clusterKernel( |
| 488 | + 'running', |
| 489 | + redisCluster(redisDown()), |
| 490 | + engineWithPrimary([{ driverName: 'pg', healthy: true }, { driverName: 'tenant_mongo', healthy: false }], primary('pg')), |
| 491 | + )); |
| 492 | + |
| 493 | + expect(responseOf(res).body.data).toEqual({ |
| 494 | + status: 'ready', |
| 495 | + state: 'running', |
| 496 | + degraded: { drivers: ['tenant_mongo'], primaryDatasource: 'pg' }, |
| 497 | + }); |
| 498 | + }); |
| 499 | + }); |
| 500 | +}); |
| 501 | + |
316 | 502 | describe('HttpDispatcher — GET /health liveness probe', () => { |
317 | 503 | // framework#3756: liveness must NOT check the database. Its failure makes the |
318 | 504 | // orchestrator restart the pod, which cannot fix an unreachable database but |
|
0 commit comments