Skip to content

Commit 84b05b1

Browse files
authored
Merge pull request #118 from escendit/fix/epm-t139-gh117-smoke-image-check-sigpipe
fix: harden the smoke image check and the context-fault tests against CI flakes
2 parents 19cf1ee + 04a33f7 commit 84b05b1

5 files changed

Lines changed: 81 additions & 46 deletions

File tree

‎.gitleaks.toml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,15 @@ paths = [
3131
]
3232

3333
# The Epic 10 context lists the firmware reference topics ("the eFuse root and HKDF purpose-key
34-
# hierarchy, ..."); the generic API key rule reads the prose as a key. Only that commit and file are
35-
# allowed.
34+
# hierarchy, ..."); the generic API key rule reads the prose as a key. Only those commits and that file
35+
# are allowed: the story 10-1 commit and the bmad-loop commit that records the story.
3636
[[allowlists]]
3737
description = "Epic 10 context names a key hierarchy, not a key"
3838
condition = "AND"
39-
commits = ["ee32e273dbf061a62ab15117f464144ea5bce61c"]
39+
commits = [
40+
"ee32e273dbf061a62ab15117f464144ea5bce61c",
41+
"41bb2582dd383ae8fb81a19a10d9c5b6bdd4ba1d",
42+
]
4043
paths = [
4144
'''^_bmad-output/implementation-artifacts/epic-10-context\.md$''',
4245
]

‎deploy/charts/smoke.sh‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,9 @@ else
198198
fi
199199
for tag in "${smoke_tags[@]}"; do
200200
docker save "${tag}" | docker exec -i "${node_container}" ctr --namespace=k8s.io images import -
201-
docker exec "${node_container}" ctr --namespace=k8s.io images ls --quiet | grep --quiet --fixed-strings --line-regexp "${tag}" \
201+
# grep reads the whole list: with pipefail, "grep --quiet" stopping at the first match can
202+
# kill ctr with SIGPIPE and fail the check on an image that is there.
203+
docker exec "${node_container}" ctr --namespace=k8s.io images ls --quiet | grep --fixed-strings --line-regexp "${tag}" >/dev/null \
202204
|| fail "image ${tag} is not in the containerd of ${node_container} after the import"
203205
done
204206

@@ -479,7 +481,7 @@ for job in temporal-schema temporal-namespace; do
479481
succeeded=$(kubectl "${ns[@]}" get job "${job}" -o jsonpath='{.status.succeeded}')
480482
[[ ${succeeded} == "1" ]] || fail "the ${job} hook Job did not complete on upgrade (succeeded=${succeeded:-0})"
481483
done
482-
kubectl "${ns[@]}" logs job/temporal-namespace | grep -q "namespace coldframe exists" \
484+
kubectl "${ns[@]}" logs job/temporal-namespace | grep "namespace coldframe exists" >/dev/null \
483485
|| fail "the namespace hook Job did not find the existing coldframe namespace"
484486
echo "ok: re-running the Temporal hooks on a current schema succeeds and keeps the namespace"
485487

@@ -590,7 +592,7 @@ kubectl "${ns[@]}" create job advance-replay --from=cronjob/server-partitions --
590592
| kubectl "${ns[@]}" apply -f -
591593
kubectl "${ns[@]}" wait job/advance-replay --for=condition=Complete --timeout=300s \
592594
|| fail "the advance-replay Job did not complete: $(kubectl "${ns[@]}" logs job/advance-replay --tail=20 2>&1)"
593-
kubectl "${ns[@]}" logs job/advance-replay | grep -q "Advanced the replay state" \
595+
kubectl "${ns[@]}" logs job/advance-replay | grep "Advanced the replay state" >/dev/null \
594596
|| fail "the advance-replay Job did not report what it advanced"
595597
echo "ok: advance-replay ran on the restored database before the apps started"
596598

‎deploy/fleet/smoke.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,9 @@ import_images() {
249249
# tarball, which leaves the upgrade's pods waiting for images that never arrive.
250250
for tag in "${tags[@]}"; do
251251
docker save "${tag}" | docker exec -i "${node_container}" ctr --namespace=k8s.io images import -
252-
docker exec "${node_container}" ctr --namespace=k8s.io images ls --quiet | grep --quiet --fixed-strings --line-regexp "${tag}" \
252+
# grep reads the whole list: with pipefail, "grep --quiet" stopping at the first match can
253+
# kill ctr with SIGPIPE and fail the check on an image that is there.
254+
docker exec "${node_container}" ctr --namespace=k8s.io images ls --quiet | grep --fixed-strings --line-regexp "${tag}" >/dev/null \
253255
|| fail "image ${tag} is not in the containerd of ${node_container} after the import"
254256
done
255257
}

‎tests/cs/server.integration/Devices/UncalibratedAlertGrainTests.cs‎

Lines changed: 30 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ public async Task AContextLostBeforeASiloRestartIsSentAgainOnceTheDeviceIsActiva
240240
var soil = node.SensorId(0, Quantity.SoilMoisture);
241241
var alertId = AlertIds.Uncalibrated(soil, 1);
242242

243-
identity.SensorFaults.FailNextContexts(int.MaxValue);
243+
identity.SensorFaults.FailContexts(SensorsOf(node));
244244

245245
try
246246
{
@@ -249,7 +249,7 @@ public async Task AContextLostBeforeASiloRestartIsSentAgainOnceTheDeviceIsActiva
249249
}
250250
finally
251251
{
252-
identity.SensorFaults.FailNextContexts(0);
252+
identity.SensorFaults.RestoreContexts();
253253
}
254254

255255
// Which Sensor confirmed the context was only in memory. The new silo activates the Device with a read
@@ -615,21 +615,21 @@ public async Task AContextThatDidNotReachTheSensorsIsSentAgainWithTheNextWake()
615615
var nodeId = node.DeviceId.ToString();
616616
var soil = node.SensorId(0, Quantity.SoilMoisture);
617617
var alertId = AlertIds.Uncalibrated(soil, 1);
618-
var failedBefore = identity.SensorFaults.FailedContexts;
618+
var failedBefore = identity.SensorFaults.FailedContexts(SensorsOf(node));
619619

620620
// No Sensor can be reached while the Node is unassigned: the unassignment answers as it always did.
621-
identity.SensorFaults.FailNextContexts(int.MaxValue);
621+
identity.SensorFaults.FailContexts(SensorsOf(node));
622622

623623
try
624624
{
625625
Assert.Equal(DeviceAssignmentOutcome.Unassigned, (await identity.Device(nodeId).Unassign(site.SiteId, Ct)).Outcome);
626-
Assert.Equal(4, identity.SensorFaults.FailedContexts - failedBefore);
626+
Assert.Equal(4, identity.SensorFaults.FailedContexts(SensorsOf(node)) - failedBefore);
627627
Assert.Null((await identity.DeviceStateAsync(nodeId)).LotId);
628628
Assert.Equal(AlertLifecycle.Open, (await identity.Alert(alertId).Describe(Ct))!.Lifecycle);
629629
}
630630
finally
631631
{
632-
identity.SensorFaults.FailNextContexts(0);
632+
identity.SensorFaults.RestoreContexts();
633633
}
634634

635635
// No Reading follows an unassignment: the Device sends the context again when it is next woken.
@@ -641,9 +641,9 @@ public async Task AContextThatDidNotReachTheSensorsIsSentAgainWithTheNextWake()
641641

642642
// Confirmed: another wake sends nothing. No Sensor is called, which its journal alone would not show.
643643
var journaled = await identity.AliasesAsync($"sensor/{soil}");
644-
var sentBefore = identity.SensorFaults.PassedContexts;
644+
var sentBefore = identity.SensorFaults.PassedContexts(SensorsOf(node));
645645
await identity.WakeDeviceAsync(nodeId);
646-
Assert.Equal(sentBefore, identity.SensorFaults.PassedContexts);
646+
Assert.Equal(sentBefore, identity.SensorFaults.PassedContexts(SensorsOf(node)));
647647
Assert.Equal(journaled, await identity.AliasesAsync($"sensor/{soil}"));
648648
}
649649

@@ -656,7 +656,7 @@ public async Task AContextThatDidNotReachTheSensorsIsSentAgainWithTheNodesNextRe
656656
var soil = node.SensorId(0, Quantity.SoilMoisture);
657657
var alertId = AlertIds.Uncalibrated(soil, 1);
658658

659-
identity.SensorFaults.FailNextContexts(int.MaxValue);
659+
identity.SensorFaults.FailContexts(SensorsOf(node));
660660

661661
try
662662
{
@@ -665,23 +665,23 @@ public async Task AContextThatDidNotReachTheSensorsIsSentAgainWithTheNodesNextRe
665665
}
666666
finally
667667
{
668-
identity.SensorFaults.FailNextContexts(0);
668+
identity.SensorFaults.RestoreContexts();
669669
}
670670

671671
// Nobody wakes the Device and no silo restarts: the Node reports. On no Lot its Readings are not
672672
// evaluated, so only the context the frame sends again can end the episode.
673-
var sentBefore = identity.SensorFaults.PassedContexts;
673+
var sentBefore = identity.SensorFaults.PassedContexts(SensorsOf(node));
674674
await ReadAsync(node);
675675

676-
Assert.Equal(4, identity.SensorFaults.PassedContexts - sentBefore);
676+
Assert.Equal(4, identity.SensorFaults.PassedContexts(SensorsOf(node)) - sentBefore);
677677
var closed = (await identity.Alert(alertId).Describe(Ct))!;
678678
Assert.Equal((AlertLifecycle.Closed, AlertCloseReason.Unassigned), (closed.Lifecycle, closed.Reason));
679679
Assert.Empty(await identity.Site(site.SiteId).OpenAlerts(Ct));
680680
Assert.Equal(new SensorContext(site.SiteId, null, false, 2), (await SensorStateAsync(soil)).Context);
681681

682682
// Confirmed: the next report sends nothing.
683683
await ReadAsync(node);
684-
Assert.Equal(4, identity.SensorFaults.PassedContexts - sentBefore);
684+
Assert.Equal(4, identity.SensorFaults.PassedContexts(SensorsOf(node)) - sentBefore);
685685
}
686686

687687
[Fact]
@@ -694,22 +694,22 @@ public async Task ASensorThatHoldsNoContextTakesItsFirstOneFromAReading()
694694

695695
// No context the Device sends reaches the Sensors, from the frame that declares them on: the Sensor is
696696
// as one journaled before Story 7.4, declared and without a context, when that frame's Reading arrives.
697-
var failedBefore = identity.SensorFaults.FailedContexts;
698-
var sentBefore = identity.SensorFaults.PassedContexts;
699-
identity.SensorFaults.FailNextContexts(int.MaxValue);
697+
var failedBefore = identity.SensorFaults.FailedContexts(SensorsOf(node));
698+
var sentBefore = identity.SensorFaults.PassedContexts(SensorsOf(node));
699+
identity.SensorFaults.FailContexts(SensorsOf(node));
700700

701701
try
702702
{
703703
await DeclareAsync(node);
704704
}
705705
finally
706706
{
707-
identity.SensorFaults.FailNextContexts(0);
707+
identity.SensorFaults.RestoreContexts();
708708
}
709709

710710
// The Reading said the Site, the Lot and the epoch.
711-
Assert.Equal(4, identity.SensorFaults.FailedContexts - failedBefore);
712-
Assert.Equal(sentBefore, identity.SensorFaults.PassedContexts);
711+
Assert.Equal(4, identity.SensorFaults.FailedContexts(SensorsOf(node)) - failedBefore);
712+
Assert.Equal(sentBefore, identity.SensorFaults.PassedContexts(SensorsOf(node)));
713713
Assert.Equal(new SensorContext(site.SiteId, lotId, false, 1), (await SensorStateAsync(soil)).Context);
714714
var open = Assert.Single(await identity.Site(site.SiteId).OpenAlerts(Ct));
715715
Assert.Equal((AlertIds.Uncalibrated(soil, 1), AlertKind.Uncalibrated, lotId), (open.AlertId, open.Kind, open.LotId));
@@ -727,7 +727,7 @@ public async Task AMoveWhoseContextWasLostHealsWithTheNextReadingBecauseReadings
727727
var soil = node.SensorId(0, Quantity.SoilMoisture);
728728
var peppers = await SeedLotAsync(site.SiteId, "Peppers");
729729

730-
identity.SensorFaults.FailNextContexts(int.MaxValue);
730+
identity.SensorFaults.FailContexts(SensorsOf(node));
731731

732732
try
733733
{
@@ -739,7 +739,7 @@ public async Task AMoveWhoseContextWasLostHealsWithTheNextReadingBecauseReadings
739739
}
740740
finally
741741
{
742-
identity.SensorFaults.FailNextContexts(0);
742+
identity.SensorFaults.RestoreContexts();
743743
}
744744

745745
var open = Assert.Single(await identity.Site(site.SiteId).OpenAlerts(Ct));
@@ -957,6 +957,15 @@ private static SpecificationSet Specifications(bool calibration)
957957
}
958958

959959
// An Active Site created by its Owner, whose User grain therefore holds the Site and is told of its Alerts.
960+
// The Node's four Sensors, whose evaluation contexts the Device sends.
961+
private static Guid[] SensorsOf(SimulatedDevice node) =>
962+
[
963+
node.SensorId(0, Quantity.SoilMoisture),
964+
node.SensorId(1, Quantity.AirTemperature),
965+
node.SensorId(2, Quantity.RelativeHumidity),
966+
node.SensorId(3, Quantity.GasResistance),
967+
];
968+
960969
private async Task<(string SiteId, string OwnerId)> SiteAsync()
961970
{
962971
var ownerId = Guid.NewGuid().ToString();

‎tests/cs/server.integration/Identity/IdentityCluster.cs‎

Lines changed: 37 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -502,27 +502,41 @@ protected override async Task CommitTransactionAsync(NpgsqlTransaction transacti
502502
public sealed class SensorFaults : IIncomingGrainCallFilter
503503
{
504504
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, bool> _failing = new(StringComparer.Ordinal);
505+
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, bool> _failingContexts = new(StringComparer.Ordinal);
506+
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, int> _failedContexts = new(StringComparer.Ordinal);
507+
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, int> _passedContexts = new(StringComparer.Ordinal);
505508
private int _evaluationFailures;
506-
private int _contextFailures;
507509

508510
/// <summary>
509-
/// How many evaluation contexts were failed (Story 7.4).
511+
/// How many evaluation contexts for <paramref name="sensorIds"/> were failed (Story 7.4). Counted per Sensor:
512+
/// the Devices of earlier tests on the same cluster send their own contexts again when they are activated.
510513
/// </summary>
511-
public int FailedContexts => _failedContexts;
514+
public int FailedContexts(IEnumerable<Guid> sensorIds) => Count(_failedContexts, sensorIds);
512515

513516
/// <summary>
514-
/// How many evaluation contexts reached a Sensor grain (Story 7.4): what the Device sent and nothing failed.
517+
/// How many evaluation contexts for <paramref name="sensorIds"/> reached their Sensor grain (Story 7.4): what
518+
/// the Device sent and nothing failed.
515519
/// </summary>
516-
public int PassedContexts => _passedContexts;
520+
public int PassedContexts(IEnumerable<Guid> sensorIds) => Count(_passedContexts, sensorIds);
517521

518-
private int _failedContexts;
519-
private int _passedContexts;
522+
/// <summary>
523+
/// Makes every evaluation context for <paramref name="sensorIds"/> fail until <see cref="RestoreContexts"/>
524+
/// (Story 7.4). The contexts of other Sensors pass, so another test's Device is left alone.
525+
/// </summary>
526+
public void FailContexts(IEnumerable<Guid> sensorIds)
527+
{
528+
ArgumentNullException.ThrowIfNull(sensorIds);
529+
530+
foreach (var sensorId in sensorIds)
531+
{
532+
_failingContexts[sensorId.ToString("D")] = true;
533+
}
534+
}
520535

521536
/// <summary>
522-
/// Makes the next <paramref name="calls"/> evaluation contexts fail, whichever Sensor they are for (Story 7.4);
523-
/// 0 lets them through again.
537+
/// Lets every evaluation context through again.
524538
/// </summary>
525-
public void FailNextContexts(int calls) => Interlocked.Exchange(ref _contextFailures, calls);
539+
public void RestoreContexts() => _failingContexts.Clear();
526540

527541
/// <summary>
528542
/// Makes the next <paramref name="calls"/> evaluations fail, whichever Sensor they are for.
@@ -552,17 +566,15 @@ public Task Invoke(IIncomingGrainCallContext context)
552566

553567
if (context.Grain is ISensorGrain && string.Equals(context.MethodName, nameof(ISensorGrain.SetContext), StringComparison.Ordinal))
554568
{
555-
int left;
556-
while ((left = Volatile.Read(ref _contextFailures)) > 0)
569+
var sensorId = context.TargetId.Key.ToString()!;
570+
571+
if (_failingContexts.ContainsKey(sensorId))
557572
{
558-
if (Interlocked.CompareExchange(ref _contextFailures, left - 1, left) == left)
559-
{
560-
Interlocked.Increment(ref _failedContexts);
561-
throw new InvalidOperationException("The test failed this evaluation context.");
562-
}
573+
_failedContexts.AddOrUpdate(sensorId, 1, static (_, count) => count + 1);
574+
throw new InvalidOperationException("The test failed this evaluation context.");
563575
}
564576

565-
Interlocked.Increment(ref _passedContexts);
577+
_passedContexts.AddOrUpdate(sensorId, 1, static (_, count) => count + 1);
566578
}
567579

568580
if (context.Grain is ISensorGrain && string.Equals(context.MethodName, nameof(ISensorGrain.Evaluate), StringComparison.Ordinal))
@@ -580,6 +592,13 @@ public Task Invoke(IIncomingGrainCallContext context)
580592

581593
return context.Invoke();
582594
}
595+
596+
private static int Count(System.Collections.Concurrent.ConcurrentDictionary<string, int> counts, IEnumerable<Guid> sensorIds)
597+
{
598+
ArgumentNullException.ThrowIfNull(sensorIds);
599+
600+
return sensorIds.Sum(sensorId => counts.GetValueOrDefault(sensorId.ToString("D")));
601+
}
583602
}
584603

585604
/// <summary>

0 commit comments

Comments
 (0)