From 2b4b81ce790af6dc77ea19f8515f5a67521fb57b Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Tue, 11 Aug 2026 12:55:18 +0300 Subject: [PATCH 01/20] Add identifier tie breaker to work poll sort to keep serial groups exclusive --- work/store/structured/mongo/mongo.go | 10 +- .../structured/mongo/mongo_suite_test.go | 11 ++ work/store/structured/mongo/mongo_test.go | 175 ++++++++++++++++++ 3 files changed, 192 insertions(+), 4 deletions(-) create mode 100644 work/store/structured/mongo/mongo_suite_test.go create mode 100644 work/store/structured/mongo/mongo_test.go diff --git a/work/store/structured/mongo/mongo.go b/work/store/structured/mongo/mongo.go index 430429a6e4..e5aba72d21 100644 --- a/work/store/structured/mongo/mongo.go +++ b/work/store/structured/mongo/mongo.go @@ -102,8 +102,10 @@ func (s *Store) Poll(ctx context.Context, poll *work.Poll) ([]*work.Work, error) bson.M{"state": "failing", "serialId": bson.M{"$exists": true}}, }}}) - // Sort by processing priority and available time - pipeline = append(pipeline, bson.M{"$sort": bson.D{bson.E{Key: "processingPriority", Value: -1}, bson.E{Key: "processingAvailableTime", Value: 1}}}) + // Sort by processing priority and available time, with _id as a tie breaker + // The _id tie breaker guarantees a total order so that, within a serial id group, the + // document already in state processing remains first and the group is reliably excluded below + pipeline = append(pipeline, bson.M{"$sort": bson.D{bson.E{Key: "processingPriority", Value: -1}, bson.E{Key: "processingAvailableTime", Value: 1}, bson.E{Key: "_id", Value: 1}}}) // Group all documents by serial id pipeline = append(pipeline, bson.M{"$group": bson.M{"_id": "$serialId", "documents": bson.M{"$push": "$$ROOT"}}}) @@ -128,8 +130,8 @@ func (s *Store) Poll(ctx context.Context, poll *work.Poll) ([]*work.Work, error) pipeline = append(pipeline, bson.M{"$unwind": "$documents"}) pipeline = append(pipeline, bson.M{"$replaceRoot": bson.M{"newRoot": "$documents"}}) - // Sort by processing priority and available time - pipeline = append(pipeline, bson.M{"$sort": bson.D{bson.E{Key: "processingPriority", Value: -1}, bson.E{Key: "processingAvailableTime", Value: 1}}}) + // Sort by processing priority and available time, with _id as a tie breaker + pipeline = append(pipeline, bson.M{"$sort": bson.D{bson.E{Key: "processingPriority", Value: -1}, bson.E{Key: "processingAvailableTime", Value: 1}, bson.E{Key: "_id", Value: 1}}}) // If one type, then just simple limit // Otherwise, group by type, limit each group by type quantity, and ungroup diff --git a/work/store/structured/mongo/mongo_suite_test.go b/work/store/structured/mongo/mongo_suite_test.go new file mode 100644 index 0000000000..478ccb4699 --- /dev/null +++ b/work/store/structured/mongo/mongo_suite_test.go @@ -0,0 +1,11 @@ +package mongo_test + +import ( + "testing" + + "github.com/tidepool-org/platform/test" +) + +func TestSuite(t *testing.T) { + test.Test(t) +} diff --git a/work/store/structured/mongo/mongo_test.go b/work/store/structured/mongo/mongo_test.go new file mode 100644 index 0000000000..a387dd311e --- /dev/null +++ b/work/store/structured/mongo/mongo_test.go @@ -0,0 +1,175 @@ +package mongo_test + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + netTest "github.com/tidepool-org/platform/net/test" + "github.com/tidepool-org/platform/pointer" + storeStructured "github.com/tidepool-org/platform/store/structured" + storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" + storeStructuredMongoTest "github.com/tidepool-org/platform/store/structured/mongo/test" + "github.com/tidepool-org/platform/test" + "github.com/tidepool-org/platform/work" + workStoreStructuredMongo "github.com/tidepool-org/platform/work/store/structured/mongo" +) + +const processingTimeout = 300 + +var _ = Describe("Mongo", func() { + var config *storeStructuredMongo.Config + var store *workStoreStructuredMongo.Store + var ctx context.Context + var typ string + + BeforeEach(func() { + config = storeStructuredMongoTest.NewConfig() + ctx = log.NewContextWithLogger(context.Background(), logTest.NewLogger()) + typ = netTest.RandomReverseDomain() + }) + + AfterEach(func() { + if store != nil { + Expect(store.Terminate(context.Background())).ToNot(HaveOccurred()) + store = nil + } + }) + + Context("with a new store", func() { + BeforeEach(func() { + var err error + store, err = workStoreStructuredMongo.NewStore(config) + Expect(err).ToNot(HaveOccurred()) + Expect(store).ToNot(BeNil()) + Expect(store.EnsureIndexes()).To(Succeed()) + }) + + Context("Poll", func() { + // These work items intentionally share an identical processing available time and + // processing priority so that only the identifier tie breaker in the Poll aggregation + // gives them a total order. Without a total order the document reported first within a + // serial id group is arbitrary, which allows a pending work item to be claimed while a + // sibling sharing its serial id is still processing. + Context("with multiple pending work items sharing a serial id and sort key", func() { + const workCount = 10 + + var serialID string + var availableTime time.Time + + BeforeEach(func() { + serialID = typ + ":" + test.RandomString() + + // Create in the future so every work item retains the exact same processing + // available time, then wait for them to become available to poll + availableTime = time.Now().Add(time.Second).UTC().Truncate(time.Millisecond) + + for range workCount { + create := &work.Create{ + Type: typ, + SerialID: pointer.FromString(serialID), + ProcessingAvailableTime: availableTime, + ProcessingTimeout: processingTimeout, + } + created, err := store.Create(ctx, create) + Expect(err).ToNot(HaveOccurred()) + Expect(created).ToNot(BeNil()) + Expect(created.ProcessingAvailableTime).To(BeTemporally("==", availableTime)) + } + + time.Sleep(time.Until(availableTime) + 100*time.Millisecond) + }) + + It("claims one work item and claims no further work item while it is processing", func() { + poll := &work.Poll{TypeQuantities: work.TypeQuantities{typ: workCount}} + + claimed, err := store.Poll(ctx, poll) + Expect(err).ToNot(HaveOccurred()) + Expect(claimed).To(HaveLen(1)) + Expect(claimed[0].State).To(Equal(work.StateProcessing)) + + for index := range 20 { + additional, err := store.Poll(ctx, poll) + Expect(err).ToNot(HaveOccurred()) + Expect(additional).To(BeEmpty(), "poll %d claimed work while a work item sharing its serial id was processing", index) + } + }) + }) + }) + + Context("Update", func() { + var created *work.Work + + BeforeEach(func() { + var err error + created, err = store.Create(ctx, &work.Create{ + Type: typ, + GroupID: pointer.FromString(typ + ":" + test.RandomString()), + SerialID: pointer.FromString(typ + ":" + test.RandomString()), + ProcessingAvailableTime: time.Now().Add(time.Hour), + ProcessingTimeout: processingTimeout, + Metadata: map[string]any{"reasons": []any{"DATA_ADDED"}}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(created).ToNot(BeNil()) + Expect(created.State).To(Equal(work.StatePending)) + Expect(created.Revision).To(Equal(1)) + }) + + // The producer coalesces repeated triggers by merging into the work item already + // pending for a group, which requires a pending to pending update that revises both the + // metadata and the processing available time while retaining the revision condition. + Context("with a pending work item updated to pending", func() { + var availableTime time.Time + var update *work.Update + + BeforeEach(func() { + availableTime = time.Now().Add(30 * time.Minute) + update = &work.Update{ + State: work.StatePending, + PendingUpdate: &work.PendingUpdate{ + ProcessingAvailableTime: availableTime, + ProcessingPriority: created.ProcessingPriority, + ProcessingTimeout: created.ProcessingTimeout, + Metadata: map[string]any{"reasons": []any{"DATA_ADDED", "UPLOAD_COMPLETED"}}, + }, + } + }) + + It("returns the updated work item with the revised metadata and processing available time", func() { + updated, err := store.Update(ctx, created.ID, &storeStructured.Condition{Revision: pointer.FromInt(created.Revision)}, update) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).ToNot(BeNil()) + Expect(updated.State).To(Equal(work.StatePending)) + Expect(updated.Revision).To(Equal(created.Revision + 1)) + Expect(updated.Metadata).To(HaveKey("reasons")) + Expect(updated.Metadata["reasons"]).To(ConsistOf("DATA_ADDED", "UPLOAD_COMPLETED")) + Expect(updated.ProcessingAvailableTime).To(BeTemporally("~", availableTime, time.Millisecond)) + Expect(updated.ProcessingTimeout).To(Equal(created.ProcessingTimeout)) + }) + + It("retains the pending time of the work item", func() { + updated, err := store.Update(ctx, created.ID, &storeStructured.Condition{Revision: pointer.FromInt(created.Revision)}, update) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).ToNot(BeNil()) + // Millisecond tolerance as the stored time is truncated to the BSON date precision + Expect(updated.PendingTime).To(BeTemporally("~", created.PendingTime, time.Millisecond)) + }) + + It("returns nil when the revision condition no longer matches", func() { + updated, err := store.Update(ctx, created.ID, &storeStructured.Condition{Revision: pointer.FromInt(created.Revision)}, update) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).ToNot(BeNil()) + + stale, err := store.Update(ctx, created.ID, &storeStructured.Condition{Revision: pointer.FromInt(created.Revision)}, update) + Expect(err).ToNot(HaveOccurred()) + Expect(stale).To(BeNil()) + }) + }) + }) + }) +}) From bb5b6d5fb5565edc8418a6e52dcd133effc6b963 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Tue, 11 Aug 2026 13:00:37 +0300 Subject: [PATCH 02/20] Add work store reaper to return expired processing work to failing --- work/store/structured/mongo/mongo.go | 65 ++++++++ work/store/structured/mongo/mongo_test.go | 190 ++++++++++++++++++++++ 2 files changed, 255 insertions(+) diff --git a/work/store/structured/mongo/mongo.go b/work/store/structured/mongo/mongo.go index e5aba72d21..829c3fbbca 100644 --- a/work/store/structured/mongo/mongo.go +++ b/work/store/structured/mongo/mongo.go @@ -67,9 +67,74 @@ func (s *Store) EnsureIndexes() error { Options: mongoOptions.Index(). SetName("ModifiedBatchId"), }, + { + Keys: bson.D{{Key: "processingTimeoutTime", Value: 1}}, + Options: mongoOptions.Index(). + SetName("ProcessingTimeoutTime"). + SetPartialFilterExpression(bson.D{{Key: "state", Value: work.StateProcessing}}), + }, }) } +// ReapExpiredProcessing transitions work that exceeded its processing timeout back to failing so +// that it is retried. A process terminated while processing work never reports its completion, +// which otherwise leaves that work in state processing indefinitely and, if it has a serial id, +// permanently prevents any other work sharing that serial id from being polled. +// +// The grace duration allows for clock skew between this process and the database, and for a worker +// that has only just exceeded its processing timeout to report its own completion. Revision is +// incremented so that any subsequent completion reported by the original worker, which is +// conditional upon the revision, no longer matches. +func (s *Store) ReapExpiredProcessing(ctx context.Context, graceDuration time.Duration) (int, error) { + if ctx == nil { + return 0, errors.New("context is missing") + } + if graceDuration < 0 { + return 0, errors.New("grace duration is invalid") + } + + lgr := log.LoggerFromContext(ctx) + + now := time.Now() + defer func() { lgr.WithField("duration", time.Since(now)/time.Microsecond).Debug("ReapExpiredProcessing") }() + + query := bson.M{ + "state": work.StateProcessing, + "processingTimeoutTime": bson.M{"$lte": now.Add(-graceDuration)}, + } + update := bson.A{ + bson.M{"$set": bson.M{ + "state": work.StateFailing, + "failingTime": now, + // Literal as the serialized error must be stored verbatim rather than evaluated as an expression + "failingError": bson.M{"$literal": errors.NewSerializable(errors.New("processing timeout expired"))}, + "failingRetryCount": bson.M{"$add": bson.A{bson.M{"$ifNull": bson.A{"$failingRetryCount", 0}}, 1}}, + // Retry immediately as the work never had the opportunity to report its completion + "failingRetryTime": now, + "processingDuration": bson.M{"$cond": bson.M{ + "if": bson.M{"$ifNull": bson.A{"$processingTime", false}}, + "then": bson.M{"$divide": bson.A{bson.M{"$subtract": bson.A{now, "$processingTime"}}, 1000}}, + "else": "$$REMOVE", + }}, + "modifiedTime": now, + "revision": bson.M{"$add": bson.A{"$revision", 1}}, + }}, + bson.M{"$unset": bson.A{"processingTimeoutTime"}}, + } + + // From this point forward, the context should not be cancelable + ctx = context.WithoutCancel(ctx) + + updateResult, err := s.UpdateMany(ctx, query, update) + lgr = lgr.WithError(err) + if err != nil { + lgr.Error("unable to reap expired processing work") + return 0, errors.Wrap(err, "unable to reap expired processing work") + } + + return int(updateResult.ModifiedCount), nil +} + func (s *Store) Poll(ctx context.Context, poll *work.Poll) ([]*work.Work, error) { if ctx == nil { return nil, errors.New("context is missing") diff --git a/work/store/structured/mongo/mongo_test.go b/work/store/structured/mongo/mongo_test.go index a387dd311e..cc7b1f18b7 100644 --- a/work/store/structured/mongo/mongo_test.go +++ b/work/store/structured/mongo/mongo_test.go @@ -6,6 +6,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + + "go.mongodb.org/mongo-driver/bson" + bsonPrimitive "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" "github.com/tidepool-org/platform/log" logTest "github.com/tidepool-org/platform/log/test" @@ -49,6 +54,191 @@ var _ = Describe("Mongo", func() { Expect(store.EnsureIndexes()).To(Succeed()) }) + Context("EnsureIndexes", func() { + It("creates an index over processing timeout time restricted to processing work", func() { + cursor, err := store.GetCollection("work").Indexes().List(ctx) + Expect(err).ToNot(HaveOccurred()) + var indexes []storeStructuredMongoTest.MongoIndex + Expect(cursor.All(ctx, &indexes)).To(Succeed()) + Expect(indexes).To(ContainElement(MatchFields(IgnoreExtras, Fields{ + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("processingTimeoutTime")), + "Name": Equal("ProcessingTimeoutTime"), + "PartialFilterExpression": Equal(bson.D{{Key: "state", Value: work.StateProcessing}}), + }))) + }) + }) + + Context("ReapExpiredProcessing", func() { + const reapGraceDuration = time.Minute + + It("returns an error when the grace duration is negative", func() { + _, err := store.ReapExpiredProcessing(ctx, -time.Second) + Expect(err).To(MatchError("grace duration is invalid")) + }) + + var collection *mongo.Collection + var poll *work.Poll + var serialID string + var claimed *work.Work + + // Expires the processing timeout time of work directly, as the alternative is to wait + // out both the processing timeout and the reap grace duration + expireProcessingTimeoutTime := func(workID string, timeoutTime time.Time) { + objectID, err := bsonPrimitive.ObjectIDFromHex(workID) + Expect(err).ToNot(HaveOccurred()) + result, err := collection.UpdateOne(ctx, bson.M{"_id": objectID}, bson.M{"$set": bson.M{"processingTimeoutTime": timeoutTime}}) + Expect(err).ToNot(HaveOccurred()) + Expect(result.ModifiedCount).To(Equal(int64(1))) + } + + BeforeEach(func() { + collection = store.GetCollection("work") + poll = &work.Poll{TypeQuantities: work.TypeQuantities{typ: 10}} + serialID = typ + ":" + test.RandomString() + + created, err := store.Create(ctx, &work.Create{ + Type: typ, + SerialID: pointer.FromString(serialID), + ProcessingTimeout: processingTimeout, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(created).ToNot(BeNil()) + + polled, err := store.Poll(ctx, poll) + Expect(err).ToNot(HaveOccurred()) + Expect(polled).To(HaveLen(1)) + claimed = polled[0] + Expect(claimed.State).To(Equal(work.StateProcessing)) + Expect(claimed.ProcessingTimeoutTime).ToNot(BeNil()) + }) + + Context("with work processing beyond the grace duration", func() { + BeforeEach(func() { + expireProcessingTimeoutTime(claimed.ID, time.Now().Add(-reapGraceDuration-time.Minute)) + }) + + It("returns the work to failing with an immediate retry", func() { + count, err := store.ReapExpiredProcessing(ctx, reapGraceDuration) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(1)) + + reaped, err := store.Get(ctx, claimed.ID, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(reaped).ToNot(BeNil()) + Expect(reaped.State).To(Equal(work.StateFailing)) + Expect(reaped.FailingTime).ToNot(BeNil()) + Expect(reaped.FailingError).ToNot(BeNil()) + Expect(reaped.FailingError.Error).To(MatchError(ContainSubstring("processing timeout expired"))) + Expect(reaped.FailingRetryCount).To(PointTo(Equal(1))) + Expect(reaped.FailingRetryTime).ToNot(BeNil()) + Expect(*reaped.FailingRetryTime).To(BeTemporally("<=", time.Now())) + Expect(reaped.Revision).To(Equal(claimed.Revision + 1)) + }) + + // State failing requires the processing timeout time to be absent and, as the work + // was processing, the processing duration to be present + It("clears the processing timeout time and records the processing duration", func() { + _, err := store.ReapExpiredProcessing(ctx, reapGraceDuration) + Expect(err).ToNot(HaveOccurred()) + + reaped, err := store.Get(ctx, claimed.ID, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(reaped).ToNot(BeNil()) + Expect(reaped.ProcessingTimeoutTime).To(BeNil()) + Expect(reaped.ProcessingDuration).ToNot(BeNil()) + Expect(*reaped.ProcessingDuration).To(BeNumerically(">=", 0)) + }) + + It("allows the reaped work to be polled again", func() { + Expect(store.ReapExpiredProcessing(ctx, reapGraceDuration)).To(Equal(1)) + + polled, err := store.Poll(ctx, poll) + Expect(err).ToNot(HaveOccurred()) + Expect(polled).To(HaveLen(1)) + Expect(polled[0].ID).To(Equal(claimed.ID)) + Expect(polled[0].State).To(Equal(work.StateProcessing)) + }) + + // The original worker may still be running and report its completion, which is + // conditional upon the revision it holds and must no longer be applied + It("prevents the completion reported with the revision held before the reap", func() { + Expect(store.ReapExpiredProcessing(ctx, reapGraceDuration)).To(Equal(1)) + + condition := &storeStructured.Condition{Revision: pointer.FromInt(claimed.Revision)} + updated, err := store.Update(ctx, claimed.ID, condition, &work.Update{ + State: work.StateSuccess, + SuccessUpdate: &work.SuccessUpdate{}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).To(BeNil()) + + deleted, err := store.Delete(ctx, claimed.ID, condition) + Expect(err).ToNot(HaveOccurred()) + Expect(deleted).To(BeNil()) + + unchanged, err := store.Get(ctx, claimed.ID, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(unchanged).ToNot(BeNil()) + Expect(unchanged.State).To(Equal(work.StateFailing)) + }) + + It("unblocks work sharing the serial id of the reaped work", func() { + sibling, err := store.Create(ctx, &work.Create{ + Type: typ, + SerialID: pointer.FromString(serialID), + ProcessingTimeout: processingTimeout, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(sibling).ToNot(BeNil()) + + blocked, err := store.Poll(ctx, poll) + Expect(err).ToNot(HaveOccurred()) + Expect(blocked).To(BeEmpty()) + + Expect(store.ReapExpiredProcessing(ctx, reapGraceDuration)).To(Equal(1)) + + polled, err := store.Poll(ctx, poll) + Expect(err).ToNot(HaveOccurred()) + Expect(polled).To(HaveLen(1)) + }) + }) + + It("does not reap work processing within the grace duration", func() { + expireProcessingTimeoutTime(claimed.ID, time.Now().Add(-time.Second)) + + count, err := store.ReapExpiredProcessing(ctx, reapGraceDuration) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(0)) + + unchanged, err := store.Get(ctx, claimed.ID, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(unchanged).ToNot(BeNil()) + Expect(unchanged.State).To(Equal(work.StateProcessing)) + Expect(unchanged.Revision).To(Equal(claimed.Revision)) + }) + + It("does not reap work that is not processing", func() { + pending, err := store.Create(ctx, &work.Create{ + Type: typ, + ProcessingAvailableTime: time.Now().Add(time.Hour), + ProcessingTimeout: processingTimeout, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(pending).ToNot(BeNil()) + expireProcessingTimeoutTime(pending.ID, time.Now().Add(-reapGraceDuration-time.Minute)) + + count, err := store.ReapExpiredProcessing(ctx, reapGraceDuration) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(0)) + + unchanged, err := store.Get(ctx, pending.ID, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(unchanged).ToNot(BeNil()) + Expect(unchanged.State).To(Equal(work.StatePending)) + Expect(unchanged.Revision).To(Equal(pending.Revision)) + }) + }) + Context("Poll", func() { // These work items intentionally share an identical processing available time and // processing priority so that only the identifier tie breaker in the Poll aggregation From e90e7065a056591f2fd282e50025e70b73b86585 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Tue, 11 Aug 2026 13:40:04 +0300 Subject: [PATCH 03/20] Reap expired processing work each time the coordinator requests work --- work/service/client.go | 8 + work/service/coordinator.go | 21 ++ work/service/coordinator_internal_test.go | 97 ++++++++ work/service/service_suite_test.go | 11 + work/service/test/client_mocks.go | 40 ++++ work/service/test/coordinator_mocks.go | 263 ++++++++++++++++++++++ 6 files changed, 440 insertions(+) create mode 100644 work/service/coordinator_internal_test.go create mode 100644 work/service/service_suite_test.go create mode 100644 work/service/test/coordinator_mocks.go diff --git a/work/service/client.go b/work/service/client.go index 8b7b3114d9..b072635073 100644 --- a/work/service/client.go +++ b/work/service/client.go @@ -2,6 +2,7 @@ package service import ( "context" + "time" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/page" @@ -14,6 +15,7 @@ import ( type Store interface { Poll(ctx context.Context, poll *work.Poll) ([]*work.Work, error) + ReapExpiredProcessing(ctx context.Context, graceDuration time.Duration) (int, error) List(ctx context.Context, filter *work.Filter, pagination *page.Pagination) ([]*work.Work, error) Create(ctx context.Context, create *work.Create) (*work.Work, error) Get(ctx context.Context, id string, condition *storeStructured.Condition) (*work.Work, error) @@ -39,6 +41,12 @@ func (c *Client) Poll(ctx context.Context, poll *work.Poll) ([]*work.Work, error return c.store.Poll(ctx, poll) } +// ReapExpiredProcessing is intentionally absent from work.Client as it is coordinator +// infrastructure rather than part of the interface offered to those that create work +func (c *Client) ReapExpiredProcessing(ctx context.Context) (int, error) { + return c.store.ReapExpiredProcessing(ctx, ReapExpiredProcessingGraceDuration) +} + func (c *Client) List(ctx context.Context, filter *work.Filter, pagination *page.Pagination) ([]*work.Work, error) { return c.store.List(ctx, filter, pagination) } diff --git a/work/service/coordinator.go b/work/service/coordinator.go index 8f69fed275..c86481bb7c 100644 --- a/work/service/coordinator.go +++ b/work/service/coordinator.go @@ -17,6 +17,8 @@ import ( workBase "github.com/tidepool-org/platform/work/base" ) +//go:generate mockgen -source=coordinator.go -destination=test/coordinator_mocks.go -package=test -typed + const ( CoordinatorFrequencyDefault = 5 * time.Minute CoordinatorDelayInitial = 1 * time.Minute @@ -24,6 +26,10 @@ const ( FailingRetryDuration = 1 * time.Minute FailingRetryDurationJitter = 5 * time.Second + + // ReapExpiredProcessingGraceDuration is the duration beyond the processing timeout time that + // must elapse before work in state processing is reaped + ReapExpiredProcessingGraceDuration = time.Minute ) type ServerSessionTokenProvider interface { @@ -32,6 +38,7 @@ type ServerSessionTokenProvider interface { type WorkClient interface { Poll(ctx context.Context, poll *work.Poll) ([]*work.Work, error) + ReapExpiredProcessing(ctx context.Context) (int, error) Update(ctx context.Context, id string, condition *request.Condition, update *work.Update) (*work.Work, error) Delete(ctx context.Context, id string, condition *request.Condition) (*work.Work, error) } @@ -201,6 +208,8 @@ func (c *Coordinator) requestAndDispatchWork() { return } + c.reapExpiredProcessingWork() + typeQuantities := c.typeQuantities.NonZero() if typeQuantities.IsEmpty() { return @@ -218,6 +227,18 @@ func (c *Coordinator) requestAndDispatchWork() { } } +// reapExpiredProcessingWork returns work abandoned by a terminated process to failing so that it is +// retried, and so that any work sharing its serial id is no longer prevented from being polled. +// Failure is reported, but does not prevent polling, as any work reaped is already delayed. +func (c *Coordinator) reapExpiredProcessingWork() { + count, err := c.workClient.ReapExpiredProcessing(context.WithoutCancel(c.managerContext)) + if err != nil { + log.LoggerFromContext(c.managerContext).WithError(err).Error("unable to reap expired processing work") + } else if count > 0 { + log.LoggerFromContext(c.managerContext).WithField("count", count).Warn("reaped expired processing work") + } +} + func (c *Coordinator) dispatchWork(ctx context.Context, wrk *work.Work) { c.typeQuantities.Decrement(wrk.Type) c.workersWaitGroup.Go(func() { diff --git a/work/service/coordinator_internal_test.go b/work/service/coordinator_internal_test.go new file mode 100644 index 0000000000..c6291d5f77 --- /dev/null +++ b/work/service/coordinator_internal_test.go @@ -0,0 +1,97 @@ +package service + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + + errorsTest "github.com/tidepool-org/platform/errors/test" + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + netTest "github.com/tidepool-org/platform/net/test" + "github.com/tidepool-org/platform/test" + workServiceTest "github.com/tidepool-org/platform/work/service/test" + workTest "github.com/tidepool-org/platform/work/test" +) + +var _ = Describe("Coordinator", func() { + var controller *gomock.Controller + var logger *logTest.Logger + var workClient *workServiceTest.MockWorkClient + var coordinator *Coordinator + + BeforeEach(func() { + controller = gomock.NewController(GinkgoT()) + logger = logTest.NewLogger() + workClient = workServiceTest.NewMockWorkClient(controller) + + var err error + coordinator, err = NewCoordinator(logger, workServiceTest.NewMockServerSessionTokenProvider(controller), workClient) + Expect(err).ToNot(HaveOccurred()) + Expect(coordinator).ToNot(BeNil()) + + // Assigned directly as the contexts are otherwise only assigned by Start, which defers the + // first request for work by CoordinatorDelayInitial + ctx := log.NewContextWithLogger(context.Background(), logger) + coordinator.workersContext = ctx + coordinator.managerContext = ctx + }) + + AfterEach(func() { + controller.Finish() + }) + + Context("requestAndDispatchWork", func() { + Context("with a registered processor factory", func() { + BeforeEach(func() { + processorFactory := workTest.NewMockProcessorFactory(controller) + processorFactory.EXPECT().Type().Return(netTest.RandomReverseDomain()).AnyTimes() + processorFactory.EXPECT().Quantity().Return(test.RandomIntFromRange(1, 4)).AnyTimes() + processorFactory.EXPECT().Frequency().Return(CoordinatorFrequencyDefault).AnyTimes() + Expect(coordinator.RegisterProcessorFactory(processorFactory)).To(Succeed()) + }) + + It("reaps expired processing work before polling for work", func() { + gomock.InOrder( + workClient.EXPECT().ReapExpiredProcessing(gomock.Any()).Return(0, nil), + workClient.EXPECT().Poll(gomock.Any(), gomock.Any()).Return(nil, nil), + ) + coordinator.requestAndDispatchWork() + }) + + // Work that was reaped is already delayed, so a failure to reap must not additionally + // prevent work that is available from being polled + It("polls for work even when reaping expired processing work fails", func() { + err := errorsTest.RandomError() + gomock.InOrder( + workClient.EXPECT().ReapExpiredProcessing(gomock.Any()).Return(0, err), + workClient.EXPECT().Poll(gomock.Any(), gomock.Any()).Return(nil, nil), + ) + coordinator.requestAndDispatchWork() + logger.AssertError("unable to reap expired processing work") + }) + + It("reports the count when expired processing work is reaped", func() { + count := test.RandomIntFromRange(1, 10) + workClient.EXPECT().ReapExpiredProcessing(gomock.Any()).Return(count, nil) + workClient.EXPECT().Poll(gomock.Any(), gomock.Any()).Return(nil, nil) + coordinator.requestAndDispatchWork() + logger.AssertWarn("reaped expired processing work", log.Fields{"count": count}) + }) + }) + + // Reaping is not specific to any processor type, so it must not be prevented by the absence + // of any processor with capacity, which stops work from being polled + It("reaps expired processing work when no processor factory is registered", func() { + workClient.EXPECT().ReapExpiredProcessing(gomock.Any()).Return(0, nil) + coordinator.requestAndDispatchWork() + }) + + It("does not reap expired processing work before the coordinator is started", func() { + coordinator.workersContext = nil + coordinator.requestAndDispatchWork() + }) + }) +}) diff --git a/work/service/service_suite_test.go b/work/service/service_suite_test.go new file mode 100644 index 0000000000..f257c20c7d --- /dev/null +++ b/work/service/service_suite_test.go @@ -0,0 +1,11 @@ +package service_test + +import ( + "testing" + + "github.com/tidepool-org/platform/test" +) + +func TestSuite(t *testing.T) { + test.Test(t) +} diff --git a/work/service/test/client_mocks.go b/work/service/test/client_mocks.go index f54d3a0bdc..5a6b50a268 100644 --- a/work/service/test/client_mocks.go +++ b/work/service/test/client_mocks.go @@ -12,6 +12,7 @@ package test import ( context "context" reflect "reflect" + time "time" gomock "go.uber.org/mock/gomock" @@ -278,6 +279,45 @@ func (c *MockStorePollCall) DoAndReturn(f func(context.Context, *work.Poll) ([]* return c } +// ReapExpiredProcessing mocks base method. +func (m *MockStore) ReapExpiredProcessing(ctx context.Context, graceDuration time.Duration) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReapExpiredProcessing", ctx, graceDuration) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReapExpiredProcessing indicates an expected call of ReapExpiredProcessing. +func (mr *MockStoreMockRecorder) ReapExpiredProcessing(ctx, graceDuration any) *MockStoreReapExpiredProcessingCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReapExpiredProcessing", reflect.TypeOf((*MockStore)(nil).ReapExpiredProcessing), ctx, graceDuration) + return &MockStoreReapExpiredProcessingCall{Call: call} +} + +// MockStoreReapExpiredProcessingCall wrap *gomock.Call +type MockStoreReapExpiredProcessingCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockStoreReapExpiredProcessingCall) Return(arg0 int, arg1 error) *MockStoreReapExpiredProcessingCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockStoreReapExpiredProcessingCall) Do(f func(context.Context, time.Duration) (int, error)) *MockStoreReapExpiredProcessingCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockStoreReapExpiredProcessingCall) DoAndReturn(f func(context.Context, time.Duration) (int, error)) *MockStoreReapExpiredProcessingCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // Update mocks base method. func (m *MockStore) Update(ctx context.Context, id string, condition *structured.Condition, update *work.Update) (*work.Work, error) { m.ctrl.T.Helper() diff --git a/work/service/test/coordinator_mocks.go b/work/service/test/coordinator_mocks.go new file mode 100644 index 0000000000..2fea213a9e --- /dev/null +++ b/work/service/test/coordinator_mocks.go @@ -0,0 +1,263 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: coordinator.go +// +// Generated by this command: +// +// mockgen -source=coordinator.go -destination=test/coordinator_mocks.go -package=test -typed +// + +// Package test is a generated GoMock package. +package test + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" + + request "github.com/tidepool-org/platform/request" + work "github.com/tidepool-org/platform/work" +) + +// MockServerSessionTokenProvider is a mock of ServerSessionTokenProvider interface. +type MockServerSessionTokenProvider struct { + ctrl *gomock.Controller + recorder *MockServerSessionTokenProviderMockRecorder + isgomock struct{} +} + +// MockServerSessionTokenProviderMockRecorder is the mock recorder for MockServerSessionTokenProvider. +type MockServerSessionTokenProviderMockRecorder struct { + mock *MockServerSessionTokenProvider +} + +// NewMockServerSessionTokenProvider creates a new mock instance. +func NewMockServerSessionTokenProvider(ctrl *gomock.Controller) *MockServerSessionTokenProvider { + mock := &MockServerSessionTokenProvider{ctrl: ctrl} + mock.recorder = &MockServerSessionTokenProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockServerSessionTokenProvider) EXPECT() *MockServerSessionTokenProviderMockRecorder { + return m.recorder +} + +// ServerSessionToken mocks base method. +func (m *MockServerSessionTokenProvider) ServerSessionToken() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ServerSessionToken") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ServerSessionToken indicates an expected call of ServerSessionToken. +func (mr *MockServerSessionTokenProviderMockRecorder) ServerSessionToken() *MockServerSessionTokenProviderServerSessionTokenCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ServerSessionToken", reflect.TypeOf((*MockServerSessionTokenProvider)(nil).ServerSessionToken)) + return &MockServerSessionTokenProviderServerSessionTokenCall{Call: call} +} + +// MockServerSessionTokenProviderServerSessionTokenCall wrap *gomock.Call +type MockServerSessionTokenProviderServerSessionTokenCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockServerSessionTokenProviderServerSessionTokenCall) Return(arg0 string, arg1 error) *MockServerSessionTokenProviderServerSessionTokenCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockServerSessionTokenProviderServerSessionTokenCall) Do(f func() (string, error)) *MockServerSessionTokenProviderServerSessionTokenCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockServerSessionTokenProviderServerSessionTokenCall) DoAndReturn(f func() (string, error)) *MockServerSessionTokenProviderServerSessionTokenCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// MockWorkClient is a mock of WorkClient interface. +type MockWorkClient struct { + ctrl *gomock.Controller + recorder *MockWorkClientMockRecorder + isgomock struct{} +} + +// MockWorkClientMockRecorder is the mock recorder for MockWorkClient. +type MockWorkClientMockRecorder struct { + mock *MockWorkClient +} + +// NewMockWorkClient creates a new mock instance. +func NewMockWorkClient(ctrl *gomock.Controller) *MockWorkClient { + mock := &MockWorkClient{ctrl: ctrl} + mock.recorder = &MockWorkClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockWorkClient) EXPECT() *MockWorkClientMockRecorder { + return m.recorder +} + +// Delete mocks base method. +func (m *MockWorkClient) Delete(ctx context.Context, id string, condition *request.Condition) (*work.Work, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, id, condition) + ret0, _ := ret[0].(*work.Work) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Delete indicates an expected call of Delete. +func (mr *MockWorkClientMockRecorder) Delete(ctx, id, condition any) *MockWorkClientDeleteCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockWorkClient)(nil).Delete), ctx, id, condition) + return &MockWorkClientDeleteCall{Call: call} +} + +// MockWorkClientDeleteCall wrap *gomock.Call +type MockWorkClientDeleteCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockWorkClientDeleteCall) Return(arg0 *work.Work, arg1 error) *MockWorkClientDeleteCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockWorkClientDeleteCall) Do(f func(context.Context, string, *request.Condition) (*work.Work, error)) *MockWorkClientDeleteCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockWorkClientDeleteCall) DoAndReturn(f func(context.Context, string, *request.Condition) (*work.Work, error)) *MockWorkClientDeleteCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Poll mocks base method. +func (m *MockWorkClient) Poll(ctx context.Context, poll *work.Poll) ([]*work.Work, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Poll", ctx, poll) + ret0, _ := ret[0].([]*work.Work) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Poll indicates an expected call of Poll. +func (mr *MockWorkClientMockRecorder) Poll(ctx, poll any) *MockWorkClientPollCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Poll", reflect.TypeOf((*MockWorkClient)(nil).Poll), ctx, poll) + return &MockWorkClientPollCall{Call: call} +} + +// MockWorkClientPollCall wrap *gomock.Call +type MockWorkClientPollCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockWorkClientPollCall) Return(arg0 []*work.Work, arg1 error) *MockWorkClientPollCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockWorkClientPollCall) Do(f func(context.Context, *work.Poll) ([]*work.Work, error)) *MockWorkClientPollCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockWorkClientPollCall) DoAndReturn(f func(context.Context, *work.Poll) ([]*work.Work, error)) *MockWorkClientPollCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// ReapExpiredProcessing mocks base method. +func (m *MockWorkClient) ReapExpiredProcessing(ctx context.Context) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReapExpiredProcessing", ctx) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReapExpiredProcessing indicates an expected call of ReapExpiredProcessing. +func (mr *MockWorkClientMockRecorder) ReapExpiredProcessing(ctx any) *MockWorkClientReapExpiredProcessingCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReapExpiredProcessing", reflect.TypeOf((*MockWorkClient)(nil).ReapExpiredProcessing), ctx) + return &MockWorkClientReapExpiredProcessingCall{Call: call} +} + +// MockWorkClientReapExpiredProcessingCall wrap *gomock.Call +type MockWorkClientReapExpiredProcessingCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockWorkClientReapExpiredProcessingCall) Return(arg0 int, arg1 error) *MockWorkClientReapExpiredProcessingCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockWorkClientReapExpiredProcessingCall) Do(f func(context.Context) (int, error)) *MockWorkClientReapExpiredProcessingCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockWorkClientReapExpiredProcessingCall) DoAndReturn(f func(context.Context) (int, error)) *MockWorkClientReapExpiredProcessingCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Update mocks base method. +func (m *MockWorkClient) Update(ctx context.Context, id string, condition *request.Condition, update *work.Update) (*work.Work, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", ctx, id, condition, update) + ret0, _ := ret[0].(*work.Work) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Update indicates an expected call of Update. +func (mr *MockWorkClientMockRecorder) Update(ctx, id, condition, update any) *MockWorkClientUpdateCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockWorkClient)(nil).Update), ctx, id, condition, update) + return &MockWorkClientUpdateCall{Call: call} +} + +// MockWorkClientUpdateCall wrap *gomock.Call +type MockWorkClientUpdateCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockWorkClientUpdateCall) Return(arg0 *work.Work, arg1 error) *MockWorkClientUpdateCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockWorkClientUpdateCall) Do(f func(context.Context, string, *request.Condition, *work.Update) (*work.Work, error)) *MockWorkClientUpdateCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockWorkClientUpdateCall) DoAndReturn(f func(context.Context, string, *request.Condition, *work.Update) (*work.Work, error)) *MockWorkClientUpdateCall { + c.Call = c.Call.DoAndReturn(f) + return c +} From c74abeefc08d086b1992b40fb37de32125c2f20c Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Wed, 26 Aug 2026 19:22:19 +0300 Subject: [PATCH 04/20] Fix work filter types validation reference --- work/work.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/work/work.go b/work/work.go index af7a540db5..c9a8c83b55 100644 --- a/work/work.go +++ b/work/work.go @@ -139,19 +139,29 @@ func (p *Poll) Validate(validator structure.Validator) { p.TypeQuantities.Validate(validator.WithReference("typeQuantities")) } +// QueueSize is the number of work items of one type in one state +type QueueSize struct { + Type string `json:"type" bson:"type"` + State string `json:"state" bson:"state"` + Count int `json:"count" bson:"count"` +} + type Filter struct { Types *[]string `json:"types,omitempty"` + State *string `json:"state,omitempty"` GroupID *string `json:"groupId,omitempty"` } func (f *Filter) Parse(parser structure.ObjectParser) { f.Types = parser.StringArray("types") f.GroupID = parser.String("groupId") + f.State = parser.String("state") } func (f *Filter) Validate(validator structure.Validator) { - validator.StringArray("type", f.Types).NotEmpty().EachUsing(net.ReverseDomainValidator).EachUnique() + validator.StringArray("types", f.Types).NotEmpty().EachUsing(net.ReverseDomainValidator).EachUnique() validator.String("groupId", f.GroupID).NotEmpty().LengthLessThanOrEqualTo(GroupIDLengthMaximum) + validator.String("state", f.State).OneOf(States()...) } type Create struct { From dddd31cf6094a3923359bcb5c4e00133835478bf Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Wed, 12 Aug 2026 17:30:47 +0300 Subject: [PATCH 05/20] Add upload postprocess work type and producer --- data/work/postprocess/enqueue.go | 80 +++++++ data/work/postprocess/enqueue_test.go | 119 ++++++++++ .../postprocess/postprocess_suite_test.go | 11 + data/work/postprocess/work.go | 102 +++++++++ data/work/postprocess/work_test.go | 214 ++++++++++++++++++ 5 files changed, 526 insertions(+) create mode 100644 data/work/postprocess/enqueue.go create mode 100644 data/work/postprocess/enqueue_test.go create mode 100644 data/work/postprocess/postprocess_suite_test.go create mode 100644 data/work/postprocess/work.go create mode 100644 data/work/postprocess/work_test.go diff --git a/data/work/postprocess/enqueue.go b/data/work/postprocess/enqueue.go new file mode 100644 index 0000000000..bde0e35952 --- /dev/null +++ b/data/work/postprocess/enqueue.go @@ -0,0 +1,80 @@ +package postprocess + +import ( + "context" + "slices" + "time" + + mapset "github.com/deckarep/golang-set/v2" + + "github.com/tidepool-org/platform/errors" + "github.com/tidepool-org/platform/log" + "github.com/tidepool-org/platform/metadata" + "github.com/tidepool-org/platform/pointer" + userWork "github.com/tidepool-org/platform/user/work" + "github.com/tidepool-org/platform/work" +) + +// Enqueue creates a work item to signal a change to the data of a user to trigger the postprocessor. +// Work is created for every change reported, rather than merged into the work already pending for the user, +// so that reporting a change is a single insert when data is uploaded. The work pending for a user is instead +// merged when it is processed. +func Enqueue(ctx context.Context, workClient work.Client, userID string, reasons ...string) error { + if ctx == nil { + return errors.New("context is missing") + } + if workClient == nil { + return errors.New("work client is missing") + } + if userID == "" { + return errors.New("user id is missing") + } + if len(reasons) == 0 { + return errors.New("reasons is missing") + } + + create, err := newCreate(userID, reasons) + if err != nil { + return err + } + + if _, err = workClient.Create(ctx, create); err != nil { + return errors.Wrap(err, "unable to create work") + } + + log.LoggerFromContext(ctx).WithFields(log.Fields{ + "userId": userID, + "reasons": reasons, + "processingAvailableTime": create.ProcessingAvailableTime, + }).Debug("created work") + return nil +} + +func newCreate(userID string, reasons []string) (*work.Create, error) { + create, err := metadata.WithMetadata( + &work.Create{ + Type: Type, + GroupID: pointer.FromString(IDFromUserID(userID)), + SerialID: pointer.FromString(IDFromUserID(userID)), + ProcessingAvailableTime: time.Now(), + ProcessingTimeout: int(ProcessingTimeout.Seconds()), + }, + &Metadata{ + Metadata: userWork.Metadata{UserID: pointer.FromString(userID)}, + Reasons: normalizeReasons(reasons), + }, + ) + if err != nil { + return nil, errors.Wrap(err, "unable to create work create") + } + return create, nil +} + +// normalizeReasons reports the reasons given, combined, without duplicates, and ordered +func normalizeReasons(reasons ...[]string) []string { + union := mapset.NewSet[string]() + for _, each := range reasons { + union.Append(each...) + } + return slices.Sorted(slices.Values(union.ToSlice())) +} diff --git a/data/work/postprocess/enqueue_test.go b/data/work/postprocess/enqueue_test.go new file mode 100644 index 0000000000..eb656e5d56 --- /dev/null +++ b/data/work/postprocess/enqueue_test.go @@ -0,0 +1,119 @@ +package postprocess_test + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + "go.uber.org/mock/gomock" + + dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" + errorsTest "github.com/tidepool-org/platform/errors/test" + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + userTest "github.com/tidepool-org/platform/user/test" + "github.com/tidepool-org/platform/work" + workTest "github.com/tidepool-org/platform/work/test" +) + +var _ = Describe("Enqueue", func() { + var controller *gomock.Controller + var workClient *workTest.MockClient + var ctx context.Context + var userID string + var id string + + BeforeEach(func() { + controller = gomock.NewController(GinkgoT()) + workClient = workTest.NewMockClient(controller) + ctx = log.NewContextWithLogger(context.Background(), logTest.NewLogger()) + userID = userTest.RandomUserID() + id = dataWorkPostprocess.IDFromUserID(userID) + }) + + AfterEach(func() { + controller.Finish() + }) + + expectCreate := func() func() *work.Create { + var created *work.Create + workClient.EXPECT().Create(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, create *work.Create) (*work.Work, error) { + created = create + return &work.Work{ID: create.Type}, nil + }) + return func() *work.Create { return created } + } + + It("creates work available now", func() { + created := expectCreate() + + Expect(dataWorkPostprocess.Enqueue(ctx, workClient, userID, dataWorkPostprocess.ReasonDataAdded)).To(Succeed()) + + Expect(created().Type).To(Equal(dataWorkPostprocess.Type)) + Expect(created().GroupID).To(PointTo(Equal(id))) + Expect(created().SerialID).To(PointTo(Equal(id))) + Expect(created().ProcessingPriority).To(Equal(0)) + Expect(created().ProcessingTimeout).To(Equal(300)) + Expect(created().ProcessingAvailableTime).To(BeTemporally("~", time.Now(), time.Second)) + Expect(created().Metadata).To(HaveKeyWithValue("userId", userID)) + Expect(created().Metadata).To(HaveKeyWithValue("reasons", ConsistOf(dataWorkPostprocess.ReasonDataAdded))) + }) + + // Work is never merged into work already pending, so that reporting a change is a single insert. + // The deduplication id is deliberately absent as it would instead discard the work reported. + It("creates work that is neither deduplicated nor merged into work already pending", func() { + created := expectCreate() + + Expect(dataWorkPostprocess.Enqueue(ctx, workClient, userID, dataWorkPostprocess.ReasonDataAdded)).To(Succeed()) + + Expect(created().DeduplicationID).To(BeNil()) + }) + + // Work is always available now. The time to defer a change until, which the legacy ingestion + // service computes from its own batching, is expected to be reported by its caller rather than + // derived here. + It("creates work available now whatever the reasons report", func() { + created := expectCreate() + + Expect(dataWorkPostprocess.Enqueue(ctx, workClient, userID, dataWorkPostprocess.ReasonLegacyDataAdded)).To(Succeed()) + + Expect(created().ProcessingAvailableTime).To(BeTemporally("~", time.Now(), time.Second)) + }) + + It("creates work reporting every reason once", func() { + created := expectCreate() + + Expect(dataWorkPostprocess.Enqueue(ctx, workClient, userID, dataWorkPostprocess.ReasonDataAdded, dataWorkPostprocess.ReasonUploadCompleted, dataWorkPostprocess.ReasonDataAdded)).To(Succeed()) + + Expect(created().Metadata).To(HaveKeyWithValue("reasons", ConsistOf(dataWorkPostprocess.ReasonDataAdded, dataWorkPostprocess.ReasonUploadCompleted))) + }) + + It("returns an error when the work cannot be created", func() { + workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil, errorsTest.RandomError()) + + err := dataWorkPostprocess.Enqueue(ctx, workClient, userID, dataWorkPostprocess.ReasonDataAdded) + Expect(err).To(MatchError(ContainSubstring("unable to create work"))) + }) + + DescribeTable("returns an error when a parameter is missing", + func(get func() (context.Context, work.Client, string, []string), expected string) { + enqueueCtx, enqueueWorkClient, enqueueUserID, enqueueReasons := get() + Expect(dataWorkPostprocess.Enqueue(enqueueCtx, enqueueWorkClient, enqueueUserID, enqueueReasons...)).To(MatchError(expected)) + }, + Entry("context", func() (context.Context, work.Client, string, []string) { + return nil, workClient, userID, []string{dataWorkPostprocess.ReasonDataAdded} + }, "context is missing"), + Entry("work client", func() (context.Context, work.Client, string, []string) { + return ctx, nil, userID, []string{dataWorkPostprocess.ReasonDataAdded} + }, "work client is missing"), + Entry("user id", func() (context.Context, work.Client, string, []string) { + return ctx, workClient, "", []string{dataWorkPostprocess.ReasonDataAdded} + }, "user id is missing"), + Entry("reasons", func() (context.Context, work.Client, string, []string) { + return ctx, workClient, userID, nil + }, "reasons is missing"), + ) +}) diff --git a/data/work/postprocess/postprocess_suite_test.go b/data/work/postprocess/postprocess_suite_test.go new file mode 100644 index 0000000000..11725fe040 --- /dev/null +++ b/data/work/postprocess/postprocess_suite_test.go @@ -0,0 +1,11 @@ +package postprocess_test + +import ( + "testing" + + "github.com/tidepool-org/platform/test" +) + +func TestSuite(t *testing.T) { + test.Test(t) +} diff --git a/data/work/postprocess/work.go b/data/work/postprocess/work.go new file mode 100644 index 0000000000..1f716b308c --- /dev/null +++ b/data/work/postprocess/work.go @@ -0,0 +1,102 @@ +package postprocess + +import ( + "fmt" + "time" + + mapset "github.com/deckarep/golang-set/v2" + + "github.com/tidepool-org/platform/structure" + userWork "github.com/tidepool-org/platform/user/work" +) + +const ( + Type = "org.tidepool.data.upload.postprocess" + + ProcessingTimeout = 5 * time.Minute + + // JellyfishBatchSize is the number of records the legacy ingestion service uploads per batch. A + // smaller batch is the final batch of an upload, which it does not otherwise report. + JellyfishBatchSize = 1000 + + // JellyfishQuietDelay defers processing so that an upload of any number of full batches is not + // processed once per batch + JellyfishQuietDelay = 90 * time.Second +) + +const ( + ReasonDataAdded = "DATA_ADDED" + + // ReasonUploadCompleted reports a data set was closed, or jellyfish uploaded a partial batch + ReasonUploadCompleted = "UPLOAD_COMPLETED" + + // ReasonLegacyDataAdded reports jellyfish uploaded a full batch + ReasonLegacyDataAdded = "LEGACY_DATA_ADDED" + + ReasonDataDeleted = "DATA_DELETED" + ReasonContinuousDataDeleted = "CONTINUOUS_DATA_DELETED" + ReasonSchemaMigration = "SCHEMA_MIGRATION" +) + +const ( + MetadataKeyReasons = "reasons" +) + +func Reasons() []string { + return []string{ + ReasonDataAdded, + ReasonUploadCompleted, + ReasonLegacyDataAdded, + ReasonDataDeleted, + ReasonContinuousDataDeleted, + ReasonSchemaMigration, + } +} + +// Data added is intentionally absent to prevent continuous data uploads from constantly triggering syncs +var ehrSyncReasons = mapset.NewSet( + ReasonUploadCompleted, + ReasonLegacyDataAdded, + ReasonDataDeleted, +) + +// Data deleted may leave no record behind reporting it was modified, so the summaries cannot be +// calculated from only the data modified since they were last calculated +var summaryResetReasons = mapset.NewSet( + ReasonDataDeleted, + ReasonContinuousDataDeleted, + ReasonSchemaMigration, +) + +func TriggersEHRSync(reasons []string) bool { + return ehrSyncReasons.ContainsAny(reasons...) +} + +func RequiresSummaryReset(reasons []string) bool { + return summaryResetReasons.ContainsAny(reasons...) +} + +// IDFromUserID returns both the serial id, which prevents the work of a user being processed +// concurrently, and the group id, which is the scope the work of a user is coalesced within. Both +// are the user, and neither is the data set, so that changes to any number of data sets, interleaved +// in any order, yield a single stream of work for the user. +func IDFromUserID(userID string) string { + return fmt.Sprintf("%s:%s", Type, userID) +} + +type Metadata struct { + userWork.Metadata `bson:",inline"` + Reasons []string `json:"reasons,omitempty" bson:"reasons,omitempty"` +} + +func (m *Metadata) Parse(parser structure.ObjectParser) { + m.Metadata.Parse(parser) + if ptr := parser.StringArray(MetadataKeyReasons); ptr != nil { + m.Reasons = *ptr + } +} + +func (m *Metadata) Validate(validator structure.Validator) { + m.Metadata.Validate(validator) + validator.StringArray(MetadataKeyReasons, &m.Reasons).NotEmpty().EachOneOf(Reasons()...).EachUnique() +} diff --git a/data/work/postprocess/work_test.go b/data/work/postprocess/work_test.go new file mode 100644 index 0000000000..8eda771c7b --- /dev/null +++ b/data/work/postprocess/work_test.go @@ -0,0 +1,214 @@ +package postprocess_test + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" + errorsTest "github.com/tidepool-org/platform/errors/test" + logTest "github.com/tidepool-org/platform/log/test" + "github.com/tidepool-org/platform/metadata" + "github.com/tidepool-org/platform/pointer" + structureParser "github.com/tidepool-org/platform/structure/parser" + structureValidator "github.com/tidepool-org/platform/structure/validator" + "github.com/tidepool-org/platform/user" + userTest "github.com/tidepool-org/platform/user/test" + userWork "github.com/tidepool-org/platform/user/work" +) + +var _ = Describe("Work", func() { + It("Type is expected", func() { + Expect(dataWorkPostprocess.Type).To(Equal("org.tidepool.data.upload.postprocess")) + }) + + It("ProcessingTimeout is expected", func() { + Expect(dataWorkPostprocess.ProcessingTimeout).To(Equal(5 * time.Minute)) + }) + + It("JellyfishBatchSize is expected", func() { + Expect(dataWorkPostprocess.JellyfishBatchSize).To(Equal(1000)) + }) + + It("JellyfishQuietDelay is expected", func() { + Expect(dataWorkPostprocess.JellyfishQuietDelay).To(Equal(90 * time.Second)) + }) + + It("Reasons is expected", func() { + Expect(dataWorkPostprocess.Reasons()).To(ConsistOf( + "DATA_ADDED", + "UPLOAD_COMPLETED", + "LEGACY_DATA_ADDED", + "DATA_DELETED", + "CONTINUOUS_DATA_DELETED", + "SCHEMA_MIGRATION", + )) + }) + + Context("TriggersEHRSync", func() { + DescribeTable("reports whether the reasons trigger a synchronization", + func(reasons []string, expected bool) { + Expect(dataWorkPostprocess.TriggersEHRSync(reasons)).To(Equal(expected)) + }, + Entry("with no reasons", nil, false), + Entry("with data added", []string{dataWorkPostprocess.ReasonDataAdded}, false), + Entry("with schema migration", []string{dataWorkPostprocess.ReasonSchemaMigration}, false), + Entry("with continuous data deleted", []string{dataWorkPostprocess.ReasonContinuousDataDeleted}, false), + Entry("with upload completed", []string{dataWorkPostprocess.ReasonUploadCompleted}, true), + Entry("with legacy data added", []string{dataWorkPostprocess.ReasonLegacyDataAdded}, true), + Entry("with data deleted", []string{dataWorkPostprocess.ReasonDataDeleted}, true), + Entry("with any reason triggering a synchronization", + []string{dataWorkPostprocess.ReasonDataAdded, dataWorkPostprocess.ReasonUploadCompleted}, true), + ) + }) + + Context("RequiresSummaryReset", func() { + DescribeTable("reports whether the reasons require the summaries to be recalculated in full", + func(reasons []string, expected bool) { + Expect(dataWorkPostprocess.RequiresSummaryReset(reasons)).To(Equal(expected)) + }, + Entry("with no reasons", nil, false), + Entry("with data added", []string{dataWorkPostprocess.ReasonDataAdded}, false), + Entry("with upload completed", []string{dataWorkPostprocess.ReasonUploadCompleted}, false), + Entry("with legacy data added", []string{dataWorkPostprocess.ReasonLegacyDataAdded}, false), + Entry("with data deleted", []string{dataWorkPostprocess.ReasonDataDeleted}, true), + Entry("with continuous data deleted", []string{dataWorkPostprocess.ReasonContinuousDataDeleted}, true), + Entry("with schema migration", []string{dataWorkPostprocess.ReasonSchemaMigration}, true), + Entry("with any reason requiring a reset", + []string{dataWorkPostprocess.ReasonDataAdded, dataWorkPostprocess.ReasonDataDeleted}, true), + ) + }) + + Context("IDFromUserID", func() { + var userID string + + BeforeEach(func() { + userID = userTest.RandomUserID() + }) + + It("returns the identifier of the work for the user", func() { + Expect(dataWorkPostprocess.IDFromUserID(userID)).To(Equal("org.tidepool.data.upload.postprocess:" + userID)) + }) + + // The work is scoped to the user rather than to any data set, so that changes to any number + // of data sets are coalesced into a single stream of work for the user + It("returns the same identifier for the user however many times it is called", func() { + Expect(dataWorkPostprocess.IDFromUserID(userID)).To(Equal(dataWorkPostprocess.IDFromUserID(userID))) + }) + + It("returns a different identifier for a different user", func() { + Expect(dataWorkPostprocess.IDFromUserID(userID)).ToNot(Equal(dataWorkPostprocess.IDFromUserID(userTest.RandomUserID()))) + }) + }) + + Context("Metadata", func() { + var userID string + + BeforeEach(func() { + userID = userTest.RandomUserID() + }) + + Context("Parse", func() { + DescribeTable("parses the metadata", + func(object map[string]any, expectedMetadata func(userID string) *dataWorkPostprocess.Metadata, expectedErrors ...error) { + if userIDValue, ok := object["userId"]; ok && userIDValue == nil { + object["userId"] = userID + } + result := &dataWorkPostprocess.Metadata{} + errorsTest.ExpectEqual(structureParser.NewObject(logTest.NewLogger(), &object).Parse(result), expectedErrors...) + Expect(result).To(Equal(expectedMetadata(userID))) + }, + Entry("with a user and a reason", + map[string]any{"userId": nil, "reasons": []any{"DATA_ADDED"}}, + func(userID string) *dataWorkPostprocess.Metadata { + return &dataWorkPostprocess.Metadata{ + Metadata: userWork.Metadata{UserID: pointer.FromString(userID)}, + Reasons: []string{"DATA_ADDED"}, + } + }, + ), + Entry("with multiple reasons", + map[string]any{"userId": nil, "reasons": []any{"LEGACY_DATA_ADDED", "UPLOAD_COMPLETED"}}, + func(userID string) *dataWorkPostprocess.Metadata { + return &dataWorkPostprocess.Metadata{ + Metadata: userWork.Metadata{UserID: pointer.FromString(userID)}, + Reasons: []string{"LEGACY_DATA_ADDED", "UPLOAD_COMPLETED"}, + } + }, + ), + Entry("with nothing", + map[string]any{}, + func(userID string) *dataWorkPostprocess.Metadata { + return &dataWorkPostprocess.Metadata{} + }, + ), + Entry("with reasons of the wrong type", + map[string]any{"userId": nil, "reasons": true}, + func(userID string) *dataWorkPostprocess.Metadata { + return &dataWorkPostprocess.Metadata{ + Metadata: userWork.Metadata{UserID: pointer.FromString(userID)}, + } + }, + errorsTest.WithPointerSource(structureParser.ErrorTypeNotArray(true), "/reasons"), + ), + ) + }) + + Context("Validate", func() { + DescribeTable("validates the metadata", + func(mutator func(metadata *dataWorkPostprocess.Metadata), expectedErrors ...error) { + datum := &dataWorkPostprocess.Metadata{ + Metadata: userWork.Metadata{UserID: pointer.FromString(userID)}, + Reasons: []string{dataWorkPostprocess.ReasonDataAdded}, + } + mutator(datum) + errorsTest.ExpectEqual(structureValidator.New(logTest.NewLogger()).Validate(datum), expectedErrors...) + }, + Entry("succeeds", func(datum *dataWorkPostprocess.Metadata) {}), + Entry("succeeds with every reason", func(datum *dataWorkPostprocess.Metadata) { + datum.Reasons = dataWorkPostprocess.Reasons() + }), + Entry("reports the user is not valid", + func(datum *dataWorkPostprocess.Metadata) { datum.UserID = pointer.FromString("invalid") }, + errorsTest.WithPointerSource(user.ErrorValueStringAsIDNotValid("invalid"), "/userId"), + ), + Entry("reports the reasons are missing", + func(datum *dataWorkPostprocess.Metadata) { datum.Reasons = nil }, + errorsTest.WithPointerSource(structureValidator.ErrorValueEmpty(), "/reasons"), + ), + Entry("reports the reasons are empty", + func(datum *dataWorkPostprocess.Metadata) { datum.Reasons = []string{} }, + errorsTest.WithPointerSource(structureValidator.ErrorValueEmpty(), "/reasons"), + ), + Entry("reports a reason is not valid", + func(datum *dataWorkPostprocess.Metadata) { datum.Reasons = []string{"INVALID"} }, + errorsTest.WithPointerSource(structureValidator.ErrorValueStringNotOneOf("INVALID", dataWorkPostprocess.Reasons()), "/reasons/0"), + ), + Entry("reports the reasons are duplicated", + func(datum *dataWorkPostprocess.Metadata) { + datum.Reasons = []string{dataWorkPostprocess.ReasonDataAdded, dataWorkPostprocess.ReasonDataAdded} + }, + errorsTest.WithPointerSource(structureValidator.ErrorValueDuplicate(), "/reasons/1"), + ), + ) + }) + + // The metadata is encoded when the work is created and decoded when it is processed + It("is unchanged by being encoded and decoded", func() { + datum := &dataWorkPostprocess.Metadata{ + Metadata: userWork.Metadata{UserID: pointer.FromString(userID)}, + Reasons: []string{dataWorkPostprocess.ReasonLegacyDataAdded, dataWorkPostprocess.ReasonUploadCompleted}, + } + + encoded, err := metadata.Encode(datum) + Expect(err).ToNot(HaveOccurred()) + Expect(encoded).ToNot(BeEmpty()) + + decoded, err := metadata.Decode[dataWorkPostprocess.Metadata](context.Background(), encoded) + Expect(err).ToNot(HaveOccurred()) + Expect(decoded).To(Equal(datum)) + }) + }) +}) From d5e7a65b285a7c04a8eb865edfcb786edd41c149 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Thu, 13 Aug 2026 14:07:20 +0300 Subject: [PATCH 06/20] Add upload postprocess work processor and wire it into the data service --- clinics/clinics_suite_test.go | 11 + clinics/service.go | 19 ++ clinics/service_test.go | 66 +++++ clinics/test/service_mocks.go | 38 +++ data/service/service/standard.go | 24 +- data/work/postprocess/factory.go | 56 ++++ data/work/postprocess/factory_test.go | 88 ++++++ data/work/postprocess/processor.go | 182 ++++++++++++ data/work/postprocess/processor_test.go | 273 ++++++++++++++++++ data/work/postprocess/summarizers.go | 39 +++ .../postprocess/test/summarizers_mocks.go | 79 +++++ data/work/postprocess/work.go | 19 +- data/work/postprocess/work_test.go | 21 -- work/store/structured/mongo/mongo.go | 3 + 14 files changed, 877 insertions(+), 41 deletions(-) create mode 100644 clinics/clinics_suite_test.go create mode 100644 clinics/service_test.go create mode 100644 data/work/postprocess/factory.go create mode 100644 data/work/postprocess/factory_test.go create mode 100644 data/work/postprocess/processor.go create mode 100644 data/work/postprocess/processor_test.go create mode 100644 data/work/postprocess/summarizers.go create mode 100644 data/work/postprocess/test/summarizers_mocks.go diff --git a/clinics/clinics_suite_test.go b/clinics/clinics_suite_test.go new file mode 100644 index 0000000000..31fcfc07ae --- /dev/null +++ b/clinics/clinics_suite_test.go @@ -0,0 +1,11 @@ +package clinics_test + +import ( + "testing" + + "github.com/tidepool-org/platform/test" +) + +func TestSuite(t *testing.T) { + test.Test(t) +} diff --git a/clinics/service.go b/clinics/service.go index 797884cfde..12a934a303 100644 --- a/clinics/service.go +++ b/clinics/service.go @@ -29,6 +29,7 @@ type Client interface { SharePatientAccount(ctx context.Context, clinicID, patientID string) (*clinic.Patient, error) ListEHREnabledClinics(ctx context.Context) ([]clinic.Clinic, error) SyncEHRData(ctx context.Context, clinicID string) error + SyncEHRDataForPatient(ctx context.Context, patientID string) error GetPatients(ctx context.Context, clinicId string, userToken string, params *clinic.ListPatientsParams, injectedParams url.Values) ([]clinic.Patient, error) GetPatient(ctx context.Context, clinicID, patientID string) (*clinic.Patient, error) } @@ -203,6 +204,24 @@ func (d *defaultClient) SyncEHRData(ctx context.Context, clinicID string) error return nil } +// SyncEHRDataForPatient reports no error when the clinic service reports the patient has no active +// subscription to any clinic enabled for an electronic health record, which it does as not found. Most +// users are not such a patient, so reporting that as a failure would fail the work of nearly every user. +func (d *defaultClient) SyncEHRDataForPatient(ctx context.Context, patientID string) error { + response, err := d.httpClient.SyncEHRDataForPatientWithResponse(ctx, clinic.PatientId(patientID)) + if err != nil { + return err + } + if response.StatusCode() != http.StatusAccepted && response.StatusCode() != http.StatusNotFound { + err = errors.Preparedf(ErrorCodeClinicClientFailure, + "Unexpected status code from clinic service", + "unexpected response status code %v from %v", response.StatusCode(), response.HTTPResponse.Request.URL) + err = errors.WithMeta(err, response.HTTPResponse) + return err + } + return nil +} + func (d *defaultClient) GetPatient(ctx context.Context, clinicID, patientID string) (*clinic.Patient, error) { response, err := d.httpClient.GetPatientWithResponse(ctx, clinic.ClinicId(clinicID), clinic.PatientId(patientID)) if err != nil { diff --git a/clinics/service_test.go b/clinics/service_test.go new file mode 100644 index 0000000000..68d04d026b --- /dev/null +++ b/clinics/service_test.go @@ -0,0 +1,66 @@ +package clinics_test + +import ( + "context" + "net/http" + "net/http/httptest" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + authTest "github.com/tidepool-org/platform/auth/test" + "github.com/tidepool-org/platform/clinics" + userTest "github.com/tidepool-org/platform/user/test" +) + +var _ = Describe("Client", func() { + Context("SyncEHRDataForPatient", func() { + var server *httptest.Server + var requestPath string + var responseStatusCode int + var client clinics.Client + var patientID string + + BeforeEach(func() { + patientID = userTest.RandomUserID() + requestPath = "" + responseStatusCode = http.StatusAccepted + + server = httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + requestPath = req.URL.Path + res.WriteHeader(responseStatusCode) + })) + GinkgoT().Setenv("TIDEPOOL_CLINIC_CLIENT_ADDRESS", server.URL) + + externalAccessor := authTest.NewExternalAccessor() + externalAccessor.ServerSessionTokenOutputs = []authTest.ServerSessionTokenOutput{{Token: authTest.NewSessionToken()}} + + var err error + client, err = clinics.NewClient(externalAccessor) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + server.Close() + }) + + It("requests a synchronization for the patient", func() { + Expect(client.SyncEHRDataForPatient(context.Background(), patientID)).To(Succeed()) + Expect(requestPath).To(Equal("/v1/patients/" + patientID + "/ehr/sync")) + }) + + // The clinic service reports a patient with no active subscription to any clinic enabled for an + // electronic health record as not found. Most users are not such a patient, so reporting that as + // a failure would fail the work of nearly every user. + It("returns no error when the patient has no active subscription", func() { + responseStatusCode = http.StatusNotFound + Expect(client.SyncEHRDataForPatient(context.Background(), patientID)).To(Succeed()) + }) + + It("returns an error when the clinic service reports an unexpected status", func() { + responseStatusCode = http.StatusInternalServerError + err := client.SyncEHRDataForPatient(context.Background(), patientID) + Expect(err).To(MatchError(ContainSubstring("unexpected response status code 500"))) + }) + }) +}) diff --git a/clinics/test/service_mocks.go b/clinics/test/service_mocks.go index 9a07b5a88b..fbb49eebc6 100644 --- a/clinics/test/service_mocks.go +++ b/clinics/test/service_mocks.go @@ -352,3 +352,41 @@ func (c *MockClientSyncEHRDataCall) DoAndReturn(f func(context.Context, string) c.Call = c.Call.DoAndReturn(f) return c } + +// SyncEHRDataForPatient mocks base method. +func (m *MockClient) SyncEHRDataForPatient(ctx context.Context, patientID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SyncEHRDataForPatient", ctx, patientID) + ret0, _ := ret[0].(error) + return ret0 +} + +// SyncEHRDataForPatient indicates an expected call of SyncEHRDataForPatient. +func (mr *MockClientMockRecorder) SyncEHRDataForPatient(ctx, patientID any) *MockClientSyncEHRDataForPatientCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncEHRDataForPatient", reflect.TypeOf((*MockClient)(nil).SyncEHRDataForPatient), ctx, patientID) + return &MockClientSyncEHRDataForPatientCall{Call: call} +} + +// MockClientSyncEHRDataForPatientCall wrap *gomock.Call +type MockClientSyncEHRDataForPatientCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockClientSyncEHRDataForPatientCall) Return(arg0 error) *MockClientSyncEHRDataForPatientCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockClientSyncEHRDataForPatientCall) Do(f func(context.Context, string) error) *MockClientSyncEHRDataForPatientCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockClientSyncEHRDataForPatientCall) DoAndReturn(f func(context.Context, string) error) *MockClientSyncEHRDataForPatientCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/data/service/service/standard.go b/data/service/service/standard.go index a309a01d5f..34042637fb 100644 --- a/data/service/service/standard.go +++ b/data/service/service/standard.go @@ -32,6 +32,7 @@ import ( dataSourceStoreStructured "github.com/tidepool-org/platform/data/source/store/structured" dataSourceStoreStructuredMongo "github.com/tidepool-org/platform/data/source/store/structured/mongo" dataStoreMongo "github.com/tidepool-org/platform/data/store/mongo" + dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/events" "github.com/tidepool-org/platform/log" @@ -94,6 +95,7 @@ type Standard struct { dataRawClient *dataRawService.Client dataSourceClient *dataSourceServiceClient.Client mailerClient mailer.Client + summarizerRegistry *summary.SummarizerRegistry summaryClient *summaryClient.Client workClient *workService.Client notificationsHistoryRecorder notificationsHistory.Recorder @@ -620,7 +622,7 @@ func (s *Standard) initializeConfirmationClient() error { func (s *Standard) initializeSummaryClient() error { s.Logger().Debug("Creating summarizer registry") - summarizerRegistry := summary.New( + s.summarizerRegistry = summary.New( s.dataStore.NewSummaryRepository().GetStore(), s.dataStore.NewBucketsRepository().GetStore(), s.dataStore.NewDataRepository(), @@ -629,7 +631,7 @@ func (s *Standard) initializeSummaryClient() error { s.Logger().Debug("Creating summary client") - clnt, err := summaryClient.New(summarizerRegistry) + clnt, err := summaryClient.New(s.summarizerRegistry) if err != nil { return errors.Wrap(err, "unable to create summary client") } @@ -808,6 +810,24 @@ func (s *Standard) initializeWorkProcessorFactories() error { processorFactories = append(processorFactories, processorFactory) } + s.Logger().Debug("Creating data upload postprocess work processor factory") + + summarizers, err := dataWorkPostprocess.NewSummarizers(s.summarizerRegistry) + if err != nil { + return errors.Wrap(err, "unable to create summarizers") + } + + if processorFactory, err := dataWorkPostprocess.NewProcessorFactory(dataWorkPostprocess.Dependencies{ + Dependencies: dependencies, + Summarizers: summarizers, + ClinicsClient: s.clinicsClient, + UserClient: s.userClient, + }); err != nil { + return errors.Wrap(err, "unable to create data upload postprocess work processor factory") + } else { + processorFactories = append(processorFactories, processorFactory) + } + if s.abbottClient != nil { s.Logger().Debug("Creating abbott processor factories") diff --git a/data/work/postprocess/factory.go b/data/work/postprocess/factory.go new file mode 100644 index 0000000000..bb7b244f7f --- /dev/null +++ b/data/work/postprocess/factory.go @@ -0,0 +1,56 @@ +package postprocess + +import ( + "time" + + "github.com/tidepool-org/platform/clinics" + "github.com/tidepool-org/platform/errors" + "github.com/tidepool-org/platform/user" + "github.com/tidepool-org/platform/work" + workBase "github.com/tidepool-org/platform/work/base" +) + +const ( + Quantity = 15 + Frequency = 30 * time.Second + + FailingRetryDuration = 1 * time.Minute + FailingRetryDurationJitter = 5 * time.Second + FailingRetryDurationMaximum = 1 * time.Hour +) + +type ( + ClinicsClient = clinics.Client + UserClient = user.Client +) + +type Dependencies struct { + workBase.Dependencies + Summarizers + ClinicsClient + UserClient +} + +func (d Dependencies) Validate() error { + if err := d.Dependencies.Validate(); err != nil { + return err + } + if d.Summarizers == nil { + return errors.New("summarizers is missing") + } + if d.ClinicsClient == nil { + return errors.New("clinics client is missing") + } + if d.UserClient == nil { + return errors.New("user client is missing") + } + return nil +} + +func NewProcessorFactory(dependencies Dependencies) (*workBase.ProcessorFactory, error) { + if err := dependencies.Validate(); err != nil { + return nil, errors.Wrap(err, "dependencies is invalid") + } + processorFactory := func() (work.Processor, error) { return NewProcessor(dependencies) } + return workBase.NewProcessorFactory(Type, Quantity, Frequency, processorFactory) +} diff --git a/data/work/postprocess/factory_test.go b/data/work/postprocess/factory_test.go new file mode 100644 index 0000000000..fa27729090 --- /dev/null +++ b/data/work/postprocess/factory_test.go @@ -0,0 +1,88 @@ +package postprocess_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + + clinicsTest "github.com/tidepool-org/platform/clinics/test" + dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" + dataWorkPostprocessTest "github.com/tidepool-org/platform/data/work/postprocess/test" + userTest "github.com/tidepool-org/platform/user/test" + workBase "github.com/tidepool-org/platform/work/base" + workTest "github.com/tidepool-org/platform/work/test" +) + +var _ = Describe("Factory", func() { + It("Quantity is expected", func() { + Expect(dataWorkPostprocess.Quantity).To(Equal(15)) + }) + + It("Frequency is expected", func() { + Expect(dataWorkPostprocess.Frequency).To(Equal(30 * time.Second)) + }) + + Context("with dependencies", func() { + var controller *gomock.Controller + var dependencies dataWorkPostprocess.Dependencies + + BeforeEach(func() { + controller = gomock.NewController(GinkgoT()) + dependencies = dataWorkPostprocess.Dependencies{ + Dependencies: workBase.Dependencies{WorkClient: workTest.NewMockClient(controller)}, + Summarizers: dataWorkPostprocessTest.NewMockSummarizers(controller), + ClinicsClient: clinicsTest.NewMockClient(controller), + UserClient: userTest.NewMockClient(controller), + } + }) + + AfterEach(func() { + controller.Finish() + }) + + DescribeTable("Validate", + func(mutator func(dependencies *dataWorkPostprocess.Dependencies), expected string) { + mutator(&dependencies) + if expected == "" { + Expect(dependencies.Validate()).To(Succeed()) + } else { + Expect(dependencies.Validate()).To(MatchError(expected)) + } + }, + Entry("succeeds", func(dependencies *dataWorkPostprocess.Dependencies) {}, ""), + Entry("reports the work client is missing", func(dependencies *dataWorkPostprocess.Dependencies) { + dependencies.WorkClient = nil + }, "work client is missing"), + Entry("reports the summarizers are missing", func(dependencies *dataWorkPostprocess.Dependencies) { + dependencies.Summarizers = nil + }, "summarizers is missing"), + Entry("reports the clinics client is missing", func(dependencies *dataWorkPostprocess.Dependencies) { + dependencies.ClinicsClient = nil + }, "clinics client is missing"), + Entry("reports the user client is missing", func(dependencies *dataWorkPostprocess.Dependencies) { + dependencies.UserClient = nil + }, "user client is missing"), + ) + + It("NewProcessorFactory reports the type, quantity and frequency of the work", func() { + processorFactory, err := dataWorkPostprocess.NewProcessorFactory(dependencies) + Expect(err).ToNot(HaveOccurred()) + Expect(processorFactory.Type()).To(Equal(dataWorkPostprocess.Type)) + Expect(processorFactory.Quantity()).To(Equal(dataWorkPostprocess.Quantity)) + Expect(processorFactory.Frequency()).To(Equal(dataWorkPostprocess.Frequency)) + + processor, err := processorFactory.New() + Expect(err).ToNot(HaveOccurred()) + Expect(processor).ToNot(BeNil()) + }) + + It("NewProcessorFactory returns an error when the dependencies are invalid", func() { + dependencies.ClinicsClient = nil + processorFactory, err := dataWorkPostprocess.NewProcessorFactory(dependencies) + Expect(err).To(MatchError(ContainSubstring("clinics client is missing"))) + Expect(processorFactory).To(BeNil()) + }) + }) +}) diff --git a/data/work/postprocess/processor.go b/data/work/postprocess/processor.go new file mode 100644 index 0000000000..42fc6b4712 --- /dev/null +++ b/data/work/postprocess/processor.go @@ -0,0 +1,182 @@ +package postprocess + +import ( + "context" + "slices" + "time" + + "github.com/tidepool-org/platform/errors" + "github.com/tidepool-org/platform/log" + "github.com/tidepool-org/platform/metadata" + "github.com/tidepool-org/platform/page" + "github.com/tidepool-org/platform/pointer" + userWork "github.com/tidepool-org/platform/user/work" + "github.com/tidepool-org/platform/work" + workBase "github.com/tidepool-org/platform/work/base" +) + +type UserMixin = userWork.MixinFromWork + +type Processor struct { + *workBase.Processor[Metadata] + UserMixin + Summarizers + ClinicsClient + + pendingBuilder *deferredPendingBuilder + wrk *work.Work +} + +func NewProcessor(dependencies Dependencies) (*Processor, error) { + if err := dependencies.Validate(); err != nil { + return nil, errors.Wrap(err, "dependencies is invalid") + } + + pendingBuilder := &deferredPendingBuilder{} + processResultBuilder := &workBase.ProcessResultBuilder{ + ProcessResultPendingBuilder: pendingBuilder, + ProcessResultFailingBuilder: &workBase.ExponentialProcessResultFailingBuilder{ + Duration: FailingRetryDuration, + DurationJitter: FailingRetryDurationJitter, + DurationMaximum: pointer.From(FailingRetryDurationMaximum), + }, + } + + processor, err := workBase.NewProcessor[Metadata](dependencies.Dependencies, processResultBuilder) + if err != nil { + return nil, errors.Wrap(err, "unable to create processor") + } + userMixin, err := userWork.NewMixinFromWork(processor, dependencies.UserClient, &processor.Metadata().Metadata) + if err != nil { + return nil, errors.Wrap(err, "unable to create user mixin") + } + + return &Processor{ + Processor: processor, + UserMixin: userMixin, + Summarizers: dependencies.Summarizers, + ClinicsClient: dependencies.ClinicsClient, + pendingBuilder: pendingBuilder, + }, nil +} + +func (p *Processor) Process(ctx context.Context, wrk *work.Work, processingUpdater work.ProcessingUpdater) *work.ProcessResult { + p.wrk = wrk + return append(p.ProcessPipeline(ctx, wrk, processingUpdater), + p.FetchUserFromWorkMetadata, + p.absorbPending, + p.updateSummaries, + p.triggerElectronicHealthRecordSync, + ).Process(p.Delete) +} + +// absorbPending absorbs reasons of other pending work items in this serial group +// +// The reasons are persisted before the work is deleted, so that a failure between the +// two leaves the reasons reported twice rather than not at all. +func (p *Processor) absorbPending() *work.ProcessResult { + filter := &work.Filter{ + Types: pointer.FromAny([]string{Type}), + State: pointer.FromAny(work.StatePending), + GroupID: pointer.FromString(IDFromUserID(*p.User().UserID)), + } + + wrks, err := page.Collect(func(pagination page.Pagination) ([]*work.Work, error) { + return p.WorkClient().List(p.Context(), filter, &pagination) + }) + if err != nil { + return p.Failing(errors.Wrap(err, "unable to list work")) + } + + var absorbed []*work.Work + var deferredUntil time.Time + reasons := p.Metadata().Reasons + for _, wrk := range wrks { + workMetadata, err := metadata.Decode[Metadata](p.Context(), wrk.Metadata) + if err != nil { + return p.Failed(errors.Wrap(err, "unable to decode metadata")) + } else if workMetadata == nil { + return p.Failed(errors.New("metadata is missing")) + } + + absorbed = append(absorbed, wrk) + reasons = normalizeReasons(reasons, workMetadata.Reasons) + if wrk.ProcessingAvailableTime.After(deferredUntil) { + deferredUntil = wrk.ProcessingAvailableTime + } + } + if len(absorbed) == 0 { + return nil + } + + p.Metadata().Reasons = reasons + if result := p.ProcessingUpdate(); result != nil { + return result + } + + for _, wrk := range absorbed { + if _, err = p.WorkClient().Delete(p.Context(), wrk.ID, nil); err != nil { + return p.Failing(errors.Wrap(err, "unable to delete work")) + } + } + + if len(absorbed) > 0 { + log.LoggerFromContext(p.Context()).WithFields(log.Fields{ + "count": len(absorbed), + "reasons": reasons, + }).Debug("absorbed pending work for the user") + } + + // An upload reporting only full batches is still in progress + if deferredUntil.After(p.Now()) && shouldDeffer(reasons) { + p.pendingBuilder.availableTime = deferredUntil + return p.Pending() + } + + return nil +} + +func (p *Processor) updateSummaries() *work.ProcessResult { + if err := p.UpdateSummaries(p.Context(), *p.User().UserID); err != nil { + return p.Failing(err) + } + + log.LoggerFromContext(p.Context()).WithField("reasons", p.Metadata().Reasons).Info("calculated the summaries of the user") + + return nil +} + +// triggerElectronicHealthRecordSync reports the data of the user to any electronic health record it is +// shared with, after the summaries it reports are calculated. It is requested at least once per change +// reported, as a request repeated reports the same data again rather than reporting it twice. +func (p *Processor) triggerElectronicHealthRecordSync() *work.ProcessResult { + if !TriggersEHRSync(p.Metadata().Reasons) { + return nil + } + + if err := p.ClinicsClient.SyncEHRDataForPatient(p.Context(), *p.User().UserID); err != nil { + return p.Failing(errors.Wrap(err, "unable to trigger EHR sync")) + } + + log.LoggerFromContext(p.Context()).Info("triggerred EHR sync") + + return nil +} + +// deferredPendingBuilder defers work until a time decided while processing, rather than by a duration +// fixed when the processor is created +type deferredPendingBuilder struct { + availableTime time.Time +} + +func (d *deferredPendingBuilder) ProcessingAvailableTime(ctx context.Context, wrk *work.Work, tm time.Time) time.Time { + if d.availableTime.After(tm) { + return d.availableTime + } + return tm +} + +// shouldDeffer returns true when jellyfish uploads a full batch +func shouldDeffer(reasons []string) bool { + return !slices.ContainsFunc(reasons, func(reason string) bool { return reason != ReasonLegacyDataAdded }) +} diff --git a/data/work/postprocess/processor_test.go b/data/work/postprocess/processor_test.go new file mode 100644 index 0000000000..e4a86f65eb --- /dev/null +++ b/data/work/postprocess/processor_test.go @@ -0,0 +1,273 @@ +package postprocess_test + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + "go.uber.org/mock/gomock" + + clinicsTest "github.com/tidepool-org/platform/clinics/test" + dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" + dataWorkPostprocessTest "github.com/tidepool-org/platform/data/work/postprocess/test" + errorsTest "github.com/tidepool-org/platform/errors/test" + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + "github.com/tidepool-org/platform/metadata" + "github.com/tidepool-org/platform/page" + "github.com/tidepool-org/platform/pointer" + "github.com/tidepool-org/platform/user" + userTest "github.com/tidepool-org/platform/user/test" + userWork "github.com/tidepool-org/platform/user/work" + "github.com/tidepool-org/platform/work" + workBase "github.com/tidepool-org/platform/work/base" + workTest "github.com/tidepool-org/platform/work/test" +) + +var _ = Describe("Processor", func() { + var controller *gomock.Controller + var workClient *workTest.MockClient + var summarizers *dataWorkPostprocessTest.MockSummarizers + var clinicsClient *clinicsTest.MockClient + var processingUpdater *workTest.MockProcessingUpdater + var userClient *userTest.MockClient + var fetchUser func() (*user.User, error) + var processor *dataWorkPostprocess.Processor + var ctx context.Context + var userID string + var wrk *work.Work + + newWork := func(state string, reasons []string, availableTime time.Time) *work.Work { + encoded, err := metadata.Encode(&dataWorkPostprocess.Metadata{ + Metadata: userWork.Metadata{UserID: pointer.FromString(userID)}, + Reasons: reasons, + }) + Expect(err).ToNot(HaveOccurred()) + return &work.Work{ + ID: workTest.RandomID(), + Type: dataWorkPostprocess.Type, + GroupID: pointer.FromString(dataWorkPostprocess.IDFromUserID(userID)), + SerialID: pointer.FromString(dataWorkPostprocess.IDFromUserID(userID)), + ProcessingAvailableTime: availableTime, + ProcessingTimeout: int(dataWorkPostprocess.ProcessingTimeout.Seconds()), + Metadata: encoded, + State: state, + Revision: 2, + } + } + + // Nothing else is pending for the user, which is the ordinary case. The work being processed is + // not reported, as only work that is pending is requested. + expectListNone := func() { + workClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, filter *work.Filter, _ *page.Pagination) ([]*work.Work, error) { + Expect(filter.Types).To(PointTo(ConsistOf(dataWorkPostprocess.Type))) + Expect(filter.State).To(PointTo(Equal(work.StatePending))) + Expect(filter.GroupID).To(PointTo(Equal(dataWorkPostprocess.IDFromUserID(userID)))) + return nil, nil + }) + } + + process := func() *work.ProcessResult { + return processor.Process(ctx, wrk, processingUpdater) + } + + BeforeEach(func() { + controller = gomock.NewController(GinkgoT()) + workClient = workTest.NewMockClient(controller) + summarizers = dataWorkPostprocessTest.NewMockSummarizers(controller) + clinicsClient = clinicsTest.NewMockClient(controller) + processingUpdater = workTest.NewMockProcessingUpdater(controller) + userClient = userTest.NewMockClient(controller) + ctx = log.NewContextWithLogger(context.Background(), logTest.NewLogger()) + userID = userTest.RandomUserID() + wrk = newWork(work.StateProcessing, []string{dataWorkPostprocess.ReasonDataAdded}, time.Now().Add(-time.Minute)) + + var err error + processor, err = dataWorkPostprocess.NewProcessor(dataWorkPostprocess.Dependencies{ + Dependencies: workBase.Dependencies{WorkClient: workClient}, + Summarizers: summarizers, + ClinicsClient: clinicsClient, + UserClient: userClient, + }) + Expect(err).ToNot(HaveOccurred()) + + // The user mixin fetches the user reported by the metadata before any step runs + fetchUser = func() (*user.User, error) { return &user.User{UserID: pointer.FromString(userID)}, nil } + userClient.EXPECT().Get(gomock.Any(), userID). + DoAndReturn(func(_ context.Context, _ string) (*user.User, error) { return fetchUser() }).AnyTimes() + }) + + AfterEach(func() { + controller.Finish() + }) + + It("calculates the summaries and deletes the work", func() { + expectListNone() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }) + + DescribeTable("requests a synchronization only for the reasons that report a change is complete", + func(reasons []string, expectSync bool) { + wrk = newWork(work.StateProcessing, reasons, time.Now().Add(-time.Minute)) + expectListNone() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + if expectSync { + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) + } + + Expect(process().Result).To(Equal(work.ResultDelete)) + }, + Entry("data added", []string{dataWorkPostprocess.ReasonDataAdded}, false), + Entry("schema migration", []string{dataWorkPostprocess.ReasonSchemaMigration}, false), + Entry("upload completed", []string{dataWorkPostprocess.ReasonUploadCompleted}, true), + Entry("legacy data added", []string{dataWorkPostprocess.ReasonLegacyDataAdded}, true), + Entry("data added and upload completed", + []string{dataWorkPostprocess.ReasonDataAdded, dataWorkPostprocess.ReasonUploadCompleted}, true), + ) + + // A synchronization reports the summaries, so it must not be requested before they are calculated + It("requests a synchronization only after the summaries are calculated", func() { + wrk = newWork(work.StateProcessing, []string{dataWorkPostprocess.ReasonUploadCompleted}, time.Now().Add(-time.Minute)) + expectListNone() + gomock.InOrder( + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil), + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil), + ) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }) + + It("fails without retrying when the work reports no user", func() { + wrk.Metadata = map[string]any{"reasons": []any{dataWorkPostprocess.ReasonDataAdded}} + + result := process() + Expect(result.Result).To(Equal(work.ResultFailed)) + Expect(result.FailedUpdate.FailedError.Error).To(MatchError(ContainSubstring("user id is missing"))) + }) + + It("fails without retrying when the user no longer exists", func() { + fetchUser = func() (*user.User, error) { return nil, nil } + + result := process() + Expect(result.Result).To(Equal(work.ResultFailed)) + Expect(result.FailedUpdate.FailedError.Error).To(MatchError(ContainSubstring("user is missing"))) + }) + + It("retries when the user cannot be fetched", func() { + fetchUser = func() (*user.User, error) { return nil, errorsTest.RandomError() } + + Expect(process().Result).To(Equal(work.ResultFailing)) + }) + + DescribeTable("retries when a step fails", + func(expect func()) { + wrk = newWork(work.StateProcessing, []string{dataWorkPostprocess.ReasonUploadCompleted}, time.Now().Add(-time.Minute)) + expect() + + result := process() + Expect(result.Result).To(Equal(work.ResultFailing)) + Expect(result.FailingUpdate.FailingRetryTime).To(BeTemporally(">", time.Now())) + }, + Entry("listing the work also pending", func() { + workClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errorsTest.RandomError()) + }), + Entry("calculating the summaries", func() { + expectListNone() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(errorsTest.RandomError()) + }), + Entry("requesting a synchronization", func() { + expectListNone() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(errorsTest.RandomError()) + }), + ) + + Context("with work also pending for the user", func() { + var sibling *work.Work + + expectListWithSibling := func() { + workClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*work.Work{sibling}, nil) + } + + // The reasons must be persisted before the work reporting them is deleted, so that a failure + // between the two leaves them reported twice rather than not at all + expectProcessingUpdateThenDelete := func() { + gomock.InOrder( + processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, update work.ProcessingUpdate) (*work.Work, error) { + Expect(update.Metadata).To(HaveKeyWithValue("reasons", + ConsistOf(dataWorkPostprocess.ReasonDataAdded, dataWorkPostprocess.ReasonUploadCompleted))) + updated := *wrk + updated.Metadata = update.Metadata + updated.Revision = wrk.Revision + 1 + return &updated, nil + }), + workClient.EXPECT().Delete(gomock.Any(), sibling.ID, gomock.Nil()).Return(sibling, nil), + ) + } + + BeforeEach(func() { + sibling = newWork(work.StatePending, []string{dataWorkPostprocess.ReasonUploadCompleted}, time.Now().Add(-time.Second)) + }) + + It("reports its reasons, deletes it, and processes once", func() { + expectListWithSibling() + expectProcessingUpdateThenDelete() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }) + + // An upload reporting only full batches is still in progress, so the summaries are not + // calculated for every batch so far. Jellyfish defers the same way. + Context("that is shouldDeffer and reports only a full batch", func() { + var deferredUntil time.Time + + BeforeEach(func() { + deferredUntil = time.Now().Add(dataWorkPostprocess.JellyfishQuietDelay) + wrk = newWork(work.StateProcessing, []string{dataWorkPostprocess.ReasonLegacyDataAdded}, time.Now().Add(-time.Minute)) + sibling = newWork(work.StatePending, []string{dataWorkPostprocess.ReasonLegacyDataAdded}, deferredUntil) + }) + + It("defers rather than calculating the summaries", func() { + expectListWithSibling() + processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).Return(wrk, nil) + workClient.EXPECT().Delete(gomock.Any(), sibling.ID, gomock.Nil()).Return(sibling, nil) + + result := process() + Expect(result.Result).To(Equal(work.ResultPending)) + Expect(result.PendingUpdate.ProcessingAvailableTime).To(BeTemporally("==", deferredUntil)) + }) + + It("calculates the summaries once a reason reports a change is complete", func() { + sibling = newWork(work.StatePending, []string{dataWorkPostprocess.ReasonUploadCompleted}, deferredUntil) + expectListWithSibling() + processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, update work.ProcessingUpdate) (*work.Work, error) { + updated := *wrk + updated.Metadata = update.Metadata + return &updated, nil + }) + workClient.EXPECT().Delete(gomock.Any(), sibling.ID, gomock.Nil()).Return(sibling, nil) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }) + }) + + It("retries when it cannot be deleted", func() { + expectListWithSibling() + processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).Return(wrk, nil) + workClient.EXPECT().Delete(gomock.Any(), sibling.ID, gomock.Nil()).Return(nil, errorsTest.RandomError()) + + Expect(process().Result).To(Equal(work.ResultFailing)) + }) + }) +}) diff --git a/data/work/postprocess/summarizers.go b/data/work/postprocess/summarizers.go new file mode 100644 index 0000000000..48cbcce073 --- /dev/null +++ b/data/work/postprocess/summarizers.go @@ -0,0 +1,39 @@ +package postprocess + +import ( + "context" + + "github.com/tidepool-org/platform/errors" + "github.com/tidepool-org/platform/summary" + summaryTypes "github.com/tidepool-org/platform/summary/types" +) + +//go:generate mockgen -source=summarizers.go -destination=test/summarizers_mocks.go -package=test -typed + +type Summarizers interface { + UpdateSummaries(ctx context.Context, userID string) error +} + +type summarizers struct { + registry *summary.SummarizerRegistry +} + +func NewSummarizers(registry *summary.SummarizerRegistry) (Summarizers, error) { + if registry == nil { + return nil, errors.New("summarizer registry is missing") + } + return &summarizers{registry: registry}, nil +} + +func (s *summarizers) UpdateSummaries(ctx context.Context, userID string) error { + if _, err := summary.GetSummarizer[*summaryTypes.CGMPeriods, *summaryTypes.GlucoseBucket](s.registry).UpdateSummary(ctx, userID); err != nil { + return errors.Wrapf(err, "unable to update %s summary", summaryTypes.SummaryTypeCGM) + } + if _, err := summary.GetSummarizer[*summaryTypes.BGMPeriods, *summaryTypes.GlucoseBucket](s.registry).UpdateSummary(ctx, userID); err != nil { + return errors.Wrapf(err, "unable to update %s summary", summaryTypes.SummaryTypeBGM) + } + if _, err := summary.GetSummarizer[*summaryTypes.ContinuousPeriods, *summaryTypes.ContinuousBucket](s.registry).UpdateSummary(ctx, userID); err != nil { + return errors.Wrapf(err, "unable to update %s summary", summaryTypes.SummaryTypeContinuous) + } + return nil +} diff --git a/data/work/postprocess/test/summarizers_mocks.go b/data/work/postprocess/test/summarizers_mocks.go new file mode 100644 index 0000000000..15c03b58ec --- /dev/null +++ b/data/work/postprocess/test/summarizers_mocks.go @@ -0,0 +1,79 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: summarizers.go +// +// Generated by this command: +// +// mockgen -source=summarizers.go -destination=test/summarizers_mocks.go -package=test -typed +// + +// Package test is a generated GoMock package. +package test + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockSummarizers is a mock of Summarizers interface. +type MockSummarizers struct { + ctrl *gomock.Controller + recorder *MockSummarizersMockRecorder + isgomock struct{} +} + +// MockSummarizersMockRecorder is the mock recorder for MockSummarizers. +type MockSummarizersMockRecorder struct { + mock *MockSummarizers +} + +// NewMockSummarizers creates a new mock instance. +func NewMockSummarizers(ctrl *gomock.Controller) *MockSummarizers { + mock := &MockSummarizers{ctrl: ctrl} + mock.recorder = &MockSummarizersMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSummarizers) EXPECT() *MockSummarizersMockRecorder { + return m.recorder +} + +// UpdateSummaries mocks base method. +func (m *MockSummarizers) UpdateSummaries(ctx context.Context, userID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateSummaries", ctx, userID) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateSummaries indicates an expected call of UpdateSummaries. +func (mr *MockSummarizersMockRecorder) UpdateSummaries(ctx, userID any) *MockSummarizersUpdateSummariesCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateSummaries", reflect.TypeOf((*MockSummarizers)(nil).UpdateSummaries), ctx, userID) + return &MockSummarizersUpdateSummariesCall{Call: call} +} + +// MockSummarizersUpdateSummariesCall wrap *gomock.Call +type MockSummarizersUpdateSummariesCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockSummarizersUpdateSummariesCall) Return(arg0 error) *MockSummarizersUpdateSummariesCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockSummarizersUpdateSummariesCall) Do(f func(context.Context, string) error) *MockSummarizersUpdateSummariesCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockSummarizersUpdateSummariesCall) DoAndReturn(f func(context.Context, string) error) *MockSummarizersUpdateSummariesCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/data/work/postprocess/work.go b/data/work/postprocess/work.go index 1f716b308c..736407f7bb 100644 --- a/data/work/postprocess/work.go +++ b/data/work/postprocess/work.go @@ -33,9 +33,7 @@ const ( // ReasonLegacyDataAdded reports jellyfish uploaded a full batch ReasonLegacyDataAdded = "LEGACY_DATA_ADDED" - ReasonDataDeleted = "DATA_DELETED" - ReasonContinuousDataDeleted = "CONTINUOUS_DATA_DELETED" - ReasonSchemaMigration = "SCHEMA_MIGRATION" + ReasonSchemaMigration = "SCHEMA_MIGRATION" ) const ( @@ -47,8 +45,6 @@ func Reasons() []string { ReasonDataAdded, ReasonUploadCompleted, ReasonLegacyDataAdded, - ReasonDataDeleted, - ReasonContinuousDataDeleted, ReasonSchemaMigration, } } @@ -57,25 +53,12 @@ func Reasons() []string { var ehrSyncReasons = mapset.NewSet( ReasonUploadCompleted, ReasonLegacyDataAdded, - ReasonDataDeleted, -) - -// Data deleted may leave no record behind reporting it was modified, so the summaries cannot be -// calculated from only the data modified since they were last calculated -var summaryResetReasons = mapset.NewSet( - ReasonDataDeleted, - ReasonContinuousDataDeleted, - ReasonSchemaMigration, ) func TriggersEHRSync(reasons []string) bool { return ehrSyncReasons.ContainsAny(reasons...) } -func RequiresSummaryReset(reasons []string) bool { - return summaryResetReasons.ContainsAny(reasons...) -} - // IDFromUserID returns both the serial id, which prevents the work of a user being processed // concurrently, and the group id, which is the scope the work of a user is coalesced within. Both // are the user, and neither is the data set, so that changes to any number of data sets, interleaved diff --git a/data/work/postprocess/work_test.go b/data/work/postprocess/work_test.go index 8eda771c7b..d7be9e06a0 100644 --- a/data/work/postprocess/work_test.go +++ b/data/work/postprocess/work_test.go @@ -41,8 +41,6 @@ var _ = Describe("Work", func() { "DATA_ADDED", "UPLOAD_COMPLETED", "LEGACY_DATA_ADDED", - "DATA_DELETED", - "CONTINUOUS_DATA_DELETED", "SCHEMA_MIGRATION", )) }) @@ -55,32 +53,13 @@ var _ = Describe("Work", func() { Entry("with no reasons", nil, false), Entry("with data added", []string{dataWorkPostprocess.ReasonDataAdded}, false), Entry("with schema migration", []string{dataWorkPostprocess.ReasonSchemaMigration}, false), - Entry("with continuous data deleted", []string{dataWorkPostprocess.ReasonContinuousDataDeleted}, false), Entry("with upload completed", []string{dataWorkPostprocess.ReasonUploadCompleted}, true), Entry("with legacy data added", []string{dataWorkPostprocess.ReasonLegacyDataAdded}, true), - Entry("with data deleted", []string{dataWorkPostprocess.ReasonDataDeleted}, true), Entry("with any reason triggering a synchronization", []string{dataWorkPostprocess.ReasonDataAdded, dataWorkPostprocess.ReasonUploadCompleted}, true), ) }) - Context("RequiresSummaryReset", func() { - DescribeTable("reports whether the reasons require the summaries to be recalculated in full", - func(reasons []string, expected bool) { - Expect(dataWorkPostprocess.RequiresSummaryReset(reasons)).To(Equal(expected)) - }, - Entry("with no reasons", nil, false), - Entry("with data added", []string{dataWorkPostprocess.ReasonDataAdded}, false), - Entry("with upload completed", []string{dataWorkPostprocess.ReasonUploadCompleted}, false), - Entry("with legacy data added", []string{dataWorkPostprocess.ReasonLegacyDataAdded}, false), - Entry("with data deleted", []string{dataWorkPostprocess.ReasonDataDeleted}, true), - Entry("with continuous data deleted", []string{dataWorkPostprocess.ReasonContinuousDataDeleted}, true), - Entry("with schema migration", []string{dataWorkPostprocess.ReasonSchemaMigration}, true), - Entry("with any reason requiring a reset", - []string{dataWorkPostprocess.ReasonDataAdded, dataWorkPostprocess.ReasonDataDeleted}, true), - ) - }) - Context("IDFromUserID", func() { var userID string diff --git a/work/store/structured/mongo/mongo.go b/work/store/structured/mongo/mongo.go index 829c3fbbca..23c5ff79ef 100644 --- a/work/store/structured/mongo/mongo.go +++ b/work/store/structured/mongo/mongo.go @@ -334,6 +334,9 @@ func (s *Store) List(ctx context.Context, filter *work.Filter, pagination *page. if filter.GroupID != nil { query["groupId"] = *filter.GroupID } + if filter.State != nil { + query["state"] = *filter.State + } opts := storeStructuredMongo.FindWithPagination(pagination). SetSort(bson.M{"createdTime": 1}) From 79840ba19b8b447b4c147674521f4d28d626e056 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Thu, 13 Aug 2026 16:06:35 +0300 Subject: [PATCH 07/20] Create upload postprocess work on upload instead of marking summaries outdated --- data/service/api/v1/datasets_data_create.go | 12 +- data/service/api/v1/datasets_update.go | 12 +- data/service/service/standard.go | 16 +- plugin/abbott/abbott/work/work.go | 3 - summary/client/client.go | 33 --- summary/summary.go | 55 ----- summary/test/summary_mocks.go | 97 --------- summary/types/uploads_test.go | 229 -------------------- 8 files changed, 10 insertions(+), 447 deletions(-) delete mode 100644 summary/client/client.go delete mode 100644 summary/types/uploads_test.go diff --git a/data/service/api/v1/datasets_data_create.go b/data/service/api/v1/datasets_data_create.go index 315d60e472..73930fd151 100644 --- a/data/service/api/v1/datasets_data_create.go +++ b/data/service/api/v1/datasets_data_create.go @@ -7,9 +7,6 @@ import ( "strconv" "strings" - "github.com/tidepool-org/platform/summary" - "github.com/tidepool-org/platform/summary/types" - "github.com/ant0ine/go-json-rest/rest" "github.com/golang-jwt/jwt/v4" @@ -17,6 +14,7 @@ import ( dataNormalizer "github.com/tidepool-org/platform/data/normalizer" dataService "github.com/tidepool-org/platform/data/service" dataTypesFactory "github.com/tidepool-org/platform/data/types/factory" + dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/permission" "github.com/tidepool-org/platform/request" @@ -120,11 +118,11 @@ func DataSetsDataCreate(dataServiceContext dataService.Context) { return } - updatesSummary := make(map[string]struct{}) - for _, datum := range datumArray { - summary.CheckDatumUpdatesSummary(updatesSummary, datum) + // Reported for every upload, whatever it contains, as data that feeds no summary is still + // postprocessed + if err = dataWorkPostprocess.Enqueue(ctx, dataServiceContext.WorkClient(), *dataSet.UserID, dataWorkPostprocess.ReasonDataAdded); err != nil { + lgr.WithError(err).Error("Unable to report data added") } - summary.MaybeUpdateSummary(ctx, dataServiceContext.SummarizerRegistry(), updatesSummary, *dataSet.UserID, types.OutdatedReasonDataAdded) if err = dataServiceContext.MetricClient().RecordMetric(ctx, "data_sets_data_create", map[string]string{"count": strconv.Itoa(len(datumArray))}); err != nil { lgr.WithError(err).Error("Unable to record metric") diff --git a/data/service/api/v1/datasets_update.go b/data/service/api/v1/datasets_update.go index 9fcf03f126..71622b5249 100644 --- a/data/service/api/v1/datasets_update.go +++ b/data/service/api/v1/datasets_update.go @@ -3,11 +3,9 @@ package v1 import ( "net/http" - "github.com/tidepool-org/platform/summary" - "github.com/tidepool-org/platform/summary/types" - "github.com/tidepool-org/platform/data" dataService "github.com/tidepool-org/platform/data/service" + dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/permission" "github.com/tidepool-org/platform/pointer" @@ -93,13 +91,9 @@ func DataSetsUpdate(dataServiceContext dataService.Context) { return } - // create map of all types, this will create redundant summaries, but will be cleaned up upon processing - updatesSummary := make(map[string]struct{}) - for _, typ := range types.AllSummaryTypes { - updatesSummary[typ] = struct{}{} + if err = dataWorkPostprocess.Enqueue(ctx, dataServiceContext.WorkClient(), *dataSet.UserID, dataWorkPostprocess.ReasonUploadCompleted); err != nil { + lgr.WithError(err).Error("Unable to report upload completed") } - - summary.MaybeUpdateSummary(ctx, dataServiceContext.SummarizerRegistry(), updatesSummary, *dataSet.UserID, types.OutdatedReasonUploadCompleted) } if err = dataServiceContext.MetricClient().RecordMetric(ctx, "data_sets_update"); err != nil { diff --git a/data/service/service/standard.go b/data/service/service/standard.go index 34042637fb..ed87893698 100644 --- a/data/service/service/standard.go +++ b/data/service/service/standard.go @@ -61,7 +61,6 @@ import ( serviceService "github.com/tidepool-org/platform/service/service" storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" "github.com/tidepool-org/platform/summary" - summaryClient "github.com/tidepool-org/platform/summary/client" synctaskStoreMongo "github.com/tidepool-org/platform/synctask/store/mongo" "github.com/tidepool-org/platform/twiist" "github.com/tidepool-org/platform/user" @@ -96,7 +95,6 @@ type Standard struct { dataSourceClient *dataSourceServiceClient.Client mailerClient mailer.Client summarizerRegistry *summary.SummarizerRegistry - summaryClient *summaryClient.Client workClient *workService.Client notificationsHistoryRecorder notificationsHistory.Recorder abbottClient *abbottClient.Client @@ -163,7 +161,7 @@ func (s *Standard) Initialize(provider application.Provider) error { if err := s.initializeUserClient(); err != nil { return err } - if err := s.initializeSummaryClient(); err != nil { + if err := s.initializeSummarizerRegistry(); err != nil { return err } if err := s.initializeConfirmationClient(); err != nil { @@ -219,7 +217,6 @@ func (s *Standard) Terminate() { s.ouraClient = nil s.abbottClient = nil s.workClient = nil - s.summaryClient = nil s.dataSourceClient = nil s.dataRawClient = nil s.dataClient = nil @@ -619,7 +616,7 @@ func (s *Standard) initializeConfirmationClient() error { return nil } -func (s *Standard) initializeSummaryClient() error { +func (s *Standard) initializeSummarizerRegistry() error { s.Logger().Debug("Creating summarizer registry") s.summarizerRegistry = summary.New( @@ -629,14 +626,6 @@ func (s *Standard) initializeSummaryClient() error { s.dataStore.GetClient(), ) - s.Logger().Debug("Creating summary client") - - clnt, err := summaryClient.New(s.summarizerRegistry) - if err != nil { - return errors.Wrap(err, "unable to create summary client") - } - s.summaryClient = clnt - return nil } @@ -836,7 +825,6 @@ func (s *Standard) initializeWorkProcessorFactories() error { DataDeduplicatorFactory: s.dataDeduplicatorFactory, DataSetClient: s.dataClient, DataSourceClient: s.dataSourceClient, - SummaryClient: s.summaryClient, ProviderSessionClient: s.AuthClient(), DataRawClient: s.dataRawClient, AbbottClient: s.abbottClient, diff --git a/plugin/abbott/abbott/work/work.go b/plugin/abbott/abbott/work/work.go index fe9de37b70..5dfa54b29c 100644 --- a/plugin/abbott/abbott/work/work.go +++ b/plugin/abbott/abbott/work/work.go @@ -13,8 +13,6 @@ type DataSetClient any type DataSourceClient any -type SummaryClient any - type ProviderSessionClient any type AbbottClient any @@ -27,7 +25,6 @@ type ProcessorDependencies struct { DataRawClient DataRawClient DataSetClient DataSetClient DataSourceClient DataSourceClient - SummaryClient SummaryClient ProviderSessionClient ProviderSessionClient AbbottClient AbbottClient } diff --git a/summary/client/client.go b/summary/client/client.go deleted file mode 100644 index 34684ca7ca..0000000000 --- a/summary/client/client.go +++ /dev/null @@ -1,33 +0,0 @@ -package client - -import ( - "context" - "errors" - - "github.com/tidepool-org/platform/data" - "github.com/tidepool-org/platform/summary" -) - -type Client struct { - summarizerRegistry *summary.SummarizerRegistry -} - -func New(summarizerRegistry *summary.SummarizerRegistry) (*Client, error) { - if summarizerRegistry == nil { - return nil, errors.New("summarizer registry missing") - } - - return &Client{ - summarizerRegistry: summarizerRegistry, - }, nil -} - -func (s *Client) CheckDataUpdatesSummary(datumArray data.Data, updatesSummary map[string]struct{}) { - for _, datum := range datumArray { - summary.CheckDatumUpdatesSummary(updatesSummary, datum) - } -} - -func (s *Client) MaybeUpdateSummary(ctx context.Context, userID string, reason string, updatesSummary map[string]struct{}) { - summary.MaybeUpdateSummary(ctx, s.summarizerRegistry, updatesSummary, userID, reason) -} diff --git a/summary/summary.go b/summary/summary.go index ea534cdd51..400f749694 100644 --- a/summary/summary.go +++ b/summary/summary.go @@ -20,11 +20,6 @@ import ( //go:generate mockgen -source=summary.go -destination=test/summary_mocks.go -package=test -typed -type Client interface { - CheckDataUpdatesSummary(datumArray data.Data, updatesSummary map[string]struct{}) - MaybeUpdateSummary(ctx context.Context, userID string, reason string, updatesSummary map[string]struct{}) -} - type SummarizerRegistry struct { summarizers map[string]any } @@ -293,56 +288,6 @@ func (gs *GlucoseSummarizer[PP, PB, P, B]) UpdateBuckets(ctx context.Context, us return nil } -func MaybeUpdateSummary(ctx context.Context, registry *SummarizerRegistry, updatesSummary map[string]struct{}, userId, reason string) map[string]*time.Time { - outdatedSinceMap := make(map[string]*time.Time) - lgr := log.LoggerFromContext(ctx) - - if _, ok := updatesSummary[types.SummaryTypeCGM]; ok { - summarizer := GetSummarizer[*types.CGMPeriods, *types.GlucoseBucket](registry) - outdatedSince, err := summarizer.SetOutdated(ctx, userId, reason) - if err != nil { - lgr.WithError(err).Error("Unable to set cgm summary outdated") - } - outdatedSinceMap[types.SummaryTypeCGM] = outdatedSince - } - - if _, ok := updatesSummary[types.SummaryTypeBGM]; ok { - summarizer := GetSummarizer[*types.BGMPeriods, *types.GlucoseBucket](registry) - outdatedSince, err := summarizer.SetOutdated(ctx, userId, reason) - if err != nil { - lgr.WithError(err).Error("Unable to set bgm summary outdated") - } - outdatedSinceMap[types.SummaryTypeBGM] = outdatedSince - } - - if _, ok := updatesSummary[types.SummaryTypeContinuous]; ok { - summarizer := GetSummarizer[*types.ContinuousPeriods, *types.ContinuousBucket](registry) - outdatedSince, err := summarizer.SetOutdated(ctx, userId, reason) - if err != nil { - lgr.WithError(err).Error("Unable to set continuous summary outdated") - } - outdatedSinceMap[types.SummaryTypeContinuous] = outdatedSince - } - - return outdatedSinceMap -} - -func CheckDatumUpdatesSummary(updatesSummary map[string]struct{}, datum data.Datum) { - twoYearsPast := time.Now().UTC().AddDate(0, -24, 0) - oneDayFuture := time.Now().UTC().AddDate(0, 0, 1) - - // we only update summaries if the data is both of a relevant type, and being uploaded as "active" - // it also must be recent enough, within the past 2 years, and no more than 1d into the future - if datum.IsActive() { - typ := datum.GetType() - if types.DeviceDataTypesSet.Contains(typ) && datum.GetTime().Before(oneDayFuture) && datum.GetTime().After(twoYearsPast) { - for _, summaryType := range types.DeviceDataToSummaryTypes[typ] { - updatesSummary[summaryType] = struct{}{} - } - } - } -} - func NewContinuousSummarizer(collection *storeStructuredMongo.Repository, bucketsCollection *storeStructuredMongo.Repository, dataFetcher fetcher.DeviceDataFetcher, mongoClient *mongo.Client) Summarizer[*types.ContinuousPeriods, *types.ContinuousBucket, types.ContinuousPeriods, types.ContinuousBucket] { return &GlucoseSummarizer[*types.ContinuousPeriods, *types.ContinuousBucket, types.ContinuousPeriods, types.ContinuousBucket]{ cursorFactory: func(c *mongo.Cursor) fetcher.DeviceDataCursor { diff --git a/summary/test/summary_mocks.go b/summary/test/summary_mocks.go index eac680f8b8..ceaafcd81c 100644 --- a/summary/test/summary_mocks.go +++ b/summary/test/summary_mocks.go @@ -17,107 +17,10 @@ import ( mongo "go.mongodb.org/mongo-driver/mongo" gomock "go.uber.org/mock/gomock" - data "github.com/tidepool-org/platform/data" page "github.com/tidepool-org/platform/page" types "github.com/tidepool-org/platform/summary/types" ) -// MockClient is a mock of Client interface. -type MockClient struct { - ctrl *gomock.Controller - recorder *MockClientMockRecorder - isgomock struct{} -} - -// MockClientMockRecorder is the mock recorder for MockClient. -type MockClientMockRecorder struct { - mock *MockClient -} - -// NewMockClient creates a new mock instance. -func NewMockClient(ctrl *gomock.Controller) *MockClient { - mock := &MockClient{ctrl: ctrl} - mock.recorder = &MockClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockClient) EXPECT() *MockClientMockRecorder { - return m.recorder -} - -// CheckDataUpdatesSummary mocks base method. -func (m *MockClient) CheckDataUpdatesSummary(datumArray data.Data, updatesSummary map[string]struct{}) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "CheckDataUpdatesSummary", datumArray, updatesSummary) -} - -// CheckDataUpdatesSummary indicates an expected call of CheckDataUpdatesSummary. -func (mr *MockClientMockRecorder) CheckDataUpdatesSummary(datumArray, updatesSummary any) *MockClientCheckDataUpdatesSummaryCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckDataUpdatesSummary", reflect.TypeOf((*MockClient)(nil).CheckDataUpdatesSummary), datumArray, updatesSummary) - return &MockClientCheckDataUpdatesSummaryCall{Call: call} -} - -// MockClientCheckDataUpdatesSummaryCall wrap *gomock.Call -type MockClientCheckDataUpdatesSummaryCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockClientCheckDataUpdatesSummaryCall) Return() *MockClientCheckDataUpdatesSummaryCall { - c.Call = c.Call.Return() - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockClientCheckDataUpdatesSummaryCall) Do(f func(data.Data, map[string]struct{})) *MockClientCheckDataUpdatesSummaryCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientCheckDataUpdatesSummaryCall) DoAndReturn(f func(data.Data, map[string]struct{})) *MockClientCheckDataUpdatesSummaryCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// MaybeUpdateSummary mocks base method. -func (m *MockClient) MaybeUpdateSummary(ctx context.Context, userID, reason string, updatesSummary map[string]struct{}) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "MaybeUpdateSummary", ctx, userID, reason, updatesSummary) -} - -// MaybeUpdateSummary indicates an expected call of MaybeUpdateSummary. -func (mr *MockClientMockRecorder) MaybeUpdateSummary(ctx, userID, reason, updatesSummary any) *MockClientMaybeUpdateSummaryCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MaybeUpdateSummary", reflect.TypeOf((*MockClient)(nil).MaybeUpdateSummary), ctx, userID, reason, updatesSummary) - return &MockClientMaybeUpdateSummaryCall{Call: call} -} - -// MockClientMaybeUpdateSummaryCall wrap *gomock.Call -type MockClientMaybeUpdateSummaryCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockClientMaybeUpdateSummaryCall) Return() *MockClientMaybeUpdateSummaryCall { - c.Call = c.Call.Return() - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockClientMaybeUpdateSummaryCall) Do(f func(context.Context, string, string, map[string]struct{})) *MockClientMaybeUpdateSummaryCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientMaybeUpdateSummaryCall) DoAndReturn(f func(context.Context, string, string, map[string]struct{})) *MockClientMaybeUpdateSummaryCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // MockSummarizer is a mock of Summarizer interface. type MockSummarizer[PP types.PeriodsPt[P, PB, B], PB types.BucketDataPt[B], P types.Periods, B types.BucketData] struct { ctrl *gomock.Controller diff --git a/summary/types/uploads_test.go b/summary/types/uploads_test.go deleted file mode 100644 index 83d2777f6f..0000000000 --- a/summary/types/uploads_test.go +++ /dev/null @@ -1,229 +0,0 @@ -package types_test - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "go.mongodb.org/mongo-driver/bson" - - dataStore "github.com/tidepool-org/platform/data/store" - dataStoreMongo "github.com/tidepool-org/platform/data/store/mongo" - "github.com/tidepool-org/platform/data/types/blood/glucose/continuous" - "github.com/tidepool-org/platform/data/types/blood/glucose/selfmonitored" - "github.com/tidepool-org/platform/data/types/food" - "github.com/tidepool-org/platform/log" - logTest "github.com/tidepool-org/platform/log/test" - storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" - "github.com/tidepool-org/platform/summary" - dataStoreSummary "github.com/tidepool-org/platform/summary/store" - . "github.com/tidepool-org/platform/summary/test" - . "github.com/tidepool-org/platform/summary/types" - userTest "github.com/tidepool-org/platform/user/test" -) - -var _ = Describe("Upload Helpers", func() { - var empty struct{} - var logger log.Logger - var ctx context.Context - var registry *summary.SummarizerRegistry - var store *dataStoreMongo.Store - var summaryRepo *storeStructuredMongo.Repository - var bucketsRepo *storeStructuredMongo.Repository - var dataRepo dataStore.DataRepository - var userId string - var cgmStore *dataStoreSummary.Summaries[*CGMPeriods, *GlucoseBucket, CGMPeriods, GlucoseBucket] - var bgmStore *dataStoreSummary.Summaries[*BGMPeriods, *GlucoseBucket, BGMPeriods, GlucoseBucket] - var continuousStore *dataStoreSummary.Summaries[*ContinuousPeriods, *ContinuousBucket, ContinuousPeriods, ContinuousBucket] - - BeforeEach(func() { - logger = logTest.NewLogger() - ctx = log.NewContextWithLogger(context.Background(), logger) - store = GetSuiteStore() - - summaryRepo = store.NewSummaryRepository().GetStore() - bucketsRepo = store.NewBucketsRepository().GetStore() - dataRepo = store.NewDataRepository() - registry = summary.New(summaryRepo, bucketsRepo, dataRepo, store.GetClient()) - userId = userTest.RandomUserID() - - cgmStore = dataStoreSummary.NewSummaries[*CGMPeriods, *GlucoseBucket](summaryRepo) - bgmStore = dataStoreSummary.NewSummaries[*BGMPeriods, *GlucoseBucket](summaryRepo) - continuousStore = dataStoreSummary.NewSummaries[*ContinuousPeriods, *ContinuousBucket](summaryRepo) - }) - - AfterEach(func() { - if summaryRepo != nil { - _, err := summaryRepo.DeleteMany(ctx, bson.D{}) - Expect(err).To(Succeed()) - } - }) - - Context("MaybeUpdateSummary", func() { - - It("with all summary types outdated", func() { - updatesSummary := map[string]struct{}{ - "cgm": empty, - "bgm": empty, - "con": empty, - } - - outdatedSinceMap := summary.MaybeUpdateSummary(ctx, registry, updatesSummary, userId, OutdatedReasonDataAdded) - Expect(outdatedSinceMap).To(HaveLen(3)) - Expect(outdatedSinceMap).To(HaveKey(SummaryTypeCGM)) - Expect(outdatedSinceMap).To(HaveKey(SummaryTypeBGM)) - Expect(outdatedSinceMap).To(HaveKey(SummaryTypeContinuous)) - - userCgmSummary, err := cgmStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(*userCgmSummary.Dates.OutdatedSince).To(Equal(*outdatedSinceMap[SummaryTypeCGM])) - - userBgmSummary, err := bgmStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(*userBgmSummary.Dates.OutdatedSince).To(Equal(*outdatedSinceMap[SummaryTypeBGM])) - - userContinuousSummary, err := continuousStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(*userContinuousSummary.Dates.OutdatedSince).To(Equal(*outdatedSinceMap[SummaryTypeContinuous])) - }) - - It("with cgm summary type outdated", func() { - updatesSummary := map[string]struct{}{ - "cgm": empty, - } - - outdatedSinceMap := summary.MaybeUpdateSummary(ctx, registry, updatesSummary, userId, OutdatedReasonDataAdded) - Expect(outdatedSinceMap).To(HaveLen(1)) - Expect(outdatedSinceMap).To(HaveKey(SummaryTypeCGM)) - - userCgmSummary, err := cgmStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(*userCgmSummary.Dates.OutdatedSince).To(Equal(*outdatedSinceMap[SummaryTypeCGM])) - - userBgmSummary, err := bgmStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(userBgmSummary).To(BeNil()) - - userContinuousSummary, err := continuousStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(userContinuousSummary).To(BeNil()) - }) - - It("with bgm summary type outdated", func() { - updatesSummary := map[string]struct{}{ - "bgm": empty, - } - - outdatedSinceMap := summary.MaybeUpdateSummary(ctx, registry, updatesSummary, userId, OutdatedReasonDataAdded) - Expect(outdatedSinceMap).To(HaveLen(1)) - Expect(outdatedSinceMap).To(HaveKey(SummaryTypeBGM)) - - userCgmSummary, err := cgmStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(userCgmSummary).To(BeNil()) - - userBgmSummary, err := bgmStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(*userBgmSummary.Dates.OutdatedSince).To(Equal(*outdatedSinceMap[SummaryTypeBGM])) - - userContinuousSummary, err := continuousStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(userContinuousSummary).To(BeNil()) - }) - - It("with continuous summary type outdated", func() { - updatesSummary := map[string]struct{}{ - "con": empty, - } - - outdatedSinceMap := summary.MaybeUpdateSummary(ctx, registry, updatesSummary, userId, OutdatedReasonDataAdded) - Expect(outdatedSinceMap).To(HaveLen(1)) - Expect(outdatedSinceMap).To(HaveKey(SummaryTypeContinuous)) - - userCgmSummary, err := cgmStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(userCgmSummary).To(BeNil()) - - userBgmSummary, err := bgmStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(userBgmSummary).To(BeNil()) - - userContinuousSummary, err := continuousStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(*userContinuousSummary.Dates.OutdatedSince).To(Equal(*outdatedSinceMap[SummaryTypeContinuous])) - }) - - It("with unknown summary type outdated", func() { - updatesSummary := map[string]struct{}{ - "food": empty, - } - - outdatedSinceMap := summary.MaybeUpdateSummary(ctx, registry, updatesSummary, userId, OutdatedReasonDataAdded) - Expect(outdatedSinceMap).To(BeEmpty()) - }) - }) - - Context("CheckDatumUpdatesSummary", func() { - It("with non-summary type", func() { - var updatesSummary map[string]struct{} - datum := NewDatum(food.Type) - - summary.CheckDatumUpdatesSummary(updatesSummary, datum) - Expect(updatesSummary).To(BeEmpty()) - }) - - It("with too old summary affecting record", func() { - updatesSummary := make(map[string]struct{}) - datum := NewOldDatum(continuous.Type) - - summary.CheckDatumUpdatesSummary(updatesSummary, datum) - Expect(updatesSummary).To(HaveLen(0)) - }) - - It("with future summary affecting record", func() { - updatesSummary := make(map[string]struct{}) - datum := NewNewDatum(continuous.Type) - - summary.CheckDatumUpdatesSummary(updatesSummary, datum) - Expect(updatesSummary).To(HaveLen(0)) - }) - - It("with CGM summary affecting record", func() { - updatesSummary := make(map[string]struct{}) - datum := NewDatum(continuous.Type) - - summary.CheckDatumUpdatesSummary(updatesSummary, datum) - Expect(updatesSummary).To(HaveLen(2)) - Expect(updatesSummary).To(HaveKey(SummaryTypeCGM)) - Expect(updatesSummary).To(HaveKey(SummaryTypeContinuous)) - }) - - It("with BGM summary affecting record", func() { - updatesSummary := make(map[string]struct{}) - datum := NewDatum(selfmonitored.Type) - - summary.CheckDatumUpdatesSummary(updatesSummary, datum) - Expect(updatesSummary).To(HaveLen(2)) - Expect(updatesSummary).To(HaveKey(SummaryTypeBGM)) - Expect(updatesSummary).To(HaveKey(SummaryTypeContinuous)) - }) - - It("with inactive BGM summary affecting record", func() { - updatesSummary := make(map[string]struct{}) - datum := NewDatum(selfmonitored.Type) - datum.Active = false - - summary.CheckDatumUpdatesSummary(updatesSummary, datum) - Expect(updatesSummary).To(HaveLen(0)) - }) - - It("with inactive CGM summary affecting record", func() { - updatesSummary := make(map[string]struct{}) - datum := NewDatum(continuous.Type) - datum.Active = false - - summary.CheckDatumUpdatesSummary(updatesSummary, datum) - Expect(updatesSummary).To(HaveLen(0)) - }) - }) -}) From f64a7e97f0a565f6aa8211b456fa1feaef4a4e79 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Fri, 14 Aug 2026 11:07:43 +0300 Subject: [PATCH 08/20] Add outdated summary sweeper replacing the summary update task runner --- data/service/service/standard.go | 23 ++ data/store/mongo/mongo_summary.go | 10 + data/store/mongo/mongo_test.go | 8 + data/work/postprocess/processor.go | 8 +- data/work/postprocess/processor_test.go | 20 +- data/work/postprocess/work.go | 13 ++ data/work/sweep/outdated/outdated.go | 143 ++++++++++++ data/work/sweep/outdated/outdated_test.go | 206 ++++++++++++++++++ .../sweep/outdated/test/outdated_mocks.go | 121 ++++++++++ summary/store/summary.go | 93 +++++++- summary/store/summary_test.go | 105 ++++++++- 11 files changed, 739 insertions(+), 11 deletions(-) create mode 100644 data/work/sweep/outdated/outdated.go create mode 100644 data/work/sweep/outdated/outdated_test.go create mode 100644 data/work/sweep/outdated/test/outdated_mocks.go diff --git a/data/service/service/standard.go b/data/service/service/standard.go index ed87893698..73cb361bdf 100644 --- a/data/service/service/standard.go +++ b/data/service/service/standard.go @@ -33,6 +33,7 @@ import ( dataSourceStoreStructuredMongo "github.com/tidepool-org/platform/data/source/store/structured/mongo" dataStoreMongo "github.com/tidepool-org/platform/data/store/mongo" dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" + dataWorkSweepOutdated "github.com/tidepool-org/platform/data/work/sweep/outdated" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/events" "github.com/tidepool-org/platform/log" @@ -61,6 +62,7 @@ import ( serviceService "github.com/tidepool-org/platform/service/service" storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" "github.com/tidepool-org/platform/summary" + summaryStore "github.com/tidepool-org/platform/summary/store" synctaskStoreMongo "github.com/tidepool-org/platform/synctask/store/mongo" "github.com/tidepool-org/platform/twiist" "github.com/tidepool-org/platform/user" @@ -95,6 +97,7 @@ type Standard struct { dataSourceClient *dataSourceServiceClient.Client mailerClient mailer.Client summarizerRegistry *summary.SummarizerRegistry + typelessSummaries *summaryStore.TypelessSummaries workClient *workService.Client notificationsHistoryRecorder notificationsHistory.Recorder abbottClient *abbottClient.Client @@ -625,6 +628,7 @@ func (s *Standard) initializeSummarizerRegistry() error { s.dataStore.NewDataRepository(), s.dataStore.GetClient(), ) + s.typelessSummaries = summaryStore.NewTypeless(s.dataStore.NewSummaryRepository().GetStore()) return nil } @@ -817,6 +821,17 @@ func (s *Standard) initializeWorkProcessorFactories() error { processorFactories = append(processorFactories, processorFactory) } + s.Logger().Debug("Creating data sweep outdated work processor factory") + + if processorFactory, err := dataWorkSweepOutdated.NewProcessorFactory(dataWorkSweepOutdated.Dependencies{ + Dependencies: dependencies, + Summaries: s.typelessSummaries, + }); err != nil { + return errors.Wrap(err, "unable to create data sweep outdated work processor factory") + } else { + processorFactories = append(processorFactories, processorFactory) + } + if s.abbottClient != nil { s.Logger().Debug("Creating abbott processor factories") @@ -943,6 +958,14 @@ func (s *Standard) initializeWorkSingletons() error { ctx, cancel := context.WithTimeout(log.NewContextWithLogger(context.Background(), s.Logger()), 10*time.Second) defer cancel() + s.Logger().Debug("Creating data sweep work") + + if workCreate, err := dataWorkSweepOutdated.NewWorkCreate(); err != nil { + return errors.Wrap(err, "unable to create data sweep outdated work create") + } else if _, err = s.workClient.Create(ctx, workCreate); err != nil { + return errors.Wrap(err, "unable to create data sweep outdated work") + } + if s.ouraClient != nil { s.Logger().Debug("Creating oura webhook subscribe work") diff --git a/data/store/mongo/mongo_summary.go b/data/store/mongo/mongo_summary.go index b1ca241788..bb68caa9f2 100644 --- a/data/store/mongo/mongo_summary.go +++ b/data/store/mongo/mongo_summary.go @@ -45,6 +45,16 @@ func (d *SummaryRepository) EnsureIndexes() error { Options: options.Index(). SetName("OutdatedAndSchemaMigration"), }, + { + // Serves the outdated sweep across types. Partial as the mark is transient — it is + // cleared as it is swept — so the index holds only the marks outstanding. + Keys: bson.D{ + {Key: "dates.outdatedSince", Value: 1}, + }, + Options: options.Index(). + SetName("OutdatedSince"). + SetPartialFilterExpression(bson.D{{Key: "dates.outdatedSince", Value: bson.M{"$exists": true}}}), + }, }) } diff --git a/data/store/mongo/mongo_test.go b/data/store/mongo/mongo_test.go index 16399add39..c8847efc49 100644 --- a/data/store/mongo/mongo_test.go +++ b/data/store/mongo/mongo_test.go @@ -422,6 +422,14 @@ var _ = Describe("Mongo", Label("mongodb", "slow", "integration"), func() { "Background": Equal(false), "Name": Equal("OutdatedAndSchemaMigration"), }), + MatchFields(IgnoreExtras, Fields{ + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("dates.outdatedSince")), + "Background": Equal(false), + "Name": Equal("OutdatedSince"), + "PartialFilterExpression": Equal(bson.D{ + {Key: "dates.outdatedSince", Value: bson.D{{Key: "$exists", Value: true}}}, + }), + }), )) }) diff --git a/data/work/postprocess/processor.go b/data/work/postprocess/processor.go index 42fc6b4712..4fec811c19 100644 --- a/data/work/postprocess/processor.go +++ b/data/work/postprocess/processor.go @@ -2,7 +2,6 @@ package postprocess import ( "context" - "slices" "time" "github.com/tidepool-org/platform/errors" @@ -128,7 +127,7 @@ func (p *Processor) absorbPending() *work.ProcessResult { } // An upload reporting only full batches is still in progress - if deferredUntil.After(p.Now()) && shouldDeffer(reasons) { + if deferredUntil.After(p.Now()) && shouldDefer(reasons) { p.pendingBuilder.availableTime = deferredUntil return p.Pending() } @@ -175,8 +174,3 @@ func (d *deferredPendingBuilder) ProcessingAvailableTime(ctx context.Context, wr } return tm } - -// shouldDeffer returns true when jellyfish uploads a full batch -func shouldDeffer(reasons []string) bool { - return !slices.ContainsFunc(reasons, func(reason string) bool { return reason != ReasonLegacyDataAdded }) -} diff --git a/data/work/postprocess/processor_test.go b/data/work/postprocess/processor_test.go index e4a86f65eb..df57516eda 100644 --- a/data/work/postprocess/processor_test.go +++ b/data/work/postprocess/processor_test.go @@ -226,7 +226,7 @@ var _ = Describe("Processor", func() { // An upload reporting only full batches is still in progress, so the summaries are not // calculated for every batch so far. Jellyfish defers the same way. - Context("that is shouldDeffer and reports only a full batch", func() { + Context("that is deferred and reports only a full batch", func() { var deferredUntil time.Time BeforeEach(func() { @@ -245,6 +245,24 @@ var _ = Describe("Processor", func() { Expect(result.PendingUpdate.ProcessingAvailableTime).To(BeTemporally("==", deferredUntil)) }) + // Deferral requires every reason to defer, which is not the complement of requesting a + // synchronization: data added requests none, yet still cancels the deferral + It("calculates the summaries when any reason does not defer, even one reporting no completion", func() { + sibling = newWork(work.StatePending, []string{dataWorkPostprocess.ReasonDataAdded}, deferredUntil) + expectListWithSibling() + processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, update work.ProcessingUpdate) (*work.Work, error) { + updated := *wrk + updated.Metadata = update.Metadata + return &updated, nil + }) + workClient.EXPECT().Delete(gomock.Any(), sibling.ID, gomock.Nil()).Return(sibling, nil) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }) + It("calculates the summaries once a reason reports a change is complete", func() { sibling = newWork(work.StatePending, []string{dataWorkPostprocess.ReasonUploadCompleted}, deferredUntil) expectListWithSibling() diff --git a/data/work/postprocess/work.go b/data/work/postprocess/work.go index 736407f7bb..78aba346df 100644 --- a/data/work/postprocess/work.go +++ b/data/work/postprocess/work.go @@ -59,6 +59,19 @@ func TriggersEHRSync(reasons []string) bool { return ehrSyncReasons.ContainsAny(reasons...) } +// Defer summary recalculation if jellyfish uploaded a full batch +var deferrableReasons = mapset.NewSet( + ReasonLegacyDataAdded, +) + +// shouldDefer reports whether reasons contains only deferrable reasons +func shouldDefer(reasons []string) bool { + if len(reasons) == 0 { + return false + } + return mapset.NewSet(reasons...).IsSubset(deferrableReasons) +} + // IDFromUserID returns both the serial id, which prevents the work of a user being processed // concurrently, and the group id, which is the scope the work of a user is coalesced within. Both // are the user, and neither is the data set, so that changes to any number of data sets, interleaved diff --git a/data/work/sweep/outdated/outdated.go b/data/work/sweep/outdated/outdated.go new file mode 100644 index 0000000000..9723b0612b --- /dev/null +++ b/data/work/sweep/outdated/outdated.go @@ -0,0 +1,143 @@ +package outdated + +import ( + "context" + "time" + + dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" + "github.com/tidepool-org/platform/errors" + "github.com/tidepool-org/platform/log" + "github.com/tidepool-org/platform/pointer" + summaryStore "github.com/tidepool-org/platform/summary/store" + "github.com/tidepool-org/platform/work" + workBase "github.com/tidepool-org/platform/work/base" +) + +//go:generate mockgen -source=outdated.go -destination=test/outdated_mocks.go -package=test -typed + +const ( + Type = "org.tidepool.data.sweep.outdated" + Quantity = 1 + Frequency = 1 * time.Minute + ProcessingTimeout = 5 * time.Minute + + // PendingAvailableDuration is how long the sweep waits before running again + PendingAvailableDuration = 1 * time.Minute + + // BatchSize is how many summaries are reported per request, and PageLimit how many requests are + // made per run, so that one run cannot hold a processor indefinitely against a large backlog + BatchSize = 500 + PageLimit = 20 + + FailingRetryDuration = 1 * time.Minute + FailingRetryDurationJitter = 5 * time.Second +) + +// Summaries is satisfied by summary/store.TypelessSummaries +type Summaries interface { + ListOutdated(ctx context.Context, limit int) ([]summaryStore.OutdatedSummary, error) + ClearOutdated(ctx context.Context, userID string, typ string, observed time.Time) error +} + +type Dependencies struct { + workBase.Dependencies + Summaries +} + +func (d Dependencies) Validate() error { + if err := d.Dependencies.Validate(); err != nil { + return err + } + if d.Summaries == nil { + return errors.New("summaries is missing") + } + return nil +} + +func NewProcessorFactory(dependencies Dependencies) (*workBase.ProcessorFactory, error) { + if err := dependencies.Validate(); err != nil { + return nil, errors.Wrap(err, "dependencies is invalid") + } + processorFactory := func() (work.Processor, error) { return NewProcessor(dependencies) } + return workBase.NewProcessorFactory(Type, Quantity, Frequency, processorFactory) +} + +func NewWorkCreate() (*work.Create, error) { + return &work.Create{ + Type: Type, + DeduplicationID: pointer.From(work.DeduplicationIDSingleton), + ProcessingTimeout: int(ProcessingTimeout.Seconds()), + }, nil +} + +type Processor struct { + *workBase.ProcessorWithoutMetadata + Summaries +} + +func NewProcessor(dependencies Dependencies) (*Processor, error) { + if err := dependencies.Validate(); err != nil { + return nil, errors.Wrap(err, "dependencies is invalid") + } + + processResultBuilder := &workBase.ProcessResultBuilder{ + ProcessResultPendingBuilder: &workBase.ConstantProcessResultPendingBuilder{ + Duration: PendingAvailableDuration, + }, + ProcessResultFailingBuilder: &workBase.ConstantProcessResultFailingBuilder{ + Duration: FailingRetryDuration, + }, + } + + processor, err := workBase.NewProcessorWithoutMetadata(dependencies.Dependencies, processResultBuilder) + if err != nil { + return nil, errors.Wrap(err, "unable to create processor") + } + + return &Processor{ + ProcessorWithoutMetadata: processor, + Summaries: dependencies.Summaries, + }, nil +} + +func (p *Processor) Process(ctx context.Context, wrk *work.Work, processingUpdater work.ProcessingUpdater) *work.ProcessResult { + return append(p.ProcessPipeline(ctx, wrk, processingUpdater), + p.sweep, + ).Process(p.Pending) +} + +// sweep reports the summaries marked outdated by the retired mechanism as work, +// so that they are calculated once the mechanism that marked them is gone. +// +// The work is created before the mark is cleared, so that a failure between the two reports the change +// twice rather than not at all. Clearing is guarded upon the time observed, so a mark made in between +// is reported by the next run rather than discarded. +func (p *Processor) sweep() *work.ProcessResult { + var swept int + + for range PageLimit { + outdated, err := p.ListOutdated(p.Context(), BatchSize) + if err != nil { + return p.Failing(errors.Wrap(err, "unable to list outdated summaries")) + } + if len(outdated) == 0 { + break + } + + for _, summary := range outdated { + if err = dataWorkPostprocess.Enqueue(p.Context(), p.WorkClient(), summary.UserID, dataWorkPostprocess.ReasonDataAdded); err != nil { + return p.Failing(errors.Wrap(err, "unable to report the change")) + } + if err = p.ClearOutdated(p.Context(), summary.UserID, summary.Type, summary.OutdatedSince); err != nil { + return p.Failing(errors.Wrap(err, "unable to clear the outdated summary")) + } + } + swept += len(outdated) + } + + if swept > 0 { + log.LoggerFromContext(p.Context()).WithField("count", swept).Info("reported outdated summaries as work") + } + + return nil +} diff --git a/data/work/sweep/outdated/outdated_test.go b/data/work/sweep/outdated/outdated_test.go new file mode 100644 index 0000000000..5c1b560a20 --- /dev/null +++ b/data/work/sweep/outdated/outdated_test.go @@ -0,0 +1,206 @@ +package outdated_test + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + + dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" + dataWorkSweepOutdated "github.com/tidepool-org/platform/data/work/sweep/outdated" + dataWorkSweepOutdatedTest "github.com/tidepool-org/platform/data/work/sweep/outdated/test" + errorsTest "github.com/tidepool-org/platform/errors/test" + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + summaryStore "github.com/tidepool-org/platform/summary/store" + summaryTypes "github.com/tidepool-org/platform/summary/types" + "github.com/tidepool-org/platform/test" + userTest "github.com/tidepool-org/platform/user/test" + "github.com/tidepool-org/platform/work" + workBase "github.com/tidepool-org/platform/work/base" + workTest "github.com/tidepool-org/platform/work/test" +) + +func TestSuite(t *testing.T) { + test.Test(t) +} + +var _ = Describe("Outdated", func() { + var controller *gomock.Controller + var workClient *workTest.MockClient + var summaries *dataWorkSweepOutdatedTest.MockSummaries + var processor *dataWorkSweepOutdated.Processor + var ctx context.Context + var wrk *work.Work + var outdatedSince time.Time + + process := func() *work.ProcessResult { + return processor.Process(ctx, wrk, workTest.NewMockProcessingUpdater(controller)) + } + + BeforeEach(func() { + controller = gomock.NewController(GinkgoT()) + workClient = workTest.NewMockClient(controller) + summaries = dataWorkSweepOutdatedTest.NewMockSummaries(controller) + ctx = log.NewContextWithLogger(context.Background(), logTest.NewLogger()) + outdatedSince = time.Now().UTC().Add(-time.Hour) + wrk = &work.Work{ + ID: workTest.RandomID(), + Type: dataWorkSweepOutdated.Type, + ProcessingTimeout: int(dataWorkSweepOutdated.ProcessingTimeout.Seconds()), + State: work.StateProcessing, + } + + var err error + processor, err = dataWorkSweepOutdated.NewProcessor(dataWorkSweepOutdated.Dependencies{ + Dependencies: workBase.Dependencies{WorkClient: workClient}, + Summaries: summaries, + }) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + controller.Finish() + }) + + It("waits to run again when nothing is outdated", func() { + summaries.EXPECT().ListOutdated(gomock.Any(), gomock.Any()).Return(nil, nil) + + result := process() + Expect(result.Result).To(Equal(work.ResultPending)) + Expect(result.PendingUpdate.ProcessingAvailableTime).To(BeTemporally("~", time.Now().Add(dataWorkSweepOutdated.PendingAvailableDuration), time.Second)) + }) + + Context("with outdated summaries", func() { + var userID string + + BeforeEach(func() { + userID = userTest.RandomUserID() + }) + + // Reported on the first request, then nothing, so the run stops rather than repeating + expectListOnce := func(outdated ...summaryStore.OutdatedSummary) { + reported := false + summaries.EXPECT().ListOutdated(gomock.Any(), dataWorkSweepOutdated.BatchSize).DoAndReturn( + func(_ context.Context, _ int) ([]summaryStore.OutdatedSummary, error) { + if reported { + return nil, nil + } + reported = true + return outdated, nil + }).Times(2) + } + + // The work is created before the mark is cleared, so that a failure between the two reports the + // change twice rather than not at all + It("reports the change as work before clearing the mark", func() { + expectListOnce(summaryStore.OutdatedSummary{UserID: userID, Type: summaryTypes.SummaryTypeCGM, OutdatedSince: outdatedSince}) + gomock.InOrder( + workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(&work.Work{}, nil), + summaries.EXPECT().ClearOutdated(gomock.Any(), userID, summaryTypes.SummaryTypeCGM, outdatedSince).Return(nil), + ) + + Expect(process().Result).To(Equal(work.ResultPending)) + }) + + // A user with more than one summary marked is reported once per mark, and every mark is + // cleared with its own report — the work created collapses into one calculation on pickup, so + // nothing groups them here + It("reports each mark of a user, and clears each", func() { + expectListOnce( + summaryStore.OutdatedSummary{UserID: userID, Type: summaryTypes.SummaryTypeCGM, OutdatedSince: outdatedSince}, + summaryStore.OutdatedSummary{UserID: userID, Type: summaryTypes.SummaryTypeBGM, OutdatedSince: outdatedSince}, + summaryStore.OutdatedSummary{UserID: userID, Type: summaryTypes.SummaryTypeContinuous, OutdatedSince: outdatedSince}, + ) + workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(&work.Work{}, nil).Times(3) + summaries.EXPECT().ClearOutdated(gomock.Any(), userID, gomock.Any(), outdatedSince).Return(nil).Times(3) + + Expect(process().Result).To(Equal(work.ResultPending)) + }) + + It("reports the change as data added, which requests no synchronization", func() { + expectListOnce(summaryStore.OutdatedSummary{UserID: userID, Type: summaryTypes.SummaryTypeCGM, OutdatedSince: outdatedSince}) + workClient.EXPECT().Create(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, create *work.Create) (*work.Work, error) { + Expect(create.Metadata).To(HaveKeyWithValue("reasons", ConsistOf(dataWorkPostprocess.ReasonDataAdded))) + Expect(create.Metadata).To(HaveKeyWithValue("userId", userID)) + return &work.Work{}, nil + }) + summaries.EXPECT().ClearOutdated(gomock.Any(), userID, gomock.Any(), gomock.Any()).Return(nil) + + Expect(process().Result).To(Equal(work.ResultPending)) + }) + + It("retries when the work cannot be created, without clearing the mark", func() { + summaries.EXPECT().ListOutdated(gomock.Any(), gomock.Any()).Return( + []summaryStore.OutdatedSummary{{UserID: userID, Type: summaryTypes.SummaryTypeCGM, OutdatedSince: outdatedSince}}, nil) + workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil, errorsTest.RandomError()) + + Expect(process().Result).To(Equal(work.ResultFailing)) + }) + + It("retries when the mark cannot be cleared", func() { + summaries.EXPECT().ListOutdated(gomock.Any(), gomock.Any()).Return( + []summaryStore.OutdatedSummary{{UserID: userID, Type: summaryTypes.SummaryTypeCGM, OutdatedSince: outdatedSince}}, nil) + workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(&work.Work{}, nil) + summaries.EXPECT().ClearOutdated(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errorsTest.RandomError()) + + Expect(process().Result).To(Equal(work.ResultFailing)) + }) + + It("retries when the summaries cannot be listed", func() { + summaries.EXPECT().ListOutdated(gomock.Any(), gomock.Any()).Return(nil, errorsTest.RandomError()) + + Expect(process().Result).To(Equal(work.ResultFailing)) + }) + + // A backlog larger than one run is left to the next, so that a run cannot hold the processor + // against it indefinitely + It("reports no more than the page limit in one run", func() { + summaries.EXPECT().ListOutdated(gomock.Any(), gomock.Any()).Return( + []summaryStore.OutdatedSummary{{UserID: userID, Type: summaryTypes.SummaryTypeCGM, OutdatedSince: outdatedSince}}, nil). + Times(dataWorkSweepOutdated.PageLimit) + workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(&work.Work{}, nil).Times(dataWorkSweepOutdated.PageLimit) + summaries.EXPECT().ClearOutdated(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(dataWorkSweepOutdated.PageLimit) + + Expect(process().Result).To(Equal(work.ResultPending)) + }) + }) + + Context("Dependencies", func() { + It("reports the work client is missing", func() { + _, err := dataWorkSweepOutdated.NewProcessorFactory(dataWorkSweepOutdated.Dependencies{ + Summaries: summaries, + }) + Expect(err).To(MatchError(ContainSubstring("work client is missing"))) + }) + + It("reports the summaries are missing", func() { + _, err := dataWorkSweepOutdated.NewProcessorFactory(dataWorkSweepOutdated.Dependencies{ + Dependencies: workBase.Dependencies{WorkClient: workClient}, + }) + Expect(err).To(MatchError(ContainSubstring("summaries is missing"))) + }) + + It("reports the type, quantity and frequency of the work", func() { + processorFactory, err := dataWorkSweepOutdated.NewProcessorFactory(dataWorkSweepOutdated.Dependencies{ + Dependencies: workBase.Dependencies{WorkClient: workClient}, + Summaries: summaries, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(processorFactory.Type()).To(Equal("org.tidepool.data.sweep.outdated")) + Expect(processorFactory.Quantity()).To(Equal(1)) + Expect(processorFactory.Frequency()).To(Equal(time.Minute)) + }) + + It("creates the work as a singleton, so that one sweep runs however many services register it", func() { + workCreate, err := dataWorkSweepOutdated.NewWorkCreate() + Expect(err).ToNot(HaveOccurred()) + Expect(workCreate.DeduplicationID).ToNot(BeNil()) + Expect(*workCreate.DeduplicationID).To(Equal(work.DeduplicationIDSingleton)) + }) + }) +}) diff --git a/data/work/sweep/outdated/test/outdated_mocks.go b/data/work/sweep/outdated/test/outdated_mocks.go new file mode 100644 index 0000000000..dcbf3e9ecf --- /dev/null +++ b/data/work/sweep/outdated/test/outdated_mocks.go @@ -0,0 +1,121 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: outdated.go +// +// Generated by this command: +// +// mockgen -source=outdated.go -destination=test/outdated_mocks.go -package=test -typed +// + +// Package test is a generated GoMock package. +package test + +import ( + context "context" + reflect "reflect" + time "time" + + gomock "go.uber.org/mock/gomock" + + store "github.com/tidepool-org/platform/summary/store" +) + +// MockSummaries is a mock of Summaries interface. +type MockSummaries struct { + ctrl *gomock.Controller + recorder *MockSummariesMockRecorder + isgomock struct{} +} + +// MockSummariesMockRecorder is the mock recorder for MockSummaries. +type MockSummariesMockRecorder struct { + mock *MockSummaries +} + +// NewMockSummaries creates a new mock instance. +func NewMockSummaries(ctrl *gomock.Controller) *MockSummaries { + mock := &MockSummaries{ctrl: ctrl} + mock.recorder = &MockSummariesMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSummaries) EXPECT() *MockSummariesMockRecorder { + return m.recorder +} + +// ClearOutdated mocks base method. +func (m *MockSummaries) ClearOutdated(ctx context.Context, userID, typ string, observed time.Time) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClearOutdated", ctx, userID, typ, observed) + ret0, _ := ret[0].(error) + return ret0 +} + +// ClearOutdated indicates an expected call of ClearOutdated. +func (mr *MockSummariesMockRecorder) ClearOutdated(ctx, userID, typ, observed any) *MockSummariesClearOutdatedCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearOutdated", reflect.TypeOf((*MockSummaries)(nil).ClearOutdated), ctx, userID, typ, observed) + return &MockSummariesClearOutdatedCall{Call: call} +} + +// MockSummariesClearOutdatedCall wrap *gomock.Call +type MockSummariesClearOutdatedCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockSummariesClearOutdatedCall) Return(arg0 error) *MockSummariesClearOutdatedCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockSummariesClearOutdatedCall) Do(f func(context.Context, string, string, time.Time) error) *MockSummariesClearOutdatedCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockSummariesClearOutdatedCall) DoAndReturn(f func(context.Context, string, string, time.Time) error) *MockSummariesClearOutdatedCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// ListOutdated mocks base method. +func (m *MockSummaries) ListOutdated(ctx context.Context, limit int) ([]store.OutdatedSummary, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListOutdated", ctx, limit) + ret0, _ := ret[0].([]store.OutdatedSummary) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListOutdated indicates an expected call of ListOutdated. +func (mr *MockSummariesMockRecorder) ListOutdated(ctx, limit any) *MockSummariesListOutdatedCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListOutdated", reflect.TypeOf((*MockSummaries)(nil).ListOutdated), ctx, limit) + return &MockSummariesListOutdatedCall{Call: call} +} + +// MockSummariesListOutdatedCall wrap *gomock.Call +type MockSummariesListOutdatedCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockSummariesListOutdatedCall) Return(arg0 []store.OutdatedSummary, arg1 error) *MockSummariesListOutdatedCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockSummariesListOutdatedCall) Do(f func(context.Context, int) ([]store.OutdatedSummary, error)) *MockSummariesListOutdatedCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockSummariesListOutdatedCall) DoAndReturn(f func(context.Context, int) ([]store.OutdatedSummary, error)) *MockSummariesListOutdatedCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/summary/store/summary.go b/summary/store/summary.go index 96d595779d..52f0b68be1 100644 --- a/summary/store/summary.go +++ b/summary/store/summary.go @@ -49,6 +49,96 @@ func NewTypeless(delegate *storeStructuredMongo.Repository) *TypelessSummaries { } } +// OutdatedSummary reports a summary marked outdated by the mechanism being retired, or by the legacy +// ingestion service, which is the summary of one type for one user +type OutdatedSummary struct { + UserID string `bson:"userId"` + Type string `bson:"type"` + OutdatedSince time.Time `bson:"-"` +} + +// ListOutdated reports the summaries marked outdated, of every type, oldest first, up to the limit. +// +// A limit rather than pagination, deliberately: the caller clears the marks it reports, so repeating +// the call reports the *next* oldest. An offset here would skip past the new head of the set as it +// shrinks, which is also why this must never be "fixed" to page as ListMigratableUserIDs does. +// +// Reported across types rather than one type at a time, as the work created for a user recalculates +// every one of their summaries, so a user whose summaries are all marked reports one user rather than +// three. GetOutdatedUserIDs is not reused as it reports no outdated time per summary, which +// ClearOutdated requires, and it records the queue metrics of the mechanism being retired. +func (r *TypelessSummaries) ListOutdated(ctx context.Context, limit int) ([]OutdatedSummary, error) { + if ctx == nil { + return nil, errors.New("context is missing") + } + if limit <= 0 { + return nil, errors.New("limit is invalid") + } + + // No filter upon the reason, as the legacy ingestion service reports its own. A summary marked + // outdated in the future is deferred by that service until its upload falls quiet, so it is + // reported only once that time passes. + selector := bson.M{"dates.outdatedSince": bson.M{"$lte": time.Now().UTC()}} + + opts := options.Find() + opts.SetSort(bson.D{{Key: "dates.outdatedSince", Value: 1}}) + opts.SetLimit(int64(limit)) + opts.SetProjection(bson.M{"userId": 1, "type": 1, "dates.outdatedSince": 1}) + + cursor, err := r.Find(ctx, selector, opts) + if err != nil { + return nil, fmt.Errorf("unable to list outdated summaries: %w", err) + } + defer cursor.Close(ctx) + + var documents []struct { + UserID string `bson:"userId"` + Type string `bson:"type"` + Dates struct { + OutdatedSince *time.Time `bson:"outdatedSince"` + } `bson:"dates"` + } + if err = cursor.All(ctx, &documents); err != nil { + return nil, fmt.Errorf("unable to decode outdated summaries: %w", err) + } + + outdated := make([]OutdatedSummary, 0, len(documents)) + for _, document := range documents { + if document.Dates.OutdatedSince == nil { + continue + } + outdated = append(outdated, OutdatedSummary{ + UserID: document.UserID, + Type: document.Type, + OutdatedSince: *document.Dates.OutdatedSince, + }) + } + + return outdated, nil +} + +// ClearOutdated clears the outdated mark of a summary, but only while it reports the outdated time +// observed, so that a mark made since is retained and reported again rather than discarded +func (r *TypelessSummaries) ClearOutdated(ctx context.Context, userID string, typ string, observed time.Time) error { + if ctx == nil { + return errors.New("context is missing") + } + if userID == "" { + return errors.New("userId is missing") + } + if typ == "" { + return errors.New("type is missing") + } + + selector := bson.M{"userId": userID, "type": typ, "dates.outdatedSince": observed} + update := bson.M{"$unset": bson.M{"dates.outdatedSince": "", "dates.outdatedReason": ""}} + + if _, err := r.UpdateOne(ctx, selector, update); err != nil { + return fmt.Errorf("unable to clear outdated summary for user %s: %w", userID, err) + } + return nil +} + func (r *Summaries[PP, PB, P, B]) GetSummary(ctx context.Context, userId string) (*types.Summary[PP, PB, P, B], error) { if ctx == nil { return nil, errors.New("context is missing") @@ -307,7 +397,6 @@ func (r *Summaries[PP, PB, P, B]) GetMigratableUserIDs(ctx context.Context, page selector := bson.M{ "type": types.GetType[PP, PB](), - "dates.outdatedSince": nil, "config.schemaVersion": bson.M{"$ne": types.SchemaVersion}, } @@ -316,7 +405,7 @@ func (r *Summaries[PP, PB, P, B]) GetMigratableUserIDs(ctx context.Context, page {Key: "dates.lastUpdatedDate", Value: 1}, }) opts.SetLimit(int64(page.Size)) - opts.SetProjection(bson.M{"stats": 0}) + opts.SetProjection(bson.M{"userId": 1}) cursor, err := r.Find(ctx, selector, opts) if errors.Is(err, mongo.ErrNoDocuments) { diff --git a/summary/store/summary_test.go b/summary/store/summary_test.go index ad93c107f4..12986d29ce 100644 --- a/summary/store/summary_test.go +++ b/summary/store/summary_test.go @@ -138,6 +138,106 @@ var _ = Describe("Summary Periods Mongo", Label("mongodb", "slow", "integration" }) }) + Context("ListOutdated and ClearOutdated", func() { + var userIdTwo string + var outdatedTime time.Time + + BeforeEach(func() { + userIdTwo = userTest.RandomUserID() + outdatedTime = time.Now().UTC().Add(-time.Hour).Truncate(time.Millisecond) + }) + + createOutdated := func(userIDs ...string) { + summaries := make([]*types.Summary[*types.ContinuousPeriods, *types.ContinuousBucket, types.ContinuousPeriods, types.ContinuousBucket], len(userIDs)) + for index, userID := range userIDs { + summaries[index] = test.RandomContinuousSummary(userID) + summaries[index].Dates.OutdatedSince = &outdatedTime + summaries[index].Dates.OutdatedReason = []string{"LEGACY_DATA_ADDED"} + } + _, err = continuousStore.CreateSummaries(ctx, summaries) + Expect(err).ToNot(HaveOccurred()) + } + + It("reports the outdated summaries with their type and the time they were marked", func() { + createOutdated(userId, userIdTwo) + + outdated, err := typelessStore.ListOutdated(ctx, 100) + Expect(err).ToNot(HaveOccurred()) + Expect(outdated).To(HaveLen(2)) + Expect(outdated[0].Type).To(Equal(types.SummaryTypeContinuous)) + Expect(outdated[0].OutdatedSince).To(BeTemporally("==", outdatedTime)) + Expect([]string{outdated[0].UserID, outdated[1].UserID}).To(ConsistOf(userId, userIdTwo)) + }) + + // The legacy ingestion service defers a full batch by marking the summary outdated in + // the future, which is reported only once that time passes, honouring its quiet window + It("does not report a summary marked outdated in the future", func() { + deferred := time.Now().UTC().Add(90 * time.Second).Truncate(time.Millisecond) + summary := test.RandomContinuousSummary(userId) + summary.Dates.OutdatedSince = &deferred + _, err = continuousStore.CreateSummaries(ctx, []*types.Summary[*types.ContinuousPeriods, *types.ContinuousBucket, types.ContinuousPeriods, types.ContinuousBucket]{summary}) + Expect(err).ToNot(HaveOccurred()) + + outdated, err := typelessStore.ListOutdated(ctx, 100) + Expect(err).ToNot(HaveOccurred()) + Expect(outdated).To(BeEmpty()) + }) + + It("does not report a summary that is not outdated", func() { + summary := test.RandomContinuousSummary(userId) + summary.Dates.OutdatedSince = nil + _, err = continuousStore.CreateSummaries(ctx, []*types.Summary[*types.ContinuousPeriods, *types.ContinuousBucket, types.ContinuousPeriods, types.ContinuousBucket]{summary}) + Expect(err).ToNot(HaveOccurred()) + + outdated, err := typelessStore.ListOutdated(ctx, 100) + Expect(err).ToNot(HaveOccurred()) + Expect(outdated).To(BeEmpty()) + }) + + It("clears the outdated mark it observed", func() { + createOutdated(userId) + + Expect(typelessStore.ClearOutdated(ctx, userId, types.SummaryTypeContinuous, outdatedTime)).To(Succeed()) + + outdated, err := typelessStore.ListOutdated(ctx, 100) + Expect(err).ToNot(HaveOccurred()) + Expect(outdated).To(BeEmpty()) + + summary, err := continuousStore.GetSummary(ctx, userId) + Expect(err).ToNot(HaveOccurred()) + Expect(summary.Dates.OutdatedReason).To(BeEmpty()) + }) + + // A mark made between the report and the clear reports a different time, and must be + // retained so that it is reported again rather than discarded + It("retains a mark made since the one it observed", func() { + createOutdated(userId) + remarked := time.Now().UTC().Truncate(time.Millisecond) + + summary, err := continuousStore.GetSummary(ctx, userId) + Expect(err).ToNot(HaveOccurred()) + summary.Dates.OutdatedSince = &remarked + Expect(continuousStore.ReplaceSummary(ctx, summary)).To(Succeed()) + + Expect(typelessStore.ClearOutdated(ctx, userId, types.SummaryTypeContinuous, outdatedTime)).To(Succeed()) + + outdated, err := typelessStore.ListOutdated(ctx, 100) + Expect(err).ToNot(HaveOccurred()) + Expect(outdated).To(HaveLen(1)) + Expect(outdated[0].OutdatedSince).To(BeTemporally("==", remarked)) + }) + + It("reports errors for missing parameters", func() { + _, err = typelessStore.ListOutdated(nil, 100) + Expect(err).To(MatchError("context is missing")) + _, err = typelessStore.ListOutdated(ctx, 0) + Expect(err).To(MatchError("limit is invalid")) + Expect(typelessStore.ClearOutdated(nil, userId, types.SummaryTypeContinuous, outdatedTime)).To(MatchError("context is missing")) + Expect(typelessStore.ClearOutdated(ctx, "", types.SummaryTypeContinuous, outdatedTime)).To(MatchError("userId is missing")) + Expect(typelessStore.ClearOutdated(ctx, userId, "", outdatedTime)).To(MatchError("type is missing")) + }) + }) + Context("GetMigratableUserIDs", func() { var userIds []string var userIdTwo string @@ -194,6 +294,9 @@ var _ = Describe("Summary Periods Mongo", Label("mongodb", "slow", "integration" Expect(userIds).To(ConsistOf([]string{userId, userIdTwo})) }) + // The outdated mark no longer divides the work of two runners between them: a summary + // that is both outdated and calculated with an outdated schema is reported by both + // sweepers, and the work each creates is absorbed into a single calculation It("With migratable and outdated CGM summaries", func() { var outdatedTime = time.Now().UTC().Truncate(time.Millisecond) var continuousSummaries = []*types.Summary[*types.ContinuousPeriods, *types.ContinuousBucket, types.ContinuousPeriods, types.ContinuousBucket]{ @@ -214,7 +317,7 @@ var _ = Describe("Summary Periods Mongo", Label("mongodb", "slow", "integration" userIds, err = continuousStore.GetMigratableUserIDs(ctx, page.NewPagination()) Expect(err).ToNot(HaveOccurred()) - Expect(userIds).To(ConsistOf([]string{userId, userIdTwo})) + Expect(userIds).To(ConsistOf([]string{userId, userIdOther, userIdTwo})) }) It("With a specific pagination size", func() { From ef83b21e9da9ae9e57137f0981d9a38396d5c244 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Tue, 18 Aug 2026 12:33:40 +0300 Subject: [PATCH 09/20] Add generic work create and get API for external producers --- .../api/v1/users_datasets_create_test.go | 15 ++ data/service/api/v1/v1.go | 1 + data/service/api/v1/work.go | 65 +++++++ data/service/api/v1/work_test.go | 178 ++++++++++++++++++ data/work/postprocess/processor_test.go | 2 +- data/work/postprocess/work.go | 8 - data/work/postprocess/work_test.go | 8 - 7 files changed, 260 insertions(+), 17 deletions(-) create mode 100644 data/service/api/v1/work.go create mode 100644 data/service/api/v1/work_test.go diff --git a/data/service/api/v1/users_datasets_create_test.go b/data/service/api/v1/users_datasets_create_test.go index a82835d608..cdc2a5a510 100644 --- a/data/service/api/v1/users_datasets_create_test.go +++ b/data/service/api/v1/users_datasets_create_test.go @@ -99,6 +99,12 @@ type mockDataServiceContext struct { // DataSetTester tests the resulting data set. DataSetTester func(testingT, *data.DataSet) + + // RestRequest, RestResponse and TestWorkClient, when set, are returned by + // Request, Response and WorkClient in place of the defaults. + RestRequest *rest.Request + RestResponse rest.ResponseWriter + TestWorkClient work.Client } func newMockDataServiceContext(t testingT) *mockDataServiceContext { @@ -119,10 +125,16 @@ func newMockDataServiceContext(t testingT) *mockDataServiceContext { } func (c *mockDataServiceContext) Response() rest.ResponseWriter { + if c.RestResponse != nil { + return c.RestResponse + } panic("not implemented") // TODO: Implement } func (c *mockDataServiceContext) Request() *rest.Request { + if c.RestRequest != nil { + return c.RestRequest + } r, err := http.NewRequest(http.MethodGet, "", nil) if err != nil { c.t.Fatalf("creating test request: %s", err) @@ -235,6 +247,9 @@ func (c *mockDataServiceContext) DataSourceClient() dataSource.Client { } func (c *mockDataServiceContext) WorkClient() work.Client { + if c.TestWorkClient != nil { + return c.TestWorkClient + } panic("not implemented") } diff --git a/data/service/api/v1/v1.go b/data/service/api/v1/v1.go index 387e43b7a6..fb0547c409 100644 --- a/data/service/api/v1/v1.go +++ b/data/service/api/v1/v1.go @@ -36,6 +36,7 @@ func Routes() []service.Route { routes = append(routes, SummaryRoutes()...) routes = append(routes, AlertsRoutes()...) routes = append(routes, NotificationsRoutes()...) + routes = append(routes, WorkRoutes()...) routes = append(routes, abbottServiceApiV1.Routes()...) routes = append(routes, ouraServiceApiV1.Routes()...) diff --git a/data/service/api/v1/work.go b/data/service/api/v1/work.go new file mode 100644 index 0000000000..f97ca96a46 --- /dev/null +++ b/data/service/api/v1/work.go @@ -0,0 +1,65 @@ +package v1 + +import ( + "net/http" + + dataService "github.com/tidepool-org/platform/data/service" + "github.com/tidepool-org/platform/request" + "github.com/tidepool-org/platform/service/api" + "github.com/tidepool-org/platform/work" +) + +func WorkRoutes() []dataService.Route { + return []dataService.Route{ + dataService.Post("/v1/work", CreateWork, api.RequireServer), + dataService.Get("/v1/work/:id", GetWork, api.RequireServer), + } +} + +func CreateWork(dataServiceContext dataService.Context) { + req := dataServiceContext.Request() + ctx := req.Context() + responder := request.MustNewResponder(dataServiceContext.Response(), req) + + create := &work.Create{} + if err := request.DecodeRequestBody(req.Request, create); err != nil { + responder.Error(http.StatusBadRequest, err) + return + } + + wrk, err := dataServiceContext.WorkClient().Create(ctx, create) + if err != nil { + responder.InternalServerError(err) + return + } + if wrk == nil { // deduplicated: an equivalent item already waits + responder.Empty(http.StatusNoContent) + return + } + + responder.Data(http.StatusCreated, wrk) +} + +func GetWork(dataServiceContext dataService.Context) { + req := dataServiceContext.Request() + ctx := req.Context() + responder := request.MustNewResponder(dataServiceContext.Response(), req) + + id := req.PathParam("id") + if id == "" { + responder.Error(http.StatusBadRequest, request.ErrorParameterMissing("id")) + return + } + + wrk, err := dataServiceContext.WorkClient().Get(ctx, id, nil) + if err != nil { + responder.InternalServerError(err) + return + } + if wrk == nil { + responder.Error(http.StatusNotFound, request.ErrorResourceNotFoundWithID(id)) + return + } + + responder.Data(http.StatusOK, wrk) +} diff --git a/data/service/api/v1/work_test.go b/data/service/api/v1/work_test.go new file mode 100644 index 0000000000..bc1b3a455b --- /dev/null +++ b/data/service/api/v1/work_test.go @@ -0,0 +1,178 @@ +package v1_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/ant0ine/go-json-rest/rest" + "go.uber.org/mock/gomock" + + v1 "github.com/tidepool-org/platform/data/service/api/v1" + errorsTest "github.com/tidepool-org/platform/errors/test" + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + testRest "github.com/tidepool-org/platform/test/rest" + "github.com/tidepool-org/platform/work" + workTest "github.com/tidepool-org/platform/work/test" +) + +var _ = Describe("Work", func() { + var controller *gomock.Controller + var workClient *workTest.MockClient + var res *testRest.ResponseWriter + var svcCtx *mockDataServiceContext + + newRestRequest := func(method string, body string, pathParams map[string]string) *rest.Request { + req := httptest.NewRequest(method, "/v1/work", strings.NewReader(body)) + req = req.WithContext(log.NewContextWithLogger(req.Context(), logTest.NewLogger())) + return &rest.Request{Request: req, PathParams: pathParams, Env: map[string]interface{}{}} + } + + statusCode := func() int { + Expect(res.WriteHeaderInputs).To(HaveLen(1)) + return res.WriteHeaderInputs[0] + } + + respondedObject := func() map[string]interface{} { + Expect(res.WriteInputs).To(HaveLen(1)) + var object map[string]interface{} + Expect(json.Unmarshal(res.WriteInputs[0], &object)).To(Succeed()) + return object + } + + BeforeEach(func() { + controller = gomock.NewController(GinkgoT()) + workClient = workTest.NewMockClient(controller) + res = testRest.NewResponseWriter() + res.HeaderOutput = &http.Header{} + res.WriteStub = func(bites []byte) (int, error) { return len(bites), nil } + svcCtx = &mockDataServiceContext{ + RestResponse: res, + TestWorkClient: workClient, + } + }) + + AfterEach(func() { + controller.Finish() + }) + + Context("CreateWork", func() { + createBody := func(availableTime *time.Time) string { + body := map[string]interface{}{ + "type": "org.tidepool.data.upload.postprocess", + "groupId": "org.tidepool.data.upload.postprocess:test-user-id", + "serialId": "org.tidepool.data.upload.postprocess:test-user-id", + "processingTimeout": 300, + "metadata": map[string]interface{}{"userId": "test-user-id", "reasons": []string{"LEGACY_DATA_ADDED"}}, + } + if availableTime != nil { + body["processingAvailableTime"] = availableTime.Format(time.RFC3339Nano) + } + bites, err := json.Marshal(body) + Expect(err).ToNot(HaveOccurred()) + return string(bites) + } + + It("creates the work and responds with the created document", func() { + availableTime := time.Now().Add(90 * time.Second) + svcCtx.RestRequest = newRestRequest(http.MethodPost, createBody(&availableTime), nil) + workClient.EXPECT().Create(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, create *work.Create) (*work.Work, error) { + Expect(create.Type).To(Equal("org.tidepool.data.upload.postprocess")) + Expect(create.GroupID).To(HaveValue(Equal("org.tidepool.data.upload.postprocess:test-user-id"))) + Expect(create.SerialID).To(HaveValue(Equal("org.tidepool.data.upload.postprocess:test-user-id"))) + Expect(create.ProcessingTimeout).To(Equal(300)) + Expect(create.ProcessingAvailableTime).To(BeTemporally("~", availableTime, time.Second)) + Expect(create.Metadata).To(HaveKeyWithValue("userId", "test-user-id")) + Expect(create.Metadata).To(HaveKeyWithValue("reasons", ConsistOf("LEGACY_DATA_ADDED"))) + return &work.Work{ID: "test-work-id", Type: create.Type}, nil + }) + + v1.CreateWork(svcCtx) + + Expect(statusCode()).To(Equal(http.StatusCreated)) + Expect(respondedObject()).To(HaveKeyWithValue("id", "test-work-id")) + }) + + It("responds with no content when the work is deduplicated", func() { + svcCtx.RestRequest = newRestRequest(http.MethodPost, createBody(nil), nil) + workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil, nil) + + v1.CreateWork(svcCtx) + + Expect(statusCode()).To(Equal(http.StatusNoContent)) + Expect(res.WriteInputs).To(BeEmpty()) + }) + + It("responds with bad request when the body is malformed", func() { + svcCtx.RestRequest = newRestRequest(http.MethodPost, "{malformed", nil) + + v1.CreateWork(svcCtx) + + Expect(statusCode()).To(Equal(http.StatusBadRequest)) + }) + + It("responds with bad request when the create is invalid", func() { + svcCtx.RestRequest = newRestRequest(http.MethodPost, "{}", nil) + + v1.CreateWork(svcCtx) + + Expect(statusCode()).To(Equal(http.StatusBadRequest)) + }) + + It("responds with internal server error when the work cannot be created", func() { + svcCtx.RestRequest = newRestRequest(http.MethodPost, createBody(nil), nil) + workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil, errorsTest.RandomError()) + + v1.CreateWork(svcCtx) + + Expect(statusCode()).To(Equal(http.StatusInternalServerError)) + }) + }) + + Context("GetWork", func() { + It("responds with the work", func() { + svcCtx.RestRequest = newRestRequest(http.MethodGet, "", map[string]string{"id": "test-work-id"}) + workClient.EXPECT().Get(gomock.Any(), "test-work-id", gomock.Nil()). + Return(&work.Work{ID: "test-work-id"}, nil) + + v1.GetWork(svcCtx) + + Expect(statusCode()).To(Equal(http.StatusOK)) + Expect(respondedObject()).To(HaveKeyWithValue("id", "test-work-id")) + }) + + It("responds with not found when the work does not exist", func() { + svcCtx.RestRequest = newRestRequest(http.MethodGet, "", map[string]string{"id": "test-work-id"}) + workClient.EXPECT().Get(gomock.Any(), "test-work-id", gomock.Nil()).Return(nil, nil) + + v1.GetWork(svcCtx) + + Expect(statusCode()).To(Equal(http.StatusNotFound)) + }) + + It("responds with bad request when the id is missing", func() { + svcCtx.RestRequest = newRestRequest(http.MethodGet, "", nil) + + v1.GetWork(svcCtx) + + Expect(statusCode()).To(Equal(http.StatusBadRequest)) + }) + + It("responds with internal server error when the work cannot be gotten", func() { + svcCtx.RestRequest = newRestRequest(http.MethodGet, "", map[string]string{"id": "test-work-id"}) + workClient.EXPECT().Get(gomock.Any(), "test-work-id", gomock.Nil()).Return(nil, errorsTest.RandomError()) + + v1.GetWork(svcCtx) + + Expect(statusCode()).To(Equal(http.StatusInternalServerError)) + }) + }) +}) diff --git a/data/work/postprocess/processor_test.go b/data/work/postprocess/processor_test.go index df57516eda..17224dda05 100644 --- a/data/work/postprocess/processor_test.go +++ b/data/work/postprocess/processor_test.go @@ -230,7 +230,7 @@ var _ = Describe("Processor", func() { var deferredUntil time.Time BeforeEach(func() { - deferredUntil = time.Now().Add(dataWorkPostprocess.JellyfishQuietDelay) + deferredUntil = time.Now().Add(90 * time.Second) wrk = newWork(work.StateProcessing, []string{dataWorkPostprocess.ReasonLegacyDataAdded}, time.Now().Add(-time.Minute)) sibling = newWork(work.StatePending, []string{dataWorkPostprocess.ReasonLegacyDataAdded}, deferredUntil) }) diff --git a/data/work/postprocess/work.go b/data/work/postprocess/work.go index 78aba346df..d54e5e0bb6 100644 --- a/data/work/postprocess/work.go +++ b/data/work/postprocess/work.go @@ -14,14 +14,6 @@ const ( Type = "org.tidepool.data.upload.postprocess" ProcessingTimeout = 5 * time.Minute - - // JellyfishBatchSize is the number of records the legacy ingestion service uploads per batch. A - // smaller batch is the final batch of an upload, which it does not otherwise report. - JellyfishBatchSize = 1000 - - // JellyfishQuietDelay defers processing so that an upload of any number of full batches is not - // processed once per batch - JellyfishQuietDelay = 90 * time.Second ) const ( diff --git a/data/work/postprocess/work_test.go b/data/work/postprocess/work_test.go index d7be9e06a0..cf941e32c5 100644 --- a/data/work/postprocess/work_test.go +++ b/data/work/postprocess/work_test.go @@ -28,14 +28,6 @@ var _ = Describe("Work", func() { Expect(dataWorkPostprocess.ProcessingTimeout).To(Equal(5 * time.Minute)) }) - It("JellyfishBatchSize is expected", func() { - Expect(dataWorkPostprocess.JellyfishBatchSize).To(Equal(1000)) - }) - - It("JellyfishQuietDelay is expected", func() { - Expect(dataWorkPostprocess.JellyfishQuietDelay).To(Equal(90 * time.Second)) - }) - It("Reasons is expected", func() { Expect(dataWorkPostprocess.Reasons()).To(ConsistOf( "DATA_ADDED", From 6897ad7731091c038cb42353a17aabce2315d37a Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Tue, 18 Aug 2026 13:14:36 +0300 Subject: [PATCH 10/20] Apply code review fixes to work polling, absorption, and reaping --- data/service/service/standard.go | 5 +- data/work/postprocess/processor.go | 11 ++- data/work/postprocess/processor_test.go | 24 +++++-- summary/store/summary.go | 14 ++-- work/service/coordinator.go | 23 +++++-- work/service/coordinator_internal_test.go | 36 ++++++++-- work/store/structured/mongo/mongo.go | 14 ++-- work/store/structured/mongo/mongo_test.go | 81 +++++++++++++++++++++++ 8 files changed, 174 insertions(+), 34 deletions(-) diff --git a/data/service/service/standard.go b/data/service/service/standard.go index 73cb361bdf..5953dbdb5c 100644 --- a/data/service/service/standard.go +++ b/data/service/service/standard.go @@ -622,13 +622,14 @@ func (s *Standard) initializeConfirmationClient() error { func (s *Standard) initializeSummarizerRegistry() error { s.Logger().Debug("Creating summarizer registry") + summaryRepositoryStore := s.dataStore.NewSummaryRepository().GetStore() s.summarizerRegistry = summary.New( - s.dataStore.NewSummaryRepository().GetStore(), + summaryRepositoryStore, s.dataStore.NewBucketsRepository().GetStore(), s.dataStore.NewDataRepository(), s.dataStore.GetClient(), ) - s.typelessSummaries = summaryStore.NewTypeless(s.dataStore.NewSummaryRepository().GetStore()) + s.typelessSummaries = summaryStore.NewTypeless(summaryRepositoryStore) return nil } diff --git a/data/work/postprocess/processor.go b/data/work/postprocess/processor.go index 4fec811c19..aaa1630007 100644 --- a/data/work/postprocess/processor.go +++ b/data/work/postprocess/processor.go @@ -9,6 +9,7 @@ import ( "github.com/tidepool-org/platform/metadata" "github.com/tidepool-org/platform/page" "github.com/tidepool-org/platform/pointer" + "github.com/tidepool-org/platform/request" userWork "github.com/tidepool-org/platform/user/work" "github.com/tidepool-org/platform/work" workBase "github.com/tidepool-org/platform/work/base" @@ -23,7 +24,6 @@ type Processor struct { ClinicsClient pendingBuilder *deferredPendingBuilder - wrk *work.Work } func NewProcessor(dependencies Dependencies) (*Processor, error) { @@ -60,7 +60,6 @@ func NewProcessor(dependencies Dependencies) (*Processor, error) { } func (p *Processor) Process(ctx context.Context, wrk *work.Work, processingUpdater work.ProcessingUpdater) *work.ProcessResult { - p.wrk = wrk return append(p.ProcessPipeline(ctx, wrk, processingUpdater), p.FetchUserFromWorkMetadata, p.absorbPending, @@ -114,9 +113,15 @@ func (p *Processor) absorbPending() *work.ProcessResult { } for _, wrk := range absorbed { - if _, err = p.WorkClient().Delete(p.Context(), wrk.ID, nil); err != nil { + deleted, err := p.WorkClient().Delete(p.Context(), wrk.ID, &request.Condition{Revision: &wrk.Revision}) + if err != nil { return p.Failing(errors.Wrap(err, "unable to delete work")) } + // The work changed since it was listed - log a warning since this shouldn't happen. + // Work items are processed serially and there's no modification path. + if deleted == nil { + log.LoggerFromContext(p.Context()).WithField("id", wrk.ID).Warn("work absorbed changed before it was deleted") + } } if len(absorbed) > 0 { diff --git a/data/work/postprocess/processor_test.go b/data/work/postprocess/processor_test.go index 17224dda05..8a1fc7c9a5 100644 --- a/data/work/postprocess/processor_test.go +++ b/data/work/postprocess/processor_test.go @@ -18,6 +18,7 @@ import ( "github.com/tidepool-org/platform/metadata" "github.com/tidepool-org/platform/page" "github.com/tidepool-org/platform/pointer" + "github.com/tidepool-org/platform/request" "github.com/tidepool-org/platform/user" userTest "github.com/tidepool-org/platform/user/test" userWork "github.com/tidepool-org/platform/user/work" @@ -207,7 +208,7 @@ var _ = Describe("Processor", func() { updated.Revision = wrk.Revision + 1 return &updated, nil }), - workClient.EXPECT().Delete(gomock.Any(), sibling.ID, gomock.Nil()).Return(sibling, nil), + workClient.EXPECT().Delete(gomock.Any(), sibling.ID, &request.Condition{Revision: pointer.FromInt(sibling.Revision)}).Return(sibling, nil), ) } @@ -238,7 +239,7 @@ var _ = Describe("Processor", func() { It("defers rather than calculating the summaries", func() { expectListWithSibling() processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).Return(wrk, nil) - workClient.EXPECT().Delete(gomock.Any(), sibling.ID, gomock.Nil()).Return(sibling, nil) + workClient.EXPECT().Delete(gomock.Any(), sibling.ID, &request.Condition{Revision: pointer.FromInt(sibling.Revision)}).Return(sibling, nil) result := process() Expect(result.Result).To(Equal(work.ResultPending)) @@ -256,7 +257,7 @@ var _ = Describe("Processor", func() { updated.Metadata = update.Metadata return &updated, nil }) - workClient.EXPECT().Delete(gomock.Any(), sibling.ID, gomock.Nil()).Return(sibling, nil) + workClient.EXPECT().Delete(gomock.Any(), sibling.ID, &request.Condition{Revision: pointer.FromInt(sibling.Revision)}).Return(sibling, nil) summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) @@ -272,7 +273,7 @@ var _ = Describe("Processor", func() { updated.Metadata = update.Metadata return &updated, nil }) - workClient.EXPECT().Delete(gomock.Any(), sibling.ID, gomock.Nil()).Return(sibling, nil) + workClient.EXPECT().Delete(gomock.Any(), sibling.ID, &request.Condition{Revision: pointer.FromInt(sibling.Revision)}).Return(sibling, nil) summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) @@ -283,9 +284,22 @@ var _ = Describe("Processor", func() { It("retries when it cannot be deleted", func() { expectListWithSibling() processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).Return(wrk, nil) - workClient.EXPECT().Delete(gomock.Any(), sibling.ID, gomock.Nil()).Return(nil, errorsTest.RandomError()) + workClient.EXPECT().Delete(gomock.Any(), sibling.ID, &request.Condition{Revision: pointer.FromInt(sibling.Revision)}).Return(nil, errorsTest.RandomError()) Expect(process().Result).To(Equal(work.ResultFailing)) }) + + // The sibling changed after it was listed — a producer may have merged another change into + // it. Deleting it would destroy that change; leaving it reprocesses the reasons already + // absorbed redundantly, which is the at-least-once contract. + It("continues without deleting it when it changed since it was listed", func() { + expectListWithSibling() + processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).Return(wrk, nil) + workClient.EXPECT().Delete(gomock.Any(), sibling.ID, &request.Condition{Revision: pointer.FromInt(sibling.Revision)}).Return(nil, nil) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }) }) }) diff --git a/summary/store/summary.go b/summary/store/summary.go index 52f0b68be1..222d66f87b 100644 --- a/summary/store/summary.go +++ b/summary/store/summary.go @@ -52,9 +52,9 @@ func NewTypeless(delegate *storeStructuredMongo.Repository) *TypelessSummaries { // OutdatedSummary reports a summary marked outdated by the mechanism being retired, or by the legacy // ingestion service, which is the summary of one type for one user type OutdatedSummary struct { - UserID string `bson:"userId"` - Type string `bson:"type"` - OutdatedSince time.Time `bson:"-"` + UserID string + Type string + OutdatedSince time.Time } // ListOutdated reports the summaries marked outdated, of every type, oldest first, up to the limit. @@ -95,7 +95,8 @@ func (r *TypelessSummaries) ListOutdated(ctx context.Context, limit int) ([]Outd UserID string `bson:"userId"` Type string `bson:"type"` Dates struct { - OutdatedSince *time.Time `bson:"outdatedSince"` + // Non-pointer, as the $lte selector cannot match a document missing the field + OutdatedSince time.Time `bson:"outdatedSince"` } `bson:"dates"` } if err = cursor.All(ctx, &documents); err != nil { @@ -104,13 +105,10 @@ func (r *TypelessSummaries) ListOutdated(ctx context.Context, limit int) ([]Outd outdated := make([]OutdatedSummary, 0, len(documents)) for _, document := range documents { - if document.Dates.OutdatedSince == nil { - continue - } outdated = append(outdated, OutdatedSummary{ UserID: document.UserID, Type: document.Type, - OutdatedSince: *document.Dates.OutdatedSince, + OutdatedSince: document.Dates.OutdatedSince, }) } diff --git a/work/service/coordinator.go b/work/service/coordinator.go index c86481bb7c..0378901e2e 100644 --- a/work/service/coordinator.go +++ b/work/service/coordinator.go @@ -27,6 +27,10 @@ const ( FailingRetryDuration = 1 * time.Minute FailingRetryDurationJitter = 5 * time.Second + // ReapExpiredProcessingInterval decouples the reap cadence from the frequency of the fastest + // processor, which drives how often work is requested + ReapExpiredProcessingInterval = time.Minute + // ReapExpiredProcessingGraceDuration is the duration beyond the processing timeout time that // must elapse before work in state processing is reaped ReapExpiredProcessingGraceDuration = time.Minute @@ -59,6 +63,7 @@ type Coordinator struct { managerCancelFunc context.CancelFunc managerWaitGroup sync.WaitGroup timer *time.Timer + lastReapTime time.Time // Testing NowFunc func() time.Time @@ -140,6 +145,12 @@ func (c *Coordinator) Start() { c.workersCompletionChannel = make(chan *coordinatorProcessingCompletion, c.typeQuantities.Total()) + c.initializeContexts() + + c.startManager() +} + +func (c *Coordinator) initializeContexts() { commonContext := log.NewContextWithLogger(context.Background(), c.logger) workersContext, workersCancelFunc := context.WithCancel(commonContext) @@ -150,8 +161,6 @@ func (c *Coordinator) Start() { managerContext, managerCancelFunc := context.WithCancel(commonContext) c.managerContext = managerContext c.managerCancelFunc = managerCancelFunc - - c.startManager() } func (c *Coordinator) Stop() { @@ -228,10 +237,14 @@ func (c *Coordinator) requestAndDispatchWork() { } // reapExpiredProcessingWork returns work abandoned by a terminated process to failing so that it is -// retried, and so that any work sharing its serial id is no longer prevented from being polled. -// Failure is reported, but does not prevent polling, as any work reaped is already delayed. +// retried. Failure is logged but does not interrupt polling. Reaping runs at most once per interval. func (c *Coordinator) reapExpiredProcessingWork() { - count, err := c.workClient.ReapExpiredProcessing(context.WithoutCancel(c.managerContext)) + if c.Now().Sub(c.lastReapTime) < ReapExpiredProcessingInterval { + return + } + c.lastReapTime = c.Now() + + count, err := c.workClient.ReapExpiredProcessing(c.managerContext) if err != nil { log.LoggerFromContext(c.managerContext).WithError(err).Error("unable to reap expired processing work") } else if count > 0 { diff --git a/work/service/coordinator_internal_test.go b/work/service/coordinator_internal_test.go index c6291d5f77..9b7aa99db3 100644 --- a/work/service/coordinator_internal_test.go +++ b/work/service/coordinator_internal_test.go @@ -1,7 +1,7 @@ package service import ( - "context" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -32,11 +32,9 @@ var _ = Describe("Coordinator", func() { Expect(err).ToNot(HaveOccurred()) Expect(coordinator).ToNot(BeNil()) - // Assigned directly as the contexts are otherwise only assigned by Start, which defers the - // first request for work by CoordinatorDelayInitial - ctx := log.NewContextWithLogger(context.Background(), logger) - coordinator.workersContext = ctx - coordinator.managerContext = ctx + // Initialized directly as the contexts are otherwise only initialized by Start, which + // defers the first request for work by CoordinatorDelayInitial + coordinator.initializeContexts() }) AfterEach(func() { @@ -71,6 +69,32 @@ var _ = Describe("Coordinator", func() { ) coordinator.requestAndDispatchWork() logger.AssertError("unable to reap expired processing work") + + // The coordinator is the single log site for the failure + count := 0 + for _, fields := range logger.SerializedFields { + if fields["message"] == "unable to reap expired processing work" { + count++ + } + } + Expect(count).To(Equal(1)) + }) + + It("does not reap expired processing work again within the reap interval", func() { + workClient.EXPECT().ReapExpiredProcessing(gomock.Any()).Return(0, nil) + workClient.EXPECT().Poll(gomock.Any(), gomock.Any()).Return(nil, nil).Times(2) + coordinator.requestAndDispatchWork() + coordinator.requestAndDispatchWork() + }) + + It("reaps expired processing work again once the reap interval passes", func() { + now := time.Now() + coordinator.NowFunc = func() time.Time { return now } + workClient.EXPECT().ReapExpiredProcessing(gomock.Any()).Return(0, nil).Times(2) + workClient.EXPECT().Poll(gomock.Any(), gomock.Any()).Return(nil, nil).Times(2) + coordinator.requestAndDispatchWork() + now = now.Add(ReapExpiredProcessingInterval) + coordinator.requestAndDispatchWork() }) It("reports the count when expired processing work is reaped", func() { diff --git a/work/store/structured/mongo/mongo.go b/work/store/structured/mongo/mongo.go index 23c5ff79ef..036f3bdfac 100644 --- a/work/store/structured/mongo/mongo.go +++ b/work/store/structured/mongo/mongo.go @@ -128,7 +128,6 @@ func (s *Store) ReapExpiredProcessing(ctx context.Context, graceDuration time.Du updateResult, err := s.UpdateMany(ctx, query, update) lgr = lgr.WithError(err) if err != nil { - lgr.Error("unable to reap expired processing work") return 0, errors.Wrap(err, "unable to reap expired processing work") } @@ -169,18 +168,23 @@ func (s *Store) Poll(ctx context.Context, poll *work.Poll) ([]*work.Work, error) // Sort by processing priority and available time, with _id as a tie breaker // The _id tie breaker guarantees a total order so that, within a serial id group, the - // document already in state processing remains first and the group is reliably excluded below + // document claimed first is deterministic pipeline = append(pipeline, bson.M{"$sort": bson.D{bson.E{Key: "processingPriority", Value: -1}, bson.E{Key: "processingAvailableTime", Value: 1}, bson.E{Key: "_id", Value: 1}}}) // Group all documents by serial id pipeline = append(pipeline, bson.M{"$group": bson.M{"_id": "$serialId", "documents": bson.M{"$push": "$$ROOT"}}}) - // Match any without a serial id or any serial id that does not have one in state processing or failing with retry time in future + // Match any without a serial id or any serial id group that contains no member in state + // processing nor in state failing with retry time in future ($elemMatch binds both failing + // conditions to the same member). Matching the whole group rather than its head is slightly + // conservative: a group is also excluded when a failing member with a future retry sorts after + // an otherwise eligible head. With uniform priority the failing member sorts first anyway; + // where priorities differ, correctness wins over throughput. pipeline = append(pipeline, bson.M{"$match": bson.M{"$or": bson.A{ bson.M{"_id": bson.M{"$exists": false}}, bson.M{"$nor": bson.A{ - bson.M{"documents.0.state": "processing"}, - bson.M{"documents.0.state": "failing", "documents.0.failingRetryTime": bson.M{"$gt": now}}, + bson.M{"documents": bson.M{"$elemMatch": bson.M{"state": "processing"}}}, + bson.M{"documents": bson.M{"$elemMatch": bson.M{"state": "failing", "failingRetryTime": bson.M{"$gt": now}}}}, }}, }}}) diff --git a/work/store/structured/mongo/mongo_test.go b/work/store/structured/mongo/mongo_test.go index cc7b1f18b7..3e971e4c16 100644 --- a/work/store/structured/mongo/mongo_test.go +++ b/work/store/structured/mongo/mongo_test.go @@ -12,6 +12,7 @@ import ( bsonPrimitive "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" + "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" logTest "github.com/tidepool-org/platform/log/test" netTest "github.com/tidepool-org/platform/net/test" @@ -289,6 +290,86 @@ var _ = Describe("Mongo", func() { } }) }) + + // The serial id group exclusion must consider every member of the group, not only the + // member that sorts first: a pending sibling with a higher processing priority sorts + // ahead of a processing or failing member, which must still block the group. + Context("with a work item claimed from a serial id group", func() { + var poll *work.Poll + var serialID string + var claimed *work.Work + + createSibling := func(processingPriority int) *work.Work { + sibling, err := store.Create(ctx, &work.Create{ + Type: typ, + SerialID: pointer.FromString(serialID), + ProcessingPriority: processingPriority, + ProcessingTimeout: processingTimeout, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(sibling).ToNot(BeNil()) + return sibling + } + + updateToFailing := func(wrk *work.Work, retryTime time.Time) { + updated, err := store.Update(ctx, wrk.ID, nil, &work.Update{ + State: work.StateFailing, + FailingUpdate: &work.FailingUpdate{ + FailingError: errors.Serializable{Error: errors.New("test failure")}, + FailingRetryCount: 1, + FailingRetryTime: retryTime, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).ToNot(BeNil()) + Expect(updated.State).To(Equal(work.StateFailing)) + } + + BeforeEach(func() { + poll = &work.Poll{TypeQuantities: work.TypeQuantities{typ: 10}} + serialID = typ + ":" + test.RandomString() + + createSibling(0) + polled, err := store.Poll(ctx, poll) + Expect(err).ToNot(HaveOccurred()) + Expect(polled).To(HaveLen(1)) + claimed = polled[0] + Expect(claimed.State).To(Equal(work.StateProcessing)) + }) + + It("claims nothing while the work item is processing, however a pending sibling sorts", func() { + createSibling(1) + + for index := range 10 { + polled, err := store.Poll(ctx, poll) + Expect(err).ToNot(HaveOccurred()) + Expect(polled).To(BeEmpty(), "poll %d claimed work while a work item sharing its serial id was processing", index) + } + }) + + It("claims nothing while the work item is failing with a future retry time, however a pending sibling sorts", func() { + updateToFailing(claimed, time.Now().Add(time.Hour)) + createSibling(1) + + for index := range 10 { + polled, err := store.Poll(ctx, poll) + Expect(err).ToNot(HaveOccurred()) + Expect(polled).To(BeEmpty(), "poll %d claimed work while a work item sharing its serial id was failing with a future retry", index) + } + }) + + It("claims the work item once its failing retry time has passed", func() { + // The store clamps the failing retry time to no earlier than the update itself, + // so requesting a past time yields a retry time that has passed by the poll + updateToFailing(claimed, time.Now().Add(-time.Minute)) + + polled, err := store.Poll(ctx, poll) + Expect(err).ToNot(HaveOccurred()) + Expect(polled).To(HaveLen(1)) + Expect(polled[0].ID).To(Equal(claimed.ID)) + Expect(polled[0].State).To(Equal(work.StateProcessing)) + }) + }) }) Context("Update", func() { From 41dfb9ca05606b5d13b8da4e419e7621e5d7cccc Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Thu, 20 Aug 2026 16:16:40 +0300 Subject: [PATCH 11/20] Update abbott plugin submodule to postprocess producer head --- private/plugin/abbott | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private/plugin/abbott b/private/plugin/abbott index d25d5a2e9b..fddb0c341e 160000 --- a/private/plugin/abbott +++ b/private/plugin/abbott @@ -1 +1 @@ -Subproject commit d25d5a2e9bc5f18eb84001fde1e98737fe4068c8 +Subproject commit fddb0c341e7b6b567a2babea8afecc51f2f4f35e From dbbae81072acdab62c875e634b8714b80dc35cac Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Mon, 24 Aug 2026 15:52:59 +0300 Subject: [PATCH 12/20] Validate work identity on pickup and skip invalid pending work when absorbing --- data/work/postprocess/processor.go | 18 ++++++-- data/work/postprocess/processor_test.go | 61 +++++++++++++++++++++++++ data/work/postprocess/work.go | 16 +++++++ 3 files changed, 92 insertions(+), 3 deletions(-) diff --git a/data/work/postprocess/processor.go b/data/work/postprocess/processor.go index aaa1630007..7fb0ba21e1 100644 --- a/data/work/postprocess/processor.go +++ b/data/work/postprocess/processor.go @@ -61,6 +61,7 @@ func NewProcessor(dependencies Dependencies) (*Processor, error) { func (p *Processor) Process(ctx context.Context, wrk *work.Work, processingUpdater work.ProcessingUpdater) *work.ProcessResult { return append(p.ProcessPipeline(ctx, wrk, processingUpdater), + func() *work.ProcessResult { return p.validateWork(wrk) }, p.FetchUserFromWorkMetadata, p.absorbPending, p.updateSummaries, @@ -68,6 +69,14 @@ func (p *Processor) Process(ctx context.Context, wrk *work.Work, processingUpdat ).Process(p.Delete) } +// validateWork fails if there is a mismatch between the user id and group/serial ids +func (p *Processor) validateWork(wrk *work.Work) *work.ProcessResult { + if err := validateIdentity(wrk.GroupID, wrk.SerialID, p.Metadata()); err != nil { + return p.Failed(errors.Wrap(err, "work is invalid")) + } + return nil +} + // absorbPending absorbs reasons of other pending work items in this serial group // // The reasons are persisted before the work is deleted, so that a failure between the @@ -91,10 +100,13 @@ func (p *Processor) absorbPending() *work.ProcessResult { reasons := p.Metadata().Reasons for _, wrk := range wrks { workMetadata, err := metadata.Decode[Metadata](p.Context(), wrk.Metadata) + if err == nil && workMetadata == nil { + err = errors.New("metadata is missing") + } + // The pending work item will fail when it's picked up if err != nil { - return p.Failed(errors.Wrap(err, "unable to decode metadata")) - } else if workMetadata == nil { - return p.Failed(errors.New("metadata is missing")) + log.LoggerFromContext(p.Context()).WithError(err).WithField("id", wrk.ID).Warn("work pending for the user has invalid metadata") + continue } absorbed = append(absorbed, wrk) diff --git a/data/work/postprocess/processor_test.go b/data/work/postprocess/processor_test.go index 8a1fc7c9a5..8106620234 100644 --- a/data/work/postprocess/processor_test.go +++ b/data/work/postprocess/processor_test.go @@ -151,6 +151,38 @@ var _ = Describe("Processor", func() { Expect(result.FailedUpdate.FailedError.Error).To(MatchError(ContainSubstring("user id is missing"))) }) + It("fails without retrying when the work has no metadata", func() { + wrk.Metadata = nil + + result := process() + Expect(result.Result).To(Equal(work.ResultFailed)) + Expect(result.FailedUpdate.FailedError.Error).To(MatchError(ContainSubstring("user id is missing"))) + }) + + // Work created outside Enqueue could otherwise mutate the user outside the serialization of + // the user + DescribeTable("fails without retrying when the work is not scoped to the user its metadata reports", + func(mutate func(wrk *work.Work)) { + mutate(wrk) + + result := process() + Expect(result.Result).To(Equal(work.ResultFailed)) + Expect(result.FailedUpdate.FailedError.Error).To(MatchError(ContainSubstring("does not match metadata user id"))) + }, + Entry("the group id names another user", func(wrk *work.Work) { + wrk.GroupID = pointer.FromString(dataWorkPostprocess.IDFromUserID(userTest.RandomUserID())) + }), + Entry("the group id is missing", func(wrk *work.Work) { + wrk.GroupID = nil + }), + Entry("the serial id names another user", func(wrk *work.Work) { + wrk.SerialID = pointer.FromString(dataWorkPostprocess.IDFromUserID(userTest.RandomUserID())) + }), + Entry("the serial id is missing", func(wrk *work.Work) { + wrk.SerialID = nil + }), + ) + It("fails without retrying when the user no longer exists", func() { fetchUser = func() (*user.User, error) { return nil, nil } @@ -225,6 +257,35 @@ var _ = Describe("Processor", func() { Expect(process().Result).To(Equal(work.ResultDelete)) }) + // A sibling left pending fails on its own pickup; failing this valid work with it would + // lose the reasons this work reports + DescribeTable("skips it and processes without failing when", + func(mutate func()) { + mutate() + expectListWithSibling() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }, + Entry("its metadata is invalid", func() { + sibling.Metadata["reasons"] = []any{"INVALID"} + }), + Entry("its metadata is missing", func() { + sibling.Metadata = nil + }), + ) + + It("absorbs the others when one of them is invalid", func() { + invalid := newWork(work.StatePending, []string{dataWorkPostprocess.ReasonUploadCompleted}, time.Now().Add(-time.Second)) + invalid.Metadata = nil + workClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*work.Work{invalid, sibling}, nil) + expectProcessingUpdateThenDelete() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }) + // An upload reporting only full batches is still in progress, so the summaries are not // calculated for every batch so far. Jellyfish defers the same way. Context("that is deferred and reports only a full batch", func() { diff --git a/data/work/postprocess/work.go b/data/work/postprocess/work.go index d54e5e0bb6..b8ceb52176 100644 --- a/data/work/postprocess/work.go +++ b/data/work/postprocess/work.go @@ -6,6 +6,7 @@ import ( mapset "github.com/deckarep/golang-set/v2" + "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/structure" userWork "github.com/tidepool-org/platform/user/work" ) @@ -72,6 +73,21 @@ func IDFromUserID(userID string) string { return fmt.Sprintf("%s:%s", Type, userID) } +// validateIdentity reports an error if the serial or group ids of the metadata don't match the expected user id +func validateIdentity(groupID *string, serialID *string, workMetadata *Metadata) error { + if workMetadata == nil || workMetadata.UserID == nil { + return errors.New("metadata user id is missing") + } + id := IDFromUserID(*workMetadata.UserID) + if groupID == nil || *groupID != id { + return errors.New("group id does not match metadata user id") + } + if serialID == nil || *serialID != id { + return errors.New("serial id does not match metadata user id") + } + return nil +} + type Metadata struct { userWork.Metadata `bson:",inline"` Reasons []string `json:"reasons,omitempty" bson:"reasons,omitempty"` From 358254e1488f8da0d63f324c8f1d72bfdc7ba0aa Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Mon, 24 Aug 2026 16:54:23 +0300 Subject: [PATCH 13/20] Rename outdated sweep index to avoid conflict with legacy index --- data/store/mongo/mongo_summary.go | 2 +- data/store/mongo/mongo_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/store/mongo/mongo_summary.go b/data/store/mongo/mongo_summary.go index bb68caa9f2..2141c7b54e 100644 --- a/data/store/mongo/mongo_summary.go +++ b/data/store/mongo/mongo_summary.go @@ -52,7 +52,7 @@ func (d *SummaryRepository) EnsureIndexes() error { {Key: "dates.outdatedSince", Value: 1}, }, Options: options.Index(). - SetName("OutdatedSince"). + SetName("OutdatedSinceSweep"). SetPartialFilterExpression(bson.D{{Key: "dates.outdatedSince", Value: bson.M{"$exists": true}}}), }, }) diff --git a/data/store/mongo/mongo_test.go b/data/store/mongo/mongo_test.go index c8847efc49..7374fa7fad 100644 --- a/data/store/mongo/mongo_test.go +++ b/data/store/mongo/mongo_test.go @@ -425,7 +425,7 @@ var _ = Describe("Mongo", Label("mongodb", "slow", "integration"), func() { MatchFields(IgnoreExtras, Fields{ "Key": Equal(storeStructuredMongoTest.MakeKeySlice("dates.outdatedSince")), "Background": Equal(false), - "Name": Equal("OutdatedSince"), + "Name": Equal("OutdatedSinceSweep"), "PartialFilterExpression": Equal(bson.D{ {Key: "dates.outdatedSince", Value: bson.D{{Key: "$exists", Value: true}}}, }), From 3967a261c6870a25e90d2b65b3c34fb1f0c189ff Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Tue, 25 Aug 2026 11:07:58 +0300 Subject: [PATCH 14/20] Remove outdated sweeper drained by the summary task runners --- data/service/service/standard.go | 23 -- data/store/mongo/mongo_summary.go | 10 - data/store/mongo/mongo_test.go | 8 - data/work/sweep/outdated/outdated.go | 143 ------------ data/work/sweep/outdated/outdated_test.go | 206 ------------------ .../sweep/outdated/test/outdated_mocks.go | 121 ---------- summary/store/summary.go | 88 -------- summary/store/summary_test.go | 100 --------- 8 files changed, 699 deletions(-) delete mode 100644 data/work/sweep/outdated/outdated.go delete mode 100644 data/work/sweep/outdated/outdated_test.go delete mode 100644 data/work/sweep/outdated/test/outdated_mocks.go diff --git a/data/service/service/standard.go b/data/service/service/standard.go index 5953dbdb5c..6b37449e8d 100644 --- a/data/service/service/standard.go +++ b/data/service/service/standard.go @@ -33,7 +33,6 @@ import ( dataSourceStoreStructuredMongo "github.com/tidepool-org/platform/data/source/store/structured/mongo" dataStoreMongo "github.com/tidepool-org/platform/data/store/mongo" dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" - dataWorkSweepOutdated "github.com/tidepool-org/platform/data/work/sweep/outdated" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/events" "github.com/tidepool-org/platform/log" @@ -62,7 +61,6 @@ import ( serviceService "github.com/tidepool-org/platform/service/service" storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" "github.com/tidepool-org/platform/summary" - summaryStore "github.com/tidepool-org/platform/summary/store" synctaskStoreMongo "github.com/tidepool-org/platform/synctask/store/mongo" "github.com/tidepool-org/platform/twiist" "github.com/tidepool-org/platform/user" @@ -97,7 +95,6 @@ type Standard struct { dataSourceClient *dataSourceServiceClient.Client mailerClient mailer.Client summarizerRegistry *summary.SummarizerRegistry - typelessSummaries *summaryStore.TypelessSummaries workClient *workService.Client notificationsHistoryRecorder notificationsHistory.Recorder abbottClient *abbottClient.Client @@ -629,7 +626,6 @@ func (s *Standard) initializeSummarizerRegistry() error { s.dataStore.NewDataRepository(), s.dataStore.GetClient(), ) - s.typelessSummaries = summaryStore.NewTypeless(summaryRepositoryStore) return nil } @@ -822,17 +818,6 @@ func (s *Standard) initializeWorkProcessorFactories() error { processorFactories = append(processorFactories, processorFactory) } - s.Logger().Debug("Creating data sweep outdated work processor factory") - - if processorFactory, err := dataWorkSweepOutdated.NewProcessorFactory(dataWorkSweepOutdated.Dependencies{ - Dependencies: dependencies, - Summaries: s.typelessSummaries, - }); err != nil { - return errors.Wrap(err, "unable to create data sweep outdated work processor factory") - } else { - processorFactories = append(processorFactories, processorFactory) - } - if s.abbottClient != nil { s.Logger().Debug("Creating abbott processor factories") @@ -959,14 +944,6 @@ func (s *Standard) initializeWorkSingletons() error { ctx, cancel := context.WithTimeout(log.NewContextWithLogger(context.Background(), s.Logger()), 10*time.Second) defer cancel() - s.Logger().Debug("Creating data sweep work") - - if workCreate, err := dataWorkSweepOutdated.NewWorkCreate(); err != nil { - return errors.Wrap(err, "unable to create data sweep outdated work create") - } else if _, err = s.workClient.Create(ctx, workCreate); err != nil { - return errors.Wrap(err, "unable to create data sweep outdated work") - } - if s.ouraClient != nil { s.Logger().Debug("Creating oura webhook subscribe work") diff --git a/data/store/mongo/mongo_summary.go b/data/store/mongo/mongo_summary.go index 2141c7b54e..b1ca241788 100644 --- a/data/store/mongo/mongo_summary.go +++ b/data/store/mongo/mongo_summary.go @@ -45,16 +45,6 @@ func (d *SummaryRepository) EnsureIndexes() error { Options: options.Index(). SetName("OutdatedAndSchemaMigration"), }, - { - // Serves the outdated sweep across types. Partial as the mark is transient — it is - // cleared as it is swept — so the index holds only the marks outstanding. - Keys: bson.D{ - {Key: "dates.outdatedSince", Value: 1}, - }, - Options: options.Index(). - SetName("OutdatedSinceSweep"). - SetPartialFilterExpression(bson.D{{Key: "dates.outdatedSince", Value: bson.M{"$exists": true}}}), - }, }) } diff --git a/data/store/mongo/mongo_test.go b/data/store/mongo/mongo_test.go index 7374fa7fad..16399add39 100644 --- a/data/store/mongo/mongo_test.go +++ b/data/store/mongo/mongo_test.go @@ -422,14 +422,6 @@ var _ = Describe("Mongo", Label("mongodb", "slow", "integration"), func() { "Background": Equal(false), "Name": Equal("OutdatedAndSchemaMigration"), }), - MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("dates.outdatedSince")), - "Background": Equal(false), - "Name": Equal("OutdatedSinceSweep"), - "PartialFilterExpression": Equal(bson.D{ - {Key: "dates.outdatedSince", Value: bson.D{{Key: "$exists", Value: true}}}, - }), - }), )) }) diff --git a/data/work/sweep/outdated/outdated.go b/data/work/sweep/outdated/outdated.go deleted file mode 100644 index 9723b0612b..0000000000 --- a/data/work/sweep/outdated/outdated.go +++ /dev/null @@ -1,143 +0,0 @@ -package outdated - -import ( - "context" - "time" - - dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" - "github.com/tidepool-org/platform/errors" - "github.com/tidepool-org/platform/log" - "github.com/tidepool-org/platform/pointer" - summaryStore "github.com/tidepool-org/platform/summary/store" - "github.com/tidepool-org/platform/work" - workBase "github.com/tidepool-org/platform/work/base" -) - -//go:generate mockgen -source=outdated.go -destination=test/outdated_mocks.go -package=test -typed - -const ( - Type = "org.tidepool.data.sweep.outdated" - Quantity = 1 - Frequency = 1 * time.Minute - ProcessingTimeout = 5 * time.Minute - - // PendingAvailableDuration is how long the sweep waits before running again - PendingAvailableDuration = 1 * time.Minute - - // BatchSize is how many summaries are reported per request, and PageLimit how many requests are - // made per run, so that one run cannot hold a processor indefinitely against a large backlog - BatchSize = 500 - PageLimit = 20 - - FailingRetryDuration = 1 * time.Minute - FailingRetryDurationJitter = 5 * time.Second -) - -// Summaries is satisfied by summary/store.TypelessSummaries -type Summaries interface { - ListOutdated(ctx context.Context, limit int) ([]summaryStore.OutdatedSummary, error) - ClearOutdated(ctx context.Context, userID string, typ string, observed time.Time) error -} - -type Dependencies struct { - workBase.Dependencies - Summaries -} - -func (d Dependencies) Validate() error { - if err := d.Dependencies.Validate(); err != nil { - return err - } - if d.Summaries == nil { - return errors.New("summaries is missing") - } - return nil -} - -func NewProcessorFactory(dependencies Dependencies) (*workBase.ProcessorFactory, error) { - if err := dependencies.Validate(); err != nil { - return nil, errors.Wrap(err, "dependencies is invalid") - } - processorFactory := func() (work.Processor, error) { return NewProcessor(dependencies) } - return workBase.NewProcessorFactory(Type, Quantity, Frequency, processorFactory) -} - -func NewWorkCreate() (*work.Create, error) { - return &work.Create{ - Type: Type, - DeduplicationID: pointer.From(work.DeduplicationIDSingleton), - ProcessingTimeout: int(ProcessingTimeout.Seconds()), - }, nil -} - -type Processor struct { - *workBase.ProcessorWithoutMetadata - Summaries -} - -func NewProcessor(dependencies Dependencies) (*Processor, error) { - if err := dependencies.Validate(); err != nil { - return nil, errors.Wrap(err, "dependencies is invalid") - } - - processResultBuilder := &workBase.ProcessResultBuilder{ - ProcessResultPendingBuilder: &workBase.ConstantProcessResultPendingBuilder{ - Duration: PendingAvailableDuration, - }, - ProcessResultFailingBuilder: &workBase.ConstantProcessResultFailingBuilder{ - Duration: FailingRetryDuration, - }, - } - - processor, err := workBase.NewProcessorWithoutMetadata(dependencies.Dependencies, processResultBuilder) - if err != nil { - return nil, errors.Wrap(err, "unable to create processor") - } - - return &Processor{ - ProcessorWithoutMetadata: processor, - Summaries: dependencies.Summaries, - }, nil -} - -func (p *Processor) Process(ctx context.Context, wrk *work.Work, processingUpdater work.ProcessingUpdater) *work.ProcessResult { - return append(p.ProcessPipeline(ctx, wrk, processingUpdater), - p.sweep, - ).Process(p.Pending) -} - -// sweep reports the summaries marked outdated by the retired mechanism as work, -// so that they are calculated once the mechanism that marked them is gone. -// -// The work is created before the mark is cleared, so that a failure between the two reports the change -// twice rather than not at all. Clearing is guarded upon the time observed, so a mark made in between -// is reported by the next run rather than discarded. -func (p *Processor) sweep() *work.ProcessResult { - var swept int - - for range PageLimit { - outdated, err := p.ListOutdated(p.Context(), BatchSize) - if err != nil { - return p.Failing(errors.Wrap(err, "unable to list outdated summaries")) - } - if len(outdated) == 0 { - break - } - - for _, summary := range outdated { - if err = dataWorkPostprocess.Enqueue(p.Context(), p.WorkClient(), summary.UserID, dataWorkPostprocess.ReasonDataAdded); err != nil { - return p.Failing(errors.Wrap(err, "unable to report the change")) - } - if err = p.ClearOutdated(p.Context(), summary.UserID, summary.Type, summary.OutdatedSince); err != nil { - return p.Failing(errors.Wrap(err, "unable to clear the outdated summary")) - } - } - swept += len(outdated) - } - - if swept > 0 { - log.LoggerFromContext(p.Context()).WithField("count", swept).Info("reported outdated summaries as work") - } - - return nil -} diff --git a/data/work/sweep/outdated/outdated_test.go b/data/work/sweep/outdated/outdated_test.go deleted file mode 100644 index 5c1b560a20..0000000000 --- a/data/work/sweep/outdated/outdated_test.go +++ /dev/null @@ -1,206 +0,0 @@ -package outdated_test - -import ( - "context" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "go.uber.org/mock/gomock" - - dataWorkPostprocess "github.com/tidepool-org/platform/data/work/postprocess" - dataWorkSweepOutdated "github.com/tidepool-org/platform/data/work/sweep/outdated" - dataWorkSweepOutdatedTest "github.com/tidepool-org/platform/data/work/sweep/outdated/test" - errorsTest "github.com/tidepool-org/platform/errors/test" - "github.com/tidepool-org/platform/log" - logTest "github.com/tidepool-org/platform/log/test" - summaryStore "github.com/tidepool-org/platform/summary/store" - summaryTypes "github.com/tidepool-org/platform/summary/types" - "github.com/tidepool-org/platform/test" - userTest "github.com/tidepool-org/platform/user/test" - "github.com/tidepool-org/platform/work" - workBase "github.com/tidepool-org/platform/work/base" - workTest "github.com/tidepool-org/platform/work/test" -) - -func TestSuite(t *testing.T) { - test.Test(t) -} - -var _ = Describe("Outdated", func() { - var controller *gomock.Controller - var workClient *workTest.MockClient - var summaries *dataWorkSweepOutdatedTest.MockSummaries - var processor *dataWorkSweepOutdated.Processor - var ctx context.Context - var wrk *work.Work - var outdatedSince time.Time - - process := func() *work.ProcessResult { - return processor.Process(ctx, wrk, workTest.NewMockProcessingUpdater(controller)) - } - - BeforeEach(func() { - controller = gomock.NewController(GinkgoT()) - workClient = workTest.NewMockClient(controller) - summaries = dataWorkSweepOutdatedTest.NewMockSummaries(controller) - ctx = log.NewContextWithLogger(context.Background(), logTest.NewLogger()) - outdatedSince = time.Now().UTC().Add(-time.Hour) - wrk = &work.Work{ - ID: workTest.RandomID(), - Type: dataWorkSweepOutdated.Type, - ProcessingTimeout: int(dataWorkSweepOutdated.ProcessingTimeout.Seconds()), - State: work.StateProcessing, - } - - var err error - processor, err = dataWorkSweepOutdated.NewProcessor(dataWorkSweepOutdated.Dependencies{ - Dependencies: workBase.Dependencies{WorkClient: workClient}, - Summaries: summaries, - }) - Expect(err).ToNot(HaveOccurred()) - }) - - AfterEach(func() { - controller.Finish() - }) - - It("waits to run again when nothing is outdated", func() { - summaries.EXPECT().ListOutdated(gomock.Any(), gomock.Any()).Return(nil, nil) - - result := process() - Expect(result.Result).To(Equal(work.ResultPending)) - Expect(result.PendingUpdate.ProcessingAvailableTime).To(BeTemporally("~", time.Now().Add(dataWorkSweepOutdated.PendingAvailableDuration), time.Second)) - }) - - Context("with outdated summaries", func() { - var userID string - - BeforeEach(func() { - userID = userTest.RandomUserID() - }) - - // Reported on the first request, then nothing, so the run stops rather than repeating - expectListOnce := func(outdated ...summaryStore.OutdatedSummary) { - reported := false - summaries.EXPECT().ListOutdated(gomock.Any(), dataWorkSweepOutdated.BatchSize).DoAndReturn( - func(_ context.Context, _ int) ([]summaryStore.OutdatedSummary, error) { - if reported { - return nil, nil - } - reported = true - return outdated, nil - }).Times(2) - } - - // The work is created before the mark is cleared, so that a failure between the two reports the - // change twice rather than not at all - It("reports the change as work before clearing the mark", func() { - expectListOnce(summaryStore.OutdatedSummary{UserID: userID, Type: summaryTypes.SummaryTypeCGM, OutdatedSince: outdatedSince}) - gomock.InOrder( - workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(&work.Work{}, nil), - summaries.EXPECT().ClearOutdated(gomock.Any(), userID, summaryTypes.SummaryTypeCGM, outdatedSince).Return(nil), - ) - - Expect(process().Result).To(Equal(work.ResultPending)) - }) - - // A user with more than one summary marked is reported once per mark, and every mark is - // cleared with its own report — the work created collapses into one calculation on pickup, so - // nothing groups them here - It("reports each mark of a user, and clears each", func() { - expectListOnce( - summaryStore.OutdatedSummary{UserID: userID, Type: summaryTypes.SummaryTypeCGM, OutdatedSince: outdatedSince}, - summaryStore.OutdatedSummary{UserID: userID, Type: summaryTypes.SummaryTypeBGM, OutdatedSince: outdatedSince}, - summaryStore.OutdatedSummary{UserID: userID, Type: summaryTypes.SummaryTypeContinuous, OutdatedSince: outdatedSince}, - ) - workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(&work.Work{}, nil).Times(3) - summaries.EXPECT().ClearOutdated(gomock.Any(), userID, gomock.Any(), outdatedSince).Return(nil).Times(3) - - Expect(process().Result).To(Equal(work.ResultPending)) - }) - - It("reports the change as data added, which requests no synchronization", func() { - expectListOnce(summaryStore.OutdatedSummary{UserID: userID, Type: summaryTypes.SummaryTypeCGM, OutdatedSince: outdatedSince}) - workClient.EXPECT().Create(gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, create *work.Create) (*work.Work, error) { - Expect(create.Metadata).To(HaveKeyWithValue("reasons", ConsistOf(dataWorkPostprocess.ReasonDataAdded))) - Expect(create.Metadata).To(HaveKeyWithValue("userId", userID)) - return &work.Work{}, nil - }) - summaries.EXPECT().ClearOutdated(gomock.Any(), userID, gomock.Any(), gomock.Any()).Return(nil) - - Expect(process().Result).To(Equal(work.ResultPending)) - }) - - It("retries when the work cannot be created, without clearing the mark", func() { - summaries.EXPECT().ListOutdated(gomock.Any(), gomock.Any()).Return( - []summaryStore.OutdatedSummary{{UserID: userID, Type: summaryTypes.SummaryTypeCGM, OutdatedSince: outdatedSince}}, nil) - workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil, errorsTest.RandomError()) - - Expect(process().Result).To(Equal(work.ResultFailing)) - }) - - It("retries when the mark cannot be cleared", func() { - summaries.EXPECT().ListOutdated(gomock.Any(), gomock.Any()).Return( - []summaryStore.OutdatedSummary{{UserID: userID, Type: summaryTypes.SummaryTypeCGM, OutdatedSince: outdatedSince}}, nil) - workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(&work.Work{}, nil) - summaries.EXPECT().ClearOutdated(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errorsTest.RandomError()) - - Expect(process().Result).To(Equal(work.ResultFailing)) - }) - - It("retries when the summaries cannot be listed", func() { - summaries.EXPECT().ListOutdated(gomock.Any(), gomock.Any()).Return(nil, errorsTest.RandomError()) - - Expect(process().Result).To(Equal(work.ResultFailing)) - }) - - // A backlog larger than one run is left to the next, so that a run cannot hold the processor - // against it indefinitely - It("reports no more than the page limit in one run", func() { - summaries.EXPECT().ListOutdated(gomock.Any(), gomock.Any()).Return( - []summaryStore.OutdatedSummary{{UserID: userID, Type: summaryTypes.SummaryTypeCGM, OutdatedSince: outdatedSince}}, nil). - Times(dataWorkSweepOutdated.PageLimit) - workClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(&work.Work{}, nil).Times(dataWorkSweepOutdated.PageLimit) - summaries.EXPECT().ClearOutdated(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(dataWorkSweepOutdated.PageLimit) - - Expect(process().Result).To(Equal(work.ResultPending)) - }) - }) - - Context("Dependencies", func() { - It("reports the work client is missing", func() { - _, err := dataWorkSweepOutdated.NewProcessorFactory(dataWorkSweepOutdated.Dependencies{ - Summaries: summaries, - }) - Expect(err).To(MatchError(ContainSubstring("work client is missing"))) - }) - - It("reports the summaries are missing", func() { - _, err := dataWorkSweepOutdated.NewProcessorFactory(dataWorkSweepOutdated.Dependencies{ - Dependencies: workBase.Dependencies{WorkClient: workClient}, - }) - Expect(err).To(MatchError(ContainSubstring("summaries is missing"))) - }) - - It("reports the type, quantity and frequency of the work", func() { - processorFactory, err := dataWorkSweepOutdated.NewProcessorFactory(dataWorkSweepOutdated.Dependencies{ - Dependencies: workBase.Dependencies{WorkClient: workClient}, - Summaries: summaries, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(processorFactory.Type()).To(Equal("org.tidepool.data.sweep.outdated")) - Expect(processorFactory.Quantity()).To(Equal(1)) - Expect(processorFactory.Frequency()).To(Equal(time.Minute)) - }) - - It("creates the work as a singleton, so that one sweep runs however many services register it", func() { - workCreate, err := dataWorkSweepOutdated.NewWorkCreate() - Expect(err).ToNot(HaveOccurred()) - Expect(workCreate.DeduplicationID).ToNot(BeNil()) - Expect(*workCreate.DeduplicationID).To(Equal(work.DeduplicationIDSingleton)) - }) - }) -}) diff --git a/data/work/sweep/outdated/test/outdated_mocks.go b/data/work/sweep/outdated/test/outdated_mocks.go deleted file mode 100644 index dcbf3e9ecf..0000000000 --- a/data/work/sweep/outdated/test/outdated_mocks.go +++ /dev/null @@ -1,121 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: outdated.go -// -// Generated by this command: -// -// mockgen -source=outdated.go -destination=test/outdated_mocks.go -package=test -typed -// - -// Package test is a generated GoMock package. -package test - -import ( - context "context" - reflect "reflect" - time "time" - - gomock "go.uber.org/mock/gomock" - - store "github.com/tidepool-org/platform/summary/store" -) - -// MockSummaries is a mock of Summaries interface. -type MockSummaries struct { - ctrl *gomock.Controller - recorder *MockSummariesMockRecorder - isgomock struct{} -} - -// MockSummariesMockRecorder is the mock recorder for MockSummaries. -type MockSummariesMockRecorder struct { - mock *MockSummaries -} - -// NewMockSummaries creates a new mock instance. -func NewMockSummaries(ctrl *gomock.Controller) *MockSummaries { - mock := &MockSummaries{ctrl: ctrl} - mock.recorder = &MockSummariesMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockSummaries) EXPECT() *MockSummariesMockRecorder { - return m.recorder -} - -// ClearOutdated mocks base method. -func (m *MockSummaries) ClearOutdated(ctx context.Context, userID, typ string, observed time.Time) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClearOutdated", ctx, userID, typ, observed) - ret0, _ := ret[0].(error) - return ret0 -} - -// ClearOutdated indicates an expected call of ClearOutdated. -func (mr *MockSummariesMockRecorder) ClearOutdated(ctx, userID, typ, observed any) *MockSummariesClearOutdatedCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearOutdated", reflect.TypeOf((*MockSummaries)(nil).ClearOutdated), ctx, userID, typ, observed) - return &MockSummariesClearOutdatedCall{Call: call} -} - -// MockSummariesClearOutdatedCall wrap *gomock.Call -type MockSummariesClearOutdatedCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSummariesClearOutdatedCall) Return(arg0 error) *MockSummariesClearOutdatedCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSummariesClearOutdatedCall) Do(f func(context.Context, string, string, time.Time) error) *MockSummariesClearOutdatedCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSummariesClearOutdatedCall) DoAndReturn(f func(context.Context, string, string, time.Time) error) *MockSummariesClearOutdatedCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// ListOutdated mocks base method. -func (m *MockSummaries) ListOutdated(ctx context.Context, limit int) ([]store.OutdatedSummary, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListOutdated", ctx, limit) - ret0, _ := ret[0].([]store.OutdatedSummary) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ListOutdated indicates an expected call of ListOutdated. -func (mr *MockSummariesMockRecorder) ListOutdated(ctx, limit any) *MockSummariesListOutdatedCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListOutdated", reflect.TypeOf((*MockSummaries)(nil).ListOutdated), ctx, limit) - return &MockSummariesListOutdatedCall{Call: call} -} - -// MockSummariesListOutdatedCall wrap *gomock.Call -type MockSummariesListOutdatedCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockSummariesListOutdatedCall) Return(arg0 []store.OutdatedSummary, arg1 error) *MockSummariesListOutdatedCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockSummariesListOutdatedCall) Do(f func(context.Context, int) ([]store.OutdatedSummary, error)) *MockSummariesListOutdatedCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSummariesListOutdatedCall) DoAndReturn(f func(context.Context, int) ([]store.OutdatedSummary, error)) *MockSummariesListOutdatedCall { - c.Call = c.Call.DoAndReturn(f) - return c -} diff --git a/summary/store/summary.go b/summary/store/summary.go index 222d66f87b..31c6066309 100644 --- a/summary/store/summary.go +++ b/summary/store/summary.go @@ -49,94 +49,6 @@ func NewTypeless(delegate *storeStructuredMongo.Repository) *TypelessSummaries { } } -// OutdatedSummary reports a summary marked outdated by the mechanism being retired, or by the legacy -// ingestion service, which is the summary of one type for one user -type OutdatedSummary struct { - UserID string - Type string - OutdatedSince time.Time -} - -// ListOutdated reports the summaries marked outdated, of every type, oldest first, up to the limit. -// -// A limit rather than pagination, deliberately: the caller clears the marks it reports, so repeating -// the call reports the *next* oldest. An offset here would skip past the new head of the set as it -// shrinks, which is also why this must never be "fixed" to page as ListMigratableUserIDs does. -// -// Reported across types rather than one type at a time, as the work created for a user recalculates -// every one of their summaries, so a user whose summaries are all marked reports one user rather than -// three. GetOutdatedUserIDs is not reused as it reports no outdated time per summary, which -// ClearOutdated requires, and it records the queue metrics of the mechanism being retired. -func (r *TypelessSummaries) ListOutdated(ctx context.Context, limit int) ([]OutdatedSummary, error) { - if ctx == nil { - return nil, errors.New("context is missing") - } - if limit <= 0 { - return nil, errors.New("limit is invalid") - } - - // No filter upon the reason, as the legacy ingestion service reports its own. A summary marked - // outdated in the future is deferred by that service until its upload falls quiet, so it is - // reported only once that time passes. - selector := bson.M{"dates.outdatedSince": bson.M{"$lte": time.Now().UTC()}} - - opts := options.Find() - opts.SetSort(bson.D{{Key: "dates.outdatedSince", Value: 1}}) - opts.SetLimit(int64(limit)) - opts.SetProjection(bson.M{"userId": 1, "type": 1, "dates.outdatedSince": 1}) - - cursor, err := r.Find(ctx, selector, opts) - if err != nil { - return nil, fmt.Errorf("unable to list outdated summaries: %w", err) - } - defer cursor.Close(ctx) - - var documents []struct { - UserID string `bson:"userId"` - Type string `bson:"type"` - Dates struct { - // Non-pointer, as the $lte selector cannot match a document missing the field - OutdatedSince time.Time `bson:"outdatedSince"` - } `bson:"dates"` - } - if err = cursor.All(ctx, &documents); err != nil { - return nil, fmt.Errorf("unable to decode outdated summaries: %w", err) - } - - outdated := make([]OutdatedSummary, 0, len(documents)) - for _, document := range documents { - outdated = append(outdated, OutdatedSummary{ - UserID: document.UserID, - Type: document.Type, - OutdatedSince: document.Dates.OutdatedSince, - }) - } - - return outdated, nil -} - -// ClearOutdated clears the outdated mark of a summary, but only while it reports the outdated time -// observed, so that a mark made since is retained and reported again rather than discarded -func (r *TypelessSummaries) ClearOutdated(ctx context.Context, userID string, typ string, observed time.Time) error { - if ctx == nil { - return errors.New("context is missing") - } - if userID == "" { - return errors.New("userId is missing") - } - if typ == "" { - return errors.New("type is missing") - } - - selector := bson.M{"userId": userID, "type": typ, "dates.outdatedSince": observed} - update := bson.M{"$unset": bson.M{"dates.outdatedSince": "", "dates.outdatedReason": ""}} - - if _, err := r.UpdateOne(ctx, selector, update); err != nil { - return fmt.Errorf("unable to clear outdated summary for user %s: %w", userID, err) - } - return nil -} - func (r *Summaries[PP, PB, P, B]) GetSummary(ctx context.Context, userId string) (*types.Summary[PP, PB, P, B], error) { if ctx == nil { return nil, errors.New("context is missing") diff --git a/summary/store/summary_test.go b/summary/store/summary_test.go index 12986d29ce..96de411010 100644 --- a/summary/store/summary_test.go +++ b/summary/store/summary_test.go @@ -138,106 +138,6 @@ var _ = Describe("Summary Periods Mongo", Label("mongodb", "slow", "integration" }) }) - Context("ListOutdated and ClearOutdated", func() { - var userIdTwo string - var outdatedTime time.Time - - BeforeEach(func() { - userIdTwo = userTest.RandomUserID() - outdatedTime = time.Now().UTC().Add(-time.Hour).Truncate(time.Millisecond) - }) - - createOutdated := func(userIDs ...string) { - summaries := make([]*types.Summary[*types.ContinuousPeriods, *types.ContinuousBucket, types.ContinuousPeriods, types.ContinuousBucket], len(userIDs)) - for index, userID := range userIDs { - summaries[index] = test.RandomContinuousSummary(userID) - summaries[index].Dates.OutdatedSince = &outdatedTime - summaries[index].Dates.OutdatedReason = []string{"LEGACY_DATA_ADDED"} - } - _, err = continuousStore.CreateSummaries(ctx, summaries) - Expect(err).ToNot(HaveOccurred()) - } - - It("reports the outdated summaries with their type and the time they were marked", func() { - createOutdated(userId, userIdTwo) - - outdated, err := typelessStore.ListOutdated(ctx, 100) - Expect(err).ToNot(HaveOccurred()) - Expect(outdated).To(HaveLen(2)) - Expect(outdated[0].Type).To(Equal(types.SummaryTypeContinuous)) - Expect(outdated[0].OutdatedSince).To(BeTemporally("==", outdatedTime)) - Expect([]string{outdated[0].UserID, outdated[1].UserID}).To(ConsistOf(userId, userIdTwo)) - }) - - // The legacy ingestion service defers a full batch by marking the summary outdated in - // the future, which is reported only once that time passes, honouring its quiet window - It("does not report a summary marked outdated in the future", func() { - deferred := time.Now().UTC().Add(90 * time.Second).Truncate(time.Millisecond) - summary := test.RandomContinuousSummary(userId) - summary.Dates.OutdatedSince = &deferred - _, err = continuousStore.CreateSummaries(ctx, []*types.Summary[*types.ContinuousPeriods, *types.ContinuousBucket, types.ContinuousPeriods, types.ContinuousBucket]{summary}) - Expect(err).ToNot(HaveOccurred()) - - outdated, err := typelessStore.ListOutdated(ctx, 100) - Expect(err).ToNot(HaveOccurred()) - Expect(outdated).To(BeEmpty()) - }) - - It("does not report a summary that is not outdated", func() { - summary := test.RandomContinuousSummary(userId) - summary.Dates.OutdatedSince = nil - _, err = continuousStore.CreateSummaries(ctx, []*types.Summary[*types.ContinuousPeriods, *types.ContinuousBucket, types.ContinuousPeriods, types.ContinuousBucket]{summary}) - Expect(err).ToNot(HaveOccurred()) - - outdated, err := typelessStore.ListOutdated(ctx, 100) - Expect(err).ToNot(HaveOccurred()) - Expect(outdated).To(BeEmpty()) - }) - - It("clears the outdated mark it observed", func() { - createOutdated(userId) - - Expect(typelessStore.ClearOutdated(ctx, userId, types.SummaryTypeContinuous, outdatedTime)).To(Succeed()) - - outdated, err := typelessStore.ListOutdated(ctx, 100) - Expect(err).ToNot(HaveOccurred()) - Expect(outdated).To(BeEmpty()) - - summary, err := continuousStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - Expect(summary.Dates.OutdatedReason).To(BeEmpty()) - }) - - // A mark made between the report and the clear reports a different time, and must be - // retained so that it is reported again rather than discarded - It("retains a mark made since the one it observed", func() { - createOutdated(userId) - remarked := time.Now().UTC().Truncate(time.Millisecond) - - summary, err := continuousStore.GetSummary(ctx, userId) - Expect(err).ToNot(HaveOccurred()) - summary.Dates.OutdatedSince = &remarked - Expect(continuousStore.ReplaceSummary(ctx, summary)).To(Succeed()) - - Expect(typelessStore.ClearOutdated(ctx, userId, types.SummaryTypeContinuous, outdatedTime)).To(Succeed()) - - outdated, err := typelessStore.ListOutdated(ctx, 100) - Expect(err).ToNot(HaveOccurred()) - Expect(outdated).To(HaveLen(1)) - Expect(outdated[0].OutdatedSince).To(BeTemporally("==", remarked)) - }) - - It("reports errors for missing parameters", func() { - _, err = typelessStore.ListOutdated(nil, 100) - Expect(err).To(MatchError("context is missing")) - _, err = typelessStore.ListOutdated(ctx, 0) - Expect(err).To(MatchError("limit is invalid")) - Expect(typelessStore.ClearOutdated(nil, userId, types.SummaryTypeContinuous, outdatedTime)).To(MatchError("context is missing")) - Expect(typelessStore.ClearOutdated(ctx, "", types.SummaryTypeContinuous, outdatedTime)).To(MatchError("userId is missing")) - Expect(typelessStore.ClearOutdated(ctx, userId, "", outdatedTime)).To(MatchError("type is missing")) - }) - }) - Context("GetMigratableUserIDs", func() { var userIds []string var userIdTwo string From ab3e8b43ca61419013bf6eba9b7c29f7da2a334d Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Tue, 25 Aug 2026 11:07:58 +0300 Subject: [PATCH 15/20] Add work queue size by type and state prometheus metrics --- work/service/client.go | 7 ++++ work/service/coordinator.go | 36 +++++++++++++++++++ work/service/coordinator_internal_test.go | 1 + work/service/test/client_mocks.go | 42 +++++++++++++++++++++-- work/service/test/coordinator_mocks.go | 42 +++++++++++++++++++++-- work/store/structured/mongo/mongo.go | 26 ++++++++++++++ work/store/structured/mongo/mongo_test.go | 30 ++++++++++++++++ 7 files changed, 180 insertions(+), 4 deletions(-) diff --git a/work/service/client.go b/work/service/client.go index b072635073..5b9a012867 100644 --- a/work/service/client.go +++ b/work/service/client.go @@ -16,6 +16,7 @@ import ( type Store interface { Poll(ctx context.Context, poll *work.Poll) ([]*work.Work, error) ReapExpiredProcessing(ctx context.Context, graceDuration time.Duration) (int, error) + QueueSizes(ctx context.Context) ([]work.QueueSize, error) List(ctx context.Context, filter *work.Filter, pagination *page.Pagination) ([]*work.Work, error) Create(ctx context.Context, create *work.Create) (*work.Work, error) Get(ctx context.Context, id string, condition *storeStructured.Condition) (*work.Work, error) @@ -47,6 +48,12 @@ func (c *Client) ReapExpiredProcessing(ctx context.Context) (int, error) { return c.store.ReapExpiredProcessing(ctx, ReapExpiredProcessingGraceDuration) } +// QueueSizes is intentionally absent from work.Client as it is coordinator +// infrastructure rather than part of the interface offered to those that create work +func (c *Client) QueueSizes(ctx context.Context) ([]work.QueueSize, error) { + return c.store.QueueSizes(ctx) +} + func (c *Client) List(ctx context.Context, filter *work.Filter, pagination *page.Pagination) ([]*work.Work, error) { return c.store.List(ctx, filter, pagination) } diff --git a/work/service/coordinator.go b/work/service/coordinator.go index 0378901e2e..ef4e302132 100644 --- a/work/service/coordinator.go +++ b/work/service/coordinator.go @@ -7,6 +7,9 @@ import ( "sync" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/tidepool-org/platform/auth" "github.com/tidepool-org/platform/crypto" "github.com/tidepool-org/platform/errors" @@ -34,8 +37,17 @@ const ( // ReapExpiredProcessingGraceDuration is the duration beyond the processing timeout time that // must elapse before work in state processing is reaped ReapExpiredProcessingGraceDuration = time.Minute + + // QueueSizeMetricsInterval decouples the metrics cadence from how often work is requested, + // the same way the reap interval does + QueueSizeMetricsInterval = time.Minute ) +var QueueSizeMetric = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "tidepool_work_queue_size", + Help: "The current number of work items by type and state", +}, []string{"type", "state"}) + type ServerSessionTokenProvider interface { ServerSessionToken() (string, error) } @@ -43,6 +55,7 @@ type ServerSessionTokenProvider interface { type WorkClient interface { Poll(ctx context.Context, poll *work.Poll) ([]*work.Work, error) ReapExpiredProcessing(ctx context.Context) (int, error) + QueueSizes(ctx context.Context) ([]work.QueueSize, error) Update(ctx context.Context, id string, condition *request.Condition, update *work.Update) (*work.Work, error) Delete(ctx context.Context, id string, condition *request.Condition) (*work.Work, error) } @@ -64,6 +77,7 @@ type Coordinator struct { managerWaitGroup sync.WaitGroup timer *time.Timer lastReapTime time.Time + lastQueueSizeMetricsTime time.Time // Testing NowFunc func() time.Time @@ -218,6 +232,7 @@ func (c *Coordinator) requestAndDispatchWork() { } c.reapExpiredProcessingWork() + c.updateQueueSizeMetrics() typeQuantities := c.typeQuantities.NonZero() if typeQuantities.IsEmpty() { @@ -252,6 +267,27 @@ func (c *Coordinator) reapExpiredProcessingWork() { } } +// updateQueueSizeMetrics reports the number of work items by type and state, at most once per +// interval. Failure is logged but does not interrupt polling. +func (c *Coordinator) updateQueueSizeMetrics() { + if c.Now().Sub(c.lastQueueSizeMetricsTime) < QueueSizeMetricsInterval { + return + } + c.lastQueueSizeMetricsTime = c.Now() + + queueSizes, err := c.workClient.QueueSizes(c.managerContext) + if err != nil { + log.LoggerFromContext(c.managerContext).WithError(err).Error("unable to count work by type and state") + return + } + + // Reset so that a type and state no longer present reports absent rather than stale + QueueSizeMetric.Reset() + for _, queueSize := range queueSizes { + QueueSizeMetric.WithLabelValues(queueSize.Type, queueSize.State).Set(float64(queueSize.Count)) + } +} + func (c *Coordinator) dispatchWork(ctx context.Context, wrk *work.Work) { c.typeQuantities.Decrement(wrk.Type) c.workersWaitGroup.Go(func() { diff --git a/work/service/coordinator_internal_test.go b/work/service/coordinator_internal_test.go index 9b7aa99db3..4333ea8fdf 100644 --- a/work/service/coordinator_internal_test.go +++ b/work/service/coordinator_internal_test.go @@ -26,6 +26,7 @@ var _ = Describe("Coordinator", func() { controller = gomock.NewController(GinkgoT()) logger = logTest.NewLogger() workClient = workServiceTest.NewMockWorkClient(controller) + workClient.EXPECT().QueueSizes(gomock.Any()).Return(nil, nil).AnyTimes() var err error coordinator, err = NewCoordinator(logger, workServiceTest.NewMockServerSessionTokenProvider(controller), workClient) diff --git a/work/service/test/client_mocks.go b/work/service/test/client_mocks.go index 5a6b50a268..039e2986dc 100644 --- a/work/service/test/client_mocks.go +++ b/work/service/test/client_mocks.go @@ -14,11 +14,10 @@ import ( reflect "reflect" time "time" - gomock "go.uber.org/mock/gomock" - page "github.com/tidepool-org/platform/page" structured "github.com/tidepool-org/platform/store/structured" work "github.com/tidepool-org/platform/work" + gomock "go.uber.org/mock/gomock" ) // MockStore is a mock of Store interface. @@ -279,6 +278,45 @@ func (c *MockStorePollCall) DoAndReturn(f func(context.Context, *work.Poll) ([]* return c } +// QueueSizes mocks base method. +func (m *MockStore) QueueSizes(ctx context.Context) ([]work.QueueSize, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "QueueSizes", ctx) + ret0, _ := ret[0].([]work.QueueSize) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueueSizes indicates an expected call of QueueSizes. +func (mr *MockStoreMockRecorder) QueueSizes(ctx any) *MockStoreQueueSizesCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueueSizes", reflect.TypeOf((*MockStore)(nil).QueueSizes), ctx) + return &MockStoreQueueSizesCall{Call: call} +} + +// MockStoreQueueSizesCall wrap *gomock.Call +type MockStoreQueueSizesCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockStoreQueueSizesCall) Return(arg0 []work.QueueSize, arg1 error) *MockStoreQueueSizesCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockStoreQueueSizesCall) Do(f func(context.Context) ([]work.QueueSize, error)) *MockStoreQueueSizesCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockStoreQueueSizesCall) DoAndReturn(f func(context.Context) ([]work.QueueSize, error)) *MockStoreQueueSizesCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // ReapExpiredProcessing mocks base method. func (m *MockStore) ReapExpiredProcessing(ctx context.Context, graceDuration time.Duration) (int, error) { m.ctrl.T.Helper() diff --git a/work/service/test/coordinator_mocks.go b/work/service/test/coordinator_mocks.go index 2fea213a9e..0148efd58c 100644 --- a/work/service/test/coordinator_mocks.go +++ b/work/service/test/coordinator_mocks.go @@ -13,10 +13,9 @@ import ( context "context" reflect "reflect" - gomock "go.uber.org/mock/gomock" - request "github.com/tidepool-org/platform/request" work "github.com/tidepool-org/platform/work" + gomock "go.uber.org/mock/gomock" ) // MockServerSessionTokenProvider is a mock of ServerSessionTokenProvider interface. @@ -184,6 +183,45 @@ func (c *MockWorkClientPollCall) DoAndReturn(f func(context.Context, *work.Poll) return c } +// QueueSizes mocks base method. +func (m *MockWorkClient) QueueSizes(ctx context.Context) ([]work.QueueSize, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "QueueSizes", ctx) + ret0, _ := ret[0].([]work.QueueSize) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueueSizes indicates an expected call of QueueSizes. +func (mr *MockWorkClientMockRecorder) QueueSizes(ctx any) *MockWorkClientQueueSizesCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueueSizes", reflect.TypeOf((*MockWorkClient)(nil).QueueSizes), ctx) + return &MockWorkClientQueueSizesCall{Call: call} +} + +// MockWorkClientQueueSizesCall wrap *gomock.Call +type MockWorkClientQueueSizesCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockWorkClientQueueSizesCall) Return(arg0 []work.QueueSize, arg1 error) *MockWorkClientQueueSizesCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockWorkClientQueueSizesCall) Do(f func(context.Context) ([]work.QueueSize, error)) *MockWorkClientQueueSizesCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockWorkClientQueueSizesCall) DoAndReturn(f func(context.Context) ([]work.QueueSize, error)) *MockWorkClientQueueSizesCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // ReapExpiredProcessing mocks base method. func (m *MockWorkClient) ReapExpiredProcessing(ctx context.Context) (int, error) { m.ctrl.T.Helper() diff --git a/work/store/structured/mongo/mongo.go b/work/store/structured/mongo/mongo.go index 036f3bdfac..28b659a261 100644 --- a/work/store/structured/mongo/mongo.go +++ b/work/store/structured/mongo/mongo.go @@ -357,6 +357,32 @@ func (s *Store) List(ctx context.Context, filter *work.Filter, pagination *page. return documents.AsWork(), nil } +func (s *Store) QueueSizes(ctx context.Context) ([]work.QueueSize, error) { + if ctx == nil { + return nil, errors.New("context is missing") + } + + // The sort seeds the planner with the TypeState index, which covers the rest of the pipeline, + // so the aggregation reads only the index and never fetches a document + pipeline := []bson.M{ + {"$sort": bson.M{"type": 1, "state": 1}}, + {"$group": bson.M{"_id": bson.M{"type": "$type", "state": "$state"}, "count": bson.M{"$sum": 1}}}, + {"$project": bson.M{"_id": 0, "type": "$_id.type", "state": "$_id.state", "count": 1}}, + } + cursor, err := s.Aggregate(ctx, pipeline) + if err != nil { + return nil, errors.Wrap(err, "unable to aggregate work by type and state") + } + defer cursor.Close(ctx) + + var queueSizes []work.QueueSize + if err = cursor.All(ctx, &queueSizes); err != nil { + return nil, errors.Wrap(err, "unable to get all work counts") + } + + return queueSizes, nil +} + func (s *Store) Create(ctx context.Context, create *work.Create) (*work.Work, error) { if ctx == nil { return nil, errors.New("context is missing") diff --git a/work/store/structured/mongo/mongo_test.go b/work/store/structured/mongo/mongo_test.go index 3e971e4c16..f9dbff9284 100644 --- a/work/store/structured/mongo/mongo_test.go +++ b/work/store/structured/mongo/mongo_test.go @@ -240,6 +240,36 @@ var _ = Describe("Mongo", func() { }) }) + Context("QueueSizes", func() { + It("reports nothing with no work present", func() { + Expect(store.QueueSizes(ctx)).To(BeEmpty()) + }) + + It("reports the number of work items by type and state", func() { + secondType := netTest.RandomReverseDomain() + for workType, count := range map[string]int{typ: 3, secondType: 2} { + for range count { + created, err := store.Create(ctx, &work.Create{ + Type: workType, + ProcessingTimeout: processingTimeout, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(created).ToNot(BeNil()) + } + } + + claimed, err := store.Poll(ctx, &work.Poll{TypeQuantities: work.TypeQuantities{typ: 1}}) + Expect(err).ToNot(HaveOccurred()) + Expect(claimed).To(HaveLen(1)) + + Expect(store.QueueSizes(ctx)).To(ConsistOf( + work.QueueSize{Type: typ, State: work.StatePending, Count: 2}, + work.QueueSize{Type: typ, State: work.StateProcessing, Count: 1}, + work.QueueSize{Type: secondType, State: work.StatePending, Count: 2}, + )) + }) + }) + Context("Poll", func() { // These work items intentionally share an identical processing available time and // processing priority so that only the identifier tie breaker in the Poll aggregation From cda6fa93bc2848c7c979a227251e35af9b03c192 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Wed, 26 Aug 2026 12:14:47 +0300 Subject: [PATCH 16/20] Regenerate mocks --- work/service/test/client_mocks.go | 3 ++- work/service/test/coordinator_mocks.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/work/service/test/client_mocks.go b/work/service/test/client_mocks.go index 039e2986dc..cbdfe4d2a6 100644 --- a/work/service/test/client_mocks.go +++ b/work/service/test/client_mocks.go @@ -14,10 +14,11 @@ import ( reflect "reflect" time "time" + gomock "go.uber.org/mock/gomock" + page "github.com/tidepool-org/platform/page" structured "github.com/tidepool-org/platform/store/structured" work "github.com/tidepool-org/platform/work" - gomock "go.uber.org/mock/gomock" ) // MockStore is a mock of Store interface. diff --git a/work/service/test/coordinator_mocks.go b/work/service/test/coordinator_mocks.go index 0148efd58c..734414975b 100644 --- a/work/service/test/coordinator_mocks.go +++ b/work/service/test/coordinator_mocks.go @@ -13,9 +13,10 @@ import ( context "context" reflect "reflect" + gomock "go.uber.org/mock/gomock" + request "github.com/tidepool-org/platform/request" work "github.com/tidepool-org/platform/work" - gomock "go.uber.org/mock/gomock" ) // MockServerSessionTokenProvider is a mock of ServerSessionTokenProvider interface. From f76caa7604177edb06e012585179d94a16edc86e Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Wed, 26 Aug 2026 16:19:50 +0300 Subject: [PATCH 17/20] Update summaries in the clinic service before triggering the EHR sync --- clinics/clinician.go | 2 +- clinics/service.go | 64 ++- clinics/service_test.go | 124 ++++-- clinics/summaries.go | 378 ++++++++++++++++++ clinics/summaries_test.go | 296 ++++++++++++++ clinics/test/clinics.go | 22 +- clinics/test/service_mocks.go | 146 +++++-- data/work/postprocess/processor.go | 82 +++- data/work/postprocess/processor_test.go | 224 ++++++++++- data/work/postprocess/summarizers.go | 69 +++- .../postprocess/test/summarizers_mocks.go | 17 +- data/work/postprocess/work.go | 41 +- data/work/postprocess/work_test.go | 45 ++- ehr/reconcile/planner_test.go | 6 +- ehr/reconcile/runner_test.go | 2 +- ehr/sync/runner_test.go | 2 +- go.mod | 2 +- go.sum | 2 + notifications/work/claims/processor.go | 2 +- prescription/api/v1.go | 2 +- prescription/api/v1_test.go | 8 +- prescription/service/service_test.go | 2 +- summary/types/summary.go | 4 + 23 files changed, 1404 insertions(+), 138 deletions(-) create mode 100644 clinics/summaries.go create mode 100644 clinics/summaries_test.go diff --git a/clinics/clinician.go b/clinics/clinician.go index 3f02c00ca1..acc3147ba3 100644 --- a/clinics/clinician.go +++ b/clinics/clinician.go @@ -6,7 +6,7 @@ import ( api "github.com/tidepool-org/clinic/client" ) -func IsPrescriber(clinician *api.Clinician) bool { +func IsPrescriber(clinician *api.ClinicianV1) bool { if clinician == nil { return false } diff --git a/clinics/service.go b/clinics/service.go index 12a934a303..6c4b2dfc61 100644 --- a/clinics/service.go +++ b/clinics/service.go @@ -23,15 +23,17 @@ const ErrorCodeClinicClientFailure = "clinic-client-failure" var ClientModule = fx.Provide(NewClient) type Client interface { - GetClinic(ctx context.Context, clinicID string) (*clinic.Clinic, error) - GetClinician(ctx context.Context, clinicID, clinicianID string) (*clinic.Clinician, error) - GetEHRSettings(ctx context.Context, clinicId string) (*clinic.EHRSettings, error) - SharePatientAccount(ctx context.Context, clinicID, patientID string) (*clinic.Patient, error) - ListEHREnabledClinics(ctx context.Context) ([]clinic.Clinic, error) + GetClinic(ctx context.Context, clinicID string) (*clinic.ClinicV1, error) + GetClinician(ctx context.Context, clinicID, clinicianID string) (*clinic.ClinicianV1, error) + GetEHRSettings(ctx context.Context, clinicId string) (*clinic.EhrSettingsV1, error) + SharePatientAccount(ctx context.Context, clinicID, patientID string) (*clinic.PatientV1, error) + ListEHREnabledClinics(ctx context.Context) ([]clinic.ClinicV1, error) SyncEHRData(ctx context.Context, clinicID string) error SyncEHRDataForPatient(ctx context.Context, patientID string) error - GetPatients(ctx context.Context, clinicId string, userToken string, params *clinic.ListPatientsParams, injectedParams url.Values) ([]clinic.Patient, error) - GetPatient(ctx context.Context, clinicID, patientID string) (*clinic.Patient, error) + GetPatients(ctx context.Context, clinicId string, userToken string, params *clinic.ListPatientsParams, injectedParams url.Values) ([]clinic.PatientV1, error) + GetPatient(ctx context.Context, clinicID, patientID string) (*clinic.PatientV1, error) + UpdatePatientSummary(ctx context.Context, patientID string, patientSummary *clinic.PatientSummaryV1) error + DeletePatientSummary(ctx context.Context, summaryID string) error } type config struct { @@ -76,7 +78,7 @@ func NewClient(authClient auth.ExternalAccessor) (Client, error) { }, nil } -func (d *defaultClient) GetClinician(ctx context.Context, clinicID, clinicianID string) (*clinic.Clinician, error) { +func (d *defaultClient) GetClinician(ctx context.Context, clinicID, clinicianID string) (*clinic.ClinicianV1, error) { response, err := d.httpClient.GetClinicianWithResponse(ctx, clinic.ClinicId(clinicID), clinic.ClinicianId(clinicianID)) if err != nil { return nil, err @@ -94,7 +96,7 @@ func (d *defaultClient) GetClinician(ctx context.Context, clinicID, clinicianID return response.JSON200, nil } -func (d *defaultClient) GetClinic(ctx context.Context, clinicID string) (*clinic.Clinic, error) { +func (d *defaultClient) GetClinic(ctx context.Context, clinicID string) (*clinic.ClinicV1, error) { response, err := d.httpClient.GetClinicWithResponse(ctx, clinic.ClinicId(clinicID)) if err != nil { return nil, err @@ -112,11 +114,11 @@ func (d *defaultClient) GetClinic(ctx context.Context, clinicID string) (*clinic return response.JSON200, nil } -func (d *defaultClient) ListEHREnabledClinics(ctx context.Context) ([]clinic.Clinic, error) { +func (d *defaultClient) ListEHREnabledClinics(ctx context.Context) ([]clinic.ClinicV1, error) { offset := 0 batchSize := 1000 - clinics := make([]clinic.Clinic, 0) + clinics := make([]clinic.ClinicV1, 0) for { response, err := d.httpClient.ListClinicsWithResponse(ctx, &clinic.ListClinicsParams{ EhrEnabled: pointer.FromBool(true), @@ -148,7 +150,7 @@ func (d *defaultClient) ListEHREnabledClinics(ctx context.Context) ([]clinic.Cli return clinics, nil } -func (d *defaultClient) GetEHRSettings(ctx context.Context, clinicId string) (*clinic.EHRSettings, error) { +func (d *defaultClient) GetEHRSettings(ctx context.Context, clinicId string) (*clinic.EhrSettingsV1, error) { response, err := d.httpClient.GetEHRSettingsWithResponse(ctx, clinicId) if err != nil { return nil, err @@ -163,10 +165,10 @@ func (d *defaultClient) GetEHRSettings(ctx context.Context, clinicId string) (*c return response.JSON200, nil } -func (d *defaultClient) SharePatientAccount(ctx context.Context, clinicID, patientID string) (*clinic.Patient, error) { +func (d *defaultClient) SharePatientAccount(ctx context.Context, clinicID, patientID string) (*clinic.PatientV1, error) { permission := make(map[string]interface{}, 0) body := clinic.CreatePatientFromUserJSONRequestBody{ - Permissions: &clinic.PatientPermissions{ + Permissions: &clinic.PatientPermissionsV1{ Note: &permission, View: &permission, }, @@ -222,7 +224,37 @@ func (d *defaultClient) SyncEHRDataForPatient(ctx context.Context, patientID str return nil } -func (d *defaultClient) GetPatient(ctx context.Context, clinicID, patientID string) (*clinic.Patient, error) { +func (d *defaultClient) UpdatePatientSummary(ctx context.Context, patientID string, patientSummary *clinic.PatientSummaryV1) error { + response, err := d.httpClient.UpdatePatientSummaryWithResponse(ctx, clinic.PatientId(patientID), *patientSummary) + if err != nil { + return err + } + if response.StatusCode() != http.StatusOK && response.StatusCode() != http.StatusNoContent && response.StatusCode() != http.StatusNotFound { + err = errors.Preparedf(ErrorCodeClinicClientFailure, + "Unexpected status code from clinic service", + "unexpected response status code %v from %v", response.StatusCode(), response.HTTPResponse.Request.URL) + err = errors.WithMeta(err, response.HTTPResponse) + return err + } + return nil +} + +func (d *defaultClient) DeletePatientSummary(ctx context.Context, summaryID string) error { + response, err := d.httpClient.DeletePatientSummaryWithResponse(ctx, clinic.SummaryId(summaryID)) + if err != nil { + return err + } + if response.StatusCode() != http.StatusOK && response.StatusCode() != http.StatusNoContent { + err = errors.Preparedf(ErrorCodeClinicClientFailure, + "Unexpected status code from clinic service", + "unexpected response status code %v from %v", response.StatusCode(), response.HTTPResponse.Request.URL) + err = errors.WithMeta(err, response.HTTPResponse) + return err + } + return nil +} + +func (d *defaultClient) GetPatient(ctx context.Context, clinicID, patientID string) (*clinic.PatientV1, error) { response, err := d.httpClient.GetPatientWithResponse(ctx, clinic.ClinicId(clinicID), clinic.PatientId(patientID)) if err != nil { return nil, err @@ -237,7 +269,7 @@ func (d *defaultClient) GetPatient(ctx context.Context, clinicID, patientID stri return response.JSON200, nil } -func (d *defaultClient) GetPatients(ctx context.Context, clinicId string, userToken string, params *clinic.ListPatientsParams, injectedParams url.Values) ([]clinic.Patient, error) { +func (d *defaultClient) GetPatients(ctx context.Context, clinicId string, userToken string, params *clinic.ListPatientsParams, injectedParams url.Values) ([]clinic.PatientV1, error) { response, err := d.httpClient.ListPatientsWithResponse(ctx, clinicId, params, func(ctx context.Context, req *http.Request) error { if len(injectedParams) != 0 { q := req.URL.Query() diff --git a/clinics/service_test.go b/clinics/service_test.go index 68d04d026b..c692fde182 100644 --- a/clinics/service_test.go +++ b/clinics/service_test.go @@ -2,46 +2,59 @@ package clinics_test import ( "context" + "encoding/json" + "io" "net/http" "net/http/httptest" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + clinic "github.com/tidepool-org/clinic/client" + "go.mongodb.org/mongo-driver/bson/primitive" authTest "github.com/tidepool-org/platform/auth/test" "github.com/tidepool-org/platform/clinics" + summaryTest "github.com/tidepool-org/platform/summary/test" userTest "github.com/tidepool-org/platform/user/test" ) var _ = Describe("Client", func() { - Context("SyncEHRDataForPatient", func() { - var server *httptest.Server - var requestPath string - var responseStatusCode int - var client clinics.Client - var patientID string - - BeforeEach(func() { - patientID = userTest.RandomUserID() - requestPath = "" - responseStatusCode = http.StatusAccepted - - server = httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - requestPath = req.URL.Path - res.WriteHeader(responseStatusCode) - })) - GinkgoT().Setenv("TIDEPOOL_CLINIC_CLIENT_ADDRESS", server.URL) + var server *httptest.Server + var requestPath string + var requestBody []byte + var responseStatusCode int + var client clinics.Client + var patientID string - externalAccessor := authTest.NewExternalAccessor() - externalAccessor.ServerSessionTokenOutputs = []authTest.ServerSessionTokenOutput{{Token: authTest.NewSessionToken()}} + BeforeEach(func() { + patientID = userTest.RandomUserID() + requestPath = "" + requestBody = nil + server = httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + requestPath = req.URL.Path var err error - client, err = clinics.NewClient(externalAccessor) + requestBody, err = io.ReadAll(req.Body) Expect(err).ToNot(HaveOccurred()) - }) + res.WriteHeader(responseStatusCode) + })) + GinkgoT().Setenv("TIDEPOOL_CLINIC_CLIENT_ADDRESS", server.URL) - AfterEach(func() { - server.Close() + externalAccessor := authTest.NewExternalAccessor() + externalAccessor.ServerSessionTokenOutputs = []authTest.ServerSessionTokenOutput{{Token: authTest.NewSessionToken()}} + + var err error + client, err = clinics.NewClient(externalAccessor) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + server.Close() + }) + + Context("SyncEHRDataForPatient", func() { + BeforeEach(func() { + responseStatusCode = http.StatusAccepted }) It("requests a synchronization for the patient", func() { @@ -63,4 +76,69 @@ var _ = Describe("Client", func() { Expect(err).To(MatchError(ContainSubstring("unexpected response status code 500"))) }) }) + + Context("UpdatePatientSummary", func() { + var patientSummary *clinic.PatientSummaryV1 + + BeforeEach(func() { + responseStatusCode = http.StatusOK + cgm := summaryTest.RandomCGMSummary(patientID) + cgm.ID = primitive.NewObjectID() + patientSummary = clinics.NewPatientSummary(cgm, nil) + }) + + It("updates the summary of the patient", func() { + Expect(client.UpdatePatientSummary(context.Background(), patientID, patientSummary)).To(Succeed()) + Expect(requestPath).To(Equal("/v1/patients/" + patientID + "/summary")) + + decoded := &clinic.PatientSummaryV1{} + Expect(json.Unmarshal(requestBody, decoded)).To(Succeed()) + Expect(decoded.CgmStats).ToNot(BeNil()) + Expect(decoded.CgmStats.Id).To(Equal(patientSummary.CgmStats.Id)) + Expect(decoded.BgmStats).To(BeNil()) + }) + + // The clinic service reports a user who is not a patient of any clinic as no change. Most + // users are not, so reporting that as a failure would fail the work of nearly every user. + It("returns no error when the user is not a patient of any clinic", func() { + responseStatusCode = http.StatusNoContent + Expect(client.UpdatePatientSummary(context.Background(), patientID, patientSummary)).To(Succeed()) + }) + + It("returns no error when the clinic service reports not found", func() { + responseStatusCode = http.StatusNotFound + Expect(client.UpdatePatientSummary(context.Background(), patientID, patientSummary)).To(Succeed()) + }) + + It("returns an error when the clinic service reports an unexpected status", func() { + responseStatusCode = http.StatusInternalServerError + err := client.UpdatePatientSummary(context.Background(), patientID, patientSummary) + Expect(err).To(MatchError(ContainSubstring("unexpected response status code 500"))) + }) + }) + + Context("DeletePatientSummary", func() { + var summaryID string + + BeforeEach(func() { + responseStatusCode = http.StatusOK + summaryID = primitive.NewObjectID().Hex() + }) + + It("deletes the summary from every patient record holding it", func() { + Expect(client.DeletePatientSummary(context.Background(), summaryID)).To(Succeed()) + Expect(requestPath).To(Equal("/v1/summaries/" + summaryID + "/clinics")) + }) + + It("returns no error when no patient record holds the summary", func() { + responseStatusCode = http.StatusNoContent + Expect(client.DeletePatientSummary(context.Background(), summaryID)).To(Succeed()) + }) + + It("returns an error when the clinic service reports an unexpected status", func() { + responseStatusCode = http.StatusInternalServerError + err := client.DeletePatientSummary(context.Background(), summaryID) + Expect(err).To(MatchError(ContainSubstring("unexpected response status code 500"))) + }) + }) }) diff --git a/clinics/summaries.go b/clinics/summaries.go new file mode 100644 index 0000000000..0694900c8d --- /dev/null +++ b/clinics/summaries.go @@ -0,0 +1,378 @@ +package clinics + +import ( + "strconv" + "strings" + "time" + + clinic "github.com/tidepool-org/clinic/client" + + "github.com/tidepool-org/platform/pointer" + "github.com/tidepool-org/platform/summary/types" +) + +// NewPatientSummary returns the summaries given as the patient summary document the clinic service +// accepts. A type given as nil is omitted from the document, which the clinic service leaves +// untouched, so each type is reported independently. Continuous summaries have no clinic +// representation. +func NewPatientSummary(cgm *types.CGMSummary, bgm *types.BGMSummary) *clinic.PatientSummaryV1 { + patientSummary := &clinic.PatientSummaryV1{} + if cgm != nil { + patientSummary.CgmStats = &clinic.CgmStatsV1{ + Id: pointer.FromString(cgm.ID.Hex()), + Config: exportSummaryConfig(cgm.Config), + Dates: exportSummaryDates(cgm.Dates), + Periods: exportCGMPeriods(cgm.Periods), + } + } + if bgm != nil { + patientSummary.BgmStats = &clinic.BgmStatsV1{ + Id: pointer.FromString(bgm.ID.Hex()), + Config: exportSummaryConfig(bgm.Config), + Dates: exportSummaryDates(bgm.Dates), + Periods: exportBGMPeriods(bgm.Periods), + } + } + return patientSummary +} + +func exportSummaryConfig(config types.Config) clinic.SummaryConfigV1 { + return clinic.SummaryConfigV1{ + SchemaVersion: config.SchemaVersion, + HighGlucoseThreshold: config.HighGlucoseThreshold, + VeryHighGlucoseThreshold: config.VeryHighGlucoseThreshold, + LowGlucoseThreshold: config.LowGlucoseThreshold, + VeryLowGlucoseThreshold: config.VeryLowGlucoseThreshold, + } +} + +func exportSummaryDates(dates types.Dates) clinic.SummaryDatesV1 { + // The reasons are reported as empty rather than absent so that the clinic service replaces any + // previously reported reasons + lastUpdatedReason := dates.LastUpdatedReason + if lastUpdatedReason == nil { + lastUpdatedReason = []string{} + } + outdatedReason := dates.OutdatedReason + if outdatedReason == nil { + outdatedReason = []string{} + } + + firstData := timeOrNil(dates.FirstData) + lastData := timeOrNil(dates.LastData) + lastUploadDate := timeOrNil(dates.LastUploadDate) + + return clinic.SummaryDatesV1{ + LastUpdatedDate: timeOrNil(dates.LastUpdatedDate), + LastUpdatedReason: &lastUpdatedReason, + OutdatedReason: &outdatedReason, + HasLastUploadDate: lastUploadDate != nil, + LastUploadDate: lastUploadDate, + HasFirstData: firstData != nil, + FirstData: firstData, + HasLastData: lastData != nil, + LastData: lastData, + HasOutdatedSince: dates.OutdatedSince != nil, + OutdatedSince: dates.OutdatedSince, + } +} + +func timeOrNil(value time.Time) *time.Time { + if value.IsZero() { + return nil + } + return pointer.FromTime(value) +} + +func exportCGMPeriods(periods *types.CGMPeriods) clinic.CgmPeriodsV1 { + exported := clinic.CgmPeriodsV1{} + if periods == nil { + return exported + } + for name, period := range periods.GlucosePeriods { + if days, ok := periodDays(name); ok && period != nil { + exported[name] = exportCGMPeriod(period, days) + } + } + return exported +} + +func exportBGMPeriods(periods *types.BGMPeriods) clinic.BgmPeriodsV1 { + exported := clinic.BgmPeriodsV1{} + if periods == nil { + return exported + } + for name, period := range periods.GlucosePeriods { + if _, ok := periodDays(name); ok && period != nil { + exported[name] = exportBGMPeriod(period) + } + } + return exported +} + +// periodDays reports the number of days of a period named 1d/7d/14d/30d +func periodDays(name string) (int, bool) { + days, err := strconv.Atoi(strings.TrimSuffix(name, "d")) + if !strings.HasSuffix(name, "d") || err != nil || days <= 0 { + return 0, false + } + return days, true +} + +func exportCGMPeriod(period *types.GlucosePeriod, days int) clinic.CgmPeriodV1 { + delta := period.Delta + if delta == nil { + delta = &types.GlucosePeriod{} + } + + exported := clinic.CgmPeriodV1{ + AverageDailyRecords: pointer.FromFloat64(period.AverageDailyRecords), + AverageDailyRecordsDelta: pointer.FromFloat64(delta.AverageDailyRecords), + DaysWithData: period.DaysWithData, + DaysWithDataDelta: delta.DaysWithData, + HasAverageDailyRecords: period.AverageDailyRecords != 0, + HasTimeCGMUseMinutes: period.Total.Minutes != 0, + HasTimeCGMUseRecords: period.Total.Records != 0, + HasTimeInAnyHighMinutes: period.AnyHigh.Minutes != 0, + HasTimeInAnyHighRecords: period.AnyHigh.Records != 0, + HasTimeInAnyLowMinutes: period.AnyLow.Minutes != 0, + HasTimeInAnyLowRecords: period.AnyLow.Records != 0, + HasTimeInExtremeHighMinutes: period.ExtremeHigh.Minutes != 0, + HasTimeInExtremeHighRecords: period.ExtremeHigh.Records != 0, + HasTimeInHighMinutes: period.High.Minutes != 0, + HasTimeInHighRecords: period.High.Records != 0, + HasTimeInLowMinutes: period.Low.Minutes != 0, + HasTimeInLowRecords: period.Low.Records != 0, + HasTimeInTargetMinutes: period.Target.Minutes != 0, + HasTimeInTargetRecords: period.Target.Records != 0, + HasTimeInVeryHighMinutes: period.VeryHigh.Minutes != 0, + HasTimeInVeryHighRecords: period.VeryHigh.Records != 0, + HasTimeInVeryLowMinutes: period.VeryLow.Minutes != 0, + HasTimeInVeryLowRecords: period.VeryLow.Records != 0, + HasTotalRecords: period.Total.Records != 0, + HoursWithData: period.HoursWithData, + HoursWithDataDelta: delta.HoursWithData, + TimeCGMUseMinutes: pointer.FromInt(period.Total.Minutes), + TimeCGMUseMinutesDelta: pointer.FromInt(delta.Total.Minutes), + TimeCGMUseRecords: pointer.FromInt(period.Total.Records), + TimeCGMUseRecordsDelta: pointer.FromInt(delta.Total.Records), + TimeInAnyHighMinutes: pointer.FromInt(period.AnyHigh.Minutes), + TimeInAnyHighMinutesDelta: pointer.FromInt(delta.AnyHigh.Minutes), + TimeInAnyHighRecords: pointer.FromInt(period.AnyHigh.Records), + TimeInAnyHighRecordsDelta: pointer.FromInt(delta.AnyHigh.Records), + TimeInAnyLowMinutes: pointer.FromInt(period.AnyLow.Minutes), + TimeInAnyLowMinutesDelta: pointer.FromInt(delta.AnyLow.Minutes), + TimeInAnyLowRecords: pointer.FromInt(period.AnyLow.Records), + TimeInAnyLowRecordsDelta: pointer.FromInt(delta.AnyLow.Records), + TimeInExtremeHighMinutes: pointer.FromInt(period.ExtremeHigh.Minutes), + TimeInExtremeHighMinutesDelta: pointer.FromInt(delta.ExtremeHigh.Minutes), + TimeInExtremeHighRecords: pointer.FromInt(period.ExtremeHigh.Records), + TimeInExtremeHighRecordsDelta: pointer.FromInt(delta.ExtremeHigh.Records), + TimeInHighMinutes: pointer.FromInt(period.High.Minutes), + TimeInHighMinutesDelta: pointer.FromInt(delta.High.Minutes), + TimeInHighRecords: pointer.FromInt(period.High.Records), + TimeInHighRecordsDelta: pointer.FromInt(delta.High.Records), + TimeInLowMinutes: pointer.FromInt(period.Low.Minutes), + TimeInLowMinutesDelta: pointer.FromInt(delta.Low.Minutes), + TimeInLowRecords: pointer.FromInt(period.Low.Records), + TimeInLowRecordsDelta: pointer.FromInt(delta.Low.Records), + TimeInTargetMinutes: pointer.FromInt(period.Target.Minutes), + TimeInTargetMinutesDelta: pointer.FromInt(delta.Target.Minutes), + TimeInTargetRecords: pointer.FromInt(period.Target.Records), + TimeInTargetRecordsDelta: pointer.FromInt(delta.Target.Records), + TimeInVeryHighMinutes: pointer.FromInt(period.VeryHigh.Minutes), + TimeInVeryHighMinutesDelta: pointer.FromInt(delta.VeryHigh.Minutes), + TimeInVeryHighRecords: pointer.FromInt(period.VeryHigh.Records), + TimeInVeryHighRecordsDelta: pointer.FromInt(delta.VeryHigh.Records), + TimeInVeryLowMinutes: pointer.FromInt(period.VeryLow.Minutes), + TimeInVeryLowMinutesDelta: pointer.FromInt(delta.VeryLow.Minutes), + TimeInVeryLowRecords: pointer.FromInt(period.VeryLow.Records), + TimeInVeryLowRecordsDelta: pointer.FromInt(delta.VeryLow.Records), + TotalRecords: pointer.FromInt(period.Total.Records), + TotalRecordsDelta: pointer.FromInt(delta.Total.Records), + Min: period.Min, + MinDelta: delta.Min, + Max: period.Max, + MaxDelta: delta.Max, + } + + // reconstruct some previous period values for comparison later + previousTotalRecords := period.Total.Records - delta.Total.Records + previousCGMUsePercent := period.Total.Percent - delta.Total.Percent + previousCGMUseMinutes := period.Total.Minutes - delta.Total.Minutes + + // The following provides concessions to allow patient list sorting and filtering according to + // certain eligibility requirements, notably: + // - TIR percent only is visible in the frontend if >1d of data, or 70% cgm use on single day metrics + // - GMI requires >70% cgm use + // - All percentages should be nil if 0 TotalRecords, as they would have been before schema v5 + // - All delta percentages should be nil if both periods do not fulfill their respective requirements above + if period.Total.Records != 0 { + exported.HasTimeCGMUsePercent = true + exported.HasAverageGlucoseMmol = true + exported.TimeCGMUsePercent = pointer.FromFloat64(period.Total.Percent) + exported.AverageGlucoseMmol = pointer.FromFloat64(period.AverageGlucose) + exported.StandardDeviation = period.StandardDeviation + exported.CoefficientOfVariation = period.CoefficientOfVariation + + if previousTotalRecords != 0 { + exported.TimeCGMUsePercentDelta = pointer.FromFloat64(delta.Total.Percent) + exported.AverageGlucoseMmolDelta = pointer.FromFloat64(delta.AverageGlucose) + exported.StandardDeviationDelta = delta.StandardDeviation + exported.CoefficientOfVariationDelta = delta.CoefficientOfVariation + } + + // if we are storing under 1d, apply 70% rule to TimeIn* + // if we are storing over 1d, check for 24h cgm use + if (days <= 1 && period.Total.Percent > 0.7) || (days > 1 && period.Total.Minutes > 1440) { + exported.HasTimeInTargetPercent = true + exported.TimeInTargetPercent = pointer.FromFloat64(period.Target.Percent) + + exported.HasTimeInLowPercent = true + exported.TimeInLowPercent = pointer.FromFloat64(period.Low.Percent) + + exported.HasTimeInVeryLowPercent = true + exported.TimeInVeryLowPercent = pointer.FromFloat64(period.VeryLow.Percent) + + exported.HasTimeInAnyLowPercent = true + exported.TimeInAnyLowPercent = pointer.FromFloat64(period.AnyLow.Percent) + + exported.HasTimeInHighPercent = true + exported.TimeInHighPercent = pointer.FromFloat64(period.High.Percent) + + exported.HasTimeInVeryHighPercent = true + exported.TimeInVeryHighPercent = pointer.FromFloat64(period.VeryHigh.Percent) + + exported.HasTimeInExtremeHighPercent = true + exported.TimeInExtremeHighPercent = pointer.FromFloat64(period.ExtremeHigh.Percent) + + exported.HasTimeInAnyHighPercent = true + exported.TimeInAnyHighPercent = pointer.FromFloat64(period.AnyHigh.Percent) + + // add deltas if delta period fulfills requirements as well + if (days <= 1 && previousCGMUsePercent > 0.7) || (days > 1 && previousCGMUseMinutes > 1440) { + exported.TimeInTargetPercentDelta = pointer.FromFloat64(delta.Target.Percent) + exported.TimeInLowPercentDelta = pointer.FromFloat64(delta.Low.Percent) + exported.TimeInVeryLowPercentDelta = pointer.FromFloat64(delta.VeryLow.Percent) + exported.TimeInAnyLowPercentDelta = pointer.FromFloat64(delta.AnyLow.Percent) + exported.TimeInHighPercentDelta = pointer.FromFloat64(delta.High.Percent) + exported.TimeInVeryHighPercentDelta = pointer.FromFloat64(delta.VeryHigh.Percent) + exported.TimeInExtremeHighPercentDelta = pointer.FromFloat64(delta.ExtremeHigh.Percent) + exported.TimeInAnyHighPercentDelta = pointer.FromFloat64(delta.AnyHigh.Percent) + } + } + + // GMI should only be present if CGM use % is >70% so that they are filtered to the bottom on GMI queries. + if period.Total.Percent > 0.7 { + exported.HasGlucoseManagementIndicator = true + exported.GlucoseManagementIndicator = pointer.FromFloat64(period.GlucoseManagementIndicator) + + // add deltas if delta period fulfills requirements as well + if previousCGMUsePercent > 0.7 { + exported.GlucoseManagementIndicatorDelta = pointer.FromFloat64(delta.GlucoseManagementIndicator) + } + } + } + + return exported +} + +func exportBGMPeriod(period *types.GlucosePeriod) clinic.BgmPeriodV1 { + delta := period.Delta + if delta == nil { + delta = &types.GlucosePeriod{} + } + + exported := clinic.BgmPeriodV1{ + AverageDailyRecords: pointer.FromFloat64(period.AverageDailyRecords), + AverageDailyRecordsDelta: pointer.FromFloat64(delta.AverageDailyRecords), + DaysWithData: period.DaysWithData, + DaysWithDataDelta: delta.DaysWithData, + HasAverageDailyRecords: period.AverageDailyRecords != 0, + HasTimeInAnyHighRecords: period.AnyHigh.Records != 0, + HasTimeInAnyLowRecords: period.AnyLow.Records != 0, + HasTimeInExtremeHighRecords: period.ExtremeHigh.Records != 0, + HasTimeInHighRecords: period.High.Records != 0, + HasTimeInLowRecords: period.Low.Records != 0, + HasTimeInTargetRecords: period.Target.Records != 0, + HasTimeInVeryHighRecords: period.VeryHigh.Records != 0, + HasTimeInVeryLowRecords: period.VeryLow.Records != 0, + HasTotalRecords: period.Total.Records != 0, + TimeInAnyHighRecords: pointer.FromInt(period.AnyHigh.Records), + TimeInAnyHighRecordsDelta: pointer.FromInt(delta.AnyHigh.Records), + TimeInAnyLowRecords: pointer.FromInt(period.AnyLow.Records), + TimeInAnyLowRecordsDelta: pointer.FromInt(delta.AnyLow.Records), + TimeInExtremeHighRecords: pointer.FromInt(period.ExtremeHigh.Records), + TimeInExtremeHighRecordsDelta: pointer.FromInt(delta.ExtremeHigh.Records), + TimeInHighRecords: pointer.FromInt(period.High.Records), + TimeInHighRecordsDelta: pointer.FromInt(delta.High.Records), + TimeInLowRecords: pointer.FromInt(period.Low.Records), + TimeInLowRecordsDelta: pointer.FromInt(delta.Low.Records), + TimeInTargetRecords: pointer.FromInt(period.Target.Records), + TimeInTargetRecordsDelta: pointer.FromInt(delta.Target.Records), + TimeInVeryHighRecords: pointer.FromInt(period.VeryHigh.Records), + TimeInVeryHighRecordsDelta: pointer.FromInt(delta.VeryHigh.Records), + TimeInVeryLowRecords: pointer.FromInt(period.VeryLow.Records), + TimeInVeryLowRecordsDelta: pointer.FromInt(delta.VeryLow.Records), + TotalRecords: pointer.FromInt(period.Total.Records), + TotalRecordsDelta: pointer.FromInt(delta.Total.Records), + Min: period.Min, + MinDelta: delta.Min, + Max: period.Max, + MaxDelta: delta.Max, + } + + // reconstruct previous period total records for comparison later + previousTotalRecords := period.Total.Records - delta.Total.Records + + // percentages should stay nil unless there is records, but schema >5 removed all optional pointers + if period.Total.Records != 0 { + exported.HasTimeInTargetPercent = true + exported.TimeInTargetPercent = pointer.FromFloat64(period.Target.Percent) + + exported.HasTimeInLowPercent = true + exported.TimeInLowPercent = pointer.FromFloat64(period.Low.Percent) + + exported.HasTimeInVeryLowPercent = true + exported.TimeInVeryLowPercent = pointer.FromFloat64(period.VeryLow.Percent) + + exported.HasTimeInAnyLowPercent = true + exported.TimeInAnyLowPercent = pointer.FromFloat64(period.AnyLow.Percent) + + exported.HasTimeInHighPercent = true + exported.TimeInHighPercent = pointer.FromFloat64(period.High.Percent) + + exported.HasTimeInVeryHighPercent = true + exported.TimeInVeryHighPercent = pointer.FromFloat64(period.VeryHigh.Percent) + + exported.HasTimeInExtremeHighPercent = true + exported.TimeInExtremeHighPercent = pointer.FromFloat64(period.ExtremeHigh.Percent) + + exported.HasTimeInAnyHighPercent = true + exported.TimeInAnyHighPercent = pointer.FromFloat64(period.AnyHigh.Percent) + + exported.HasAverageGlucoseMmol = true + exported.AverageGlucoseMmol = pointer.FromFloat64(period.AverageGlucose) + + if previousTotalRecords != 0 { + exported.TimeInTargetPercentDelta = pointer.FromFloat64(delta.Target.Percent) + exported.TimeInLowPercentDelta = pointer.FromFloat64(delta.Low.Percent) + exported.TimeInVeryLowPercentDelta = pointer.FromFloat64(delta.VeryLow.Percent) + exported.TimeInAnyLowPercentDelta = pointer.FromFloat64(delta.AnyLow.Percent) + exported.TimeInHighPercentDelta = pointer.FromFloat64(delta.High.Percent) + exported.TimeInVeryHighPercentDelta = pointer.FromFloat64(delta.VeryHigh.Percent) + exported.TimeInExtremeHighPercentDelta = pointer.FromFloat64(delta.ExtremeHigh.Percent) + exported.TimeInAnyHighPercentDelta = pointer.FromFloat64(delta.AnyHigh.Percent) + exported.AverageGlucoseMmolDelta = pointer.FromFloat64(delta.AverageGlucose) + } + } + + if period.Total.Records >= 30 && period.DaysWithData >= 7 { + exported.StandardDeviation = pointer.FromFloat64(period.StandardDeviation) + exported.StandardDeviationDelta = pointer.FromFloat64(delta.StandardDeviation) + exported.CoefficientOfVariation = pointer.FromFloat64(period.CoefficientOfVariation) + exported.CoefficientOfVariationDelta = pointer.FromFloat64(delta.CoefficientOfVariation) + } + + return exported +} diff --git a/clinics/summaries_test.go b/clinics/summaries_test.go new file mode 100644 index 0000000000..21e05d149f --- /dev/null +++ b/clinics/summaries_test.go @@ -0,0 +1,296 @@ +package clinics_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/tidepool-org/platform/clinics" + summaryTest "github.com/tidepool-org/platform/summary/test" + "github.com/tidepool-org/platform/summary/types" + userTest "github.com/tidepool-org/platform/user/test" +) + +var _ = Describe("NewPatientSummary", func() { + var userID string + var cgm *types.CGMSummary + var bgm *types.BGMSummary + + BeforeEach(func() { + userID = userTest.RandomUserID() + cgm = summaryTest.RandomCGMSummary(userID) + cgm.ID = primitive.NewObjectID() + bgm = summaryTest.RandomBGMSummary(userID) + bgm.ID = primitive.NewObjectID() + }) + + It("omits the stats of a summary not given", func() { + patientSummary := clinics.NewPatientSummary(cgm, nil) + Expect(patientSummary.CgmStats).ToNot(BeNil()) + Expect(patientSummary.BgmStats).To(BeNil()) + + patientSummary = clinics.NewPatientSummary(nil, bgm) + Expect(patientSummary.CgmStats).To(BeNil()) + Expect(patientSummary.BgmStats).ToNot(BeNil()) + }) + + It("reports the id, config, and dates of each summary", func() { + patientSummary := clinics.NewPatientSummary(cgm, bgm) + + Expect(patientSummary.CgmStats.Id).To(PointTo(Equal(cgm.ID.Hex()))) + Expect(patientSummary.BgmStats.Id).To(PointTo(Equal(bgm.ID.Hex()))) + + Expect(patientSummary.CgmStats.Config.SchemaVersion).To(Equal(cgm.Config.SchemaVersion)) + Expect(patientSummary.CgmStats.Config.HighGlucoseThreshold).To(Equal(cgm.Config.HighGlucoseThreshold)) + Expect(patientSummary.CgmStats.Config.VeryHighGlucoseThreshold).To(Equal(cgm.Config.VeryHighGlucoseThreshold)) + Expect(patientSummary.CgmStats.Config.LowGlucoseThreshold).To(Equal(cgm.Config.LowGlucoseThreshold)) + Expect(patientSummary.CgmStats.Config.VeryLowGlucoseThreshold).To(Equal(cgm.Config.VeryLowGlucoseThreshold)) + + dates := patientSummary.CgmStats.Dates + Expect(dates.LastUpdatedDate).To(PointTo(BeTemporally("==", cgm.Dates.LastUpdatedDate))) + Expect(dates.FirstData).To(PointTo(BeTemporally("==", cgm.Dates.FirstData))) + Expect(dates.HasFirstData).To(BeTrue()) + Expect(dates.LastData).To(PointTo(BeTemporally("==", cgm.Dates.LastData))) + Expect(dates.HasLastData).To(BeTrue()) + Expect(dates.LastUploadDate).To(PointTo(BeTemporally("==", cgm.Dates.LastUploadDate))) + Expect(dates.HasLastUploadDate).To(BeTrue()) + Expect(dates.OutdatedSince).To(Equal(cgm.Dates.OutdatedSince)) + Expect(dates.HasOutdatedSince).To(BeTrue()) + Expect(dates.LastUpdatedReason).To(PointTo(Equal(cgm.Dates.LastUpdatedReason))) + Expect(dates.OutdatedReason).To(PointTo(Equal(cgm.Dates.OutdatedReason))) + }) + + It("omits the dates the summary does not have", func() { + cgm.Dates = types.Dates{} + + dates := clinics.NewPatientSummary(cgm, nil).CgmStats.Dates + Expect(dates.LastUpdatedDate).To(BeNil()) + Expect(dates.FirstData).To(BeNil()) + Expect(dates.HasFirstData).To(BeFalse()) + Expect(dates.LastData).To(BeNil()) + Expect(dates.HasLastData).To(BeFalse()) + Expect(dates.LastUploadDate).To(BeNil()) + Expect(dates.HasLastUploadDate).To(BeFalse()) + Expect(dates.OutdatedSince).To(BeNil()) + Expect(dates.HasOutdatedSince).To(BeFalse()) + }) + + // The clinic service replaces only what the report carries, so reasons no longer held must be + // reported as empty rather than omitted + It("reports absent reasons as empty", func() { + cgm.Dates.LastUpdatedReason = nil + cgm.Dates.OutdatedReason = nil + + dates := clinics.NewPatientSummary(cgm, nil).CgmStats.Dates + Expect(dates.LastUpdatedReason).To(PointTo(BeEmpty())) + Expect(dates.OutdatedReason).To(PointTo(BeEmpty())) + }) + + It("drops a period not named as a number of days", func() { + cgm.Periods.GlucosePeriods["invalid"] = summaryTest.RandomGlucosePeriod(true) + + periods := clinics.NewPatientSummary(cgm, nil).CgmStats.Periods + Expect(periods).To(HaveLen(4)) + Expect(periods).ToNot(HaveKey("invalid")) + Expect(periods).To(HaveKey("30d")) + }) + + Context("with a cgm period", func() { + var period *types.GlucosePeriod + + BeforeEach(func() { + period = &types.GlucosePeriod{ + GlucoseRanges: types.GlucoseRanges{ + Total: types.Range{Records: 100, Minutes: 500, Percent: 0.8}, + Target: types.Range{Records: 60, Minutes: 300, Percent: 0.6}, + Low: types.Range{Records: 10, Minutes: 50, Percent: 0.1}, + High: types.Range{Records: 30, Minutes: 150, Percent: 0.3}, + }, + MinMax: types.MinMax{Min: 2.2, Max: 19.1}, + HoursWithData: 20, + DaysWithData: 1, + AverageGlucose: 7.5, + GlucoseManagementIndicator: 6.9, + StandardDeviation: 1.2, + CoefficientOfVariation: 0.3, + AverageDailyRecords: 288, + Delta: &types.GlucosePeriod{ + GlucoseRanges: types.GlucoseRanges{ + Total: types.Range{Records: 40, Minutes: 200, Percent: 0.05}, + Target: types.Range{Records: 20, Minutes: 100, Percent: 0.02}, + }, + AverageGlucose: 0.5, + }, + } + cgm.Periods = &types.CGMPeriods{GlucosePeriods: types.GlucosePeriods{"1d": period}} + }) + + It("reports the ranges and their deltas", func() { + exported := clinics.NewPatientSummary(cgm, nil).CgmStats.Periods["1d"] + Expect(exported.TotalRecords).To(PointTo(Equal(100))) + Expect(exported.TotalRecordsDelta).To(PointTo(Equal(40))) + Expect(exported.TimeCGMUseMinutes).To(PointTo(Equal(500))) + Expect(exported.TimeCGMUsePercent).To(PointTo(Equal(0.8))) + Expect(exported.TimeInTargetRecords).To(PointTo(Equal(60))) + Expect(exported.TimeInTargetRecordsDelta).To(PointTo(Equal(20))) + Expect(exported.TimeInTargetMinutes).To(PointTo(Equal(300))) + Expect(exported.TimeInLowRecords).To(PointTo(Equal(10))) + Expect(exported.TimeInHighRecords).To(PointTo(Equal(30))) + Expect(exported.TimeInVeryLowRecords).To(PointTo(Equal(0))) + Expect(exported.HasTotalRecords).To(BeTrue()) + Expect(exported.HasTimeInTargetRecords).To(BeTrue()) + Expect(exported.HasTimeInVeryLowRecords).To(BeFalse()) + Expect(exported.AverageGlucoseMmol).To(PointTo(Equal(7.5))) + Expect(exported.AverageGlucoseMmolDelta).To(PointTo(Equal(0.5))) + Expect(exported.StandardDeviation).To(Equal(1.2)) + Expect(exported.CoefficientOfVariation).To(Equal(0.3)) + Expect(exported.AverageDailyRecords).To(PointTo(Equal(288.0))) + Expect(exported.DaysWithData).To(Equal(1)) + Expect(exported.HoursWithData).To(Equal(20)) + Expect(exported.Min).To(Equal(2.2)) + Expect(exported.Max).To(Equal(19.1)) + }) + + It("reports the time-in-range percentages of a single day only above 70 percent use", func() { + exported := clinics.NewPatientSummary(cgm, nil).CgmStats.Periods["1d"] + Expect(exported.HasTimeInTargetPercent).To(BeTrue()) + Expect(exported.TimeInTargetPercent).To(PointTo(Equal(0.6))) + + period.Total.Percent = 0.5 + exported = clinics.NewPatientSummary(cgm, nil).CgmStats.Periods["1d"] + Expect(exported.HasTimeInTargetPercent).To(BeFalse()) + Expect(exported.TimeInTargetPercent).To(BeNil()) + }) + + It("reports the time-in-range percentages of a longer period only above a day of use", func() { + cgm.Periods = &types.CGMPeriods{GlucosePeriods: types.GlucosePeriods{"7d": period}} + period.Total.Percent = 0.2 + + period.Total.Minutes = 1441 + exported := clinics.NewPatientSummary(cgm, nil).CgmStats.Periods["7d"] + Expect(exported.HasTimeInTargetPercent).To(BeTrue()) + + period.Total.Minutes = 1440 + exported = clinics.NewPatientSummary(cgm, nil).CgmStats.Periods["7d"] + Expect(exported.HasTimeInTargetPercent).To(BeFalse()) + }) + + It("reports the glucose management indicator only above 70 percent use", func() { + exported := clinics.NewPatientSummary(cgm, nil).CgmStats.Periods["1d"] + Expect(exported.HasGlucoseManagementIndicator).To(BeTrue()) + Expect(exported.GlucoseManagementIndicator).To(PointTo(Equal(6.9))) + + period.Total.Percent = 0.7 + exported = clinics.NewPatientSummary(cgm, nil).CgmStats.Periods["1d"] + Expect(exported.HasGlucoseManagementIndicator).To(BeFalse()) + Expect(exported.GlucoseManagementIndicator).To(BeNil()) + }) + + // The delta reconstructs the previous period, which must fulfill the same requirements + // before its percentages are compared against + It("reports the percentage deltas only when the previous period also qualifies", func() { + // previous use is 0.8 - 0.05 = 0.75 + exported := clinics.NewPatientSummary(cgm, nil).CgmStats.Periods["1d"] + Expect(exported.TimeInTargetPercentDelta).To(PointTo(Equal(0.02))) + Expect(exported.GlucoseManagementIndicatorDelta).ToNot(BeNil()) + + // previous use is 0.8 - 0.2 = 0.6 + period.Delta.Total.Percent = 0.2 + exported = clinics.NewPatientSummary(cgm, nil).CgmStats.Periods["1d"] + Expect(exported.TimeInTargetPercentDelta).To(BeNil()) + Expect(exported.GlucoseManagementIndicatorDelta).To(BeNil()) + }) + + It("reports no percentages without records", func() { + period.Total = types.Range{} + exported := clinics.NewPatientSummary(cgm, nil).CgmStats.Periods["1d"] + Expect(exported.HasTimeCGMUsePercent).To(BeFalse()) + Expect(exported.TimeCGMUsePercent).To(BeNil()) + Expect(exported.HasAverageGlucoseMmol).To(BeFalse()) + Expect(exported.AverageGlucoseMmol).To(BeNil()) + Expect(exported.HasTimeInTargetPercent).To(BeFalse()) + Expect(exported.HasGlucoseManagementIndicator).To(BeFalse()) + }) + + It("reports a period without a delta as changed from nothing", func() { + period.Delta = nil + exported := clinics.NewPatientSummary(cgm, nil).CgmStats.Periods["1d"] + Expect(exported.TotalRecordsDelta).To(PointTo(Equal(0))) + Expect(exported.TimeInTargetRecordsDelta).To(PointTo(Equal(0))) + }) + }) + + Context("with a bgm period", func() { + var period *types.GlucosePeriod + + BeforeEach(func() { + period = &types.GlucosePeriod{ + GlucoseRanges: types.GlucoseRanges{ + Total: types.Range{Records: 30, Percent: 1}, + Target: types.Range{Records: 20, Percent: 0.67}, + }, + DaysWithData: 7, + AverageGlucose: 8.1, + StandardDeviation: 1.4, + CoefficientOfVariation: 0.4, + Delta: &types.GlucosePeriod{ + GlucoseRanges: types.GlucoseRanges{ + Total: types.Range{Records: 10, Percent: 0.1}, + Target: types.Range{Records: 5, Percent: 0.07}, + }, + StandardDeviation: 0.2, + CoefficientOfVariation: 0.1, + }, + } + bgm.Periods = &types.BGMPeriods{GlucosePeriods: types.GlucosePeriods{"30d": period}} + }) + + It("reports the records, percentages, and their deltas", func() { + exported := clinics.NewPatientSummary(nil, bgm).BgmStats.Periods["30d"] + Expect(exported.TotalRecords).To(PointTo(Equal(30))) + Expect(exported.TotalRecordsDelta).To(PointTo(Equal(10))) + Expect(exported.TimeInTargetRecords).To(PointTo(Equal(20))) + Expect(exported.TimeInTargetRecordsDelta).To(PointTo(Equal(5))) + Expect(exported.HasTimeInTargetPercent).To(BeTrue()) + Expect(exported.TimeInTargetPercent).To(PointTo(Equal(0.67))) + Expect(exported.TimeInTargetPercentDelta).To(PointTo(Equal(0.07))) + Expect(exported.AverageGlucoseMmol).To(PointTo(Equal(8.1))) + }) + + It("reports no percentages without records", func() { + period.Total.Records = 0 + exported := clinics.NewPatientSummary(nil, bgm).BgmStats.Periods["30d"] + Expect(exported.HasTimeInTargetPercent).To(BeFalse()) + Expect(exported.TimeInTargetPercent).To(BeNil()) + Expect(exported.AverageGlucoseMmol).To(BeNil()) + }) + + It("reports the deviation measures only with at least 30 records over at least 7 days", func() { + exported := clinics.NewPatientSummary(nil, bgm).BgmStats.Periods["30d"] + Expect(exported.StandardDeviation).To(PointTo(Equal(1.4))) + Expect(exported.StandardDeviationDelta).To(PointTo(Equal(0.2))) + Expect(exported.CoefficientOfVariation).To(PointTo(Equal(0.4))) + Expect(exported.CoefficientOfVariationDelta).To(PointTo(Equal(0.1))) + + period.Total.Records = 29 + exported = clinics.NewPatientSummary(nil, bgm).BgmStats.Periods["30d"] + Expect(exported.StandardDeviation).To(BeNil()) + Expect(exported.CoefficientOfVariation).To(BeNil()) + + period.Total.Records = 30 + period.DaysWithData = 6 + exported = clinics.NewPatientSummary(nil, bgm).BgmStats.Periods["30d"] + Expect(exported.StandardDeviation).To(BeNil()) + }) + }) + + It("round-trips a zero-valued outdated since", func() { + cgm.Dates.OutdatedSince = &time.Time{} + dates := clinics.NewPatientSummary(cgm, nil).CgmStats.Dates + Expect(dates.OutdatedSince).To(PointTo(BeTemporally("==", time.Time{}))) + Expect(dates.HasOutdatedSince).To(BeTrue()) + }) +}) diff --git a/clinics/test/clinics.go b/clinics/test/clinics.go index 225f2cae55..8aa0b094ef 100644 --- a/clinics/test/clinics.go +++ b/clinics/test/clinics.go @@ -9,19 +9,19 @@ import ( "github.com/tidepool-org/platform/test" ) -func NewRandomClinic() api.Clinic { - return api.Clinic{ +func NewRandomClinic() api.ClinicV1 { + return api.ClinicV1{ Address: pointer.FromAny(faker.Address().StreetAddress()), CanMigrate: pointer.FromAny(test.RandomBool()), City: pointer.FromAny(faker.Address().City()), - ClinicType: pointer.FromAny(test.RandomChoice([]api.ClinicClinicType{api.HealthcareSystem, api.VeterinaryClinic, api.Other})), + ClinicType: pointer.FromAny(test.RandomChoice([]api.ClinicV1ClinicType{api.ClinicV1ClinicTypeHealthcareSystem, api.ClinicV1ClinicTypeVeterinaryClinic, api.ClinicV1ClinicTypeOther})), Country: pointer.FromAny(faker.Address().Country()), CreatedTime: pointer.FromAny(test.RandomTime()), Id: pointer.FromAny(primitive.NewObjectIDFromTimestamp(test.RandomTime()).Hex()), Name: faker.Company().Name(), - PhoneNumbers: pointer.FromAny([]api.PhoneNumber{{Number: faker.PhoneNumber().PhoneNumber()}}), + PhoneNumbers: pointer.FromAny([]api.PhoneNumberV1{{Number: faker.PhoneNumber().PhoneNumber()}}), PostalCode: pointer.FromAny(faker.Address().ZipCode()), - PreferredBgUnits: test.RandomChoice([]api.ClinicPreferredBgUnits{api.MgdL, api.MmolL}), + PreferredBgUnits: test.RandomChoice([]api.ClinicV1PreferredBgUnits{api.ClinicV1PreferredBgUnitsMgdL, api.ClinicV1PreferredBgUnitsMmolL}), ShareCode: pointer.FromAny(faker.RandomString(15)), State: pointer.FromAny(faker.Address().State()), Tier: pointer.FromAny(test.RandomChoice([]string{"tier1000", "tier2000"})), @@ -31,26 +31,26 @@ func NewRandomClinic() api.Clinic { } } -func NewRandomEHRSettings() *api.EHRSettings { - return &api.EHRSettings{ - DestinationIds: &api.EHRDestinationIds{ +func NewRandomEHRSettings() *api.EhrSettingsV1 { + return &api.EhrSettingsV1{ + DestinationIds: &api.EhrDestinationsV1{ Flowsheet: faker.RandomString(16), Notes: faker.RandomString(16), Results: faker.RandomString(16), }, Enabled: true, MrnIdType: "MRN", - ProcedureCodes: api.EHRProcedureCodes{ + ProcedureCodes: api.EhrProceduresV1{ CreateAccount: pointer.FromAny(faker.RandomString(5)), CreateAccountAndEnableReports: pointer.FromAny(faker.RandomString(5)), DisableSummaryReports: pointer.FromAny(faker.RandomString(5)), EnableSummaryReports: pointer.FromAny(faker.RandomString(5)), }, Provider: "redox", - ScheduledReports: api.ScheduledReports{ + ScheduledReports: api.ScheduledReportsV1{ Cadence: api.N14d, OnUploadEnabled: true, - OnUploadNoteEventType: pointer.FromAny(api.ScheduledReportsOnUploadNoteEventTypeNew), + OnUploadNoteEventType: pointer.FromAny(api.ScheduledReportsV1OnUploadNoteEventTypeNew), }, SourceId: faker.RandomString(16), } diff --git a/clinics/test/service_mocks.go b/clinics/test/service_mocks.go index fbb49eebc6..c7dbc4375b 100644 --- a/clinics/test/service_mocks.go +++ b/clinics/test/service_mocks.go @@ -42,11 +42,49 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { return m.recorder } +// DeletePatientSummary mocks base method. +func (m *MockClient) DeletePatientSummary(ctx context.Context, summaryID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeletePatientSummary", ctx, summaryID) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeletePatientSummary indicates an expected call of DeletePatientSummary. +func (mr *MockClientMockRecorder) DeletePatientSummary(ctx, summaryID any) *MockClientDeletePatientSummaryCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeletePatientSummary", reflect.TypeOf((*MockClient)(nil).DeletePatientSummary), ctx, summaryID) + return &MockClientDeletePatientSummaryCall{Call: call} +} + +// MockClientDeletePatientSummaryCall wrap *gomock.Call +type MockClientDeletePatientSummaryCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockClientDeletePatientSummaryCall) Return(arg0 error) *MockClientDeletePatientSummaryCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockClientDeletePatientSummaryCall) Do(f func(context.Context, string) error) *MockClientDeletePatientSummaryCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockClientDeletePatientSummaryCall) DoAndReturn(f func(context.Context, string) error) *MockClientDeletePatientSummaryCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // GetClinic mocks base method. -func (m *MockClient) GetClinic(ctx context.Context, clinicID string) (*client.Clinic, error) { +func (m *MockClient) GetClinic(ctx context.Context, clinicID string) (*client.ClinicV1, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetClinic", ctx, clinicID) - ret0, _ := ret[0].(*client.Clinic) + ret0, _ := ret[0].(*client.ClinicV1) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -64,28 +102,28 @@ type MockClientGetClinicCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockClientGetClinicCall) Return(arg0 *client.Clinic, arg1 error) *MockClientGetClinicCall { +func (c *MockClientGetClinicCall) Return(arg0 *client.ClinicV1, arg1 error) *MockClientGetClinicCall { c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockClientGetClinicCall) Do(f func(context.Context, string) (*client.Clinic, error)) *MockClientGetClinicCall { +func (c *MockClientGetClinicCall) Do(f func(context.Context, string) (*client.ClinicV1, error)) *MockClientGetClinicCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientGetClinicCall) DoAndReturn(f func(context.Context, string) (*client.Clinic, error)) *MockClientGetClinicCall { +func (c *MockClientGetClinicCall) DoAndReturn(f func(context.Context, string) (*client.ClinicV1, error)) *MockClientGetClinicCall { c.Call = c.Call.DoAndReturn(f) return c } // GetClinician mocks base method. -func (m *MockClient) GetClinician(ctx context.Context, clinicID, clinicianID string) (*client.Clinician, error) { +func (m *MockClient) GetClinician(ctx context.Context, clinicID, clinicianID string) (*client.ClinicianV1, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetClinician", ctx, clinicID, clinicianID) - ret0, _ := ret[0].(*client.Clinician) + ret0, _ := ret[0].(*client.ClinicianV1) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -103,28 +141,28 @@ type MockClientGetClinicianCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockClientGetClinicianCall) Return(arg0 *client.Clinician, arg1 error) *MockClientGetClinicianCall { +func (c *MockClientGetClinicianCall) Return(arg0 *client.ClinicianV1, arg1 error) *MockClientGetClinicianCall { c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockClientGetClinicianCall) Do(f func(context.Context, string, string) (*client.Clinician, error)) *MockClientGetClinicianCall { +func (c *MockClientGetClinicianCall) Do(f func(context.Context, string, string) (*client.ClinicianV1, error)) *MockClientGetClinicianCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientGetClinicianCall) DoAndReturn(f func(context.Context, string, string) (*client.Clinician, error)) *MockClientGetClinicianCall { +func (c *MockClientGetClinicianCall) DoAndReturn(f func(context.Context, string, string) (*client.ClinicianV1, error)) *MockClientGetClinicianCall { c.Call = c.Call.DoAndReturn(f) return c } // GetEHRSettings mocks base method. -func (m *MockClient) GetEHRSettings(ctx context.Context, clinicId string) (*client.EHRSettings, error) { +func (m *MockClient) GetEHRSettings(ctx context.Context, clinicId string) (*client.EhrSettingsV1, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetEHRSettings", ctx, clinicId) - ret0, _ := ret[0].(*client.EHRSettings) + ret0, _ := ret[0].(*client.EhrSettingsV1) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -142,28 +180,28 @@ type MockClientGetEHRSettingsCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockClientGetEHRSettingsCall) Return(arg0 *client.EHRSettings, arg1 error) *MockClientGetEHRSettingsCall { +func (c *MockClientGetEHRSettingsCall) Return(arg0 *client.EhrSettingsV1, arg1 error) *MockClientGetEHRSettingsCall { c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockClientGetEHRSettingsCall) Do(f func(context.Context, string) (*client.EHRSettings, error)) *MockClientGetEHRSettingsCall { +func (c *MockClientGetEHRSettingsCall) Do(f func(context.Context, string) (*client.EhrSettingsV1, error)) *MockClientGetEHRSettingsCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientGetEHRSettingsCall) DoAndReturn(f func(context.Context, string) (*client.EHRSettings, error)) *MockClientGetEHRSettingsCall { +func (c *MockClientGetEHRSettingsCall) DoAndReturn(f func(context.Context, string) (*client.EhrSettingsV1, error)) *MockClientGetEHRSettingsCall { c.Call = c.Call.DoAndReturn(f) return c } // GetPatient mocks base method. -func (m *MockClient) GetPatient(ctx context.Context, clinicID, patientID string) (*client.Patient, error) { +func (m *MockClient) GetPatient(ctx context.Context, clinicID, patientID string) (*client.PatientV1, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetPatient", ctx, clinicID, patientID) - ret0, _ := ret[0].(*client.Patient) + ret0, _ := ret[0].(*client.PatientV1) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -181,28 +219,28 @@ type MockClientGetPatientCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockClientGetPatientCall) Return(arg0 *client.Patient, arg1 error) *MockClientGetPatientCall { +func (c *MockClientGetPatientCall) Return(arg0 *client.PatientV1, arg1 error) *MockClientGetPatientCall { c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockClientGetPatientCall) Do(f func(context.Context, string, string) (*client.Patient, error)) *MockClientGetPatientCall { +func (c *MockClientGetPatientCall) Do(f func(context.Context, string, string) (*client.PatientV1, error)) *MockClientGetPatientCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientGetPatientCall) DoAndReturn(f func(context.Context, string, string) (*client.Patient, error)) *MockClientGetPatientCall { +func (c *MockClientGetPatientCall) DoAndReturn(f func(context.Context, string, string) (*client.PatientV1, error)) *MockClientGetPatientCall { c.Call = c.Call.DoAndReturn(f) return c } // GetPatients mocks base method. -func (m *MockClient) GetPatients(ctx context.Context, clinicId, userToken string, params *client.ListPatientsParams, injectedParams url.Values) ([]client.Patient, error) { +func (m *MockClient) GetPatients(ctx context.Context, clinicId, userToken string, params *client.ListPatientsParams, injectedParams url.Values) ([]client.PatientV1, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetPatients", ctx, clinicId, userToken, params, injectedParams) - ret0, _ := ret[0].([]client.Patient) + ret0, _ := ret[0].([]client.PatientV1) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -220,28 +258,28 @@ type MockClientGetPatientsCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockClientGetPatientsCall) Return(arg0 []client.Patient, arg1 error) *MockClientGetPatientsCall { +func (c *MockClientGetPatientsCall) Return(arg0 []client.PatientV1, arg1 error) *MockClientGetPatientsCall { c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockClientGetPatientsCall) Do(f func(context.Context, string, string, *client.ListPatientsParams, url.Values) ([]client.Patient, error)) *MockClientGetPatientsCall { +func (c *MockClientGetPatientsCall) Do(f func(context.Context, string, string, *client.ListPatientsParams, url.Values) ([]client.PatientV1, error)) *MockClientGetPatientsCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientGetPatientsCall) DoAndReturn(f func(context.Context, string, string, *client.ListPatientsParams, url.Values) ([]client.Patient, error)) *MockClientGetPatientsCall { +func (c *MockClientGetPatientsCall) DoAndReturn(f func(context.Context, string, string, *client.ListPatientsParams, url.Values) ([]client.PatientV1, error)) *MockClientGetPatientsCall { c.Call = c.Call.DoAndReturn(f) return c } // ListEHREnabledClinics mocks base method. -func (m *MockClient) ListEHREnabledClinics(ctx context.Context) ([]client.Clinic, error) { +func (m *MockClient) ListEHREnabledClinics(ctx context.Context) ([]client.ClinicV1, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListEHREnabledClinics", ctx) - ret0, _ := ret[0].([]client.Clinic) + ret0, _ := ret[0].([]client.ClinicV1) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -259,28 +297,28 @@ type MockClientListEHREnabledClinicsCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockClientListEHREnabledClinicsCall) Return(arg0 []client.Clinic, arg1 error) *MockClientListEHREnabledClinicsCall { +func (c *MockClientListEHREnabledClinicsCall) Return(arg0 []client.ClinicV1, arg1 error) *MockClientListEHREnabledClinicsCall { c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockClientListEHREnabledClinicsCall) Do(f func(context.Context) ([]client.Clinic, error)) *MockClientListEHREnabledClinicsCall { +func (c *MockClientListEHREnabledClinicsCall) Do(f func(context.Context) ([]client.ClinicV1, error)) *MockClientListEHREnabledClinicsCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientListEHREnabledClinicsCall) DoAndReturn(f func(context.Context) ([]client.Clinic, error)) *MockClientListEHREnabledClinicsCall { +func (c *MockClientListEHREnabledClinicsCall) DoAndReturn(f func(context.Context) ([]client.ClinicV1, error)) *MockClientListEHREnabledClinicsCall { c.Call = c.Call.DoAndReturn(f) return c } // SharePatientAccount mocks base method. -func (m *MockClient) SharePatientAccount(ctx context.Context, clinicID, patientID string) (*client.Patient, error) { +func (m *MockClient) SharePatientAccount(ctx context.Context, clinicID, patientID string) (*client.PatientV1, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SharePatientAccount", ctx, clinicID, patientID) - ret0, _ := ret[0].(*client.Patient) + ret0, _ := ret[0].(*client.PatientV1) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -298,19 +336,19 @@ type MockClientSharePatientAccountCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockClientSharePatientAccountCall) Return(arg0 *client.Patient, arg1 error) *MockClientSharePatientAccountCall { +func (c *MockClientSharePatientAccountCall) Return(arg0 *client.PatientV1, arg1 error) *MockClientSharePatientAccountCall { c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockClientSharePatientAccountCall) Do(f func(context.Context, string, string) (*client.Patient, error)) *MockClientSharePatientAccountCall { +func (c *MockClientSharePatientAccountCall) Do(f func(context.Context, string, string) (*client.PatientV1, error)) *MockClientSharePatientAccountCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientSharePatientAccountCall) DoAndReturn(f func(context.Context, string, string) (*client.Patient, error)) *MockClientSharePatientAccountCall { +func (c *MockClientSharePatientAccountCall) DoAndReturn(f func(context.Context, string, string) (*client.PatientV1, error)) *MockClientSharePatientAccountCall { c.Call = c.Call.DoAndReturn(f) return c } @@ -390,3 +428,41 @@ func (c *MockClientSyncEHRDataForPatientCall) DoAndReturn(f func(context.Context c.Call = c.Call.DoAndReturn(f) return c } + +// UpdatePatientSummary mocks base method. +func (m *MockClient) UpdatePatientSummary(ctx context.Context, patientID string, patientSummary *client.PatientSummaryV1) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdatePatientSummary", ctx, patientID, patientSummary) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdatePatientSummary indicates an expected call of UpdatePatientSummary. +func (mr *MockClientMockRecorder) UpdatePatientSummary(ctx, patientID, patientSummary any) *MockClientUpdatePatientSummaryCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdatePatientSummary", reflect.TypeOf((*MockClient)(nil).UpdatePatientSummary), ctx, patientID, patientSummary) + return &MockClientUpdatePatientSummaryCall{Call: call} +} + +// MockClientUpdatePatientSummaryCall wrap *gomock.Call +type MockClientUpdatePatientSummaryCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockClientUpdatePatientSummaryCall) Return(arg0 error) *MockClientUpdatePatientSummaryCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockClientUpdatePatientSummaryCall) Do(f func(context.Context, string, *client.PatientSummaryV1) error) *MockClientUpdatePatientSummaryCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockClientUpdatePatientSummaryCall) DoAndReturn(f func(context.Context, string, *client.PatientSummaryV1) error) *MockClientUpdatePatientSummaryCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/data/work/postprocess/processor.go b/data/work/postprocess/processor.go index 7fb0ba21e1..86118f9ff3 100644 --- a/data/work/postprocess/processor.go +++ b/data/work/postprocess/processor.go @@ -2,14 +2,17 @@ package postprocess import ( "context" + "slices" "time" + "github.com/tidepool-org/platform/clinics" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/metadata" "github.com/tidepool-org/platform/page" "github.com/tidepool-org/platform/pointer" "github.com/tidepool-org/platform/request" + summaryTypes "github.com/tidepool-org/platform/summary/types" userWork "github.com/tidepool-org/platform/user/work" "github.com/tidepool-org/platform/work" workBase "github.com/tidepool-org/platform/work/base" @@ -23,7 +26,8 @@ type Processor struct { Summarizers ClinicsClient - pendingBuilder *deferredPendingBuilder + pendingBuilder *deferredPendingBuilder + summariesUpdate SummariesUpdate } func NewProcessor(dependencies Dependencies) (*Processor, error) { @@ -65,6 +69,7 @@ func (p *Processor) Process(ctx context.Context, wrk *work.Work, processingUpdat p.FetchUserFromWorkMetadata, p.absorbPending, p.updateSummaries, + p.updateClinicSummaries, p.triggerElectronicHealthRecordSync, ).Process(p.Delete) } @@ -103,9 +108,14 @@ func (p *Processor) absorbPending() *work.ProcessResult { if err == nil && workMetadata == nil { err = errors.New("metadata is missing") } + // The work must be scoped to the same user. This check should never fail under normal circumstances + // given we are filtering by the group id when retrieving pending work items + if err == nil { + err = validateIdentity(wrk.GroupID, wrk.SerialID, workMetadata) + } // The pending work item will fail when it's picked up if err != nil { - log.LoggerFromContext(p.Context()).WithError(err).WithField("id", wrk.ID).Warn("work pending for the user has invalid metadata") + log.LoggerFromContext(p.Context()).WithError(err).WithField("id", wrk.ID).Warn("work pending for the user is invalid") continue } @@ -153,18 +163,73 @@ func (p *Processor) absorbPending() *work.ProcessResult { } func (p *Processor) updateSummaries() *work.ProcessResult { - if err := p.UpdateSummaries(p.Context(), *p.User().UserID); err != nil { + var err error + p.summariesUpdate, err = p.UpdateSummaries(p.Context(), *p.User().UserID) + + // The changes made are recorded in the metadata before they are synced to the clinic service, + // so that a failure between the two retries the update + changed := p.Metadata().recordSummariesUpdate(p.summariesUpdate) + if err != nil { return p.Failing(err) } + if changed { + if result := p.ProcessingUpdate(); result != nil { + return result + } + log.LoggerFromContext(p.Context()).WithFields(log.Fields{ + "reasons": p.Metadata().Reasons, + "updated": p.summariesUpdate.UpdatedTypes, + "deleted": p.summariesUpdate.Deleted, + }).Info("updated user summaries") + } + + return nil +} + +func (p *Processor) updateClinicSummaries() *work.ProcessResult { + workMetadata := p.Metadata() + if len(workMetadata.PendingSummaryUpdates) == 0 && len(workMetadata.PendingSummaryDeletes) == 0 { + return nil + } + + for _, summaryID := range workMetadata.PendingSummaryDeletes { + if err := p.ClinicsClient.DeletePatientSummary(p.Context(), summaryID); err != nil { + return p.Failing(errors.Wrap(err, "unable to delete patient summary")) + } + } + if len(workMetadata.PendingSummaryDeletes) > 0 { + log.LoggerFromContext(p.Context()).WithFields(log.Fields{ + "deleted": workMetadata.PendingSummaryDeletes, + }).Debug("deleted clinic service summaries") + } + + var cgm *summaryTypes.CGMSummary + var bgm *summaryTypes.BGMSummary + if slices.Contains(workMetadata.PendingSummaryUpdates, summaryTypes.SummaryTypeCGM) { + cgm = p.summariesUpdate.CGM + } + if slices.Contains(workMetadata.PendingSummaryUpdates, summaryTypes.SummaryTypeBGM) { + bgm = p.summariesUpdate.BGM + } + if cgm != nil || bgm != nil { + if err := p.ClinicsClient.UpdatePatientSummary(p.Context(), *p.User().UserID, clinics.NewPatientSummary(cgm, bgm)); err != nil { + return p.Failing(errors.Wrap(err, "unable to update patient summary")) + } + log.LoggerFromContext(p.Context()).WithFields(log.Fields{ + "updated": workMetadata.PendingSummaryUpdates, + }).Debug("updated clinic service summaries") + } - log.LoggerFromContext(p.Context()).WithField("reasons", p.Metadata().Reasons).Info("calculated the summaries of the user") + log.LoggerFromContext(p.Context()).WithFields(log.Fields{ + "updated": workMetadata.PendingSummaryUpdates, + "deleted": workMetadata.PendingSummaryDeletes, + }).Info("synced user summaries with the clinic service") return nil } -// triggerElectronicHealthRecordSync reports the data of the user to any electronic health record it is -// shared with, after the summaries it reports are calculated. It is requested at least once per change -// reported, as a request repeated reports the same data again rather than reporting it twice. +// triggerElectronicHealthRecordSync reports the data of the user to any electronic health record it +// is shared with. Repeating the request reports the same data again, not twice, so retries are safe. func (p *Processor) triggerElectronicHealthRecordSync() *work.ProcessResult { if !TriggersEHRSync(p.Metadata().Reasons) { return nil @@ -179,8 +244,7 @@ func (p *Processor) triggerElectronicHealthRecordSync() *work.ProcessResult { return nil } -// deferredPendingBuilder defers work until a time decided while processing, rather than by a duration -// fixed when the processor is created +// deferredPendingBuilder defers work until a time decided during processing type deferredPendingBuilder struct { availableTime time.Time } diff --git a/data/work/postprocess/processor_test.go b/data/work/postprocess/processor_test.go index 8106620234..2b07205879 100644 --- a/data/work/postprocess/processor_test.go +++ b/data/work/postprocess/processor_test.go @@ -7,6 +7,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" + clinic "github.com/tidepool-org/clinic/client" + "go.mongodb.org/mongo-driver/bson/primitive" "go.uber.org/mock/gomock" clinicsTest "github.com/tidepool-org/platform/clinics/test" @@ -19,6 +21,8 @@ import ( "github.com/tidepool-org/platform/page" "github.com/tidepool-org/platform/pointer" "github.com/tidepool-org/platform/request" + summaryTest "github.com/tidepool-org/platform/summary/test" + summaryTypes "github.com/tidepool-org/platform/summary/types" "github.com/tidepool-org/platform/user" userTest "github.com/tidepool-org/platform/user/test" userWork "github.com/tidepool-org/platform/user/work" @@ -59,6 +63,29 @@ var _ = Describe("Processor", func() { } } + newSummariesUpdate := func(updatedTypes ...string) dataWorkPostprocess.SummariesUpdate { + update := dataWorkPostprocess.SummariesUpdate{ + CGM: summaryTest.RandomCGMSummary(userID), + BGM: summaryTest.RandomBGMSummary(userID), + UpdatedTypes: updatedTypes, + } + update.CGM.ID = primitive.NewObjectID() + update.BGM.ID = primitive.NewObjectID() + return update + } + + // The processing update persists the metadata of the request, so it is echoed back the way the + // store does + expectProcessingUpdate := func() { + processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, update work.ProcessingUpdate) (*work.Work, error) { + updated := *wrk + updated.Metadata = update.Metadata + updated.Revision = wrk.Revision + 1 + return &updated, nil + }) + } + // Nothing else is pending for the user, which is the ordinary case. The work being processed is // not reported, as only work that is pending is requested. expectListNone := func() { @@ -107,7 +134,7 @@ var _ = Describe("Processor", func() { It("calculates the summaries and deletes the work", func() { expectListNone() - summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(dataWorkPostprocess.SummariesUpdate{}, nil) Expect(process().Result).To(Equal(work.ResultDelete)) }) @@ -116,7 +143,7 @@ var _ = Describe("Processor", func() { func(reasons []string, expectSync bool) { wrk = newWork(work.StateProcessing, reasons, time.Now().Add(-time.Minute)) expectListNone() - summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(dataWorkPostprocess.SummariesUpdate{}, nil) if expectSync { clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) } @@ -136,7 +163,7 @@ var _ = Describe("Processor", func() { wrk = newWork(work.StateProcessing, []string{dataWorkPostprocess.ReasonUploadCompleted}, time.Now().Add(-time.Minute)) expectListNone() gomock.InOrder( - summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil), + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(dataWorkPostprocess.SummariesUpdate{}, nil), clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil), ) @@ -211,15 +238,185 @@ var _ = Describe("Processor", func() { }), Entry("calculating the summaries", func() { expectListNone() - summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(errorsTest.RandomError()) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(dataWorkPostprocess.SummariesUpdate{}, errorsTest.RandomError()) }), Entry("requesting a synchronization", func() { expectListNone() - summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(dataWorkPostprocess.SummariesUpdate{}, nil) clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(errorsTest.RandomError()) }), + Entry("recording the changed summaries", func() { + expectListNone() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(newSummariesUpdate(summaryTypes.SummaryTypeCGM), nil) + processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).Return(nil, errorsTest.RandomError()) + }), + Entry("reporting a summary update to the clinic service", func() { + expectListNone() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(newSummariesUpdate(summaryTypes.SummaryTypeCGM), nil) + expectProcessingUpdate() + clinicsClient.EXPECT().UpdatePatientSummary(gomock.Any(), userID, gomock.Any()).Return(errorsTest.RandomError()) + }), + Entry("reporting a summary deletion to the clinic service", func() { + expectListNone() + update := dataWorkPostprocess.SummariesUpdate{Deleted: map[string]string{summaryTypes.SummaryTypeCGM: primitive.NewObjectID().Hex()}} + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(update, nil) + expectProcessingUpdate() + clinicsClient.EXPECT().DeletePatientSummary(gomock.Any(), gomock.Any()).Return(errorsTest.RandomError()) + }), ) + // The clinic service stores a copy of the summaries of its patients, and the synchronization + // reports from that copy, so a change must reach it first, and a summary that did not change must + // not be reported at all + Context("when the calculation changes the summaries", func() { + var summariesUpdate dataWorkPostprocess.SummariesUpdate + + BeforeEach(func() { + summariesUpdate = newSummariesUpdate(summaryTypes.SummaryTypeCGM, summaryTypes.SummaryTypeBGM) + wrk = newWork(work.StateProcessing, []string{dataWorkPostprocess.ReasonUploadCompleted}, time.Now().Add(-time.Minute)) + }) + + // The changes are recorded in the metadata before they are reported, so that a failure + // between the two reports them again rather than not at all + It("records them, reports them to the clinic service, and requests a synchronization, in that order", func() { + expectListNone() + gomock.InOrder( + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(summariesUpdate, nil), + processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, update work.ProcessingUpdate) (*work.Work, error) { + Expect(update.Metadata).To(HaveKeyWithValue("pendingSummaryUpdates", + ConsistOf(summaryTypes.SummaryTypeCGM, summaryTypes.SummaryTypeBGM))) + updated := *wrk + updated.Metadata = update.Metadata + updated.Revision = wrk.Revision + 1 + return &updated, nil + }), + clinicsClient.EXPECT().UpdatePatientSummary(gomock.Any(), userID, gomock.Any()).DoAndReturn( + func(_ context.Context, _ string, patientSummary *clinic.PatientSummaryV1) error { + Expect(patientSummary.CgmStats).ToNot(BeNil()) + Expect(patientSummary.CgmStats.Id).To(PointTo(Equal(summariesUpdate.CGM.ID.Hex()))) + Expect(patientSummary.BgmStats).ToNot(BeNil()) + Expect(patientSummary.BgmStats.Id).To(PointTo(Equal(summariesUpdate.BGM.ID.Hex()))) + return nil + }), + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil), + ) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }) + + It("reports only the summaries that changed", func() { + summariesUpdate = newSummariesUpdate(summaryTypes.SummaryTypeCGM) + expectListNone() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(summariesUpdate, nil) + expectProcessingUpdate() + clinicsClient.EXPECT().UpdatePatientSummary(gomock.Any(), userID, gomock.Any()).DoAndReturn( + func(_ context.Context, _ string, patientSummary *clinic.PatientSummaryV1) error { + Expect(patientSummary.CgmStats).ToNot(BeNil()) + Expect(patientSummary.BgmStats).To(BeNil()) + return nil + }) + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }) + + // The summaries returned without a change recorded were recalculated without change, or not + // at all, and reporting them would store them again for every no-op work of the user + It("reports nothing when the calculation reports the summaries did not change", func() { + summariesUpdate = newSummariesUpdate() + expectListNone() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(summariesUpdate, nil) + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }) + + It("reports a deletion, before any update", func() { + deletedSummaryID := primitive.NewObjectID().Hex() + summariesUpdate = newSummariesUpdate(summaryTypes.SummaryTypeBGM) + summariesUpdate.CGM = nil + summariesUpdate.Deleted = map[string]string{summaryTypes.SummaryTypeCGM: deletedSummaryID} + expectListNone() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(summariesUpdate, nil) + processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, update work.ProcessingUpdate) (*work.Work, error) { + Expect(update.Metadata).To(HaveKeyWithValue("pendingSummaryDeletes", ConsistOf(deletedSummaryID))) + updated := *wrk + updated.Metadata = update.Metadata + updated.Revision = wrk.Revision + 1 + return &updated, nil + }) + gomock.InOrder( + clinicsClient.EXPECT().DeletePatientSummary(gomock.Any(), deletedSummaryID).Return(nil), + clinicsClient.EXPECT().UpdatePatientSummary(gomock.Any(), userID, gomock.Any()).DoAndReturn( + func(_ context.Context, _ string, patientSummary *clinic.PatientSummaryV1) error { + Expect(patientSummary.CgmStats).To(BeNil()) + Expect(patientSummary.BgmStats).ToNot(BeNil()) + return nil + }), + ) + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }) + + // A change recorded by an earlier attempt was calculated but not yet reported; the retried + // calculation reports no further change, so the record alone drives the report + It("reports the changes recorded by an earlier attempt even though this calculation reports none", func() { + deletedSummaryID := primitive.NewObjectID().Hex() + encoded, err := metadata.Encode(&dataWorkPostprocess.Metadata{ + Metadata: userWork.Metadata{UserID: pointer.FromString(userID)}, + Reasons: []string{dataWorkPostprocess.ReasonUploadCompleted}, + PendingSummaryUpdates: []string{summaryTypes.SummaryTypeCGM}, + PendingSummaryDeletes: []string{deletedSummaryID}, + }) + Expect(err).ToNot(HaveOccurred()) + wrk.Metadata = encoded + + summariesUpdate = newSummariesUpdate() + expectListNone() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(summariesUpdate, nil) + clinicsClient.EXPECT().DeletePatientSummary(gomock.Any(), deletedSummaryID).Return(nil) + clinicsClient.EXPECT().UpdatePatientSummary(gomock.Any(), userID, gomock.Any()).DoAndReturn( + func(_ context.Context, _ string, patientSummary *clinic.PatientSummaryV1) error { + Expect(patientSummary.CgmStats).ToNot(BeNil()) + Expect(patientSummary.CgmStats.Id).To(PointTo(Equal(summariesUpdate.CGM.ID.Hex()))) + Expect(patientSummary.BgmStats).To(BeNil()) + return nil + }) + clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) + + Expect(process().Result).To(Equal(work.ResultDelete)) + }) + + // The changes ride along on the failing update, so the retry still knows a report is owed + It("keeps them recorded when reporting them fails", func() { + expectListNone() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(summariesUpdate, nil) + expectProcessingUpdate() + clinicsClient.EXPECT().UpdatePatientSummary(gomock.Any(), userID, gomock.Any()).Return(errorsTest.RandomError()) + + result := process() + Expect(result.Result).To(Equal(work.ResultFailing)) + Expect(result.FailingUpdate.Metadata).To(HaveKeyWithValue("pendingSummaryUpdates", + ConsistOf(summaryTypes.SummaryTypeCGM, summaryTypes.SummaryTypeBGM))) + }) + + // A change calculated before the failure is recorded on the failing update, so the retry + // reports it even though its own calculation reports no further change + It("records a change calculated before the calculation of another summary fails", func() { + summariesUpdate = newSummariesUpdate(summaryTypes.SummaryTypeCGM) + expectListNone() + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(summariesUpdate, errorsTest.RandomError()) + + result := process() + Expect(result.Result).To(Equal(work.ResultFailing)) + Expect(result.FailingUpdate.Metadata).To(HaveKeyWithValue("pendingSummaryUpdates", + ConsistOf(summaryTypes.SummaryTypeCGM))) + }) + }) + Context("with work also pending for the user", func() { var sibling *work.Work @@ -251,7 +448,7 @@ var _ = Describe("Processor", func() { It("reports its reasons, deletes it, and processes once", func() { expectListWithSibling() expectProcessingUpdateThenDelete() - summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(dataWorkPostprocess.SummariesUpdate{}, nil) clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) Expect(process().Result).To(Equal(work.ResultDelete)) @@ -263,7 +460,7 @@ var _ = Describe("Processor", func() { func(mutate func()) { mutate() expectListWithSibling() - summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(dataWorkPostprocess.SummariesUpdate{}, nil) Expect(process().Result).To(Equal(work.ResultDelete)) }, @@ -273,6 +470,11 @@ var _ = Describe("Processor", func() { Entry("its metadata is missing", func() { sibling.Metadata = nil }), + // A mislabeled row listed by group belongs to the user its own metadata names; absorbing + // it would merge and destroy another user's change + Entry("it is not scoped to the user its metadata reports", func() { + sibling.Metadata["userId"] = userTest.RandomUserID() + }), ) It("absorbs the others when one of them is invalid", func() { @@ -280,7 +482,7 @@ var _ = Describe("Processor", func() { invalid.Metadata = nil workClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*work.Work{invalid, sibling}, nil) expectProcessingUpdateThenDelete() - summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(dataWorkPostprocess.SummariesUpdate{}, nil) clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) Expect(process().Result).To(Equal(work.ResultDelete)) @@ -319,7 +521,7 @@ var _ = Describe("Processor", func() { return &updated, nil }) workClient.EXPECT().Delete(gomock.Any(), sibling.ID, &request.Condition{Revision: pointer.FromInt(sibling.Revision)}).Return(sibling, nil) - summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(dataWorkPostprocess.SummariesUpdate{}, nil) clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) Expect(process().Result).To(Equal(work.ResultDelete)) @@ -335,7 +537,7 @@ var _ = Describe("Processor", func() { return &updated, nil }) workClient.EXPECT().Delete(gomock.Any(), sibling.ID, &request.Condition{Revision: pointer.FromInt(sibling.Revision)}).Return(sibling, nil) - summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(dataWorkPostprocess.SummariesUpdate{}, nil) clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) Expect(process().Result).To(Equal(work.ResultDelete)) @@ -357,7 +559,7 @@ var _ = Describe("Processor", func() { expectListWithSibling() processingUpdater.EXPECT().ProcessingUpdate(gomock.Any(), gomock.Any()).Return(wrk, nil) workClient.EXPECT().Delete(gomock.Any(), sibling.ID, &request.Condition{Revision: pointer.FromInt(sibling.Revision)}).Return(nil, nil) - summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(nil) + summarizers.EXPECT().UpdateSummaries(gomock.Any(), userID).Return(dataWorkPostprocess.SummariesUpdate{}, nil) clinicsClient.EXPECT().SyncEHRDataForPatient(gomock.Any(), userID).Return(nil) Expect(process().Result).To(Equal(work.ResultDelete)) diff --git a/data/work/postprocess/summarizers.go b/data/work/postprocess/summarizers.go index 48cbcce073..8cad7bdcce 100644 --- a/data/work/postprocess/summarizers.go +++ b/data/work/postprocess/summarizers.go @@ -11,7 +11,18 @@ import ( //go:generate mockgen -source=summarizers.go -destination=test/summarizers_mocks.go -package=test -typed type Summarizers interface { - UpdateSummaries(ctx context.Context, userID string) error + UpdateSummaries(ctx context.Context, userID string) (SummariesUpdate, error) +} + +type SummariesUpdate struct { + CGM *summaryTypes.CGMSummary + BGM *summaryTypes.BGMSummary + + // UpdatedTypes reports each summary type the update created or recalculated + UpdatedTypes []string + + // Deleted reports the id of each summary the update deleted, by summary type + Deleted map[string]string } type summarizers struct { @@ -25,15 +36,55 @@ func NewSummarizers(registry *summary.SummarizerRegistry) (Summarizers, error) { return &summarizers{registry: registry}, nil } -func (s *summarizers) UpdateSummaries(ctx context.Context, userID string) error { - if _, err := summary.GetSummarizer[*summaryTypes.CGMPeriods, *summaryTypes.GlucoseBucket](s.registry).UpdateSummary(ctx, userID); err != nil { - return errors.Wrapf(err, "unable to update %s summary", summaryTypes.SummaryTypeCGM) +// UpdateSummaries returns the update made so far even when it reports an error, so that a change +// calculated before the error is recorded rather than lost to the retry. +func (s *summarizers) UpdateSummaries(ctx context.Context, userID string) (SummariesUpdate, error) { + update := SummariesUpdate{} + var err error + if update.CGM, err = updateSummary(ctx, summary.GetSummarizer[*summaryTypes.CGMPeriods, *summaryTypes.GlucoseBucket](s.registry), userID, &update); err != nil { + return update, err + } + if update.BGM, err = updateSummary(ctx, summary.GetSummarizer[*summaryTypes.BGMPeriods, *summaryTypes.GlucoseBucket](s.registry), userID, &update); err != nil { + return update, err } - if _, err := summary.GetSummarizer[*summaryTypes.BGMPeriods, *summaryTypes.GlucoseBucket](s.registry).UpdateSummary(ctx, userID); err != nil { - return errors.Wrapf(err, "unable to update %s summary", summaryTypes.SummaryTypeBGM) + if _, err = summary.GetSummarizer[*summaryTypes.ContinuousPeriods, *summaryTypes.ContinuousBucket](s.registry).UpdateSummary(ctx, userID); err != nil { + return update, errors.Wrapf(err, "unable to update %s summary", summaryTypes.SummaryTypeContinuous) + } + return update, nil +} + +// updateSummary calculates the summary of the user and records the change made in the given update +func updateSummary[PP summaryTypes.PeriodsPt[P, PB, B], PB summaryTypes.BucketDataPt[B], P summaryTypes.Periods, B summaryTypes.BucketData](ctx context.Context, summarizer summary.Summarizer[PP, PB, P, B], userID string, update *SummariesUpdate) (*summaryTypes.Summary[PP, PB, P, B], error) { + summaryType := summaryTypes.GetType[PP, PB]() + + before, err := summarizer.GetSummary(ctx, userID) + if err != nil { + return nil, errors.Wrapf(err, "unable to get %s summary", summaryType) } - if _, err := summary.GetSummarizer[*summaryTypes.ContinuousPeriods, *summaryTypes.ContinuousBucket](s.registry).UpdateSummary(ctx, userID); err != nil { - return errors.Wrapf(err, "unable to update %s summary", summaryTypes.SummaryTypeContinuous) + after, err := summarizer.UpdateSummary(ctx, userID) + if err != nil { + return nil, errors.Wrapf(err, "unable to update %s summary", summaryType) + } + + if after == nil { + if before != nil { + update.recordDeleted(summaryType, before.ID.Hex()) + } + return nil, nil + } + if before == nil || !before.Dates.LastUpdatedDate.Equal(after.Dates.LastUpdatedDate) { + update.recordUpdated(summaryType) + } + return after, nil +} + +func (u *SummariesUpdate) recordUpdated(summaryType string) { + u.UpdatedTypes = append(u.UpdatedTypes, summaryType) +} + +func (u *SummariesUpdate) recordDeleted(summaryType string, summaryID string) { + if u.Deleted == nil { + u.Deleted = map[string]string{} } - return nil + u.Deleted[summaryType] = summaryID } diff --git a/data/work/postprocess/test/summarizers_mocks.go b/data/work/postprocess/test/summarizers_mocks.go index 15c03b58ec..2ab7f90add 100644 --- a/data/work/postprocess/test/summarizers_mocks.go +++ b/data/work/postprocess/test/summarizers_mocks.go @@ -14,6 +14,8 @@ import ( reflect "reflect" gomock "go.uber.org/mock/gomock" + + postprocess "github.com/tidepool-org/platform/data/work/postprocess" ) // MockSummarizers is a mock of Summarizers interface. @@ -41,11 +43,12 @@ func (m *MockSummarizers) EXPECT() *MockSummarizersMockRecorder { } // UpdateSummaries mocks base method. -func (m *MockSummarizers) UpdateSummaries(ctx context.Context, userID string) error { +func (m *MockSummarizers) UpdateSummaries(ctx context.Context, userID string) (postprocess.SummariesUpdate, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UpdateSummaries", ctx, userID) - ret0, _ := ret[0].(error) - return ret0 + ret0, _ := ret[0].(postprocess.SummariesUpdate) + ret1, _ := ret[1].(error) + return ret0, ret1 } // UpdateSummaries indicates an expected call of UpdateSummaries. @@ -61,19 +64,19 @@ type MockSummarizersUpdateSummariesCall struct { } // Return rewrite *gomock.Call.Return -func (c *MockSummarizersUpdateSummariesCall) Return(arg0 error) *MockSummarizersUpdateSummariesCall { - c.Call = c.Call.Return(arg0) +func (c *MockSummarizersUpdateSummariesCall) Return(arg0 postprocess.SummariesUpdate, arg1 error) *MockSummarizersUpdateSummariesCall { + c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *MockSummarizersUpdateSummariesCall) Do(f func(context.Context, string) error) *MockSummarizersUpdateSummariesCall { +func (c *MockSummarizersUpdateSummariesCall) Do(f func(context.Context, string) (postprocess.SummariesUpdate, error)) *MockSummarizersUpdateSummariesCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockSummarizersUpdateSummariesCall) DoAndReturn(f func(context.Context, string) error) *MockSummarizersUpdateSummariesCall { +func (c *MockSummarizersUpdateSummariesCall) DoAndReturn(f func(context.Context, string) (postprocess.SummariesUpdate, error)) *MockSummarizersUpdateSummariesCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/data/work/postprocess/work.go b/data/work/postprocess/work.go index b8ceb52176..7a7c4532d8 100644 --- a/data/work/postprocess/work.go +++ b/data/work/postprocess/work.go @@ -2,12 +2,14 @@ package postprocess import ( "fmt" + "slices" "time" mapset "github.com/deckarep/golang-set/v2" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/structure" + summaryTypes "github.com/tidepool-org/platform/summary/types" userWork "github.com/tidepool-org/platform/user/work" ) @@ -30,7 +32,9 @@ const ( ) const ( - MetadataKeyReasons = "reasons" + MetadataKeyReasons = "reasons" + MetadataKeyPendingSummaryUpdates = "pendingSummaryUpdates" + MetadataKeyPendingSummaryDeletes = "pendingSummaryDeletes" ) func Reasons() []string { @@ -91,6 +95,10 @@ func validateIdentity(groupID *string, serialID *string, workMetadata *Metadata) type Metadata struct { userWork.Metadata `bson:",inline"` Reasons []string `json:"reasons,omitempty" bson:"reasons,omitempty"` + + // Summary changes already calculated by this work item but not yet reported to the clinic service. + PendingSummaryUpdates []string `json:"pendingSummaryUpdates,omitempty" bson:"pendingSummaryUpdates,omitempty"` + PendingSummaryDeletes []string `json:"pendingSummaryDeletes,omitempty" bson:"pendingSummaryDeletes,omitempty"` } func (m *Metadata) Parse(parser structure.ObjectParser) { @@ -98,9 +106,40 @@ func (m *Metadata) Parse(parser structure.ObjectParser) { if ptr := parser.StringArray(MetadataKeyReasons); ptr != nil { m.Reasons = *ptr } + if ptr := parser.StringArray(MetadataKeyPendingSummaryUpdates); ptr != nil { + m.PendingSummaryUpdates = *ptr + } + if ptr := parser.StringArray(MetadataKeyPendingSummaryDeletes); ptr != nil { + m.PendingSummaryDeletes = *ptr + } } func (m *Metadata) Validate(validator structure.Validator) { m.Metadata.Validate(validator) validator.StringArray(MetadataKeyReasons, &m.Reasons).NotEmpty().EachOneOf(Reasons()...).EachUnique() + validator.StringArray(MetadataKeyPendingSummaryUpdates, &m.PendingSummaryUpdates).EachOneOf(summaryTypes.SummaryTypeCGM, summaryTypes.SummaryTypeBGM).EachUnique() + validator.StringArray(MetadataKeyPendingSummaryDeletes, &m.PendingSummaryDeletes).EachNotEmpty().EachUnique() +} + +// recordSummariesUpdate updates the pending/deleted summaries and return true if any changes were made +// as part of this update cycle +func (m *Metadata) recordSummariesUpdate(update SummariesUpdate) bool { + changed := false + for _, summaryType := range update.UpdatedTypes { + if !slices.Contains(m.PendingSummaryUpdates, summaryType) { + m.PendingSummaryUpdates = append(m.PendingSummaryUpdates, summaryType) + changed = true + } + } + for _, summaryID := range update.Deleted { + if !slices.Contains(m.PendingSummaryDeletes, summaryID) { + m.PendingSummaryDeletes = append(m.PendingSummaryDeletes, summaryID) + changed = true + } + } + if changed { + slices.Sort(m.PendingSummaryUpdates) + slices.Sort(m.PendingSummaryDeletes) + } + return changed } diff --git a/data/work/postprocess/work_test.go b/data/work/postprocess/work_test.go index cf941e32c5..1dc3cdd6bc 100644 --- a/data/work/postprocess/work_test.go +++ b/data/work/postprocess/work_test.go @@ -14,6 +14,7 @@ import ( "github.com/tidepool-org/platform/pointer" structureParser "github.com/tidepool-org/platform/structure/parser" structureValidator "github.com/tidepool-org/platform/structure/validator" + summaryTypes "github.com/tidepool-org/platform/summary/types" "github.com/tidepool-org/platform/user" userTest "github.com/tidepool-org/platform/user/test" userWork "github.com/tidepool-org/platform/user/work" @@ -124,6 +125,22 @@ var _ = Describe("Work", func() { }, errorsTest.WithPointerSource(structureParser.ErrorTypeNotArray(true), "/reasons"), ), + Entry("with pending summary changes", + map[string]any{ + "userId": nil, + "reasons": []any{"DATA_ADDED"}, + "pendingSummaryUpdates": []any{"cgm"}, + "pendingSummaryDeletes": []any{"68ad3f439bd2caa1a5758a9c"}, + }, + func(userID string) *dataWorkPostprocess.Metadata { + return &dataWorkPostprocess.Metadata{ + Metadata: userWork.Metadata{UserID: pointer.FromString(userID)}, + Reasons: []string{"DATA_ADDED"}, + PendingSummaryUpdates: []string{"cgm"}, + PendingSummaryDeletes: []string{"68ad3f439bd2caa1a5758a9c"}, + } + }, + ), ) }) @@ -163,14 +180,38 @@ var _ = Describe("Work", func() { }, errorsTest.WithPointerSource(structureValidator.ErrorValueDuplicate(), "/reasons/1"), ), + Entry("succeeds with pending summary changes", func(datum *dataWorkPostprocess.Metadata) { + datum.PendingSummaryUpdates = []string{summaryTypes.SummaryTypeBGM, summaryTypes.SummaryTypeCGM} + datum.PendingSummaryDeletes = []string{"68ad3f439bd2caa1a5758a9c"} + }), + Entry("reports a pending summary update the clinic service has no representation of", + func(datum *dataWorkPostprocess.Metadata) { + datum.PendingSummaryUpdates = []string{summaryTypes.SummaryTypeContinuous} + }, + errorsTest.WithPointerSource(structureValidator.ErrorValueStringNotOneOf(summaryTypes.SummaryTypeContinuous, []string{summaryTypes.SummaryTypeCGM, summaryTypes.SummaryTypeBGM}), "/pendingSummaryUpdates/0"), + ), + Entry("reports the pending summary updates are duplicated", + func(datum *dataWorkPostprocess.Metadata) { + datum.PendingSummaryUpdates = []string{summaryTypes.SummaryTypeCGM, summaryTypes.SummaryTypeCGM} + }, + errorsTest.WithPointerSource(structureValidator.ErrorValueDuplicate(), "/pendingSummaryUpdates/1"), + ), + Entry("reports a pending summary delete is empty", + func(datum *dataWorkPostprocess.Metadata) { + datum.PendingSummaryDeletes = []string{""} + }, + errorsTest.WithPointerSource(structureValidator.ErrorValueEmpty(), "/pendingSummaryDeletes/0"), + ), ) }) // The metadata is encoded when the work is created and decoded when it is processed It("is unchanged by being encoded and decoded", func() { datum := &dataWorkPostprocess.Metadata{ - Metadata: userWork.Metadata{UserID: pointer.FromString(userID)}, - Reasons: []string{dataWorkPostprocess.ReasonLegacyDataAdded, dataWorkPostprocess.ReasonUploadCompleted}, + Metadata: userWork.Metadata{UserID: pointer.FromString(userID)}, + Reasons: []string{dataWorkPostprocess.ReasonLegacyDataAdded, dataWorkPostprocess.ReasonUploadCompleted}, + PendingSummaryUpdates: []string{summaryTypes.SummaryTypeCGM}, + PendingSummaryDeletes: []string{"68ad3f439bd2caa1a5758a9c"}, } encoded, err := metadata.Encode(datum) diff --git a/ehr/reconcile/planner_test.go b/ehr/reconcile/planner_test.go index a8c73c686d..d83869834b 100644 --- a/ehr/reconcile/planner_test.go +++ b/ehr/reconcile/planner_test.go @@ -33,7 +33,7 @@ var _ = Describe("Planner", func() { }) Context("With random data", func() { - var clinics []api.Clinic + var clinics []api.ClinicV1 var tasks map[string]task.Task BeforeEach(func() { @@ -104,7 +104,7 @@ var _ = Describe("Planner", func() { It("returns multiple clinics for deletion when multiple tasks don't exist", func() { firstDeleted := clinics[1] secondDeleted := clinics[2] - clinics = []api.Clinic{clinics[0]} + clinics = []api.ClinicV1{clinics[0]} clinicsClient.EXPECT().ListEHREnabledClinics(gomock.Any()).Return(clinics, nil) setupEHRSettingsForClinics(clinicsClient, clinics) @@ -170,7 +170,7 @@ var _ = Describe("Planner", func() { }) }) -func setupEHRSettingsForClinics(clinicsClient *clinicsTest.MockClient, clinics []api.Clinic) { +func setupEHRSettingsForClinics(clinicsClient *clinicsTest.MockClient, clinics []api.ClinicV1) { for _, clinic := range clinics { clinicsClient.EXPECT().GetEHRSettings(gomock.Any(), *clinic.Id).Return(clinicsTest.NewRandomEHRSettings(), nil) } diff --git a/ehr/reconcile/runner_test.go b/ehr/reconcile/runner_test.go index 4ec44b2d17..0ce08b36f9 100644 --- a/ehr/reconcile/runner_test.go +++ b/ehr/reconcile/runner_test.go @@ -36,7 +36,7 @@ var _ = Describe("Runner", func() { }) Context("With random data", func() { - var clinics []api.Clinic + var clinics []api.ClinicV1 var tasks map[string]task.Task BeforeEach(func() { diff --git a/ehr/sync/runner_test.go b/ehr/sync/runner_test.go index 8ff6ffa26d..81a493323d 100644 --- a/ehr/sync/runner_test.go +++ b/ehr/sync/runner_test.go @@ -35,7 +35,7 @@ var _ = Describe("Runner", func() { Describe("Run", func() { var tsk task.Task - var clinic api.Clinic + var clinic api.ClinicV1 BeforeEach(func() { clinic = clinicsTest.NewRandomClinic() diff --git a/go.mod b/go.mod index 1da9663fad..729c780ac4 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/prometheus/client_model v0.6.1 github.com/rinchsan/device-check-go v1.3.0 github.com/solworktech/md2pdf/v2 v2.2.18 - github.com/tidepool-org/clinic/client v0.0.0-20250122123230-f89e2b1540dc + github.com/tidepool-org/clinic/client v0.0.0-20260814105914-911a077531db github.com/tidepool-org/devices/api v0.0.0-20241122210913-d66c72510ddb github.com/tidepool-org/go-common v0.12.3-0.20250812104912-8c5789d87f55 github.com/tidepool-org/hydrophone/client v0.0.0-20260311102224-0a387435e093 diff --git a/go.sum b/go.sum index 197434ff44..94ee739fcc 100644 --- a/go.sum +++ b/go.sum @@ -260,6 +260,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidepool-org/clinic/client v0.0.0-20250122123230-f89e2b1540dc h1:VXaiFR+UNbEjUsEQ4Ma6ewZb24/HtW2Tk3WRo52xErk= github.com/tidepool-org/clinic/client v0.0.0-20250122123230-f89e2b1540dc/go.mod h1:7BpAdFdGJNB3aw/xvCz5XnWjSWRoUtWIX4xcMc4Bsko= +github.com/tidepool-org/clinic/client v0.0.0-20260814105914-911a077531db h1:ho1g3pSCr3qsj+v+3g5aHl0F5GMQrEe0htPsB5f1arQ= +github.com/tidepool-org/clinic/client v0.0.0-20260814105914-911a077531db/go.mod h1:k23OfUIbA30QCPs7gBF9DHDjOXJ7faS0x3amXlI2XsE= github.com/tidepool-org/devices/api v0.0.0-20241122210913-d66c72510ddb h1:SgtVs9wCnat4M4ELRTsOS390ZPThbmgYQWf0ULgLjEM= github.com/tidepool-org/devices/api v0.0.0-20241122210913-d66c72510ddb/go.mod h1:xuQ8k0mLR1ZyEmwe/m0v2BuXctqQuCZeR43urSQpTUM= github.com/tidepool-org/go-common v0.12.3-0.20250812104912-8c5789d87f55 h1:xgYwNbURhgZRig2gTyNyt/4KDiF/PT0Qlmkc76Usz2M= diff --git a/notifications/work/claims/processor.go b/notifications/work/claims/processor.go index 795029b625..43228f2bb9 100644 --- a/notifications/work/claims/processor.go +++ b/notifications/work/claims/processor.go @@ -47,7 +47,7 @@ func (m *Metadata) Validate(validator structure.Validator) { type Processor struct { *workBase.Processor[Metadata] Dependencies - patient *clinicClient.Patient + patient *clinicClient.PatientV1 } func NewProcessor(dependencies Dependencies) (*Processor, error) { diff --git a/prescription/api/v1.go b/prescription/api/v1.go index 0e3071d700..a72db179c7 100644 --- a/prescription/api/v1.go +++ b/prescription/api/v1.go @@ -331,7 +331,7 @@ func (r *Router) canAccessPrescriptionsForRequestUserID(details request.AuthDeta return details.IsService() || currentUserID == requestedUserID } -func (r *Router) getClinicianOrRespondWithError(ctx context2.Context, clinicID, clinicianID string, responder *request.Responder) *clinic.Clinician { +func (r *Router) getClinicianOrRespondWithError(ctx context2.Context, clinicID, clinicianID string, responder *request.Responder) *clinic.ClinicianV1 { clinician, err := r.clinicsClient.GetClinician(ctx, clinicID, clinicianID) if err != nil { responder.InternalServerError(err) diff --git a/prescription/api/v1_test.go b/prescription/api/v1_test.go index 29af5149e8..6eca8492b2 100644 --- a/prescription/api/v1_test.go +++ b/prescription/api/v1_test.go @@ -118,14 +118,14 @@ var _ = Describe("V1", func() { Context("with patient and clinician", func() { var userID string var clinicID string - var clinician *clinic.Clinician + var clinician *clinic.ClinicianV1 BeforeEach(func() { userID = userTest.RandomUserID() - clinicianID := clinic.TidepoolUserId(userID) - clinician = &clinic.Clinician{ + clinicianID := clinic.Tidepooluserid(userID) + clinician = &clinic.ClinicianV1{ Id: &clinicianID, - Roles: clinic.ClinicianRoles{"PRESCRIBER"}, + Roles: clinic.ClinicianRolesV1{"PRESCRIBER"}, } clinicID = faker.Number().Hexadecimal(24) }) diff --git a/prescription/service/service_test.go b/prescription/service/service_test.go index 1a83ca3871..072ca8fe43 100644 --- a/prescription/service/service_test.go +++ b/prescription/service/service_test.go @@ -107,7 +107,7 @@ var _ = Describe("PrescriptionService", func() { Context("Claim Prescription", func() { It("uses the clinic service to share the patient account with the clinic", func() { prescr := prescriptionTest.RandomPrescription() - patient := clinic.Patient{Id: &prescr.PatientUserID} + patient := clinic.PatientV1{Id: &prescr.PatientUserID} claim := &prescription.Claim{ PatientID: prescr.PatientUserID, AccessCode: prescr.AccessCode, diff --git a/summary/types/summary.go b/summary/types/summary.go index 940d1781df..5874bb6311 100644 --- a/summary/types/summary.go +++ b/summary/types/summary.go @@ -122,6 +122,10 @@ type Summary[PP PeriodsPt[P, PB, B], PB BucketDataPt[B], P Periods, B BucketData Periods PP `json:"periods" bson:"periods"` } +type CGMSummary = Summary[*CGMPeriods, *GlucoseBucket, CGMPeriods, GlucoseBucket] +type BGMSummary = Summary[*BGMPeriods, *GlucoseBucket, BGMPeriods, GlucoseBucket] +type ContinuousSummary = Summary[*ContinuousPeriods, *ContinuousBucket, ContinuousPeriods, ContinuousBucket] + func NewConfig() Config { return Config{ SchemaVersion: SchemaVersion, From 02888a93e29b27eabf3708c568fe761238ccfc91 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Thu, 27 Aug 2026 12:28:54 +0300 Subject: [PATCH 18/20] Clean up logging and improve comments --- data/work/postprocess/enqueue.go | 10 ++-------- data/work/postprocess/processor.go | 21 ++++++--------------- work/store/structured/mongo/mongo.go | 13 +++++++------ 3 files changed, 15 insertions(+), 29 deletions(-) diff --git a/data/work/postprocess/enqueue.go b/data/work/postprocess/enqueue.go index bde0e35952..8cb7d2f1ac 100644 --- a/data/work/postprocess/enqueue.go +++ b/data/work/postprocess/enqueue.go @@ -8,7 +8,6 @@ import ( mapset "github.com/deckarep/golang-set/v2" "github.com/tidepool-org/platform/errors" - "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/metadata" "github.com/tidepool-org/platform/pointer" userWork "github.com/tidepool-org/platform/user/work" @@ -17,8 +16,8 @@ import ( // Enqueue creates a work item to signal a change to the data of a user to trigger the postprocessor. // Work is created for every change reported, rather than merged into the work already pending for the user, -// so that reporting a change is a single insert when data is uploaded. The work pending for a user is instead -// merged when it is processed. +// so that reporting a change is a single insert when data is uploaded. If there are multiple pending work items +// for the same user, they are merged during processing. func Enqueue(ctx context.Context, workClient work.Client, userID string, reasons ...string) error { if ctx == nil { return errors.New("context is missing") @@ -42,11 +41,6 @@ func Enqueue(ctx context.Context, workClient work.Client, userID string, reasons return errors.Wrap(err, "unable to create work") } - log.LoggerFromContext(ctx).WithFields(log.Fields{ - "userId": userID, - "reasons": reasons, - "processingAvailableTime": create.ProcessingAvailableTime, - }).Debug("created work") return nil } diff --git a/data/work/postprocess/processor.go b/data/work/postprocess/processor.go index 86118f9ff3..efc008a14f 100644 --- a/data/work/postprocess/processor.go +++ b/data/work/postprocess/processor.go @@ -70,7 +70,7 @@ func (p *Processor) Process(ctx context.Context, wrk *work.Work, processingUpdat p.absorbPending, p.updateSummaries, p.updateClinicSummaries, - p.triggerElectronicHealthRecordSync, + p.triggerEHRSync, ).Process(p.Delete) } @@ -167,7 +167,9 @@ func (p *Processor) updateSummaries() *work.ProcessResult { p.summariesUpdate, err = p.UpdateSummaries(p.Context(), *p.User().UserID) // The changes made are recorded in the metadata before they are synced to the clinic service, - // so that a failure between the two retries the update + // so that a failure between the two retries the update. They are recorded even when the + // calculation fails partway (a partial update is returned alongside the error), riding along + // on the failing update so the retry still reports them changed := p.Metadata().recordSummariesUpdate(p.summariesUpdate) if err != nil { return p.Failing(err) @@ -197,11 +199,6 @@ func (p *Processor) updateClinicSummaries() *work.ProcessResult { return p.Failing(errors.Wrap(err, "unable to delete patient summary")) } } - if len(workMetadata.PendingSummaryDeletes) > 0 { - log.LoggerFromContext(p.Context()).WithFields(log.Fields{ - "deleted": workMetadata.PendingSummaryDeletes, - }).Debug("deleted clinic service summaries") - } var cgm *summaryTypes.CGMSummary var bgm *summaryTypes.BGMSummary @@ -215,9 +212,6 @@ func (p *Processor) updateClinicSummaries() *work.ProcessResult { if err := p.ClinicsClient.UpdatePatientSummary(p.Context(), *p.User().UserID, clinics.NewPatientSummary(cgm, bgm)); err != nil { return p.Failing(errors.Wrap(err, "unable to update patient summary")) } - log.LoggerFromContext(p.Context()).WithFields(log.Fields{ - "updated": workMetadata.PendingSummaryUpdates, - }).Debug("updated clinic service summaries") } log.LoggerFromContext(p.Context()).WithFields(log.Fields{ @@ -228,9 +222,8 @@ func (p *Processor) updateClinicSummaries() *work.ProcessResult { return nil } -// triggerElectronicHealthRecordSync reports the data of the user to any electronic health record it -// is shared with. Repeating the request reports the same data again, not twice, so retries are safe. -func (p *Processor) triggerElectronicHealthRecordSync() *work.ProcessResult { +// triggerEHRSync triggers report and flowsheet upload for patients with active subscriptions +func (p *Processor) triggerEHRSync() *work.ProcessResult { if !TriggersEHRSync(p.Metadata().Reasons) { return nil } @@ -239,8 +232,6 @@ func (p *Processor) triggerElectronicHealthRecordSync() *work.ProcessResult { return p.Failing(errors.Wrap(err, "unable to trigger EHR sync")) } - log.LoggerFromContext(p.Context()).Info("triggerred EHR sync") - return nil } diff --git a/work/store/structured/mongo/mongo.go b/work/store/structured/mongo/mongo.go index 28b659a261..20a93feb62 100644 --- a/work/store/structured/mongo/mongo.go +++ b/work/store/structured/mongo/mongo.go @@ -174,12 +174,13 @@ func (s *Store) Poll(ctx context.Context, poll *work.Poll) ([]*work.Work, error) // Group all documents by serial id pipeline = append(pipeline, bson.M{"$group": bson.M{"_id": "$serialId", "documents": bson.M{"$push": "$$ROOT"}}}) - // Match any without a serial id or any serial id group that contains no member in state - // processing nor in state failing with retry time in future ($elemMatch binds both failing - // conditions to the same member). Matching the whole group rather than its head is slightly - // conservative: a group is also excluded when a failing member with a future retry sorts after - // an otherwise eligible head. With uniform priority the failing member sorts first anyway; - // where priorities differ, correctness wins over throughput. + // Match documents without a serial id, and serial id groups with no member processing or + // failing with a future retry time ($elemMatch binds both failing conditions to the same + // member). The whole group is checked, not just its head, because the sort is by priority + // first: a lower-priority processing or failing member can sort behind an eligible sibling, + // and a head-only check would dispatch that sibling — running two members of a serial group + // at once or retrying out of order. Blocking the group whenever such a member exists is + // slightly conservative, but keeps the serial guarantees unconditional. pipeline = append(pipeline, bson.M{"$match": bson.M{"$or": bson.A{ bson.M{"_id": bson.M{"$exists": false}}, bson.M{"$nor": bson.A{ From 96fc6908c3a9af724dbe2ff4f54ccd4758334de8 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Thu, 27 Aug 2026 14:00:59 +0300 Subject: [PATCH 19/20] Allow fetching custodial users without an email --- user/user.go | 2 +- user/user_test.go | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/user/user.go b/user/user.go index d35407c306..9df5ab9d5c 100644 --- a/user/user.go +++ b/user/user.go @@ -78,7 +78,7 @@ func (u *User) Parse(parser structure.ObjectParser) { func (u *User) Validate(validator structure.Validator) { validator.String("userid", u.UserID).Exists().Using(IDValidator) - validator.String("username", u.Username).Exists().NotEmpty() + validator.String("username", u.Username).NotEmpty() validator.String("termsAccepted", u.TermsAccepted).AsTime(time.RFC3339Nano).NotZero() validator.StringArray("roles", u.Roles).EachOneOf(Roles()...).EachUnique() } diff --git a/user/user_test.go b/user/user_test.go index ec067fb752..c687cc1894 100644 --- a/user/user_test.go +++ b/user/user_test.go @@ -192,7 +192,6 @@ var _ = Describe("User", func() { ), Entry("username missing", func(datum *user.User) { datum.Username = nil }, - errorsTest.WithPointerSource(structureValidator.ErrorValueNotExists(), "/username"), ), Entry("username empty", func(datum *user.User) { datum.Username = pointer.FromString("") }, @@ -244,7 +243,6 @@ var _ = Describe("User", func() { datum.Roles = pointer.FromStringArray([]string{user.RoleClinic, "invalid"}) }, errorsTest.WithPointerSource(structureValidator.ErrorValueNotExists(), "/userid"), - errorsTest.WithPointerSource(structureValidator.ErrorValueNotExists(), "/username"), errorsTest.WithPointerSource(structureValidator.ErrorValueStringAsTimeNotValid("", time.RFC3339Nano), "/termsAccepted"), errorsTest.WithPointerSource(structureValidator.ErrorValueStringNotOneOf("invalid", user.Roles()), "/roles/1"), ), From 5b20992055685ee682e6985719ff261c6fe0a7b5 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Mon, 31 Aug 2026 13:00:27 +0300 Subject: [PATCH 20/20] Tolerate not found responses when deleting patient summaries --- clinics/service.go | 5 ++++- clinics/service_test.go | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/clinics/service.go b/clinics/service.go index 6c4b2dfc61..cda07faacd 100644 --- a/clinics/service.go +++ b/clinics/service.go @@ -239,12 +239,15 @@ func (d *defaultClient) UpdatePatientSummary(ctx context.Context, patientID stri return nil } +// DeletePatientSummary reports no error when the clinic service reports the summary as not found, +// so that a delete resent by retried work targeting an already deleted summary does not fail the +// work forever. func (d *defaultClient) DeletePatientSummary(ctx context.Context, summaryID string) error { response, err := d.httpClient.DeletePatientSummaryWithResponse(ctx, clinic.SummaryId(summaryID)) if err != nil { return err } - if response.StatusCode() != http.StatusOK && response.StatusCode() != http.StatusNoContent { + if response.StatusCode() != http.StatusOK && response.StatusCode() != http.StatusNoContent && response.StatusCode() != http.StatusNotFound { err = errors.Preparedf(ErrorCodeClinicClientFailure, "Unexpected status code from clinic service", "unexpected response status code %v from %v", response.StatusCode(), response.HTTPResponse.Request.URL) diff --git a/clinics/service_test.go b/clinics/service_test.go index c692fde182..5a76bb448e 100644 --- a/clinics/service_test.go +++ b/clinics/service_test.go @@ -135,6 +135,12 @@ var _ = Describe("Client", func() { Expect(client.DeletePatientSummary(context.Background(), summaryID)).To(Succeed()) }) + // A delete resent by retried work may target a summary the clinic service already deleted + It("returns no error when the summary is not found", func() { + responseStatusCode = http.StatusNotFound + Expect(client.DeletePatientSummary(context.Background(), summaryID)).To(Succeed()) + }) + It("returns an error when the clinic service reports an unexpected status", func() { responseStatusCode = http.StatusInternalServerError err := client.DeletePatientSummary(context.Background(), summaryID)