diff --git a/entsoe/api_client.go b/entsoe/api_client.go index e6bc324..868b761 100644 --- a/entsoe/api_client.go +++ b/entsoe/api_client.go @@ -49,7 +49,12 @@ type DownloadOptions struct { // DownloadPublicationMarketData downloads and decodes publication market data for the current and next day if needed. // fetchNextDay indicates whether to also download data for the next day (e.g. when it is past 13:30). // cache is an optional XMLDocumentCache; when non-nil, cached entries are returned instead of fetching from the network. -func DownloadPublicationMarketData(ctx context.Context, securityToken string, urlFormat string, location *time.Location, fetchNextDay bool, cache *XMLDocumentCache) (*PublicationMarketData, error) { +// +// The second return value reports whether the returned document already covers the next day. +// It is false when fetchNextDay is true but ENTSO-E has not published tomorrow's prices yet, +// which lets callers shorten their cache lifetime and retry soon instead of caching a +// partial (today-only) document for the rest of the day. +func DownloadPublicationMarketData(ctx context.Context, securityToken string, urlFormat string, location *time.Location, fetchNextDay bool, cache *XMLDocumentCache) (*PublicationMarketData, bool, error) { now := time.Now().In(location) client := NewAPIClient() @@ -74,7 +79,7 @@ func DownloadPublicationMarketData(ctx context.Context, securityToken string, ur var rawXML []byte marketDocument, rawXML, err = downloadWithTimeout(ctx, url, opts) if err != nil { - return nil, err + return nil, false, err } if cache != nil { cache.StoreDocumentWithRaw(todayKey, marketDocument, rawXML, CacheSourceDownload) @@ -106,7 +111,7 @@ func DownloadPublicationMarketData(ctx context.Context, securityToken string, ur fmt.Printf("No market data published yet for %s: %v\n", tomorrowKey, err) marketDocumentNextDay = nil } else { - return nil, err + return nil, false, err } } else if cache != nil { cache.StoreDocumentWithRaw(tomorrowKey, marketDocumentNextDay, rawXMLNextDay, CacheSourceDownload) @@ -118,7 +123,10 @@ func DownloadPublicationMarketData(ctx context.Context, securityToken string, ur marketDocument = mergePublicationMarketData(marketDocument, marketDocumentNextDay) } - return marketDocument, nil + // The result is complete unless the caller asked for the next day and it is still missing. + nextDayIncluded := !fetchNextDay || marketDocumentNextDay != nil + + return marketDocument, nextDayIncluded, nil } // requestTimeout bounds a single ENTSO-E API call. The Transparency Platform can be diff --git a/scheduler/pricing.go b/scheduler/pricing.go index e022bef..7aff6a1 100644 --- a/scheduler/pricing.go +++ b/scheduler/pricing.go @@ -8,6 +8,12 @@ import ( "github.com/devskill-org/ems/entsoe" ) +// marketDataRetryInterval is how long a partial (today-only) market data document +// is cached when the next day's prices have not been published by ENTSO-E yet. +// It is long enough to avoid hammering the Transparency Platform every minute, +// and short enough to pick up the prices soon after they appear. +const marketDataRetryInterval = 15 * time.Minute + // StoreMarketDataXML parses the given XML bytes and stores the resulting document // in the XML document cache under the given date key (YYYY-MM-DD format). func (s *MinerScheduler) StoreMarketDataXML(date string, xmlData []byte) error { @@ -99,11 +105,19 @@ func (s *MinerScheduler) GetMarketData(ctx context.Context) (*entsoe.Publication // Perform the network download WITHOUT holding the lock so other goroutines // (GetConfig, runStateCheck, etc.) are never blocked during I/O. - newDoc, err := entsoe.DownloadPublicationMarketData(ctx, s.config.SecurityToken, s.config.URLFormat, location, fetchNextDay, s.xmlCache) + newDoc, nextDayIncluded, err := entsoe.DownloadPublicationMarketData(ctx, s.config.SecurityToken, s.config.URLFormat, location, fetchNextDay, s.xmlCache) if err != nil { return nil, fmt.Errorf("failed to download PublicationMarketData: %w", err) } + // ENTSO-E has not published tomorrow's day-ahead prices yet. Caching this + // partial (today-only) document until tomorrow 13:30 would suppress every + // subsequent refresh for ~24h, so expire it quickly and retry instead. + if !nextDayIncluded { + nextExpiry = now.Add(marketDataRetryInterval) + s.logger.Printf("Next-day prices not published yet, retrying at %s", nextExpiry.Format(time.RFC3339)) + } + // Re-acquire the write lock only to store the result. // Double-check: another goroutine may have already refreshed while we were // downloading; keep whichever copy is fresher.