fix: next day market data fetch (#20)

Author: gmlexx

entsoe/api_client.go

@@ -49,7 +49,12 @@ type DownloadOptions struct {
 // DownloadPublicationMarketData downloads and decodes publication market data for the current and next day if needed.
 // fetchNextDay indicates whether to also download data for the next day (e.g. when it is past 13:30).
 // cache is an optional XMLDocumentCache; when non-nil, cached entries are returned instead of fetching from the network.
-func DownloadPublicationMarketData(ctx context.Context, securityToken string, urlFormat string, location *time.Location, fetchNextDay bool, cache *XMLDocumentCache) (*PublicationMarketData, error) {
+//
+// The second return value reports whether the returned document already covers the next day.
+// It is false when fetchNextDay is true but ENTSO-E has not published tomorrow's prices yet,
+// which lets callers shorten their cache lifetime and retry soon instead of caching a
+// partial (today-only) document for the rest of the day.
+func DownloadPublicationMarketData(ctx context.Context, securityToken string, urlFormat string, location *time.Location, fetchNextDay bool, cache *XMLDocumentCache) (*PublicationMarketData, bool, error) {
 
 	now := time.Now().In(location)
 	client := NewAPIClient()
@@ -74,7 +79,7 @@ func DownloadPublicationMarketData(ctx context.Context, securityToken string, ur
 		var rawXML []byte
 		marketDocument, rawXML, err = downloadWithTimeout(ctx, url, opts)
 		if err != nil {
-			return nil, err
+			return nil, false, err
 		}
 		if cache != nil {
 			cache.StoreDocumentWithRaw(todayKey, marketDocument, rawXML, CacheSourceDownload)
@@ -106,7 +111,7 @@ func DownloadPublicationMarketData(ctx context.Context, securityToken string, ur
 				fmt.Printf("No market data published yet for %s: %v\n", tomorrowKey, err)
 				marketDocumentNextDay = nil
 			} else {
-				return nil, err
+				return nil, false, err
 			}
 		} else if cache != nil {
 			cache.StoreDocumentWithRaw(tomorrowKey, marketDocumentNextDay, rawXMLNextDay, CacheSourceDownload)
@@ -118,7 +123,10 @@ func DownloadPublicationMarketData(ctx context.Context, securityToken string, ur
 		marketDocument = mergePublicationMarketData(marketDocument, marketDocumentNextDay)
 	}
 
-	return marketDocument, nil
+	// The result is complete unless the caller asked for the next day and it is still missing.
+	nextDayIncluded := !fetchNextDay || marketDocumentNextDay != nil
+
+	return marketDocument, nextDayIncluded, nil
 }
 
 // requestTimeout bounds a single ENTSO-E API call. The Transparency Platform can be

scheduler/pricing.go

@@ -8,6 +8,12 @@ import (
 	"github.com/devskill-org/ems/entsoe"
 )
 
+// marketDataRetryInterval is how long a partial (today-only) market data document
+// is cached when the next day's prices have not been published by ENTSO-E yet.
+// It is long enough to avoid hammering the Transparency Platform every minute,
+// and short enough to pick up the prices soon after they appear.
+const marketDataRetryInterval = 15 * time.Minute
+
 // StoreMarketDataXML parses the given XML bytes and stores the resulting document
 // in the XML document cache under the given date key (YYYY-MM-DD format).
 func (s *MinerScheduler) StoreMarketDataXML(date string, xmlData []byte) error {
@@ -99,11 +105,19 @@ func (s *MinerScheduler) GetMarketData(ctx context.Context) (*entsoe.Publication
 
 	// Perform the network download WITHOUT holding the lock so other goroutines
 	// (GetConfig, runStateCheck, etc.) are never blocked during I/O.
-	newDoc, err := entsoe.DownloadPublicationMarketData(ctx, s.config.SecurityToken, s.config.URLFormat, location, fetchNextDay, s.xmlCache)
+	newDoc, nextDayIncluded, err := entsoe.DownloadPublicationMarketData(ctx, s.config.SecurityToken, s.config.URLFormat, location, fetchNextDay, s.xmlCache)
 	if err != nil {
 		return nil, fmt.Errorf("failed to download PublicationMarketData: %w", err)
 	}
 
+	// ENTSO-E has not published tomorrow's day-ahead prices yet. Caching this
+	// partial (today-only) document until tomorrow 13:30 would suppress every
+	// subsequent refresh for ~24h, so expire it quickly and retry instead.
+	if !nextDayIncluded {
+		nextExpiry = now.Add(marketDataRetryInterval)
+		s.logger.Printf("Next-day prices not published yet, retrying at %s", nextExpiry.Format(time.RFC3339))
+	}
+
 	// Re-acquire the write lock only to store the result.
 	// Double-check: another goroutine may have already refreshed while we were
 	// downloading; keep whichever copy is fresher.