feat: download XML document with market data

Author: gmlexx

entsoe/api_client.go

@@ -2,8 +2,10 @@
 package entsoe
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"time"
 
@@ -51,6 +53,9 @@ func DownloadPublicationMarketData(ctx context.Context, securityToken string, ur
 
 	now := time.Now().In(location)
 	client := NewAPIClient()
+	opts := &DownloadOptions{
+		UserAgent: client.userAgent,
+	}
 	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()
 
@@ -68,40 +73,42 @@ func DownloadPublicationMarketData(ctx context.Context, securityToken string, ur
 		url := buildPublicationMarketDataURL(securityToken, urlFormat, now)
 		fmt.Println(url)
 		var err error
-		marketDocument, err = client.DownloadPublicationMarketData(ctx, url)
+		var rawXML []byte
+		marketDocument, rawXML, err = DownloadPublicationMarketDataWithRaw(ctx, url, opts)
 		if err != nil {
 			return nil, err
 		}
 		if cache != nil {
-			cache.StoreDocument(todayKey, marketDocument, CacheSourceDownload)
+			cache.StoreDocumentWithRaw(todayKey, marketDocument, rawXML, CacheSourceDownload)
 		}
 	}
 
-	// If instructed, also retrieve data for the next day.
-	if fetchNextDay {
-		tomorrow := now.AddDate(0, 0, 1)
-		tomorrowKey := tomorrow.Format("2006-01-02")
+	// Retrieve data for the next day – use cache when available, or fetch if fetchNextDay is true.
+	tomorrow := now.AddDate(0, 0, 1)
+	tomorrowKey := tomorrow.Format("2006-01-02")
 
-		var marketDocumentNextDay *PublicationMarketData
-		if cache != nil {
-			if cached, ok := cache.Get(tomorrowKey); ok {
-				fmt.Printf("Using cached market data for %s\n", tomorrowKey)
-				marketDocumentNextDay = cached
-			}
+	var marketDocumentNextDay *PublicationMarketData
+	if cache != nil {
+		if cached, ok := cache.Get(tomorrowKey); ok {
+			fmt.Printf("Using cached market data for %s\n", tomorrowKey)
+			marketDocumentNextDay = cached
 		}
+	}
 
-		if marketDocumentNextDay == nil {
-			urlNextDay := buildPublicationMarketDataURL(securityToken, urlFormat, tomorrow)
-			var err error
-			marketDocumentNextDay, err = client.DownloadPublicationMarketData(ctx, urlNextDay)
-			if err != nil {
-				return nil, err
-			}
-			if cache != nil {
-				cache.StoreDocument(tomorrowKey, marketDocumentNextDay, CacheSourceDownload)
-			}
+	if marketDocumentNextDay == nil && fetchNextDay {
+		urlNextDay := buildPublicationMarketDataURL(securityToken, urlFormat, tomorrow)
+		var err error
+		var rawXMLNextDay []byte
+		marketDocumentNextDay, rawXMLNextDay, err = DownloadPublicationMarketDataWithRaw(ctx, urlNextDay, opts)
+		if err != nil {
+			return nil, err
 		}
+		if cache != nil {
+			cache.StoreDocumentWithRaw(tomorrowKey, marketDocumentNextDay, rawXMLNextDay, CacheSourceDownload)
+		}
+	}
 
+	if marketDocumentNextDay != nil {
 		// Merge the data from both days
 		marketDocument = mergePublicationMarketData(marketDocument, marketDocumentNextDay)
 	}
@@ -144,50 +151,63 @@ func mergePublicationMarketData(first *PublicationMarketData, second *Publicatio
 
 // DownloadPublicationMarketDataWithOptions downloads and decodes a PublicationMarketData with custom options
 func DownloadPublicationMarketDataWithOptions(ctx context.Context, apiURL string, opts *DownloadOptions) (*PublicationMarketData, error) {
+	doc, _, err := DownloadPublicationMarketDataWithRaw(ctx, apiURL, opts)
+	return doc, err
+}
+
+// DownloadPublicationMarketDataWithRaw downloads and decodes a PublicationMarketData along with its raw XML bytes
+func DownloadPublicationMarketDataWithRaw(ctx context.Context, apiURL string, opts *DownloadOptions) (*PublicationMarketData, []byte, error) {
 	if apiURL == "" {
-		return nil, fmt.Errorf("API URL cannot be empty")
+		return nil, nil, fmt.Errorf("API URL cannot be empty")
 	}
 
 	client := &http.Client{}
 
 	// Create HTTP request with context
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create HTTP request: %w", err)
+		return nil, nil, fmt.Errorf("failed to create HTTP request: %w", err)
 	}
 
 	// Set default headers
 	userAgent := "entsoe-go-client/1.0"
-	if opts.UserAgent != "" {
+	if opts != nil && opts.UserAgent != "" {
 		userAgent = opts.UserAgent
 	}
 	req.Header.Set("User-Agent", userAgent)
 	req.Header.Set("Accept", "application/xml, text/xml")
 
 	// Set custom headers
-	for key, value := range opts.Headers {
-		req.Header.Set(key, value)
+	if opts != nil {
+		for key, value := range opts.Headers {
+			req.Header.Set(key, value)
+		}
 	}
 
 	// Execute the request
 	resp, err := client.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("failed to execute HTTP request: %w", err)
+		return nil, nil, fmt.Errorf("failed to execute HTTP request: %w", err)
 	}
 	defer resp.Body.Close()
 
 	// Check HTTP status code
 	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("HTTP request failed with status %d: %s", resp.StatusCode, resp.Status)
+		return nil, nil, fmt.Errorf("HTTP request failed with status %d: %s", resp.StatusCode, resp.Status)
+	}
+
+	rawBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to read response body: %w", err)
 	}
 
 	// Decode the XML response using the existing decoder
-	doc, err := DecodeEnergyPricesXML(resp.Body)
+	doc, err := DecodeEnergyPricesXML(bytes.NewReader(rawBytes))
 	if err != nil {
-		return nil, fmt.Errorf("failed to decode XML response: %w", err)
+		return nil, nil, fmt.Errorf("failed to decode XML response: %w", err)
 	}
 
-	return doc, nil
+	return doc, rawBytes, nil
 }
 
 // ValidateAPIURL performs basic validation on the API URL

entsoe/energy_prices_decoder_test.go

@@ -628,3 +628,67 @@ func TestDocumentDecode(t *testing.T) {
 		t.Errorf("Returned price: %f, want %f", price, 57.73)
 	}
 }
+
+func TestDocumentDecode_20260902(t *testing.T) {
+	file, err := os.Open("../test_data/Energy_Prices_202609012200-202609022200.xml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+
+	doc, err := DecodeEnergyPricesXML(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(doc.TimeSeries) != 1 {
+		t.Fatalf("expected 1 time series, got %d", len(doc.TimeSeries))
+	}
+
+	period := doc.TimeSeries[0].Period
+	if len(period.Points) != 96 {
+		t.Fatalf("expected 96 points, got %d", len(period.Points))
+	}
+
+	// Verify position 1 (01:00-01:15) = 139.05
+	if period.Points[0].PriceAmount != 139.05 {
+		t.Errorf("pos 1 price = %f, want 139.05", period.Points[0].PriceAmount)
+	}
+
+	// Verify position 96 (00:45-01:00) = 26.08
+	if period.Points[95].PriceAmount != 26.08 {
+		t.Errorf("pos 96 price = %f, want 26.08", period.Points[95].PriceAmount)
+	}
+}
+
+func TestDocumentDecode_20260901(t *testing.T) {
+	file, err := os.Open("../test_data/Energy_Prices_202608312200-202609012200.xml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+
+	doc, err := DecodeEnergyPricesXML(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(doc.TimeSeries) != 1 {
+		t.Fatalf("expected 1 time series, got %d", len(doc.TimeSeries))
+	}
+
+	period := doc.TimeSeries[0].Period
+	if len(period.Points) != 96 {
+		t.Fatalf("expected 96 points, got %d", len(period.Points))
+	}
+
+	// Verify position 1 (01:00-01:15) = 180.41
+	if period.Points[0].PriceAmount != 180.41 {
+		t.Errorf("pos 1 price = %f, want 180.41", period.Points[0].PriceAmount)
+	}
+
+	// Verify position 96 (00:45-01:00) = 73.23
+	if period.Points[95].PriceAmount != 73.23 {
+		t.Errorf("pos 96 price = %f, want 73.23", period.Points[95].PriceAmount)
+	}
+}

entsoe/xml_cache.go

@@ -24,6 +24,9 @@ type CacheEntry struct {
 	// Document is the parsed market data.
 	Document *PublicationMarketData
 
+	// RawXML holds the original raw XML document bytes.
+	RawXML []byte
+
 	// UploadedAt is the time the document was stored in the cache.
 	UploadedAt time.Time
 
@@ -58,25 +61,44 @@ func (c *XMLDocumentCache) Store(date string, xmlData []byte) error {
 		return fmt.Errorf("failed to parse XML for date %s: %w", date, err)
 	}
 
-	c.StoreDocument(date, doc, CacheSourceUpload)
+	c.StoreDocumentWithRaw(date, doc, xmlData, CacheSourceUpload)
 	return nil
 }
 
 // StoreDocument stores an already-parsed PublicationMarketData document under
 // the given date key (expected format: YYYY-MM-DD).
 // Any existing entry for that date is overwritten.
 func (c *XMLDocumentCache) StoreDocument(date string, doc *PublicationMarketData, source CacheSource) {
+	c.StoreDocumentWithRaw(date, doc, nil, source)
+}
+
+// StoreDocumentWithRaw stores an already-parsed PublicationMarketData document alongside
+// its raw XML bytes under the given date key (expected format: YYYY-MM-DD).
+func (c *XMLDocumentCache) StoreDocumentWithRaw(date string, doc *PublicationMarketData, rawXML []byte, source CacheSource) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
 	c.entries[date] = &CacheEntry{
 		Document:   doc,
+		RawXML:     rawXML,
 		UploadedAt: time.Now(),
 		Date:       date,
 		Source:     source,
 	}
 }
 
+// GetRaw returns the raw XML bytes for the given date key, if available.
+func (c *XMLDocumentCache) GetRaw(date string) ([]byte, bool) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	entry, ok := c.entries[date]
+	if !ok || len(entry.RawXML) == 0 {
+		return nil, false
+	}
+	return entry.RawXML, true
+}
+
 // Get returns the cached PublicationMarketData for the given date key.
 // Returns (nil, false) when no entry exists for that date.
 func (c *XMLDocumentCache) Get(date string) (*PublicationMarketData, bool) {

scheduler/mpc.go

@@ -19,17 +19,24 @@ func (s *MinerScheduler) RunMPCOptimize(ctx context.Context) error {
 
 	config := s.GetConfig()
 
-	// Check if Plant Modbus Address is configured
-	if config.PlantModbusAddress == "" {
-		s.logger.Printf("MPC optimization skipped: PlantModbusAddress not configured")
-		return nil
-	}
-
-	// Step 1: Read plant running info from inverter
-	plantInfo, err := s.readPlantRunningInfo(ctx, config)
-	if err != nil {
-		s.logger.Printf("Error reading plant running info from inverter: %v", err)
-		return err
+	// Step 1: Read plant running info from inverter if configured, or use default plant info
+	var plantInfo *sigenergy.PlantRunningInfo
+	if config.PlantModbusAddress != "" {
+		var err error
+		plantInfo, err = s.readPlantRunningInfo(ctx, config)
+		if err != nil {
+			s.logger.Printf("Warning: Error reading plant running info from inverter: %v, using default plant info", err)
+			plantInfo = &sigenergy.PlantRunningInfo{
+				ESSSOC:                 50.0,
+				ESSAvgCellTemperature: 20.0,
+			}
+		}
+	} else {
+		s.logger.Printf("PlantModbusAddress not configured, using default plant info for MPC optimization")
+		plantInfo = &sigenergy.PlantRunningInfo{
+			ESSSOC:                 50.0,
+			ESSAvgCellTemperature: 20.0,
+		}
 	}
 
 	// Extract initial SOC from plant info

scheduler/pricing.go

@@ -11,12 +11,37 @@ import (
 // StoreMarketDataXML parses the given XML bytes and stores the resulting document
 // in the XML document cache under the given date key (YYYY-MM-DD format).
 func (s *MinerScheduler) StoreMarketDataXML(date string, xmlData []byte) error {
-	return s.xmlCache.Store(date, xmlData)
+	if err := s.xmlCache.Store(date, xmlData); err != nil {
+		return err
+	}
+	go s.refreshMarketDataAndMPC()
+	return nil
 }
 
 // DeleteMarketDataXML removes the cached XML document for the given date key.
 func (s *MinerScheduler) DeleteMarketDataXML(date string) {
 	s.xmlCache.Delete(date)
+	go s.refreshMarketDataAndMPC()
+}
+
+// refreshMarketDataAndMPC invalidates cached prices market data and triggers GetMarketData
+// followed by RunMPCOptimize to ensure MPC optimization decisions are updated immediately.
+func (s *MinerScheduler) refreshMarketDataAndMPC() {
+	s.mu.Lock()
+	s.pricesMarketData = nil
+	s.pricesMarketDataExpiry = time.Time{}
+	s.mu.Unlock()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if _, err := s.GetMarketData(ctx); err != nil {
+		s.logger.Printf("Warning: failed to refresh market data after cache update: %v", err)
+	}
+
+	if err := s.RunMPCOptimize(ctx); err != nil {
+		s.logger.Printf("Warning: failed to run MPC optimization after market data cache update: %v", err)
+	}
 }
 
 // GetXMLCacheEntries returns a snapshot of all entries currently held in the