@@ -1119,27 +1119,230 @@ def download_octopus_free_func(self, url):
11191119
11201120 def download_octopus_free (self , url ):
11211121 """
1122- Download octopus free session data directly from a URL and process the data
1122+ Download octopus free session data.
1123+ If response is JSON, parse as Go API response. Otherwise, use legacy HTML parsing.
11231124 """
1125+ free_sessions = []
1126+ pdata = self .download_octopus_free_func (url )
1127+ if not pdata :
1128+ return free_sessions
1129+
1130+ # Check if response is JSON (Go API) or HTML (legacy)
1131+ try :
1132+ import json
1133+
1134+ data = json .loads (pdata )
1135+ sessions = data .get ("sessions" , [])
1136+
1137+ # Convert Go API format to PredBat format
1138+ for session in sessions :
1139+ if "session_start" in session and "session_end" in session :
1140+ from datetime import datetime
1141+ from config import TIME_FORMAT
1142+
1143+ start_time = datetime .fromisoformat (session ["session_start" ].replace ("Z" , "+00:00" ))
1144+ end_time = datetime .fromisoformat (session ["session_end" ].replace ("Z" , "+00:00" ))
11241145
1146+ start_local = start_time .astimezone (self .local_tz )
1147+ end_local = end_time .astimezone (self .local_tz )
1148+
1149+ predbat_session = {"start" : start_local .strftime (TIME_FORMAT ), "end" : end_local .strftime (TIME_FORMAT ), "rate" : 0.0 }
1150+ free_sessions .append (predbat_session )
1151+
1152+ return free_sessions
1153+
1154+ except json .JSONDecodeError :
1155+ # Not JSON, use legacy HTML parsing
1156+ return self .download_octopus_free_legacy (url )
1157+
1158+ def download_octopus_free_legacy (self , url ):
1159+ """
1160+ Legacy method: Download and parse HTML directly (fallback only).
1161+ Kept for backward compatibility when Go API is unavailable.
1162+ """
11251163 free_sessions = []
11261164 pdata = self .download_octopus_free_func (url )
11271165 if not pdata :
11281166 return free_sessions
11291167
1168+ # Legacy parsing logic - basic pattern matching
11301169 for line in pdata .split ("\n " ):
1131- if "Past sessions" in line :
1132- future_line = line .split ("<p data-block-key" )
1133- for fline in future_line :
1134- for this_line in fline .split ("<br/>" ):
1135- res = re .search (r"<i>\s*(\S+)\s+(\d+)(\S+)\s+(\S+)\s+(\S+)-(\S+)" , this_line )
1136- self .octopus_free_line (res , free_sessions )
1170+ # Look for the most common current format
1171+ # "Last Free Electricity Session: DOUBLE Session 12-2pm, Sunday 7th September"
1172+ if "Free Electricity Session:" in line or "Last Free Electricity:" in line :
1173+ # Extract session time and date with regex
1174+ match = re .search (r"(\d{1,2})-(\d{1,2})(am|pm),?\s+(\w+day)\s+(\d{1,2})(?:st|nd|rd|th)\s+(\w+)" , line , re .IGNORECASE )
1175+ if match :
1176+ start_hour = int (match .group (1 ))
1177+ end_hour = int (match .group (2 ))
1178+ period = match .group (3 ).lower ()
1179+ day_of_week = match .group (4 )
1180+ day_num = int (match .group (5 ))
1181+ month = match .group (6 )
1182+
1183+ session = self .create_free_session_simple (start_hour , end_hour , period , day_num , month )
1184+ if session :
1185+ free_sessions .append (session )
1186+ self .log (f"Legacy parser found session: { session ['start' ]} to { session ['end' ]} " )
1187+
1188+ # Legacy format support for older patterns
11371189 if "Free Electricity:" in line :
1138- # Free Electricity: Sunday 24th November 7-9am
11391190 res = re .search (r"Free Electricity:\s+(\S+)\s+(\d+)(\S+)\s+(\S+)\s+(\S+)-(\S+)" , line )
11401191 self .octopus_free_line (res , free_sessions )
1192+
11411193 return free_sessions
11421194
1195+ def create_free_session_simple (self , start_hour , end_hour , period , day_num , month ):
1196+ """
1197+ Create a free session from basic components (simplified legacy method).
1198+ """
1199+ try :
1200+ # Adjust hours for AM/PM
1201+ if period == "pm" and start_hour != 12 :
1202+ start_hour += 12
1203+ end_hour += 12
1204+ elif period == "am" and start_hour == 12 :
1205+ start_hour = 0
1206+ if end_hour == 12 :
1207+ end_hour = 0
1208+
1209+ # Simple month parsing
1210+ month_names = ["january" , "february" , "march" , "april" , "may" , "june" , "july" , "august" , "september" , "october" , "november" , "december" ]
1211+ month_num = None
1212+ for i , month_name in enumerate (month_names , 1 ):
1213+ if month .lower ().startswith (month_name [:3 ]):
1214+ month_num = i
1215+ break
1216+
1217+ if not month_num :
1218+ return None
1219+
1220+ # Create session
1221+ from datetime import datetime
1222+ from config import TIME_FORMAT
1223+
1224+ year = datetime .now ().year
1225+ start_time = datetime (year , month_num , day_num , start_hour , 0 )
1226+ end_time = datetime (year , month_num , day_num , end_hour , 0 )
1227+
1228+ return {"start" : start_time .strftime (TIME_FORMAT ), "end" : end_time .strftime (TIME_FORMAT ), "rate" : 0.0 }
1229+
1230+ except Exception as e :
1231+ self .log (f"Error in create_free_session_simple: { e } " )
1232+ return None
1233+
1234+ def html_to_text (self , html ):
1235+ """
1236+ Convert HTML to human-readable text by removing tags and normalizing whitespace.
1237+ Simple text extraction that preserves line breaks for human-like reading.
1238+ """
1239+ # Remove HTML tags but preserve some structure
1240+ text = html
1241+
1242+ # Replace block elements with newlines for better text flow
1243+ block_elements = ["</h1>" , "</h2>" , "</h3>" , "</h4>" , "</h5>" , "</h6>" , "</p>" , "</div>" , "</section>" , "</article>" , "</li>" , "</br>" , "<br/>" , "<br>" ]
1244+ for element in block_elements :
1245+ text = text .replace (element , "\n " )
1246+
1247+ # Remove remaining HTML tags
1248+ text = re .sub (r"<[^>]+>" , "" , text )
1249+
1250+ # Decode HTML entities
1251+ text = text .replace ("&" , "&" ).replace ("<" , "<" ).replace (">" , ">" ).replace (""" , '"' ).replace ("'" , "'" )
1252+
1253+ # Normalize whitespace but preserve line breaks
1254+ lines = []
1255+ for line in text .split ("\n " ):
1256+ # Clean up each line
1257+ cleaned = re .sub (r"\s+" , " " , line .strip ())
1258+ if cleaned : # Only keep non-empty lines
1259+ lines .append (cleaned )
1260+
1261+ return "\n " .join (lines )
1262+
1263+ def create_free_session (self , time_slot , day_of_week , day_num , month , original_line ):
1264+ """
1265+ Create a free session dictionary from parsed components.
1266+ Returns None if the session cannot be properly parsed.
1267+ """
1268+ try :
1269+ # Parse the time slot (e.g., "12-2pm", "7-9am")
1270+ time_match = re .match (r"(\d{1,2})(?::(\d{2}))?(?:am|pm)?-(\d{1,2})(?::(\d{2}))?(am|pm)" , time_slot .lower ())
1271+ if not time_match :
1272+ self .log (f"Warning: Cannot parse time slot '{ time_slot } ' in: { original_line [:100 ]} " )
1273+ return None
1274+
1275+ start_hour = int (time_match .group (1 ))
1276+ start_min = int (time_match .group (2 ) or 0 )
1277+ end_hour = int (time_match .group (3 ))
1278+ end_min = int (time_match .group (4 ) or 0 )
1279+ period = time_match .group (5 ) # am or pm
1280+
1281+ # Adjust for PM times
1282+ if period == "pm" and start_hour != 12 :
1283+ start_hour += 12
1284+ end_hour += 12
1285+ elif period == "am" and start_hour == 12 :
1286+ start_hour = 0
1287+ if end_hour == 12 :
1288+ end_hour = 0
1289+
1290+ # Handle cases like "11pm-1am" (crosses midnight)
1291+ if end_hour < start_hour :
1292+ end_hour += 24
1293+
1294+ # Parse date components
1295+ day = int (re .sub (r"[^\d]" , "" , day_num )) # Remove "st", "nd", "rd", "th"
1296+
1297+ # Estimate year (current year or next year if date has passed)
1298+ from datetime import datetime
1299+
1300+ now = datetime .now ()
1301+ year = now .year
1302+
1303+ # Try to parse the month
1304+ month_names = ["january" , "february" , "march" , "april" , "may" , "june" , "july" , "august" , "september" , "october" , "november" , "december" ]
1305+ month_num = None
1306+ for i , month_name in enumerate (month_names , 1 ):
1307+ if month .lower ().startswith (month_name [:3 ]): # Match "sep", "sept", "september"
1308+ month_num = i
1309+ break
1310+
1311+ if not month_num :
1312+ self .log (f"Warning: Cannot parse month '{ month } ' in: { original_line [:100 ]} " )
1313+ return None
1314+
1315+ # Create datetime objects
1316+ try :
1317+ start_time = datetime (year , month_num , day , start_hour , start_min )
1318+ end_time = datetime (year , month_num , day , end_hour % 24 , end_min )
1319+
1320+ # If end time crosses midnight, adjust the date
1321+ if end_hour >= 24 :
1322+ from datetime import timedelta
1323+
1324+ end_time += timedelta (days = 1 )
1325+
1326+ # If the date is in the past, assume it's next year
1327+ if start_time < now :
1328+ start_time = start_time .replace (year = year + 1 )
1329+ end_time = end_time .replace (year = year + 1 )
1330+
1331+ except ValueError as e :
1332+ self .log (f"Warning: Cannot create datetime from { day } /{ month_num } /{ year } { start_hour } :{ start_min } : { e } " )
1333+ return None
1334+
1335+ # Format for PredBat (uses TIME_FORMAT from config.py)
1336+ from config import TIME_FORMAT
1337+
1338+ session = {"start" : start_time .strftime (TIME_FORMAT ), "end" : end_time .strftime (TIME_FORMAT ), "rate" : 0.0 } # Free electricity
1339+
1340+ return session
1341+
1342+ except Exception as e :
1343+ self .log (f"Error creating free session from '{ time_slot } ' '{ day_of_week } ' '{ day_num } ' '{ month } ': { e } " )
1344+ return None
1345+
11431346 def download_octopus_rates (self , url ):
11441347 """
11451348 Download octopus rates directly from a URL or return from cache if recent
0 commit comments