Skip to content

Commit 985d0e5

Browse files
committed
Fix Octopus free electricity session detection
- Replace DOM-dependent HTML parsing with Go API integration - Use dedicated octopus-free-sessions service for reliable parsing - Add fallback to legacy HTML parsing for backward compatibility - Improve session data format conversion with proper timezone handling - This resolves issues where free sessions weren't being detected due to changes in Octopus Energy website HTML structure The Go API uses goquery for better DOM parsing and will be upgraded to use proper CSS rendering in the future for maximum reliability.
1 parent 914e335 commit 985d0e5

1 file changed

Lines changed: 211 additions & 8 deletions

File tree

‎apps/predbat/octopus.py‎

Lines changed: 211 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1119,27 +1119,230 @@ def download_octopus_free_func(self, url):
11191119

11201120
def download_octopus_free(self, url):
11211121
"""
1122-
Download octopus free session data directly from a URL and process the data
1122+
Download octopus free session data.
1123+
If response is JSON, parse as Go API response. Otherwise, use legacy HTML parsing.
11231124
"""
1125+
free_sessions = []
1126+
pdata = self.download_octopus_free_func(url)
1127+
if not pdata:
1128+
return free_sessions
1129+
1130+
# Check if response is JSON (Go API) or HTML (legacy)
1131+
try:
1132+
import json
1133+
1134+
data = json.loads(pdata)
1135+
sessions = data.get("sessions", [])
1136+
1137+
# Convert Go API format to PredBat format
1138+
for session in sessions:
1139+
if "session_start" in session and "session_end" in session:
1140+
from datetime import datetime
1141+
from config import TIME_FORMAT
1142+
1143+
start_time = datetime.fromisoformat(session["session_start"].replace("Z", "+00:00"))
1144+
end_time = datetime.fromisoformat(session["session_end"].replace("Z", "+00:00"))
11241145

1146+
start_local = start_time.astimezone(self.local_tz)
1147+
end_local = end_time.astimezone(self.local_tz)
1148+
1149+
predbat_session = {"start": start_local.strftime(TIME_FORMAT), "end": end_local.strftime(TIME_FORMAT), "rate": 0.0}
1150+
free_sessions.append(predbat_session)
1151+
1152+
return free_sessions
1153+
1154+
except json.JSONDecodeError:
1155+
# Not JSON, use legacy HTML parsing
1156+
return self.download_octopus_free_legacy(url)
1157+
1158+
def download_octopus_free_legacy(self, url):
1159+
"""
1160+
Legacy method: Download and parse HTML directly (fallback only).
1161+
Kept for backward compatibility when Go API is unavailable.
1162+
"""
11251163
free_sessions = []
11261164
pdata = self.download_octopus_free_func(url)
11271165
if not pdata:
11281166
return free_sessions
11291167

1168+
# Legacy parsing logic - basic pattern matching
11301169
for line in pdata.split("\n"):
1131-
if "Past sessions" in line:
1132-
future_line = line.split("<p data-block-key")
1133-
for fline in future_line:
1134-
for this_line in fline.split("<br/>"):
1135-
res = re.search(r"<i>\s*(\S+)\s+(\d+)(\S+)\s+(\S+)\s+(\S+)-(\S+)", this_line)
1136-
self.octopus_free_line(res, free_sessions)
1170+
# Look for the most common current format
1171+
# "Last Free Electricity Session: DOUBLE Session 12-2pm, Sunday 7th September"
1172+
if "Free Electricity Session:" in line or "Last Free Electricity:" in line:
1173+
# Extract session time and date with regex
1174+
match = re.search(r"(\d{1,2})-(\d{1,2})(am|pm),?\s+(\w+day)\s+(\d{1,2})(?:st|nd|rd|th)\s+(\w+)", line, re.IGNORECASE)
1175+
if match:
1176+
start_hour = int(match.group(1))
1177+
end_hour = int(match.group(2))
1178+
period = match.group(3).lower()
1179+
day_of_week = match.group(4)
1180+
day_num = int(match.group(5))
1181+
month = match.group(6)
1182+
1183+
session = self.create_free_session_simple(start_hour, end_hour, period, day_num, month)
1184+
if session:
1185+
free_sessions.append(session)
1186+
self.log(f"Legacy parser found session: {session['start']} to {session['end']}")
1187+
1188+
# Legacy format support for older patterns
11371189
if "Free Electricity:" in line:
1138-
# Free Electricity: Sunday 24th November 7-9am
11391190
res = re.search(r"Free Electricity:\s+(\S+)\s+(\d+)(\S+)\s+(\S+)\s+(\S+)-(\S+)", line)
11401191
self.octopus_free_line(res, free_sessions)
1192+
11411193
return free_sessions
11421194

1195+
def create_free_session_simple(self, start_hour, end_hour, period, day_num, month):
1196+
"""
1197+
Create a free session from basic components (simplified legacy method).
1198+
"""
1199+
try:
1200+
# Adjust hours for AM/PM
1201+
if period == "pm" and start_hour != 12:
1202+
start_hour += 12
1203+
end_hour += 12
1204+
elif period == "am" and start_hour == 12:
1205+
start_hour = 0
1206+
if end_hour == 12:
1207+
end_hour = 0
1208+
1209+
# Simple month parsing
1210+
month_names = ["january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"]
1211+
month_num = None
1212+
for i, month_name in enumerate(month_names, 1):
1213+
if month.lower().startswith(month_name[:3]):
1214+
month_num = i
1215+
break
1216+
1217+
if not month_num:
1218+
return None
1219+
1220+
# Create session
1221+
from datetime import datetime
1222+
from config import TIME_FORMAT
1223+
1224+
year = datetime.now().year
1225+
start_time = datetime(year, month_num, day_num, start_hour, 0)
1226+
end_time = datetime(year, month_num, day_num, end_hour, 0)
1227+
1228+
return {"start": start_time.strftime(TIME_FORMAT), "end": end_time.strftime(TIME_FORMAT), "rate": 0.0}
1229+
1230+
except Exception as e:
1231+
self.log(f"Error in create_free_session_simple: {e}")
1232+
return None
1233+
1234+
def html_to_text(self, html):
1235+
"""
1236+
Convert HTML to human-readable text by removing tags and normalizing whitespace.
1237+
Simple text extraction that preserves line breaks for human-like reading.
1238+
"""
1239+
# Remove HTML tags but preserve some structure
1240+
text = html
1241+
1242+
# Replace block elements with newlines for better text flow
1243+
block_elements = ["</h1>", "</h2>", "</h3>", "</h4>", "</h5>", "</h6>", "</p>", "</div>", "</section>", "</article>", "</li>", "</br>", "<br/>", "<br>"]
1244+
for element in block_elements:
1245+
text = text.replace(element, "\n")
1246+
1247+
# Remove remaining HTML tags
1248+
text = re.sub(r"<[^>]+>", "", text)
1249+
1250+
# Decode HTML entities
1251+
text = text.replace("&amp;", "&").replace("&lt;", "<").replace("&gt;", ">").replace("&quot;", '"').replace("&#39;", "'")
1252+
1253+
# Normalize whitespace but preserve line breaks
1254+
lines = []
1255+
for line in text.split("\n"):
1256+
# Clean up each line
1257+
cleaned = re.sub(r"\s+", " ", line.strip())
1258+
if cleaned: # Only keep non-empty lines
1259+
lines.append(cleaned)
1260+
1261+
return "\n".join(lines)
1262+
1263+
def create_free_session(self, time_slot, day_of_week, day_num, month, original_line):
1264+
"""
1265+
Create a free session dictionary from parsed components.
1266+
Returns None if the session cannot be properly parsed.
1267+
"""
1268+
try:
1269+
# Parse the time slot (e.g., "12-2pm", "7-9am")
1270+
time_match = re.match(r"(\d{1,2})(?::(\d{2}))?(?:am|pm)?-(\d{1,2})(?::(\d{2}))?(am|pm)", time_slot.lower())
1271+
if not time_match:
1272+
self.log(f"Warning: Cannot parse time slot '{time_slot}' in: {original_line[:100]}")
1273+
return None
1274+
1275+
start_hour = int(time_match.group(1))
1276+
start_min = int(time_match.group(2) or 0)
1277+
end_hour = int(time_match.group(3))
1278+
end_min = int(time_match.group(4) or 0)
1279+
period = time_match.group(5) # am or pm
1280+
1281+
# Adjust for PM times
1282+
if period == "pm" and start_hour != 12:
1283+
start_hour += 12
1284+
end_hour += 12
1285+
elif period == "am" and start_hour == 12:
1286+
start_hour = 0
1287+
if end_hour == 12:
1288+
end_hour = 0
1289+
1290+
# Handle cases like "11pm-1am" (crosses midnight)
1291+
if end_hour < start_hour:
1292+
end_hour += 24
1293+
1294+
# Parse date components
1295+
day = int(re.sub(r"[^\d]", "", day_num)) # Remove "st", "nd", "rd", "th"
1296+
1297+
# Estimate year (current year or next year if date has passed)
1298+
from datetime import datetime
1299+
1300+
now = datetime.now()
1301+
year = now.year
1302+
1303+
# Try to parse the month
1304+
month_names = ["january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"]
1305+
month_num = None
1306+
for i, month_name in enumerate(month_names, 1):
1307+
if month.lower().startswith(month_name[:3]): # Match "sep", "sept", "september"
1308+
month_num = i
1309+
break
1310+
1311+
if not month_num:
1312+
self.log(f"Warning: Cannot parse month '{month}' in: {original_line[:100]}")
1313+
return None
1314+
1315+
# Create datetime objects
1316+
try:
1317+
start_time = datetime(year, month_num, day, start_hour, start_min)
1318+
end_time = datetime(year, month_num, day, end_hour % 24, end_min)
1319+
1320+
# If end time crosses midnight, adjust the date
1321+
if end_hour >= 24:
1322+
from datetime import timedelta
1323+
1324+
end_time += timedelta(days=1)
1325+
1326+
# If the date is in the past, assume it's next year
1327+
if start_time < now:
1328+
start_time = start_time.replace(year=year + 1)
1329+
end_time = end_time.replace(year=year + 1)
1330+
1331+
except ValueError as e:
1332+
self.log(f"Warning: Cannot create datetime from {day}/{month_num}/{year} {start_hour}:{start_min}: {e}")
1333+
return None
1334+
1335+
# Format for PredBat (uses TIME_FORMAT from config.py)
1336+
from config import TIME_FORMAT
1337+
1338+
session = {"start": start_time.strftime(TIME_FORMAT), "end": end_time.strftime(TIME_FORMAT), "rate": 0.0} # Free electricity
1339+
1340+
return session
1341+
1342+
except Exception as e:
1343+
self.log(f"Error creating free session from '{time_slot}' '{day_of_week}' '{day_num}' '{month}': {e}")
1344+
return None
1345+
11431346
def download_octopus_rates(self, url):
11441347
"""
11451348
Download octopus rates directly from a URL or return from cache if recent

0 commit comments

Comments
 (0)