Skip to content

Commit e975460

Browse files
committed
fix(mastery): persist retry backoff across restarts
A failed profile fetch now records a consecutive-failure counter to disk (mastery-fail-<account_id>.json) and checks it before the next attempt, skipping the network call while its backoff cooldown hasn't elapsed. Reuses the same wf_data::poll backoff helper eaef86e gave the worldstate/riven-price pollers. Without this, a short-lived process (any CLI subcommand, or the tray relaunched every few minutes) re-hit the currently DE-blocked profile endpoint on every single launch with no memory of the prior failure.
1 parent 20c9ceb commit e975460

1 file changed

Lines changed: 124 additions & 16 deletions

File tree

crates/wf-relic/src/mastery.rs

Lines changed: 124 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -159,9 +159,26 @@ pub async fn fetch_display_name(
159159
Ok(name)
160160
}
161161

162+
/// Base cooldown before retrying a failed profile fetch, doubling per
163+
/// consecutive failure via [`wf_data::poll::backoff_interval`] (the same
164+
/// helper `eaef86e` gave the worldstate/riven-price polling loops) and
165+
/// capped at [`MASTERY_RETRY_CAP`].
166+
const MASTERY_RETRY_BASE: std::time::Duration = std::time::Duration::from_secs(15 * 60);
167+
/// Ceiling on [`MASTERY_RETRY_BASE`]'s backoff — long enough to stop
168+
/// hammering a blocked/down endpoint, short enough to notice it come back
169+
/// within a session.
170+
const MASTERY_RETRY_CAP: std::time::Duration = std::time::Duration::from_secs(6 * 3600);
171+
162172
/// Load the mastered set from a disk cache when fresh (younger than `ttl`),
163-
/// otherwise refetch. Falls back to a stale cache on network failure, and to an
164-
/// empty set if there is nothing cached and the fetch fails.
173+
/// otherwise refetch. Falls back to a stale cache on network failure (or an
174+
/// empty set if there is nothing cached), same as before — but a failed
175+
/// fetch now also persists a retry-backoff marker (see
176+
/// [`retry_cooldown_remaining`]) to disk, so a *process restart* within the
177+
/// cooldown window skips the network call instead of re-attempting it: this
178+
/// function is called fresh on every CLI subcommand invocation and every
179+
/// tray relaunch, so without persisting the backoff to disk, a user
180+
/// restarting every few minutes during a DE-side outage would silently
181+
/// re-hit a blocked endpoint every single time.
165182
pub async fn load_cached(
166183
client: &reqwest::Client,
167184
account_id: &str,
@@ -176,34 +193,78 @@ pub async fn load_cached(
176193
// under the old bug doesn't keep hiding a mastered sentinel weapon
177194
// between 450k-900k XP (issue #65).
178195
let file = format!("mastery-v5-{account_id}.json");
179-
if let Some(cached) = wf_cache::load_blob::<MasterySet>(&file) {
196+
let cached = wf_cache::load_blob::<MasterySet>(&file);
197+
if let Some(cached) = &cached {
180198
if cached.age() < ttl {
181199
tracing::info!("mastery from cache ({} items)", cached.value.len());
182-
return cached.value;
183-
}
184-
match fetch(client, account_id).await {
185-
Ok(set) => {
186-
let _ = wf_cache::save_blob(&file, &set);
187-
return set;
188-
}
189-
Err(e) => {
190-
tracing::warn!("mastery refresh failed ({e:#}); using stale cache");
191-
return cached.value;
192-
}
200+
return cached.value.clone();
193201
}
194202
}
203+
204+
if let Some(wait) = retry_cooldown_remaining(account_id) {
205+
tracing::info!("mastery refresh skipped (retrying in {wait:?}); using stale cache");
206+
return cached.map(|c| c.value).unwrap_or_default();
207+
}
208+
195209
match fetch(client, account_id).await {
196210
Ok(set) => {
197211
let _ = wf_cache::save_blob(&file, &set);
212+
clear_retry_state(account_id);
198213
set
199214
}
200215
Err(e) => {
201-
tracing::warn!("mastery fetch failed: {e:#}");
202-
MasterySet::default()
216+
let failures = record_retry_failure(account_id);
217+
tracing::warn!("mastery refresh failed (failure #{failures}: {e:#}); using stale cache");
218+
cached.map(|c| c.value).unwrap_or_default()
203219
}
204220
}
205221
}
206222

223+
/// `<cache_dir>/mastery-fail-<account_id>.json`'s name: a persisted
224+
/// consecutive-failure counter for `account_id`'s profile fetch, stamped
225+
/// with when it was last written (via [`wf_cache::save_blob`]'s
226+
/// [`wf_cache::Stamped`] wrapper) — that timestamp doubles as "when did the
227+
/// most recent failure happen," which is all [`retry_cooldown_remaining`]
228+
/// needs.
229+
fn retry_file(account_id: &str) -> String {
230+
format!("mastery-fail-{account_id}.json")
231+
}
232+
233+
/// How much longer to wait before retrying a failed profile fetch, if a
234+
/// previous failure's backoff cooldown hasn't elapsed yet. `None` means
235+
/// either there's no recorded failure, or its cooldown has already passed —
236+
/// either way, safe to attempt a fetch now.
237+
fn retry_cooldown_remaining(account_id: &str) -> Option<std::time::Duration> {
238+
let stamped = wf_cache::load_blob::<u32>(&retry_file(account_id))?;
239+
// `failures - 1` so the *first* failure gets `MASTERY_RETRY_BASE`
240+
// (not `2×`) — `backoff_interval` treats `0` as "no failures yet."
241+
let cooldown = wf_data::poll::backoff_interval(
242+
MASTERY_RETRY_BASE,
243+
stamped.value.saturating_sub(1),
244+
MASTERY_RETRY_CAP,
245+
);
246+
let elapsed = stamped.age();
247+
(elapsed < cooldown).then(|| cooldown - elapsed)
248+
}
249+
250+
/// Bump and persist `account_id`'s consecutive-failure count, returning the
251+
/// new count.
252+
fn record_retry_failure(account_id: &str) -> u32 {
253+
let file = retry_file(account_id);
254+
let failures = wf_cache::load_blob::<u32>(&file).map(|s| s.value).unwrap_or(0) + 1;
255+
let _ = wf_cache::save_blob(&file, &failures);
256+
failures
257+
}
258+
259+
/// Clear `account_id`'s retry-backoff state after a successful fetch, so the
260+
/// next failure (whenever it happens) starts its backoff from scratch
261+
/// instead of picking up where a since-resolved outage left off.
262+
fn clear_retry_state(account_id: &str) {
263+
if let Ok(dir) = wf_cache::cache_dir() {
264+
let _ = std::fs::remove_file(dir.join(retry_file(account_id)));
265+
}
266+
}
267+
207268
#[derive(Deserialize)]
208269
struct ProfileResponse {
209270
#[serde(rename = "Results", default)]
@@ -766,4 +827,51 @@ mod tests {
766827
)]);
767828
assert!(!set.is_mastered_by_path("/Lotus/Weapons/Tenno/LongGuns/BratonPrime"));
768829
}
830+
831+
/// Unique per test (and cleaned up after) since these tests hit the real
832+
/// on-disk cache dir, same isolation approach as `wf_cache`'s own tests.
833+
fn test_account_id(case: &str) -> String {
834+
format!("test-{}-{case}", std::process::id())
835+
}
836+
837+
fn cleanup_retry_state(account_id: &str) {
838+
clear_retry_state(account_id);
839+
}
840+
841+
#[test]
842+
fn retry_cooldown_is_none_with_no_recorded_failure() {
843+
let id = test_account_id("no-failure");
844+
assert!(retry_cooldown_remaining(&id).is_none());
845+
}
846+
847+
#[test]
848+
fn a_recorded_failure_starts_a_cooldown_of_roughly_the_base_interval() {
849+
let id = test_account_id("one-failure");
850+
assert_eq!(record_retry_failure(&id), 1);
851+
let wait = retry_cooldown_remaining(&id).expect("cooldown active right after a failure");
852+
// Freshly recorded, so almost the full base interval should remain.
853+
assert!(wait > MASTERY_RETRY_BASE - std::time::Duration::from_secs(2));
854+
assert!(wait <= MASTERY_RETRY_BASE);
855+
cleanup_retry_state(&id);
856+
}
857+
858+
#[test]
859+
fn consecutive_failures_double_the_cooldown() {
860+
let id = test_account_id("two-failures");
861+
record_retry_failure(&id);
862+
assert_eq!(record_retry_failure(&id), 2);
863+
let wait = retry_cooldown_remaining(&id).expect("cooldown active after a second failure");
864+
assert!(wait > MASTERY_RETRY_BASE);
865+
assert!(wait <= MASTERY_RETRY_BASE * 2);
866+
cleanup_retry_state(&id);
867+
}
868+
869+
#[test]
870+
fn clearing_retry_state_lifts_the_cooldown() {
871+
let id = test_account_id("cleared");
872+
record_retry_failure(&id);
873+
assert!(retry_cooldown_remaining(&id).is_some());
874+
clear_retry_state(&id);
875+
assert!(retry_cooldown_remaining(&id).is_none());
876+
}
769877
}

0 commit comments

Comments
 (0)