fix: wire retry env vars into helm configmap, dedupe terminal retry log

helm/templates/env-configmap.yaml never listed RETRY_ATTEMPTS and
RETRY_BACKOFF_MS even though values.yaml gained them, so --set
env.RETRY_ATTEMPTS=N was silently ignored by Kubernetes deployments.
Add both keys in the same explicit style as the existing entries.

src/utils/retry.rs logged its own "failed after N attempts" error on
exhaustion, on top of the terminal log each call site already writes,
producing two error entries per failure. Worse, it changed a log
level: FileLock::acquire's "backup_already_in_progress" bails through
the combinator, which now logged it as error before runner.rs got a
chance to reclassify it as the routine warn it always was. A manual
backup colliding with a scheduled one would show up as a hard error
on the dashboard instead of the harmless warn it used to be, breaking
the "fails exactly as it does today" guarantee for job records.

Drop the combinator's terminal error log and give download_backup its
own terminal error log so all three call sites (runner, uploader,
downloader) own their failure logging uniformly. Update the two tests
that asserted the removed message to assert the new behavior instead.
This commit is contained in:
charles-gauthereau
2026-08-27 22:46:49 +02:00
parent b6a120fcbf
commit 87f33af772
5 changed files with 15 additions and 12 deletions
+3 -1
View File
@@ -7,4 +7,6 @@ data:
TZ: {{ .Values.env.TZ | quote }}
POLLING: {{ .Values.env.POLLING | quote }}
APP_ENV: {{ .Values.env.APP_ENV | quote }}
LOG: {{ .Values.env.LOG | quote }}
LOG: {{ .Values.env.LOG | quote }}
RETRY_ATTEMPTS: {{ .Values.env.RETRY_ATTEMPTS | quote }}
RETRY_BACKOFF_MS: {{ .Values.env.RETRY_BACKOFF_MS | quote }}
+8 -2
View File
@@ -32,7 +32,7 @@ impl RestoreService {
let logger_ref = &logger;
retry("Backup download", &logger, &policy, move |_| {
let outcome = retry("Backup download", &logger, &policy, move |_| {
let expected = expected_size.clone();
async move {
@@ -40,7 +40,13 @@ impl RestoreService {
.await
}
})
.await
.await;
if let Err(e) = &outcome {
logger.log("error", format!("Download failed: {e}"));
}
outcome
}
pub async fn download_once(
+2 -2
View File
@@ -51,8 +51,8 @@ async fn a_failing_backup_is_retried_and_leaves_no_attempt_directory() {
assert!(
entries
.iter()
.any(|e| e.level == "error" && e.message.starts_with("Database backup failed after 3 attempts")),
"expected a single terminal error naming the attempt count"
.any(|e| e.level == "error" && e.message.starts_with("Backup failed:")),
"expected a single terminal error from the runner"
);
let leftovers: Vec<_> = std::fs::read_dir(tmp_path)
+2 -6
View File
@@ -88,7 +88,7 @@ async fn retries_until_success_and_logs_each_attempt() {
}
#[tokio::test]
async fn exhausts_attempts_and_logs_a_single_error() {
async fn exhausts_attempts_and_logs_no_terminal_error() {
init_tracing_for_test();
let logger = JobLogger::new();
let calls = AtomicU32::new(0);
@@ -105,11 +105,7 @@ async fn exhausts_attempts_and_logs_a_single_error() {
let entries = logger.into_entries();
assert_eq!(entries.iter().filter(|e| e.level == "warn").count(), 2);
assert_eq!(entries.iter().filter(|e| e.level == "error").count(), 1);
assert_eq!(
entries.iter().find(|e| e.level == "error").unwrap().message,
"Test op failed after 3 attempts: always"
);
assert_eq!(entries.iter().filter(|e| e.level == "error").count(), 0);
}
#[tokio::test]
-1
View File
@@ -68,7 +68,6 @@ where
attempt += 1;
}
Err(e) => {
logger.log("error", format!("{op} failed after {total} attempts: {e}"));
return Err(e);
}
}