From e042c56decf880ada23fb7695367b602d0a0a4e6 Mon Sep 17 00:00:00 2001 From: Jas Kalayan Date: Fri, 6 Nov 2026 18:47:50 +0000 Subject: [PATCH 1/2] docs(webform): update instructions for webform --- biosimdb_interface/schema/webform.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/biosimdb_interface/schema/webform.py b/biosimdb_interface/schema/webform.py index e325595..240b50d 100644 --- a/biosimdb_interface/schema/webform.py +++ b/biosimdb_interface/schema/webform.py @@ -59,10 +59,12 @@ def get_simulation_metadata(): { "label": "Please note:", "options": [ - "Further instructions for using this weform are available " + "Instructions for using this deposition form are available " " here', + "Sessions are lost if the browser tab is closed.", + "If your files are larger than 100~GB, please contact us first.", "This deposition form is in active development, with new features " "being added, there may be a few bugs 🐞 that have crept " "in!", From dd294d111de2e69bcbf2f5301b2926c542804d04 Mon Sep 17 00:00:00 2001 From: Jas Kalayan Date: Fri, 6 Nov 2026 18:54:27 +0000 Subject: [PATCH 2/2] Show step-by-step upload progress and add cancel to extraction Replace spinners with a live step list for extraction and Invenio submission. The server writes progress to the workflow tmpdir, which the loading page polls via a new /progress route. The extraction overlay gains a Cancel button. --- biosimdb_interface/core/progress_state.py | 66 +++++++++++++++++++ biosimdb_interface/invenio/client.py | 19 +++++- biosimdb_interface/invenio/multipart.py | 43 +++++++++--- biosimdb_interface/invenio/progress.py | 7 +- biosimdb_interface/static/js/core/ui.js | 42 +++++++++++- .../static/js/extract/extract.js | 59 ++++++++++++++--- .../static/js/uploads/chunked_upload.js | 7 +- biosimdb_interface/submit/routes.py | 20 +++++- .../templates/form/loading.html | 39 ++++++++++- biosimdb_interface/uploads/to_invenio.py | 26 +++++++- tests/test_core/test_progress_state.py | 14 ++++ 11 files changed, 312 insertions(+), 30 deletions(-) create mode 100644 biosimdb_interface/core/progress_state.py create mode 100644 tests/test_core/test_progress_state.py diff --git a/biosimdb_interface/core/progress_state.py b/biosimdb_interface/core/progress_state.py new file mode 100644 index 0000000..1285d5c --- /dev/null +++ b/biosimdb_interface/core/progress_state.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +"""File-backed submission progress, shared between requests and upload threads. + +Progress is stored in the workflow's temporary directory (like the +cancellation flag) so a polling request can read what a blocking +``do_submit`` request is doing, even across worker processes. +""" + +import json +import os +import threading +import time + +PROGRESS_FILENAME = "progress.json" + + +class ProgressReporter: + """Write the ordered step list and the current step's progress.""" + + def __init__(self, tmpdir, min_interval=0.5): + """Create a reporter writing to ``tmpdir``. + + Args: + tmpdir (str): Workflow temporary directory. + min_interval (float): Minimum seconds between writes for updates + within the same step. + """ + self.path = os.path.join(tmpdir, PROGRESS_FILENAME) + self.min_interval = min_interval + self.state = {"steps": [], "current": None, "pct": None, "detail": ""} + self.lock = threading.Lock() + self.last_write = 0.0 + + def set_steps(self, steps): + """Set the ordered steps as ``[{"id": ..., "label": ...}, ...]``.""" + with self.lock: + self.state["steps"] = steps + self._write() + + def update(self, step, pct=None, detail=""): + """Record progress for ``step``; ``pct`` is 0-1 or None if unknown.""" + with self.lock: + changed = step != self.state["current"] + self.state.update(current=step, pct=pct, detail=detail) + if ( + changed + or pct == 1 + or time.monotonic() - self.last_write >= self.min_interval + ): + self._write() + + def _write(self): + tmp = f"{self.path}.{os.getpid()}.tmp" + with open(tmp, "w") as f: + json.dump(self.state, f) + os.replace(tmp, self.path) # atomic: readers never see partial JSON + self.last_write = time.monotonic() + + +def read_progress(tmpdir): + """Return the saved progress state, or an empty state if none exists.""" + try: + with open(os.path.join(tmpdir, PROGRESS_FILENAME)) as f: + return json.load(f) + except (OSError, ValueError): + return {"steps": [], "current": None, "pct": None, "detail": ""} diff --git a/biosimdb_interface/invenio/client.py b/biosimdb_interface/invenio/client.py index f96121f..3d4ccf0 100644 --- a/biosimdb_interface/invenio/client.py +++ b/biosimdb_interface/invenio/client.py @@ -16,7 +16,13 @@ def run_record_upload( - api_url, api_key, metadata_path, files, community, cancelled=lambda: False + api_url, + api_key, + metadata_path, + files, + community, + cancelled=lambda: False, + report=None, ): """Create a draft, upload its files, and submit it to a community. @@ -33,6 +39,8 @@ def run_record_upload( community (str): Community identifier to submit the draft to. cancelled (Callable[[], bool]): Callback that returns ``True`` when the upload should be cancelled. + report (Callable | None): Called as ``report(step, pct, detail)`` + with steps ``draft``, ``file:`` and ``review``. Returns: tuple: ``(None, draft_id)`` for the created record. @@ -51,6 +59,8 @@ def run_record_upload( with Path(metadata_path).open("r", encoding="utf8") as file: data = json.load(file) + report = report or (lambda *args, **kwargs: None) + report("draft", None, "Creating draft record") r = s.post(f"{api}/api/records", json=data) r.raise_for_status() draft_id = r.json()["id"] @@ -60,6 +70,8 @@ def run_record_upload( if cancelled(): raise UploadCancelled path = Path(file) + step = f"file:{path.name}" + report(step, 0, "Starting transfer") commit_url = upload_file( s, api, @@ -69,12 +81,17 @@ def run_record_upload( path.stat().st_size, PART_SIZE, cancelled=cancelled, + on_update=lambda p, step=step: report( + step, p.sent / p.size if p.size else 1, p.status() + ), ) + report(step, 1, "Finalising file") r = s.post(commit_url) r.raise_for_status() if cancelled(): raise UploadCancelled + report("review", None, "Submitting to the BioSimDB community") r = s.get(f"{api}/api/communities/{community}") r.raise_for_status() r = s.put( diff --git a/biosimdb_interface/invenio/multipart.py b/biosimdb_interface/invenio/multipart.py index 152f034..5a1dab4 100644 --- a/biosimdb_interface/invenio/multipart.py +++ b/biosimdb_interface/invenio/multipart.py @@ -59,7 +59,9 @@ def check(r, expected, what): raise Exception(f"{what} failed: HTTP {r.status_code}{hint}\n{r.text[:2000]}") -def upload_single(s, api, draft_id, path, key, size, cancelled=lambda: False): +def upload_single( + s, api, draft_id, path, key, size, cancelled=lambda: False, on_update=None +): """Initialize and stream a file using a single content PUT. Args: @@ -70,6 +72,7 @@ def upload_single(s, api, draft_id, path, key, size, cancelled=lambda: False): key (str): File key to create in the draft. size (int): File size in bytes. cancelled (Callable[[], bool]): Callback indicating cancellation. + on_update (Callable[[Progress], None] | None): Progress callback. Returns: str: File commit URL. @@ -78,7 +81,7 @@ def upload_single(s, api, draft_id, path, key, size, cancelled=lambda: False): check(r, 201, "File init") links = next(e["links"] for e in r.json()["entries"] if e["key"] == key) - progress = Progress(size) + progress = Progress(size, on_update=on_update) current_app.logger.info( f"Uploading {key} ({size / 1e9:.2f} GB) in one PUT at " f"{time.strftime('%H:%M:%S %Z')}" @@ -157,7 +160,16 @@ def put_part(s3, path, part, url, part_size, size, progress, cancelled): def upload_multipart( - s, api, draft_id, path, key, size, chunk_mb, workers, cancelled=lambda: False + s, + api, + draft_id, + path, + key, + size, + chunk_mb, + workers, + cancelled=lambda: False, + on_update=None, ): """Initialize multipart transfer, upload parts, and return the commit URL. @@ -171,6 +183,7 @@ def upload_multipart( chunk_mb (int): Requested part size in MiB, subject to S3 limits. workers (int): Maximum number of concurrent part uploads. cancelled (Callable[[], bool]): Callback indicating cancellation. + on_update (Callable[[Progress], None] | None): Progress callback. Returns: str: File commit URL. @@ -204,7 +217,7 @@ def upload_multipart( current_app.logger.error(message) raise RuntimeError(message) - progress = Progress(size) + progress = Progress(size, on_update=on_update) current_app.logger.debug( f"Uploading {key} ({size / 1e9:.2f} GB) as {parts} x " f"{part_size / 2**20:.0f} MiB parts, {workers} at a time, at " @@ -248,7 +261,17 @@ def upload_multipart( return links["commit"] -def upload_file(s, api, draft_id, path, key, size, chunk_mb, cancelled=lambda: False): +def upload_file( + s, + api, + draft_id, + path, + key, + size, + chunk_mb, + cancelled=lambda: False, + on_update=None, +): """Upload one file, using multipart transfer with a slow-upload fallback. Large files use multipart transfer. Smaller files also try multipart @@ -263,13 +286,15 @@ def upload_file(s, api, draft_id, path, key, size, chunk_mb, cancelled=lambda: F size (int): File size in bytes. chunk_mb (int): Requested multipart part size in MiB. cancelled (Callable[[], bool]): Callback indicating cancellation. + on_update (Callable[[Progress], None] | None): Progress callback. Returns: str: File commit URL. """ + extra = {"on_update": on_update} if on_update else {} if size > SINGLE_MAX: return upload_multipart( - s, api, draft_id, path, key, size, chunk_mb, 1, cancelled + s, api, draft_id, path, key, size, chunk_mb, 1, cancelled, **extra ) deadline = time.monotonic() + SLOW_AFTER @@ -282,7 +307,9 @@ def stop(): return False try: - return upload_multipart(s, api, draft_id, path, key, size, chunk_mb, 1, stop) + return upload_multipart( + s, api, draft_id, path, key, size, chunk_mb, 1, stop, **extra + ) except MultipartTooSlow: current_app.logger.info( f"Multipart for {key} exceeded {SLOW_AFTER}s; falling back to single PUT", @@ -293,4 +320,4 @@ def stop(): f"{api}/api/records/{draft_id}/draft/files/{quote(key, safe='')}", timeout=30, ) - return upload_single(s, api, draft_id, path, key, size, cancelled) + return upload_single(s, api, draft_id, path, key, size, cancelled, **extra) diff --git a/biosimdb_interface/invenio/progress.py b/biosimdb_interface/invenio/progress.py index 9b54600..8700859 100644 --- a/biosimdb_interface/invenio/progress.py +++ b/biosimdb_interface/invenio/progress.py @@ -14,12 +14,14 @@ class Progress: """Thread-safe byte counter that logs progress at most every `every` seconds.""" - def __init__(self, size, every=30): + def __init__(self, size, every=30, on_update=None): """Initialize a progress counter. Args: size (int): Total upload size in bytes. every (int): Minimum seconds between progress log updates. + on_update (Callable[[Progress], None] | None): Called after each + ``add``, e.g. to report progress to the UI. """ self.size = size self.sent = 0 @@ -27,6 +29,7 @@ def __init__(self, size, every=30): self.every = every self.last_log = self.start self.done_at = None + self.on_update = on_update self.lock = threading.Lock() def add(self, n): @@ -38,6 +41,8 @@ def add(self, n): self.done_at = now if now - self.last_log >= self.every or self.sent >= self.size: self.last_log = now + if self.on_update: + self.on_update(self) def status(self): """Return a human-readable summary of bytes, percentage, elapsed time, and rate. diff --git a/biosimdb_interface/static/js/core/ui.js b/biosimdb_interface/static/js/core/ui.js index 2656f23..289c1d3 100644 --- a/biosimdb_interface/static/js/core/ui.js +++ b/biosimdb_interface/static/js/core/ui.js @@ -43,8 +43,10 @@ export function escapeHtml(s) { * Shows a full-page wait overlay and puts a button into a disabled "working" state. * @param {HTMLElement} button - The button that triggered the action. * @param {string} loadingText - Text to show on the button while working. + * @param {boolean} [withSteps=false] - Show an empty panel for a stepper instead of a spinner. + * @returns {HTMLElement|null} The panel to render a stepper into when `withSteps` is true. */ -export function showLoadingOverlay(button, loadingText) { +export function showLoadingOverlay(button, loadingText, withSteps = false) { button.dataset.originalText = button.textContent; button.disabled = true; button.innerHTML = `${loadingText}`; @@ -52,8 +54,44 @@ export function showLoadingOverlay(button, loadingText) { const overlay = document.createElement('div'); overlay.id = 'loading-overlay'; overlay.style.cssText = 'position:fixed; inset:0; z-index:2000; display:flex; align-items:center; justify-content:center; background:rgba(255,255,255,0.5);'; - overlay.innerHTML = '
Loading...
'; + overlay.innerHTML = withSteps + ? '
' + : '
Loading...
'; document.body.appendChild(overlay); + return withSteps ? overlay.firstElementChild : null; +} + +/** + * Render a list of progress steps into a container. + * @param {HTMLElement} container - Element to render into. + * @param {{id: string, label: string}[]} steps - Ordered steps. + * @returns {function(string, string, ?number, string=): void} update(id, state, fraction, detail) + * where state is 'pending' | 'active' | 'done' | 'error'; fraction is 0-1 or null + * for an indeterminate bar. + */ +export function createStepper(container, steps) { + container.innerHTML = steps.map(s => ` +
+ ${escapeHtml(s.label)} +
+
+
+
+
`).join(''); + + const icons = { pending: '○', active: '⏳', done: '✅', error: '❌' }; + return (id, state, fraction = null, detail = '') => { + const el = [...container.querySelectorAll('.step')].find(e => e.dataset.step === id); + if (!el) return; + el.classList.toggle('text-muted', state === 'pending'); + el.querySelector('.icon').textContent = icons[state]; + el.querySelector('.detail').textContent = state === 'active' ? detail : ''; + el.querySelector('.progress').classList.toggle('d-none', state !== 'active'); + const bar = el.querySelector('.progress-bar'); + bar.classList.toggle('progress-bar-striped', fraction == null); + bar.classList.toggle('progress-bar-animated', fraction == null); + bar.style.width = fraction == null ? '100%' : `${Math.round(fraction * 100)}%`; + }; } /** diff --git a/biosimdb_interface/static/js/extract/extract.js b/biosimdb_interface/static/js/extract/extract.js index 4f4c9fa..c0f64a4 100644 --- a/biosimdb_interface/static/js/extract/extract.js +++ b/biosimdb_interface/static/js/extract/extract.js @@ -1,28 +1,48 @@ import { ChunkedUploader } from '../uploads/chunked_upload.js'; -import { addWorkflowId, getWorkflowId } from '../core/workflow.js'; -import { escapeHtml, hideLoadingOverlay, showAlert, showLoadingOverlay } from '../core/ui.js'; +import { addWorkflowId, clearWorkflow, getWorkflowId } from '../core/workflow.js'; +import { createStepper, escapeHtml, hideLoadingOverlay, showAlert, showLoadingOverlay } from '../core/ui.js'; import { populateFields, setFieldsDisabled } from '../webform/fields.js'; import { saveExtracted, saveFormState } from '../webform/state.js'; import { showValidationErrors } from '../webform/validation.js'; /** Upload the selected files in chunks, then request metadata extraction. */ -async function uploadAndExtract() { +async function uploadAndExtract(panel, uploader) { const topology = document.querySelector('input[name="topology"]').files[0]; const trajectories = Array.from( document.querySelectorAll('input[name="trajectory[]"]') ).flatMap(input => Array.from(input.files)); - const uploader = new ChunkedUploader(getWorkflowId()); + const files = [topology, ...trajectories]; + const steps = files.map((f, i) => ({ id: `file${i}`, label: `Uploading ${f.name}` })); + steps.push({ id: 'extract', label: 'Extracting metadata' }); + const update = createStepper(panel, steps); + steps.forEach(s => update(s.id, 'pending')); + const formData = addWorkflowId(new FormData()); - formData.append('topology_name', await uploader.upload(topology)); - for (const file of trajectories) { - formData.append('trajectory_names[]', await uploader.upload(file)); + const names = []; + for (const [i, file] of files.entries()) { + const id = `file${i}`; + const show = f => update(id, 'active', f, `${(f * 100).toFixed(0)}% of ${(file.size / 1e6).toFixed(1)} MB`); + show(0); + try { + names.push(await uploader.upload(file, show)); + } catch (err) { + update(id, 'error'); + throw err; + } + update(id, 'done'); } + formData.append('topology_name', names[0]); + names.slice(1).forEach(n => formData.append('trajectory_names[]', n)); + + update('extract', 'active', null, 'Reading topology and trajectories on the server...'); const r = await fetch(window.APPLICATION_BASE + '/extract_metadata', { method: 'POST', body: formData, + signal: uploader.signal, }); + update('extract', r.ok ? 'done' : 'error'); const ct = r.headers.get('content-type') || ''; if (!ct.includes('application/json')) { throw new Error(`Unexpected server response (${r.status}).`); @@ -58,11 +78,30 @@ function handleExtraction(data) { * @param {HTMLElement} button - The clicked extract button. */ export function extractMetadata(button) { - showLoadingOverlay(button, 'Extracting...'); + const panel = showLoadingOverlay(button, 'Extracting...', true); setFieldsDisabled(true); - return uploadAndExtract() + const controller = new AbortController(); + const uploader = new ChunkedUploader(getWorkflowId(), { signal: controller.signal }); + const cancelBtn = document.createElement('button'); + cancelBtn.type = 'button'; + cancelBtn.className = 'btn btn-outline-secondary mt-2'; + cancelBtn.textContent = 'Cancel'; + cancelBtn.addEventListener('click', () => { + controller.abort(); + clearWorkflow(); // discard files already uploaded to the server + }); + panel.after(cancelBtn); + panel.parentElement.classList.add('flex-column'); + + return uploadAndExtract(panel, uploader) .then(handleExtraction) - .catch(err => showAlert(`Error: ${err.message}`, 'danger', 10000)) + .catch(err => { + if (err.name === 'AbortError') { + showAlert('Extraction cancelled.', 'info', 5000); + return; + } + showAlert(`Error: ${err.message}`, 'danger', 10000); + }) .finally(() => hideLoadingOverlay(button)); } diff --git a/biosimdb_interface/static/js/uploads/chunked_upload.js b/biosimdb_interface/static/js/uploads/chunked_upload.js index 6ecc842..6d1fa72 100644 --- a/biosimdb_interface/static/js/uploads/chunked_upload.js +++ b/biosimdb_interface/static/js/uploads/chunked_upload.js @@ -1,9 +1,10 @@ /** Uploads files to the server in parallel, resumable chunks. */ export class ChunkedUploader { - constructor(workflowId, { concurrency = 4, retries = 4 } = {}) { + constructor(workflowId, { concurrency = 4, retries = 4, signal } = {}) { this.workflowId = workflowId; this.concurrency = concurrency; this.retries = retries; + this.signal = signal; this.base = `${window.APPLICATION_BASE}/uploads`; } @@ -13,6 +14,7 @@ export class ChunkedUploader { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ workflow_id: this.workflowId, ...payload }), + signal: this.signal, }); const data = await r.json(); if (!r.ok) throw new Error(data.error || 'Upload failed.'); @@ -32,13 +34,14 @@ export class ChunkedUploader { method: 'PUT', headers: { 'Content-Type': 'application/octet-stream' }, body, + signal: this.signal, }); if (r.ok) return; if (r.status < 500) { throw Object.assign(new Error((await r.json()).error), { fatal: true }); } } catch (err) { - if (err.fatal || attempt >= this.retries) throw err; + if (err.fatal || err.name === 'AbortError' || attempt >= this.retries) throw err; } await new Promise(res => setTimeout(res, 500 * 2 ** attempt)); } diff --git a/biosimdb_interface/submit/routes.py b/biosimdb_interface/submit/routes.py index 9848fe6..0c60dae 100644 --- a/biosimdb_interface/submit/routes.py +++ b/biosimdb_interface/submit/routes.py @@ -34,6 +34,7 @@ is_submission_cancelled, mark_submission_cancelled, ) +from ..core.progress_state import ProgressReporter, read_progress from ..core.workflows import WorkflowNotFound, get_workflow from ..uploads.to_invenio import prepare_for_invenio from ..uploads.utils import SIM_METADATA_FILENAME @@ -120,8 +121,10 @@ def do_submit(): try: token = session.get("access_token") + reporter = ProgressReporter(tmpdir) + reporter.update("invite", None, "Checking BioSimDB community membership") invite_status = invite_user("biosimdb", token) - draft_id = prepare_for_invenio(sim_metadata, tmpdir) + draft_id = prepare_for_invenio(sim_metadata, tmpdir, reporter) if not draft_id: current_app.logger.error("No draft ID was created, upload failed") @@ -202,6 +205,21 @@ def do_submit(): return redirect(url_for("submit.submit_success")) +@submit_bp.route("/progress") +def progress(): + """Return the current submission progress for polling by the loading page. + + Returns: + Response: JSON with ``steps``, ``current``, ``pct`` and ``detail``, or + JSON 400 response if the workflow ID cannot be resolved. + """ + try: + workflow = get_workflow() + except WorkflowNotFound as exc: + return jsonify({"validation_errors": [str(exc)]}), 400 + return jsonify(read_progress(workflow.tmpdir or os.devnull)) + + @submit_bp.route("/cancel_submit", methods=["POST"]) def cancel_submit(): """Signal cancellation of the current submission. diff --git a/biosimdb_interface/templates/form/loading.html b/biosimdb_interface/templates/form/loading.html index bfd289d..68366e1 100644 --- a/biosimdb_interface/templates/form/loading.html +++ b/biosimdb_interface/templates/form/loading.html @@ -5,28 +5,61 @@ Behaviour: - Submits to form.do_submit via fetch so it can be cancelled client-side. - On completion, navigates to the final response URL (success or webform). + - Polls submit.progress and shows each step (invite, draft, per-file upload, + review) with a progress bar and live detail. - Cancel aborts the fetch and resets session state via form.cancel_submit. #} {% extends "main/base.html" %} {% block content %}
-

Submitting to BioSimDB, please wait...

+
{% endblock %} {% block scripts %} {{ super() }} -