Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions biosimdb_interface/core/progress_state.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#!/usr/bin/env python
"""File-backed submission progress, shared between requests and upload threads.

Progress is stored in the workflow's temporary directory (like the
cancellation flag) so a polling request can read what a blocking
``do_submit`` request is doing, even across worker processes.
"""

import json
import os
import threading
import time

PROGRESS_FILENAME = "progress.json"


class ProgressReporter:
"""Write the ordered step list and the current step's progress."""

def __init__(self, tmpdir, min_interval=0.5):
"""Create a reporter writing to ``tmpdir``.

Args:
tmpdir (str): Workflow temporary directory.
min_interval (float): Minimum seconds between writes for updates
within the same step.
"""
self.path = os.path.join(tmpdir, PROGRESS_FILENAME)
self.min_interval = min_interval
self.state = {"steps": [], "current": None, "pct": None, "detail": ""}
self.lock = threading.Lock()
self.last_write = 0.0

def set_steps(self, steps):
"""Set the ordered steps as ``[{"id": ..., "label": ...}, ...]``."""
with self.lock:
self.state["steps"] = steps
self._write()

def update(self, step, pct=None, detail=""):
"""Record progress for ``step``; ``pct`` is 0-1 or None if unknown."""
with self.lock:
changed = step != self.state["current"]
self.state.update(current=step, pct=pct, detail=detail)
if (
changed
or pct == 1
or time.monotonic() - self.last_write >= self.min_interval
):
self._write()

def _write(self):
tmp = f"{self.path}.{os.getpid()}.tmp"
with open(tmp, "w") as f:
json.dump(self.state, f)
os.replace(tmp, self.path) # atomic: readers never see partial JSON
self.last_write = time.monotonic()


def read_progress(tmpdir):
"""Return the saved progress state, or an empty state if none exists."""
try:
with open(os.path.join(tmpdir, PROGRESS_FILENAME)) as f:
return json.load(f)
except (OSError, ValueError):
return {"steps": [], "current": None, "pct": None, "detail": ""}
19 changes: 18 additions & 1 deletion biosimdb_interface/invenio/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,13 @@


def run_record_upload(
api_url, api_key, metadata_path, files, community, cancelled=lambda: False
api_url,
api_key,
metadata_path,
files,
community,
cancelled=lambda: False,
report=None,
):
"""Create a draft, upload its files, and submit it to a community.

Expand All @@ -33,6 +39,8 @@ def run_record_upload(
community (str): Community identifier to submit the draft to.
cancelled (Callable[[], bool]): Callback that returns ``True`` when
the upload should be cancelled.
report (Callable | None): Called as ``report(step, pct, detail)``
with steps ``draft``, ``file:<name>`` and ``review``.

Returns:
tuple: ``(None, draft_id)`` for the created record.
Expand All @@ -51,6 +59,8 @@ def run_record_upload(
with Path(metadata_path).open("r", encoding="utf8") as file:
data = json.load(file)

report = report or (lambda *args, **kwargs: None)
report("draft", None, "Creating draft record")
r = s.post(f"{api}/api/records", json=data)
r.raise_for_status()
draft_id = r.json()["id"]
Expand All @@ -60,6 +70,8 @@ def run_record_upload(
if cancelled():
raise UploadCancelled
path = Path(file)
step = f"file:{path.name}"
report(step, 0, "Starting transfer")
commit_url = upload_file(
s,
api,
Expand All @@ -69,12 +81,17 @@ def run_record_upload(
path.stat().st_size,
PART_SIZE,
cancelled=cancelled,
on_update=lambda p, step=step: report(
step, p.sent / p.size if p.size else 1, p.status()
),
)
report(step, 1, "Finalising file")
r = s.post(commit_url)
r.raise_for_status()

if cancelled():
raise UploadCancelled
report("review", None, "Submitting to the BioSimDB community")
r = s.get(f"{api}/api/communities/{community}")
r.raise_for_status()
r = s.put(
Expand Down
43 changes: 35 additions & 8 deletions biosimdb_interface/invenio/multipart.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@ def check(r, expected, what):
raise Exception(f"{what} failed: HTTP {r.status_code}{hint}\n{r.text[:2000]}")


def upload_single(s, api, draft_id, path, key, size, cancelled=lambda: False):
def upload_single(
s, api, draft_id, path, key, size, cancelled=lambda: False, on_update=None
):
"""Initialize and stream a file using a single content PUT.

Args:
Expand All @@ -70,6 +72,7 @@ def upload_single(s, api, draft_id, path, key, size, cancelled=lambda: False):
key (str): File key to create in the draft.
size (int): File size in bytes.
cancelled (Callable[[], bool]): Callback indicating cancellation.
on_update (Callable[[Progress], None] | None): Progress callback.

Returns:
str: File commit URL.
Expand All @@ -78,7 +81,7 @@ def upload_single(s, api, draft_id, path, key, size, cancelled=lambda: False):
check(r, 201, "File init")
links = next(e["links"] for e in r.json()["entries"] if e["key"] == key)

progress = Progress(size)
progress = Progress(size, on_update=on_update)
current_app.logger.info(
f"Uploading {key} ({size / 1e9:.2f} GB) in one PUT at "
f"{time.strftime('%H:%M:%S %Z')}"
Expand Down Expand Up @@ -157,7 +160,16 @@ def put_part(s3, path, part, url, part_size, size, progress, cancelled):


def upload_multipart(
s, api, draft_id, path, key, size, chunk_mb, workers, cancelled=lambda: False
s,
api,
draft_id,
path,
key,
size,
chunk_mb,
workers,
cancelled=lambda: False,
on_update=None,
):
"""Initialize multipart transfer, upload parts, and return the commit URL.

Expand All @@ -171,6 +183,7 @@ def upload_multipart(
chunk_mb (int): Requested part size in MiB, subject to S3 limits.
workers (int): Maximum number of concurrent part uploads.
cancelled (Callable[[], bool]): Callback indicating cancellation.
on_update (Callable[[Progress], None] | None): Progress callback.

Returns:
str: File commit URL.
Expand Down Expand Up @@ -204,7 +217,7 @@ def upload_multipart(
current_app.logger.error(message)
raise RuntimeError(message)

progress = Progress(size)
progress = Progress(size, on_update=on_update)
current_app.logger.debug(
f"Uploading {key} ({size / 1e9:.2f} GB) as {parts} x "
f"{part_size / 2**20:.0f} MiB parts, {workers} at a time, at "
Expand Down Expand Up @@ -248,7 +261,17 @@ def upload_multipart(
return links["commit"]


def upload_file(s, api, draft_id, path, key, size, chunk_mb, cancelled=lambda: False):
def upload_file(
s,
api,
draft_id,
path,
key,
size,
chunk_mb,
cancelled=lambda: False,
on_update=None,
):
"""Upload one file, using multipart transfer with a slow-upload fallback.

Large files use multipart transfer. Smaller files also try multipart
Expand All @@ -263,13 +286,15 @@ def upload_file(s, api, draft_id, path, key, size, chunk_mb, cancelled=lambda: F
size (int): File size in bytes.
chunk_mb (int): Requested multipart part size in MiB.
cancelled (Callable[[], bool]): Callback indicating cancellation.
on_update (Callable[[Progress], None] | None): Progress callback.

Returns:
str: File commit URL.
"""
extra = {"on_update": on_update} if on_update else {}
if size > SINGLE_MAX:
return upload_multipart(
s, api, draft_id, path, key, size, chunk_mb, 1, cancelled
s, api, draft_id, path, key, size, chunk_mb, 1, cancelled, **extra
)

deadline = time.monotonic() + SLOW_AFTER
Expand All @@ -282,7 +307,9 @@ def stop():
return False

try:
return upload_multipart(s, api, draft_id, path, key, size, chunk_mb, 1, stop)
return upload_multipart(
s, api, draft_id, path, key, size, chunk_mb, 1, stop, **extra
)
except MultipartTooSlow:
current_app.logger.info(
f"Multipart for {key} exceeded {SLOW_AFTER}s; falling back to single PUT",
Expand All @@ -293,4 +320,4 @@ def stop():
f"{api}/api/records/{draft_id}/draft/files/{quote(key, safe='')}",
timeout=30,
)
return upload_single(s, api, draft_id, path, key, size, cancelled)
return upload_single(s, api, draft_id, path, key, size, cancelled, **extra)
7 changes: 6 additions & 1 deletion biosimdb_interface/invenio/progress.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,22 @@
class Progress:
"""Thread-safe byte counter that logs progress at most every `every` seconds."""

def __init__(self, size, every=30):
def __init__(self, size, every=30, on_update=None):
"""Initialize a progress counter.

Args:
size (int): Total upload size in bytes.
every (int): Minimum seconds between progress log updates.
on_update (Callable[[Progress], None] | None): Called after each
``add``, e.g. to report progress to the UI.
"""
self.size = size
self.sent = 0
self.start = time.monotonic()
self.every = every
self.last_log = self.start
self.done_at = None
self.on_update = on_update
self.lock = threading.Lock()

def add(self, n):
Expand All @@ -38,6 +41,8 @@ def add(self, n):
self.done_at = now
if now - self.last_log >= self.every or self.sent >= self.size:
self.last_log = now
if self.on_update:
self.on_update(self)

def status(self):
"""Return a human-readable summary of bytes, percentage, elapsed time, and rate.
Expand Down
4 changes: 3 additions & 1 deletion biosimdb_interface/schema/webform.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,12 @@ def get_simulation_metadata():
{
"label": "Please note:",
"options": [
"Further <b>instructions</b> for using this weform are available "
"<b>Instructions</b> for using this deposition form are available "
"<a "
'href="https://biosimdb-interface.readthedocs.io/en/latest/biosimdb_intro.html"'
' target="_blank" class="text-reset"> here</a>',
"Sessions are lost if the browser tab is closed.",
"If your files are larger than 100~GB, please contact us first.",
"This deposition form is in active development, with new features "
"being added, there may be a few bugs &#128030; that have crept "
"in!",
Expand Down
42 changes: 40 additions & 2 deletions biosimdb_interface/static/js/core/ui.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,17 +43,55 @@ export function escapeHtml(s) {
* Shows a full-page wait overlay and puts a button into a disabled "working" state.
* @param {HTMLElement} button - The button that triggered the action.
* @param {string} loadingText - Text to show on the button while working.
* @param {boolean} [withSteps=false] - Show an empty panel for a stepper instead of a spinner.
* @returns {HTMLElement|null} The panel to render a stepper into when `withSteps` is true.
*/
export function showLoadingOverlay(button, loadingText) {
export function showLoadingOverlay(button, loadingText, withSteps = false) {
button.dataset.originalText = button.textContent;
button.disabled = true;
button.innerHTML = `<span class="spinner-border spinner-border-sm me-2" role="status"></span>${loadingText}`;

const overlay = document.createElement('div');
overlay.id = 'loading-overlay';
overlay.style.cssText = 'position:fixed; inset:0; z-index:2000; display:flex; align-items:center; justify-content:center; background:rgba(255,255,255,0.5);';
overlay.innerHTML = '<div class="spinner-border text-primary" role="status"><span class="visually-hidden">Loading...</span></div>';
overlay.innerHTML = withSteps
? '<div id="loading-steps" class="card shadow p-3" style="min-width:360px; max-width:600px;"></div>'
: '<div class="spinner-border text-primary" role="status"><span class="visually-hidden">Loading...</span></div>';
document.body.appendChild(overlay);
return withSteps ? overlay.firstElementChild : null;
}

/**
* Render a list of progress steps into a container.
* @param {HTMLElement} container - Element to render into.
* @param {{id: string, label: string}[]} steps - Ordered steps.
* @returns {function(string, string, ?number, string=): void} update(id, state, fraction, detail)
* where state is 'pending' | 'active' | 'done' | 'error'; fraction is 0-1 or null
* for an indeterminate bar.
*/
export function createStepper(container, steps) {
container.innerHTML = steps.map(s => `
<div class="step mb-2 text-start" data-step="${escapeHtml(s.id)}">
<span class="icon me-2"></span><span class="label">${escapeHtml(s.label)}</span>
<div class="detail small text-muted ms-4"></div>
<div class="progress ms-4 mt-1 d-none" style="height:6px">
<div class="progress-bar"></div>
</div>
</div>`).join('');

const icons = { pending: '○', active: '⏳', done: '✅', error: '❌' };
return (id, state, fraction = null, detail = '') => {
const el = [...container.querySelectorAll('.step')].find(e => e.dataset.step === id);
if (!el) return;
el.classList.toggle('text-muted', state === 'pending');
el.querySelector('.icon').textContent = icons[state];
el.querySelector('.detail').textContent = state === 'active' ? detail : '';
el.querySelector('.progress').classList.toggle('d-none', state !== 'active');
const bar = el.querySelector('.progress-bar');
bar.classList.toggle('progress-bar-striped', fraction == null);
bar.classList.toggle('progress-bar-animated', fraction == null);
bar.style.width = fraction == null ? '100%' : `${Math.round(fraction * 100)}%`;
};
}

/**
Expand Down
Loading
Loading