feat: Implement job abandonment logic for reconnected workers in bot services. Introduce a mechanism to cancel pending and running jobs when a worker reconnects, ensuring a fresh start for new jobs. Update tests to verify job cancellation behavior upon worker re-registration. Enhance UI components in desktop application for better user experience and clarity on bot status.

This commit is contained in:
erik
2026-08-12 16:37:12 -05:00
parent e9b766281a
commit 343ffb8ebe
9 changed files with 271 additions and 127 deletions
+66 -1
View File
@@ -27,6 +27,7 @@ from infrastructure.database import (
from infrastructure.storage_minio import reports_prefix
HEARTBEAT_OFFLINE_SECONDS = 45
_FRESH_WORKER_GRACE_SECONDS = 45
def _parse_iso(value: str | None) -> datetime | None:
@@ -44,6 +45,7 @@ class JobStore:
def __init__(self, db_path: str | None = None):
# db_path ignorado (compat); usa DATABASE_URL.
init_db()
self._fresh_until: dict[str, datetime] = {}
def _row_worker(self, row: DigitadorWorkerRow) -> Worker:
worker = Worker(
@@ -120,7 +122,67 @@ class JobStore:
)
session.add(row)
session.flush()
return self._row_worker(row)
parent_ids = self._abandon_assigned_jobs(
session,
worker_id,
now,
"Agent reconectado: job anterior abandonado. Lanza una corrida nueva desde el desktop.",
)
session.flush()
worker = self._row_worker(row)
self._fresh_until[worker_id] = now + timedelta(seconds=_FRESH_WORKER_GRACE_SECONDS)
for parent_id in parent_ids:
try:
self.refresh_parent_from_shards(parent_id)
except Exception:
pass
return worker
def _abandon_assigned_jobs(
self, session, worker_id: str, now: datetime, message: str
) -> list[str]:
"""Cancela pending/running de este worker para que un agent nuevo arranque idle."""
rows = session.scalars(
select(DigitadorJobRow).where(
DigitadorJobRow.worker_id == worker_id,
DigitadorJobRow.status.in_((JobStatus.PENDING.value, JobStatus.RUNNING.value)),
)
).all()
parent_ids: set[str] = set()
for row in rows:
is_parent = (
row.parent_digitador_job_id is None
and row.shard_count is not None
and int(row.shard_count) > 1
)
if is_parent:
continue
row.status = JobStatus.CANCELLED.value
row.cancel_requested = True
row.finished_at = now
row.error = message
self._sync_task(session, row, status=JobStatus.CANCELLED.value, error=message)
session.add(
DigitadorJobEventRow(
digitador_job_id=row.id,
event_type="warning",
message=message,
timestamp=now,
metadata_json={"abandoned_on_register": True, "worker_id": worker_id},
)
)
if row.parent_digitador_job_id:
parent_ids.add(row.parent_digitador_job_id)
session.add(
DigitadorJobEventRow(
digitador_job_id=row.parent_digitador_job_id,
event_type="warning",
message=f"Shard {row.shard_index} abandonado (agent reconectado)",
timestamp=now,
metadata_json={"shard_id": row.id, "worker_id": worker_id},
)
)
return list(parent_ids)
def heartbeat(self, worker_id: str) -> Optional[Worker]:
with session_scope() as session:
@@ -609,6 +671,9 @@ class JobStore:
w.current_job_id = None
w.status = WorkerStatus.OFFLINE.value
elif not w.current_job_id:
fresh_until = self._fresh_until.get(w.id)
if fresh_until is not None and now < fresh_until:
continue
online_free.append(w)
if not offline_ids: