feat: Implement job abandonment logic for reconnected workers in bot services. Introduce a mechanism to cancel pending and running jobs when a worker reconnects, ensuring a fresh start for new jobs. Update tests to verify job cancellation behavior upon worker re-registration. Enhance UI components in desktop application for better user experience and clarity on bot status.
This commit is contained in:
@@ -27,6 +27,7 @@ from infrastructure.database import (
|
||||
from infrastructure.storage_minio import reports_prefix
|
||||
|
||||
HEARTBEAT_OFFLINE_SECONDS = 45
|
||||
_FRESH_WORKER_GRACE_SECONDS = 45
|
||||
|
||||
|
||||
def _parse_iso(value: str | None) -> datetime | None:
|
||||
@@ -44,6 +45,7 @@ class JobStore:
|
||||
def __init__(self, db_path: str | None = None):
|
||||
# db_path ignorado (compat); usa DATABASE_URL.
|
||||
init_db()
|
||||
self._fresh_until: dict[str, datetime] = {}
|
||||
|
||||
def _row_worker(self, row: DigitadorWorkerRow) -> Worker:
|
||||
worker = Worker(
|
||||
@@ -120,7 +122,67 @@ class JobStore:
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return self._row_worker(row)
|
||||
parent_ids = self._abandon_assigned_jobs(
|
||||
session,
|
||||
worker_id,
|
||||
now,
|
||||
"Agent reconectado: job anterior abandonado. Lanza una corrida nueva desde el desktop.",
|
||||
)
|
||||
session.flush()
|
||||
worker = self._row_worker(row)
|
||||
self._fresh_until[worker_id] = now + timedelta(seconds=_FRESH_WORKER_GRACE_SECONDS)
|
||||
for parent_id in parent_ids:
|
||||
try:
|
||||
self.refresh_parent_from_shards(parent_id)
|
||||
except Exception:
|
||||
pass
|
||||
return worker
|
||||
|
||||
def _abandon_assigned_jobs(
|
||||
self, session, worker_id: str, now: datetime, message: str
|
||||
) -> list[str]:
|
||||
"""Cancela pending/running de este worker para que un agent nuevo arranque idle."""
|
||||
rows = session.scalars(
|
||||
select(DigitadorJobRow).where(
|
||||
DigitadorJobRow.worker_id == worker_id,
|
||||
DigitadorJobRow.status.in_((JobStatus.PENDING.value, JobStatus.RUNNING.value)),
|
||||
)
|
||||
).all()
|
||||
parent_ids: set[str] = set()
|
||||
for row in rows:
|
||||
is_parent = (
|
||||
row.parent_digitador_job_id is None
|
||||
and row.shard_count is not None
|
||||
and int(row.shard_count) > 1
|
||||
)
|
||||
if is_parent:
|
||||
continue
|
||||
row.status = JobStatus.CANCELLED.value
|
||||
row.cancel_requested = True
|
||||
row.finished_at = now
|
||||
row.error = message
|
||||
self._sync_task(session, row, status=JobStatus.CANCELLED.value, error=message)
|
||||
session.add(
|
||||
DigitadorJobEventRow(
|
||||
digitador_job_id=row.id,
|
||||
event_type="warning",
|
||||
message=message,
|
||||
timestamp=now,
|
||||
metadata_json={"abandoned_on_register": True, "worker_id": worker_id},
|
||||
)
|
||||
)
|
||||
if row.parent_digitador_job_id:
|
||||
parent_ids.add(row.parent_digitador_job_id)
|
||||
session.add(
|
||||
DigitadorJobEventRow(
|
||||
digitador_job_id=row.parent_digitador_job_id,
|
||||
event_type="warning",
|
||||
message=f"Shard {row.shard_index} abandonado (agent reconectado)",
|
||||
timestamp=now,
|
||||
metadata_json={"shard_id": row.id, "worker_id": worker_id},
|
||||
)
|
||||
)
|
||||
return list(parent_ids)
|
||||
|
||||
def heartbeat(self, worker_id: str) -> Optional[Worker]:
|
||||
with session_scope() as session:
|
||||
@@ -609,6 +671,9 @@ class JobStore:
|
||||
w.current_job_id = None
|
||||
w.status = WorkerStatus.OFFLINE.value
|
||||
elif not w.current_job_id:
|
||||
fresh_until = self._fresh_until.get(w.id)
|
||||
if fresh_until is not None and now < fresh_until:
|
||||
continue
|
||||
online_free.append(w)
|
||||
|
||||
if not offline_ids:
|
||||
|
||||
Reference in New Issue
Block a user