feat: Update default models for extraction and normalization to GPT-5.6 Luna across configuration files and documentation. Remove deprecated code and improve pricing details in the UI components. Add tests to ensure correct model settings and pricing calculations.

2026-08-16 15:39:01 -05:00
parent 046f26c20c
commit 3b01456875
38 changed files with 615 additions and 639 deletions
@@ -100,59 +100,43 @@ def _service_for_model(base_service: InvoiceProcessingService, model: str | None
  )


-def process_desktop_ingest(
-  service: InvoiceProcessingService,
+def persist_desktop_ingest_result(
  session: Session,
  *,
  task: Task,
  batch_id: str,
-  temp_path: str,
  radicado: str,
  filename: str,
  doc_type: str | None,
-  instruction: str | None,
-  tiff_page_range: tuple[int, int] | None,
-  model: str | None = None,
+  request: dict[str, Any],
+  normalized_obj: Any,
 ) -> dict[str, Any]:
-  service = _service_for_model(service, model)
-  request = dict(task.request or {})
+  """Persiste una extracción desktop y contabiliza una vez por radicado.
+
+  Cuando una tarea separa los campos por tipo documental, cada documento se
+  guarda primero en ``partials``. El radicado se materializa y ``processed`` se
+  incrementa únicamente cuando llegaron todos sus tipos esperados.
+  """
  fields = list(request.get("fields") or [])
  alias_map = dict(request.get("alias_map") or {})
  structure_mode = str(request.get("structure_mode") or "radicados")
  doc_types = list(request.get("doc_types") or [])
-  prof = get_profile("invoice")
-
-  selected_paths = _paths_for_type(fields, doc_type if structure_mode == "radicados" and doc_types else None)
-  if doc_type and structure_mode == "radicados" and doc_types:
-    try:
-      prompt = PromptBuilder.build_for_type(doc_type, selected_paths, {}, profile_name=prof.get("name", "invoice"))
-    except Exception:
-      prompt = PromptBuilder.build(selected_paths, {}, profile_name=prof.get("name", "invoice"))
-  else:
-    prompt = instruction or PromptBuilder.build(selected_paths, {}, profile_name=prof.get("name", "invoice"))
-
-  data, _usage = service.extract_full_from_pdf(
-    temp_path,
-    instruction=prompt,
-    tiff_page_range=tiff_page_range,
-  )
-  raw_data = data if isinstance(data, dict) else {"data": data}
-  if isinstance(data, dict):
-    data["radicado"] = radicado
-  else:
-    data = {"radicado": radicado, "data": data}
-
-  norm = normalize_to_schema(
-    data,
-    paths=selected_paths,
-    profile_name=prof.get("name", "invoice"),
-  )
-  normalized_obj = data
-  if isinstance(norm, dict):
-    normalized_obj = norm.get("normalized") or data
-
  result = dict(task.result or {})
-  errors = list(result.get("errors") or [])
+  failed_radicados = {str(value) for value in result.get("failed_radicados") or []}
+
+  # Otro documento del mismo radicado puede terminar después de que uno de sus
+  # hermanos haya fallado. No debe recrear parciales ni contabilizarlo otra vez.
+  if radicado in failed_radicados:
+    return {
+      "radicado": radicado,
+      "filename": filename,
+      "doc_type": doc_type,
+      "finalized": False,
+      "skipped": True,
+      "reason": "radicado_failed",
+      "processed": int(result.get("processed") or 0),
+      "total": int(result.get("total") or 0),
+    }

  if structure_mode == "radicados" and doc_types and doc_type:
    partials = dict(result.get("partials") or {})
@@ -216,6 +200,113 @@ def process_desktop_ingest(
  }


+def record_desktop_ingest_error(
+  session: Session,
+  *,
+  task: Task,
+  radicado: str,
+  filename: str,
+  doc_type: str | None,
+  error: Exception,
+) -> dict[str, Any]:
+  """Registra un fallo y da por procesado el radicado exactamente una vez."""
+  result = dict(task.result or {})
+  errors = list(result.get("errors") or [])
+  errors.append({
+    "radicado": radicado,
+    "filename": filename,
+    "doc_type": doc_type,
+    "error": str(error),
+  })
+  result["errors"] = errors
+
+  failed_radicados = {str(value) for value in result.get("failed_radicados") or []}
+  first_failure = radicado not in failed_radicados
+  if first_failure:
+    failed_radicados.add(radicado)
+    result["processed"] = int(result.get("processed") or 0) + 1
+
+  # Un radicado fallido ya no puede completarse con los parciales restantes.
+  partials = dict(result.get("partials") or {})
+  partials.pop(radicado, None)
+  result["partials"] = partials
+  result["failed_radicados"] = sorted(failed_radicados)
+  task.result = result
+  task.updated_at = utcnow()
+  session.flush()
+
+  return {
+    "radicado": radicado,
+    "filename": filename,
+    "doc_type": doc_type,
+    "error": str(error),
+    "first_failure": first_failure,
+    "processed": int(result.get("processed") or 0),
+    "total": int(result.get("total") or 0),
+  }
+
+
+def process_desktop_ingest(
+  service: InvoiceProcessingService,
+  session: Session,
+  *,
+  task: Task,
+  batch_id: str,
+  temp_path: str,
+  radicado: str,
+  filename: str,
+  doc_type: str | None,
+  instruction: str | None,
+  tiff_page_range: tuple[int, int] | None,
+  model: str | None = None,
+) -> dict[str, Any]:
+  service = _service_for_model(service, model)
+  request = dict(task.request or {})
+  fields = list(request.get("fields") or [])
+  structure_mode = str(request.get("structure_mode") or "radicados")
+  doc_types = list(request.get("doc_types") or [])
+  prof = get_profile("invoice")
+
+  selected_paths = _paths_for_type(fields, doc_type if structure_mode == "radicados" and doc_types else None)
+  if doc_type and structure_mode == "radicados" and doc_types:
+    try:
+      prompt = PromptBuilder.build_for_type(doc_type, selected_paths, {}, profile_name=prof.get("name", "invoice"))
+    except Exception:
+      prompt = PromptBuilder.build(selected_paths, {}, profile_name=prof.get("name", "invoice"))
+  else:
+    prompt = instruction or PromptBuilder.build(selected_paths, {}, profile_name=prof.get("name", "invoice"))
+
+  data, _usage = service.extract_full_from_pdf(
+    temp_path,
+    instruction=prompt,
+    tiff_page_range=tiff_page_range,
+  )
+  if isinstance(data, dict):
+    data["radicado"] = radicado
+  else:
+    data = {"radicado": radicado, "data": data}
+
+  norm = normalize_to_schema(
+    data,
+    paths=selected_paths,
+    profile_name=prof.get("name", "invoice"),
+  )
+  normalized_obj = data
+  if isinstance(norm, dict):
+    normalized_obj = norm.get("normalized") or data
+
+  return persist_desktop_ingest_result(
+    session,
+    task=task,
+    batch_id=batch_id,
+    radicado=radicado,
+    filename=filename,
+    doc_type=doc_type,
+    request=request,
+    normalized_obj=normalized_obj,
+  )
+
+
 def mark_task_completed_if_done(session: Session, task: Task, batch_id: str) -> None:
  result = dict(task.result or {})
  processed = int(result.get("processed") or 0)
@@ -20,7 +20,11 @@ from fastapi import BackgroundTasks, FastAPI, File, Form, HTTPException, Respons
 from pydantic import BaseModel, Field, model_validator
 from sqlalchemy import delete, select

-from api.desktop_ingest import mark_task_completed_if_done, process_desktop_ingest
+from api.desktop_ingest import (
+  mark_task_completed_if_done,
+  persist_desktop_ingest_result,
+  record_desktop_ingest_error,
+)

 from domain.batch_processor import BatchProcessor
 from domain.config import extractor_llm_config, normalizer_llm_config, settings
@@ -612,10 +616,6 @@ async def _process_sync(
  import asyncio as _asyncio
  import logging as _logging
  import tempfile as _tempfile
-  from domain.service import InvoiceProcessingService
-  from domain.strategies import OpenAICompatibleExtractor
-  from domain.batch_processor import BatchProcessor
-  from domain.estimator import Estimator
  from api.desktop_ingest import _service_for_model, _paths_for_type
  from domain.profiles import get_profile
  from domain.prompts import PromptBuilder
@@ -678,58 +678,61 @@ async def _process_sync(

    # ── Fase 2: persistir resultado (con FOR UPDATE para serializar) ───
    def _persist():
-      from infrastructure.serialization import flatten_extraction_result
-      from infrastructure.database import ExtractionRow, ExtractionBatch, utcnow
-      req = dict(request)
-      fields = list(req.get("fields") or [])
-      alias_map = dict(req.get("alias_map") or {})
      with session_scope() as session:
        task = session.get(Task, task_id, with_for_update=True)
        if task is None:
          return {"error": "task_not_found"}
-        result = dict(task.result or {})
-        fused = {"radicado": radicado, "_source_pdf": effective_name}
-        if doc_type:
-          fused[doc_type] = normalized_obj or {}
-        else:
-          fused.update(
-            normalized_obj if isinstance(normalized_obj, dict)
-            else {"data": normalized_obj}
-          )
-        row_index = int(result.get("row_count") or 0)
-        for row in flatten_extraction_result(fused, fields, alias_map):
-          row_index += 1
-          session.add(ExtractionRow(
-            batch_id=batch_id,
-            radicado=radicado,
-            source_pdf=effective_name,
-            row_index=row_index,
-            row_json=row,
-            raw_json=fused,
-          ))
-        result["row_count"] = row_index
-        processed_now = int(result.get("processed") or 0) + 1
-        result["processed"] = processed_now
-        task.result = result
-        task.updated_at = utcnow()
-        session.flush()
+        payload = persist_desktop_ingest_result(
+          session,
+          task=task,
+          batch_id=batch_id,
+          radicado=radicado,
+          filename=effective_name,
+          doc_type=doc_type,
+          request=request,
+          normalized_obj=normalized_obj,
+        )
        mark_task_completed_if_done(session, task, batch_id)
        session.flush()  # persiste status=completed / finished_at si aplica
+        processed_now = int((task.result or {}).get("processed") or 0)
        _log.info(
          "✓ guardado radicado=%s processed=%d/%d status=%s",
-          radicado, processed_now, int(result.get("total") or 0), task.status,
+          radicado, processed_now, int((task.result or {}).get("total") or 0), task.status,
        )
        session.refresh(task)
-        return {
-          "radicado": radicado,
-          "filename": effective_name,
-          "processed": processed_now,
-          "total": int(result.get("total") or 0),
-          "task": serialize_task(task),
-        }
+        return {**payload, "task": serialize_task(task)}

    return await _asyncio.to_thread(_persist)

+  except Exception as exc:
+    # La llamada HTTP conserva el error, pero el progreso debe quedar persistido
+    # para que un documento fallido no deje la tarea eternamente en running.
+    def _record_error(error: Exception) -> None:
+      with session_scope() as session:
+        task = session.get(Task, task_id, with_for_update=True)
+        if task is None:
+          return
+        record_desktop_ingest_error(
+          session,
+          task=task,
+          radicado=radicado,
+          filename=effective_name,
+          doc_type=doc_type,
+          error=error,
+        )
+        mark_task_completed_if_done(session, task, batch_id)
+        session.flush()
+
+    try:
+      await _asyncio.to_thread(_record_error, exc)
+    except Exception:
+      _log.exception(
+        "No se pudo registrar el error de radicado=%s filename=%s",
+        radicado,
+        effective_name,
+      )
+    raise
+
  finally:
    try:
      os.unlink(temp_path)