349 lines
10 KiB
Python
349 lines
10 KiB
Python
"""Procesamiento de ingestas individuales desde la app desktop."""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Any
|
|
|
|
from domain.normalizer import normalize_to_schema
|
|
from domain.profiles import get_profile
|
|
from domain.prompts import PromptBuilder
|
|
from domain.service import InvoiceProcessingService
|
|
from domain.config import settings
|
|
from domain.usage_tracking import record_api_usage
|
|
from infrastructure.database import ExtractionRow, Task, utcnow
|
|
from infrastructure.serialization import flatten_extraction_result
|
|
from sqlalchemy.orm import Session
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _paths_for_type(fields: list[str], doc_type: str | None) -> list[str]:
|
|
if not doc_type:
|
|
return list(fields or [])
|
|
prefix = f"{doc_type}/"
|
|
return [path.split("/", 1)[1] for path in fields if path.startswith(prefix)]
|
|
|
|
|
|
def _expected_doc_types_for_radicado(request: dict[str, Any], radicado: str) -> set[str]:
|
|
expected: set[str] = set()
|
|
for item in request.get("documents") or []:
|
|
if str(item.get("radicado") or "") != radicado:
|
|
continue
|
|
doc_type = str(item.get("doc_type") or item.get("docType") or "").strip()
|
|
if doc_type:
|
|
expected.add(doc_type)
|
|
return expected
|
|
|
|
|
|
def _finalize_radicado_if_ready(
|
|
session: Session,
|
|
*,
|
|
task: Task,
|
|
batch_id: str,
|
|
radicado: str,
|
|
request: dict[str, Any],
|
|
fields: list[str],
|
|
alias_map: dict[str, str],
|
|
) -> bool:
|
|
partials = dict((task.result or {}).get("partials") or {})
|
|
radicado_partials = dict(partials.get(radicado) or {})
|
|
expected = _expected_doc_types_for_radicado(request, radicado)
|
|
if expected and not expected.issubset(set(radicado_partials.keys())):
|
|
return False
|
|
|
|
fused: dict[str, Any] = {"radicado": radicado, **radicado_partials}
|
|
source_pdf = None
|
|
for item in request.get("documents") or []:
|
|
if str(item.get("radicado") or "") == radicado:
|
|
source_pdf = item.get("filename")
|
|
break
|
|
if source_pdf:
|
|
fused["_source_pdf"] = source_pdf
|
|
|
|
row_index = int((task.result or {}).get("row_count") or 0)
|
|
for row in flatten_extraction_result(fused, fields, alias_map):
|
|
row_index += 1
|
|
session.add(
|
|
ExtractionRow(
|
|
batch_id=batch_id,
|
|
radicado=radicado,
|
|
source_pdf=source_pdf,
|
|
row_index=row_index,
|
|
row_json=row,
|
|
raw_json=fused,
|
|
)
|
|
)
|
|
|
|
partials.pop(radicado, None)
|
|
result = dict(task.result or {})
|
|
result["partials"] = partials
|
|
result["row_count"] = row_index
|
|
result["processed"] = int(result.get("processed") or 0) + 1
|
|
task.result = result
|
|
task.updated_at = utcnow()
|
|
return True
|
|
|
|
|
|
def _service_for_model(base_service: InvoiceProcessingService, model: str | None) -> InvoiceProcessingService:
|
|
"""Devuelve un servicio con el modelo solicitado; reutiliza base_service si coincide."""
|
|
if not model:
|
|
return base_service
|
|
current_model = getattr(base_service.extractor, "model", None)
|
|
if current_model == model:
|
|
return base_service
|
|
from domain.strategies import OpenAICompatibleExtractor
|
|
from domain.batch_processor import BatchProcessor
|
|
from domain.estimator import Estimator
|
|
extractor = OpenAICompatibleExtractor(model=model)
|
|
return InvoiceProcessingService(
|
|
extractor,
|
|
batch_processor_factory=lambda s: BatchProcessor(s),
|
|
estimator_factory=lambda s: Estimator(s),
|
|
)
|
|
|
|
|
|
def persist_desktop_ingest_result(
|
|
session: Session,
|
|
*,
|
|
task: Task,
|
|
batch_id: str,
|
|
radicado: str,
|
|
filename: str,
|
|
doc_type: str | None,
|
|
request: dict[str, Any],
|
|
normalized_obj: Any,
|
|
) -> dict[str, Any]:
|
|
"""Persiste una extracción desktop y contabiliza una vez por radicado.
|
|
|
|
Cuando una tarea separa los campos por tipo documental, cada documento se
|
|
guarda primero en ``partials``. El radicado se materializa y ``processed`` se
|
|
incrementa únicamente cuando llegaron todos sus tipos esperados.
|
|
"""
|
|
fields = list(request.get("fields") or [])
|
|
alias_map = dict(request.get("alias_map") or {})
|
|
structure_mode = str(request.get("structure_mode") or "radicados")
|
|
doc_types = list(request.get("doc_types") or [])
|
|
result = dict(task.result or {})
|
|
failed_radicados = {str(value) for value in result.get("failed_radicados") or []}
|
|
|
|
# Otro documento del mismo radicado puede terminar después de que uno de sus
|
|
# hermanos haya fallado. No debe recrear parciales ni contabilizarlo otra vez.
|
|
if radicado in failed_radicados:
|
|
return {
|
|
"radicado": radicado,
|
|
"filename": filename,
|
|
"doc_type": doc_type,
|
|
"finalized": False,
|
|
"skipped": True,
|
|
"reason": "radicado_failed",
|
|
"processed": int(result.get("processed") or 0),
|
|
"total": int(result.get("total") or 0),
|
|
}
|
|
|
|
if structure_mode == "radicados" and doc_types and doc_type:
|
|
partials = dict(result.get("partials") or {})
|
|
radicado_partials = dict(partials.get(radicado) or {})
|
|
radicado_partials[doc_type] = normalized_obj or {}
|
|
partials[radicado] = radicado_partials
|
|
result["partials"] = partials
|
|
task.result = result
|
|
session.flush()
|
|
finalized = _finalize_radicado_if_ready(
|
|
session,
|
|
task=task,
|
|
batch_id=batch_id,
|
|
radicado=radicado,
|
|
request=request,
|
|
fields=fields,
|
|
alias_map=alias_map,
|
|
)
|
|
return {
|
|
"radicado": radicado,
|
|
"filename": filename,
|
|
"doc_type": doc_type,
|
|
"finalized": finalized,
|
|
"processed": int((task.result or {}).get("processed") or 0),
|
|
"total": int((task.result or {}).get("total") or 0),
|
|
}
|
|
|
|
fused = {"radicado": radicado, "_source_pdf": filename}
|
|
if doc_type:
|
|
fused[doc_type] = normalized_obj or {}
|
|
else:
|
|
fused.update(normalized_obj if isinstance(normalized_obj, dict) else {"data": normalized_obj})
|
|
|
|
row_index = int(result.get("row_count") or 0)
|
|
for row in flatten_extraction_result(fused, fields, alias_map):
|
|
row_index += 1
|
|
session.add(
|
|
ExtractionRow(
|
|
batch_id=batch_id,
|
|
radicado=radicado,
|
|
source_pdf=filename,
|
|
row_index=row_index,
|
|
row_json=row,
|
|
raw_json=fused,
|
|
)
|
|
)
|
|
|
|
result["row_count"] = row_index
|
|
result["processed"] = int(result.get("processed") or 0) + 1
|
|
task.result = result
|
|
task.updated_at = utcnow()
|
|
session.flush()
|
|
|
|
return {
|
|
"radicado": radicado,
|
|
"filename": filename,
|
|
"doc_type": doc_type,
|
|
"finalized": True,
|
|
"processed": int(result.get("processed") or 0),
|
|
"total": int(result.get("total") or 0),
|
|
}
|
|
|
|
|
|
def record_desktop_ingest_error(
|
|
session: Session,
|
|
*,
|
|
task: Task,
|
|
radicado: str,
|
|
filename: str,
|
|
doc_type: str | None,
|
|
error: Exception,
|
|
) -> dict[str, Any]:
|
|
"""Registra un fallo y da por procesado el radicado exactamente una vez."""
|
|
result = dict(task.result or {})
|
|
errors = list(result.get("errors") or [])
|
|
errors.append({
|
|
"radicado": radicado,
|
|
"filename": filename,
|
|
"doc_type": doc_type,
|
|
"error": str(error),
|
|
})
|
|
result["errors"] = errors
|
|
|
|
failed_radicados = {str(value) for value in result.get("failed_radicados") or []}
|
|
first_failure = radicado not in failed_radicados
|
|
if first_failure:
|
|
failed_radicados.add(radicado)
|
|
result["processed"] = int(result.get("processed") or 0) + 1
|
|
|
|
# Un radicado fallido ya no puede completarse con los parciales restantes.
|
|
partials = dict(result.get("partials") or {})
|
|
partials.pop(radicado, None)
|
|
result["partials"] = partials
|
|
result["failed_radicados"] = sorted(failed_radicados)
|
|
task.result = result
|
|
task.updated_at = utcnow()
|
|
session.flush()
|
|
|
|
return {
|
|
"radicado": radicado,
|
|
"filename": filename,
|
|
"doc_type": doc_type,
|
|
"error": str(error),
|
|
"first_failure": first_failure,
|
|
"processed": int(result.get("processed") or 0),
|
|
"total": int(result.get("total") or 0),
|
|
}
|
|
|
|
|
|
def process_desktop_ingest(
|
|
service: InvoiceProcessingService,
|
|
session: Session,
|
|
*,
|
|
task: Task,
|
|
batch_id: str,
|
|
temp_path: str,
|
|
radicado: str,
|
|
filename: str,
|
|
doc_type: str | None,
|
|
instruction: str | None,
|
|
tiff_page_range: tuple[int, int] | None,
|
|
model: str | None = None,
|
|
) -> dict[str, Any]:
|
|
service = _service_for_model(service, model)
|
|
request = dict(task.request or {})
|
|
fields = list(request.get("fields") or [])
|
|
structure_mode = str(request.get("structure_mode") or "radicados")
|
|
doc_types = list(request.get("doc_types") or [])
|
|
prof = get_profile("invoice")
|
|
|
|
selected_paths = _paths_for_type(fields, doc_type if structure_mode == "radicados" and doc_types else None)
|
|
if doc_type and structure_mode == "radicados" and doc_types:
|
|
try:
|
|
prompt = PromptBuilder.build_for_type(doc_type, selected_paths, {}, profile_name=prof.get("name", "invoice"))
|
|
except Exception:
|
|
prompt = PromptBuilder.build(selected_paths, {}, profile_name=prof.get("name", "invoice"))
|
|
else:
|
|
prompt = instruction or PromptBuilder.build(selected_paths, {}, profile_name=prof.get("name", "invoice"))
|
|
|
|
data, extraction_usage = service.extract_full_from_pdf(
|
|
temp_path,
|
|
instruction=prompt,
|
|
tiff_page_range=tiff_page_range,
|
|
)
|
|
if isinstance(data, dict):
|
|
data["radicado"] = radicado
|
|
else:
|
|
data = {"radicado": radicado, "data": data}
|
|
|
|
norm = normalize_to_schema(
|
|
data,
|
|
paths=selected_paths,
|
|
profile_name=prof.get("name", "invoice"),
|
|
)
|
|
normalized_obj = data
|
|
normalizer_usage = None
|
|
if isinstance(norm, dict):
|
|
normalized_obj = norm.get("normalized") or data
|
|
normalizer_usage = norm.get("_usage")
|
|
|
|
usage_result = record_api_usage(
|
|
dict(task.result or {}),
|
|
phase="extract",
|
|
model=str(getattr(service.extractor, "model", model or "unknown")),
|
|
usage=extraction_usage,
|
|
radicado=radicado,
|
|
filename=filename,
|
|
)
|
|
usage_result = record_api_usage(
|
|
usage_result,
|
|
phase="normalize",
|
|
model=str(settings.api.normalizer_model),
|
|
usage=normalizer_usage,
|
|
radicado=radicado,
|
|
filename=filename,
|
|
)
|
|
task.result = usage_result
|
|
|
|
return persist_desktop_ingest_result(
|
|
session,
|
|
task=task,
|
|
batch_id=batch_id,
|
|
radicado=radicado,
|
|
filename=filename,
|
|
doc_type=doc_type,
|
|
request=request,
|
|
normalized_obj=normalized_obj,
|
|
)
|
|
|
|
|
|
def mark_task_completed_if_done(session: Session, task: Task, batch_id: str) -> None:
|
|
result = dict(task.result or {})
|
|
processed = int(result.get("processed") or 0)
|
|
total = int(result.get("total") or 0)
|
|
partials = dict(result.get("partials") or {})
|
|
if processed < total or partials:
|
|
return
|
|
|
|
from infrastructure.database import ExtractionBatch
|
|
|
|
task.status = "completed"
|
|
task.finished_at = utcnow()
|
|
task.updated_at = utcnow()
|
|
batch = session.get(ExtractionBatch, batch_id)
|
|
if batch is not None:
|
|
batch.status = "completed"
|
|
batch.updated_at = utcnow()
|