365 lines
16 KiB
Python
365 lines
16 KiB
Python
"""Servicios de alto nivel para el procesamiento de facturas.
|
|
|
|
Orquesta la extracción usando una `InvoiceExtractorStrategy`, la conversión de
|
|
documentos (PDF/TIFF) a imágenes base64 y el parseo del bloque JSON.
|
|
Incluye un flujo por lotes con selección de campos y exportación incremental a CSV.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Dict, List, Optional, Tuple, Callable, Awaitable
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
import asyncio
|
|
from typing import List, Dict, Any, Optional, Tuple
|
|
from typing import Callable
|
|
|
|
from .strategies import InvoiceExtractorStrategy
|
|
from .prompts import PromptBuilder
|
|
from .profiles import get_profile
|
|
from .config import settings
|
|
from .normalizer import normalize_to_schema
|
|
from .logging_config import log_execution_time
|
|
from .file_discovery import get_sorted_radicados, find_file_for_type_in_radicados, get_files_as_radicados
|
|
from .batch_processor import BatchProcessor
|
|
from .estimator import Estimator
|
|
from core.pdf_utils import document_to_images_bs64
|
|
from core.text_utils import extract_json_from_text, build_schema
|
|
from core.utils import find_first_matching_file
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class InvoiceProcessingService:
|
|
"""Fachada de negocio para extraer y transformar datos de facturas."""
|
|
def __init__(self, extractor: InvoiceExtractorStrategy) -> None:
|
|
self.extractor = extractor
|
|
# Componentes delegados
|
|
self.batch_processor = BatchProcessor(self)
|
|
self.estimator = Estimator(self)
|
|
|
|
# Lock para escrituras concurrentes al mismo CSV desde múltiples hilos
|
|
# (Mantenido por retrocompatibilidad si se accede directo, aunque BatchProcessor usa su propio CSVManager)
|
|
try:
|
|
import threading
|
|
self._csv_lock = threading.Lock()
|
|
except Exception:
|
|
self._csv_lock = None
|
|
|
|
@log_execution_time(logger=logger)
|
|
def extract_full_from_pdf(
|
|
self,
|
|
pdf_path: str,
|
|
instruction: Optional[str] = None,
|
|
*,
|
|
tiff_page_range: Optional[Tuple[int, int]] = None,
|
|
history: Optional[List[Dict[str, Any]]] = None,
|
|
) -> Dict[str, Any]:
|
|
"""Extrae un JSON completo desde un documento (PDF/TIFF) usando el extractor.
|
|
|
|
Estrategia: intenta con configuración por defecto. Si el backend responde
|
|
con un error (p. ej., 500 por payload demasiado grande), reintenta con
|
|
menor DPI y recortando a las primeras 5 páginas.
|
|
"""
|
|
filename = os.path.basename(pdf_path)
|
|
logger.info(f"Iniciando extracción de documento: {filename}", extra={"pdf": pdf_path})
|
|
|
|
t0 = time.time()
|
|
# Preparar imágenes según configuración
|
|
dpi_cfg = settings.imaging.pdf_dpi
|
|
thr_cfg = settings.imaging.pdf_thread_count
|
|
images_t0 = time.time()
|
|
tiff_pages = getattr(settings.imaging, 'tiff_pages', (2, 3))
|
|
|
|
try:
|
|
images_b64 = document_to_images_bs64(
|
|
pdf_path,
|
|
dpi=dpi_cfg,
|
|
thread_count=thr_cfg,
|
|
tiff_pages=tiff_pages,
|
|
tiff_page_range=tiff_page_range,
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Error al convertir documento a imágenes: {filename}", extra={"pdf": pdf_path, "error": str(e)})
|
|
raise
|
|
|
|
images_ms = int((time.time() - images_t0) * 1000)
|
|
logger.debug(f"Imágenes procesadas para {filename} (count={len(images_b64)}, dpi={dpi_cfg}, threads={thr_cfg}, ms={images_ms})", extra={"pdf": pdf_path, "dpi": dpi_cfg})
|
|
|
|
# Selección de instrucción por defecto según el contexto
|
|
profile = get_profile('invoice')
|
|
text_instruction = instruction if instruction is not None else profile["default_instruction"]
|
|
# Métricas del prompt (sin registrar el contenido completo por privacidad)
|
|
inst_preview = (text_instruction[:180] + "…") if isinstance(text_instruction, str) and len(text_instruction) > 180 else text_instruction
|
|
inst_len = len(text_instruction) if isinstance(text_instruction, str) else 0
|
|
hist_len = len(history) if history else 0
|
|
|
|
logger.debug(f"Enviando a extractor {filename} (images={len(images_b64)}, prompt_len={inst_len}, history_len={hist_len})", extra={"pdf": pdf_path, "dpi": dpi_cfg})
|
|
|
|
try:
|
|
call_t0 = time.time()
|
|
model_output = self.extractor.extract(images_b64, text_instruction, history=history)
|
|
call_ms = int((time.time() - call_t0) * 1000)
|
|
logger.debug(f"Extractor OK para {filename} (ms={call_ms})", extra={"pdf": pdf_path, "dpi": dpi_cfg})
|
|
except Exception as e:
|
|
logger.warning(f"Fallo en extracción por defecto para {filename}. Reintentando con DPI reducido.", extra={"pdf": pdf_path, "error": str(e)})
|
|
# Fallback: DPI reducido y limitar páginas
|
|
fb_dpi = 400
|
|
tiff_pages_fb = getattr(settings.imaging, 'tiff_pages', (2, 3))
|
|
# En fallback NO aplicamos `tiff_page_range`; se usa rango propio (pages<=5).
|
|
images_b64_fb = document_to_images_bs64(pdf_path, dpi=fb_dpi, tiff_pages=tiff_pages_fb)
|
|
# Limitar a 5 imágenes en fallback para controlar tamaño
|
|
if len(images_b64_fb) > 5:
|
|
images_b64_fb = images_b64_fb[:5]
|
|
logger.debug(f"Fallback activo para {filename} (images={len(images_b64_fb)}, dpi={fb_dpi})", extra={"pdf": pdf_path, "dpi": fb_dpi})
|
|
call_t0 = time.time()
|
|
model_output = self.extractor.extract(images_b64_fb, text_instruction, history=history)
|
|
call_ms = int((time.time() - call_t0) * 1000)
|
|
logger.debug(f"Extractor OK tras fallback para {filename} (ms={call_ms})", extra={"pdf": pdf_path, "dpi": fb_dpi})
|
|
|
|
raw_text = model_output.get("raw", "")
|
|
usage = model_output.get("usage")
|
|
json_text = extract_json_from_text(raw_text)
|
|
if not json_text:
|
|
logger.error(f"No se pudo extraer JSON válido de la respuesta para {filename}", extra={"pdf": pdf_path})
|
|
raise ValueError(f"No se pudo extraer un bloque JSON del texto de respuesta para {filename}.")
|
|
|
|
data = json.loads(json_text)
|
|
# Resumen de claves extraídas (sin volcar todo el JSON)
|
|
try:
|
|
if isinstance(data, dict):
|
|
top_keys = list(data.keys())
|
|
preview_keys = ", ".join(top_keys[:12]) + ("…" if len(top_keys) > 12 else "")
|
|
logger.debug(f"Datos extraídos de {filename} (keys={len(top_keys)}: {preview_keys})", extra={"pdf": pdf_path, "dpi": dpi_cfg})
|
|
else:
|
|
logger.debug(f"Datos extraídos de {filename} (tipo={type(data).__name__})", extra={"pdf": pdf_path, "dpi": dpi_cfg})
|
|
except Exception:
|
|
pass
|
|
|
|
total_ms = int((time.time() - t0) * 1000)
|
|
logger.info(f"Extracción finalizada exitosamente: {filename} (total_ms={total_ms})", extra={"pdf": pdf_path})
|
|
return data, usage
|
|
|
|
@log_execution_time(logger=logger)
|
|
async def extract_full_from_pdf_async(
|
|
self,
|
|
pdf_path: str,
|
|
instruction: Optional[str] = None,
|
|
*,
|
|
tiff_page_range: Optional[Tuple[int, int]] = None,
|
|
history: Optional[List[Dict[str, Any]]] = None,
|
|
) -> Dict[str, Any]:
|
|
"""Versión asíncrona de extract_full_from_pdf."""
|
|
filename = os.path.basename(pdf_path)
|
|
logger.info(f"Iniciando extracción asíncrona de documento: {filename}", extra={"pdf": pdf_path})
|
|
|
|
t0 = time.time()
|
|
# Preparar imágenes según configuración
|
|
dpi_cfg = settings.imaging.pdf_dpi
|
|
thr_cfg = settings.imaging.pdf_thread_count
|
|
images_t0 = time.time()
|
|
tiff_pages = getattr(settings.imaging, 'tiff_pages', (2, 3))
|
|
|
|
try:
|
|
# Ejecutar conversión de imágenes en thread pool
|
|
images_b64 = await asyncio.to_thread(
|
|
document_to_images_bs64,
|
|
pdf_path,
|
|
dpi=dpi_cfg,
|
|
thread_count=thr_cfg,
|
|
tiff_pages=tiff_pages,
|
|
tiff_page_range=tiff_page_range,
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Error al convertir documento a imágenes: {filename}", extra={"pdf": pdf_path, "error": str(e)})
|
|
raise
|
|
|
|
images_ms = int((time.time() - images_t0) * 1000)
|
|
logger.debug(f"Imágenes procesadas para {filename} (count={len(images_b64)}, dpi={dpi_cfg}, threads={thr_cfg}, ms={images_ms})", extra={"pdf": pdf_path, "dpi": dpi_cfg})
|
|
|
|
# Selección de instrucción por defecto según el contexto
|
|
profile = get_profile('invoice')
|
|
text_instruction = instruction if instruction is not None else profile["default_instruction"]
|
|
inst_len = len(text_instruction) if isinstance(text_instruction, str) else 0
|
|
hist_len = len(history) if history else 0
|
|
|
|
logger.debug(f"Enviando a extractor {filename} (images={len(images_b64)}, prompt_len={inst_len}, history_len={hist_len})", extra={"pdf": pdf_path, "dpi": dpi_cfg})
|
|
|
|
try:
|
|
call_t0 = time.time()
|
|
model_output = await self.extractor.extract_async(images_b64, text_instruction, history=history)
|
|
call_ms = int((time.time() - call_t0) * 1000)
|
|
logger.debug(f"Extractor OK para {filename} (ms={call_ms})", extra={"pdf": pdf_path, "dpi": dpi_cfg})
|
|
except Exception as e:
|
|
logger.warning(f"Fallo en extracción por defecto para {filename}. Reintentando con DPI reducido.", extra={"pdf": pdf_path, "error": str(e)})
|
|
# Fallback: DPI reducido y limitar páginas
|
|
fb_dpi = 400
|
|
tiff_pages_fb = getattr(settings.imaging, 'tiff_pages', (2, 3))
|
|
|
|
images_b64_fb = await asyncio.to_thread(
|
|
document_to_images_bs64,
|
|
pdf_path,
|
|
dpi=fb_dpi,
|
|
tiff_pages=tiff_pages_fb
|
|
)
|
|
|
|
# Limitar a 5 imágenes en fallback para controlar tamaño
|
|
if len(images_b64_fb) > 5:
|
|
images_b64_fb = images_b64_fb[:5]
|
|
logger.debug(f"Fallback activo para {filename} (images={len(images_b64_fb)}, dpi={fb_dpi})", extra={"pdf": pdf_path, "dpi": fb_dpi})
|
|
call_t0 = time.time()
|
|
model_output = await self.extractor.extract_async(images_b64_fb, text_instruction, history=history)
|
|
call_ms = int((time.time() - call_t0) * 1000)
|
|
logger.debug(f"Extractor OK tras fallback para {filename} (ms={call_ms})", extra={"pdf": pdf_path, "dpi": fb_dpi})
|
|
|
|
raw_text = model_output.get("raw", "")
|
|
usage = model_output.get("usage")
|
|
json_text = extract_json_from_text(raw_text)
|
|
if not json_text:
|
|
logger.error(f"No se pudo extraer JSON válido de la respuesta para {filename}", extra={"pdf": pdf_path})
|
|
raise ValueError(f"No se pudo extraer un bloque JSON del texto de respuesta para {filename}.")
|
|
|
|
data = json.loads(json_text)
|
|
# Resumen de claves extraídas (sin volcar todo el JSON)
|
|
try:
|
|
if isinstance(data, dict):
|
|
top_keys = list(data.keys())
|
|
preview_keys = ", ".join(top_keys[:12]) + ("…" if len(top_keys) > 12 else "")
|
|
logger.debug(f"Datos extraídos de {filename} (keys={len(top_keys)}: {preview_keys})", extra={"pdf": pdf_path, "dpi": dpi_cfg})
|
|
else:
|
|
logger.debug(f"Datos extraídos de {filename} (tipo={type(data).__name__})", extra={"pdf": pdf_path, "dpi": dpi_cfg})
|
|
except Exception:
|
|
pass
|
|
|
|
total_ms = int((time.time() - t0) * 1000)
|
|
logger.info(f"Extracción finalizada exitosamente: {filename} (total_ms={total_ms})", extra={"pdf": pdf_path})
|
|
return data, usage
|
|
|
|
def build_schema_from_first(
|
|
self,
|
|
pdf_path: str,
|
|
instruction: Optional[str] = None,
|
|
*,
|
|
tiff_page_range: Optional[Tuple[int, int]] = None,
|
|
) -> Tuple[Dict[str, str], Dict[str, Any]]:
|
|
"""Devuelve (schema, data) generados a partir del primer documento."""
|
|
data, _usage = self.extract_full_from_pdf(pdf_path, instruction=instruction, tiff_page_range=tiff_page_range)
|
|
schema = build_schema(data)
|
|
try:
|
|
logger.debug(f"Schema generado para {os.path.basename(pdf_path)} (total_paths={len(schema)})", extra={"pdf": pdf_path})
|
|
except Exception:
|
|
pass
|
|
return schema, data
|
|
|
|
def build_schema_from_first_for_types(
|
|
self,
|
|
root_path: str,
|
|
doc_types: List[Dict[str, str]],
|
|
instruction: Optional[str] = None,
|
|
*,
|
|
tiff_page_range: Optional[Tuple[int, int]] = None,
|
|
structure_mode: str = "radicados",
|
|
) -> Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Any]], str | None]:
|
|
"""Genera schema y muestra para cada Tipo de Documento buscando el primer archivo disponible."""
|
|
if structure_mode == "general":
|
|
radicados = get_files_as_radicados(root_path)
|
|
else:
|
|
radicados = get_sorted_radicados(root_path)
|
|
|
|
if not radicados:
|
|
raise ValueError(f"No se encontraron documentos/radicados en: {root_path}")
|
|
|
|
schema_by_type: Dict[str, Dict[str, str]] = {}
|
|
first_full_by_type: Dict[str, Dict[str, Any]] = {}
|
|
prof = get_profile('invoice')
|
|
inst0 = instruction if instruction is not None else prof["default_instruction"]
|
|
|
|
first_radicado_found = None
|
|
|
|
for dt in (doc_types or []):
|
|
tname = str(dt.get("name") or "").strip() or "Tipo"
|
|
kw = str(dt.get("keyword") or "").strip()
|
|
|
|
pdf_path, rad_used = find_file_for_type_in_radicados(root_path, radicados, kw)
|
|
|
|
if not pdf_path:
|
|
logger.info(f"No se encontró archivo para tipo '{tname}' con keyword '{kw}' en ningún radicado")
|
|
schema_by_type[tname] = {}
|
|
first_full_by_type[tname] = {}
|
|
continue
|
|
|
|
if first_radicado_found is None:
|
|
first_radicado_found = rad_used
|
|
|
|
try:
|
|
logger.info(
|
|
f"Usando archivo para schema por tipo '{tname}': {os.path.basename(pdf_path)} (Radicado: {rad_used})",
|
|
extra={"tipo": tname, "radicado": rad_used, "pdf": pdf_path},
|
|
)
|
|
sch, dat = self.build_schema_from_first(pdf_path, instruction=inst0, tiff_page_range=tiff_page_range)
|
|
schema_by_type[tname] = sch or {}
|
|
first_full_by_type[tname] = dat or {}
|
|
except Exception as e:
|
|
logger.exception(f"Error generando schema para tipo '{tname}'", extra={"tipo": tname, "radicado": rad_used})
|
|
schema_by_type[tname] = {}
|
|
first_full_by_type[tname] = {"_error": str(e)}
|
|
|
|
return schema_by_type, first_full_by_type, (first_radicado_found or radicados[0])
|
|
|
|
# --- Delegación a BatchProcessor y Estimator ---
|
|
|
|
@log_execution_time(logger=logger)
|
|
def process_root_with_field_selection(
|
|
self,
|
|
root_path: str,
|
|
keywords: List[str] | None = None,
|
|
*,
|
|
doc_types: Optional[List[Dict[str, str]]] = None,
|
|
fields: Optional[List[str]] = None,
|
|
on_progress: Optional[callable] = None,
|
|
csv_output_path: Optional[str] = None,
|
|
csv_header_override: Optional[List[str]] = None,
|
|
instruction_override: Optional[str] = None,
|
|
on_usage: Optional[Callable[[str, str, Dict[str, Any] | None, Dict[str, Any] | None], None]] = None,
|
|
on_error_retry: Optional[Callable[[Exception], Awaitable[bool]]] = None,
|
|
alias_map: Optional[Dict[str, str]] = None,
|
|
tiff_page_range: Optional[Tuple[int, int]] = None,
|
|
structure_mode: str = "radicados",
|
|
) -> List[Dict[str, Any]]:
|
|
"""Procesa radicados con selección de campos y CSV incremental. (Delegado a BatchProcessor)"""
|
|
return self.batch_processor.process_root(
|
|
root_path, keywords,
|
|
doc_types=doc_types, fields=fields, on_progress=on_progress,
|
|
csv_output_path=csv_output_path, csv_header_override=csv_header_override,
|
|
instruction_override=instruction_override, on_usage=on_usage,
|
|
on_error_retry=on_error_retry,
|
|
alias_map=alias_map, tiff_page_range=tiff_page_range,
|
|
structure_mode=structure_mode
|
|
)
|
|
|
|
def estimate_tokens_and_cost(
|
|
self,
|
|
root_path: str,
|
|
keywords: List[str] | None = None,
|
|
schema: Dict[str, str] | None = None,
|
|
selected_fields: List[str] | None = None,
|
|
*,
|
|
doc_types: Optional[List[Dict[str, str]]] = None,
|
|
schema_by_type: Optional[Dict[str, Dict[str, str]]] = None,
|
|
selected_paths: Optional[List[str]] = None,
|
|
structure_mode: str = "radicados",
|
|
) -> Dict[str, Any]:
|
|
"""Calcula estimado de tokens y costos. (Delegado a Estimator)"""
|
|
return self.estimator.estimate(
|
|
root_path, keywords, schema, selected_fields,
|
|
doc_types=doc_types, schema_by_type=schema_by_type, selected_paths=selected_paths,
|
|
structure_mode=structure_mode
|
|
)
|
|
|
|
# CSV legacy helper (por si algún test lo llama directamente)
|
|
def _append_rows_to_csv(self, csv_path: str, rows: List[Dict[str, Any]], header: List[str]) -> None:
|
|
"""DEPRECATED: Use CSVManager directly or via BatchProcessor."""
|
|
# Delegar temporalmente a un CSVManager local para mantener compatibilidad sin duplicar código
|
|
# o simplemente usar la instancia del batch processor
|
|
self.batch_processor.csv_manager.append_rows(csv_path, rows, header)
|