881 lines
36 KiB
Python
881 lines
36 KiB
Python
"""Controlador de la aplicación GUI: coordina vista y servicio.
|
|
|
|
Gestiona selección de carpetas, ejecución en hilo de procesamiento, selector
|
|
interactivo de campos y actualización de configuración del extractor.
|
|
"""
|
|
import json
|
|
import logging
|
|
import threading
|
|
import asyncio
|
|
import tkinter.filedialog as fd
|
|
import tkinter.messagebox as mb
|
|
import os
|
|
|
|
from core.utils import parse_comma_separated, collect_radicado_docs, collect_documents_flat
|
|
from application.logging_config import configure_logging
|
|
from gui.field_selector import FieldSelectorDialog
|
|
from application.preclassification import PreclassificationService
|
|
from application.strategies import OllamaExtractor
|
|
from application.database import DatabaseManager
|
|
from application.config import settings
|
|
from application.costs import compute_cost_from_usage, MODEL_PRICING, sum_costs
|
|
import pandas as pd
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class AppController:
|
|
"""Controlador principal de la GUI que orquesta la vista y el servicio.
|
|
|
|
Se encarga de:
|
|
- Vincular callbacks de la vista.
|
|
- Disparar el flujo de procesamiento de facturas.
|
|
- Abrir el diálogo de selección de campos cuando el servicio lo solicite.
|
|
"""
|
|
def __init__(self, view, service):
|
|
"""Inicializa el controlador y vincula handlers de la vista.
|
|
|
|
Args:
|
|
view: Instancia de vista (por ejemplo, `InvoiceAppView`).
|
|
service: Servicio de facturas compatible con `process_root_with_field_selection`.
|
|
"""
|
|
# asegurar logging configurado
|
|
try:
|
|
configure_logging()
|
|
except Exception:
|
|
pass
|
|
self.view = view
|
|
self.service = service
|
|
# Acumuladores de uso
|
|
self._total_prompt_tokens = 0
|
|
self._total_cached_tokens = 0
|
|
self._total_completion_tokens = 0
|
|
self._total_usd = 0.0
|
|
# Estado para dashboard
|
|
self._hist_requests = [] # list[dict]: {radicado, phase, prompt, cached, completion, cost_usd}
|
|
self._per_doc_costs = {} # radicado -> {"extract": float|None, "normalize": float|None}
|
|
self._completed_doc_count = 0
|
|
self._phase_tokens = {
|
|
"extract": {"input": 0, "cached": 0, "output": 0, "usd": 0.0},
|
|
"normalize": {"input": 0, "cached": 0, "output": 0, "usd": 0.0},
|
|
}
|
|
# Guard para evitar doble lanzamiento de worker2 (por doble click o condiciones de carrera)
|
|
self._worker2_running = False
|
|
|
|
# Estado Preclasificación
|
|
self._last_run_id = None
|
|
self.db_manager = DatabaseManager() # Para pestaña resultados
|
|
|
|
self.view.set_on_select_root(self.on_select_root)
|
|
try:
|
|
self.view.set_on_select_preclass_root(self.on_select_preclass_root)
|
|
self.view.set_on_run_preclassification(self.on_run_preclassification)
|
|
self.view.set_on_export_excel(self.on_export_excel)
|
|
self.view.set_on_organize_files(self.on_organize_files)
|
|
|
|
# Resultados
|
|
self.view.set_on_res_search(self.on_res_search)
|
|
self.view.set_on_res_export(self.on_res_export)
|
|
self.view.set_on_res_refresh(self.on_res_refresh)
|
|
self.view.set_on_res_organize(self.on_res_organize_run)
|
|
self.view.set_on_res_select(self.on_res_select)
|
|
|
|
# Carga inicial
|
|
self.on_res_refresh()
|
|
except Exception:
|
|
pass
|
|
self.view.set_on_process(self.on_process)
|
|
self.view.set_on_select_csv(self.on_select_csv)
|
|
self.view.set_on_save_settings(self.on_save_settings)
|
|
|
|
# Mapeo de nombres de UI a IDs de modelo (y viceversa)
|
|
self.ui_to_model = {
|
|
"5.1": "gpt-5.1",
|
|
"4.1": "gpt-4.1",
|
|
"4.1 mini": "gpt-4.1-mini",
|
|
"4.1 nano": "gpt-4.1-nano"
|
|
}
|
|
self.model_to_ui = {v: k for k, v in self.ui_to_model.items()}
|
|
|
|
# Inicializar vista con configuración actual
|
|
try:
|
|
# Llenar tabla de precios de modelos
|
|
pricing_data = self._get_model_pricing_data()
|
|
self.view.update_comparison_table(pricing_data)
|
|
|
|
current_model = getattr(self.service.extractor, "model", "")
|
|
# Intentar mapear modelo backend -> nombre UI, si no existe usar raw
|
|
ui_name = self.model_to_ui.get(current_model, current_model)
|
|
if ui_name:
|
|
self.view.set_selected_model(ui_name)
|
|
|
|
# Cargar concurrencia
|
|
self.view.set_concurrency_limit(settings.processing.concurrency_limit)
|
|
except Exception:
|
|
pass
|
|
|
|
def _get_model_pricing_data(self):
|
|
"""Obtiene las tarifas por 1M de tokens para cada modelo."""
|
|
results = []
|
|
for model_id, rates in MODEL_PRICING.items():
|
|
results.append({
|
|
"model": self.model_to_ui.get(model_id, model_id),
|
|
"input": rates["input"],
|
|
"cached_input": rates["cached_input"],
|
|
"output": rates["output"]
|
|
})
|
|
# Ordenar por precio de input descendente
|
|
results.sort(key=lambda x: x["input"], reverse=True)
|
|
return results
|
|
|
|
def on_save_settings(self):
|
|
"""Actualiza la configuración del servicio (modelo LLM y concurrencia)."""
|
|
selected_name = self.view.get_selected_model()
|
|
concurrency = self.view.get_concurrency_limit()
|
|
|
|
if not selected_name:
|
|
mb.showwarning("Configuración", "El modelo no puede estar vacío.")
|
|
return
|
|
|
|
if concurrency <= 0:
|
|
mb.showwarning("Configuración", "El límite de concurrencia debe ser mayor a 0.")
|
|
return
|
|
|
|
# Traducir nombre UI -> ID modelo real
|
|
real_model_id = self.ui_to_model.get(selected_name, selected_name)
|
|
|
|
try:
|
|
# Actualizar extractor
|
|
if hasattr(self.service.extractor, "update_config"):
|
|
self.service.extractor.update_config(model=real_model_id)
|
|
|
|
# Actualizar settings globales
|
|
settings.processing.concurrency_limit = concurrency
|
|
|
|
mb.showinfo("Configuración", f"Configuración actualizada.\nModelo: {selected_name}\nConcurrencia: {concurrency}")
|
|
self.view.append_output_safe(f"Configuración actualizada: Modelo={real_model_id}, Concurrencia={concurrency}")
|
|
except Exception as e:
|
|
logger.exception("Error actualizando configuración")
|
|
mb.showerror("Error", f"No se pudo actualizar la configuración: {e}")
|
|
|
|
def on_select_root(self):
|
|
"""Abre un selector de carpeta y establece la ruta raíz en la vista.
|
|
|
|
Además, si el primer documento detectado es TIFF, muestra los controles
|
|
de rango TIFF para que el usuario pueda ajustar el índice antes de procesar.
|
|
"""
|
|
selected = fd.askdirectory()
|
|
if not selected:
|
|
return
|
|
self.view.root_entry.delete(0, 'end')
|
|
self.view.root_entry.insert(0, selected)
|
|
|
|
# Detectar si el primer documento a procesar es TIFF para esta carpeta
|
|
try:
|
|
keywords_str = self.view.get_keywords()
|
|
keywords = parse_comma_separated(keywords_str) or ["Factura_O_Documento_Equivalente", "factura"]
|
|
except Exception:
|
|
keywords = ["Factura_O_Documento_Equivalente", "factura"]
|
|
has_tiff_first = False
|
|
try:
|
|
preview_entries = collect_radicado_docs(selected, keywords)
|
|
if preview_entries:
|
|
first_path = str(preview_entries[0].get("path") or "").lower()
|
|
has_tiff_first = first_path.endswith((".tif", ".tiff"))
|
|
except Exception:
|
|
has_tiff_first = False
|
|
try:
|
|
self.view.show_tiff_range_controls(has_tiff_first)
|
|
except Exception:
|
|
pass
|
|
|
|
def on_select_preclass_root(self):
|
|
"""Abre un selector de carpeta y establece la ruta raíz para preclasificación."""
|
|
selected = fd.askdirectory()
|
|
if not selected:
|
|
return
|
|
try:
|
|
self.view.preclass_root_entry.delete(0, 'end')
|
|
self.view.preclass_root_entry.insert(0, selected)
|
|
except Exception:
|
|
# fallback si cambia el nombre del widget
|
|
try:
|
|
setter = getattr(self.view, "set_preclass_root_path", None)
|
|
if callable(setter):
|
|
setter(selected)
|
|
except Exception:
|
|
pass
|
|
|
|
def on_run_preclassification(self):
|
|
"""Ejecuta la preclasificación en un hilo separado."""
|
|
# 1. Leer inputs de la UI
|
|
root = self.view.get_preclass_root_path()
|
|
mode = getattr(self.view, "get_preclass_mode", lambda: "radicados")()
|
|
patterns = self.view.get_preclass_patterns()
|
|
target_info = self.view.get_preclass_target_info()
|
|
page_range = self.view.get_preclass_range() # (enabled, start, end)
|
|
mode = self.view.get_preclass_mode()
|
|
|
|
# Validaciones básicas
|
|
if not root or not os.path.isdir(root):
|
|
mb.showwarning("Error", "Seleccione una carpeta raíz válida.")
|
|
return
|
|
|
|
if not patterns:
|
|
mb.showwarning("Error", "Debe agregar al menos un patrón.")
|
|
return
|
|
|
|
if target_info["enabled"] and not target_info["pattern"]:
|
|
mb.showwarning("Error", "Si habilita archivo objetivo, debe especificar un patrón de archivo.")
|
|
return
|
|
|
|
# Bloquear UI
|
|
self.view.set_processing(True)
|
|
self.view.enable_preclass_actions(False)
|
|
self.view.append_preclass_output("Iniciando preclasificación...")
|
|
self.view.append_preclass_output(f"Raíz: {root}")
|
|
self.view.append_preclass_output(f"Modo: {mode}")
|
|
self.view.append_preclass_output(f"Patrones: {len(patterns)}")
|
|
if target_info["enabled"]:
|
|
self.view.append_preclass_output(f"Filtro: {target_info['pattern']}")
|
|
|
|
# Instanciar servicio (usando el extractor actual)
|
|
# Nota: PreclassificationService maneja su propia instancia de DB
|
|
pre_service = PreclassificationService(self.service.extractor)
|
|
|
|
def worker():
|
|
try:
|
|
def on_progress(radicado, idx, total):
|
|
pct = int((idx / total) * 100)
|
|
msg = f"[{idx}/{total}] Procesando radicado: {radicado} ({pct}%)"
|
|
self.view.append_preclass_output_safe(msg)
|
|
|
|
run_id = pre_service.run_preclassification(
|
|
root_path=root,
|
|
mode=mode,
|
|
patterns=patterns,
|
|
target_info=target_info,
|
|
page_range=page_range,
|
|
on_progress=on_progress
|
|
)
|
|
|
|
self.view.append_preclass_output_safe(f"\n--- Ejecución finalizada (ID: {run_id}) ---")
|
|
|
|
# Guardar ID
|
|
self._last_run_id = run_id
|
|
def _refresh_results_tab():
|
|
# Recargar runs, seleccionar el run recién ejecutado y listar sus resultados
|
|
try:
|
|
self.on_res_refresh()
|
|
self.view.set_res_run(run_id)
|
|
self.on_res_search()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
self.view.enable_preclass_actions(True)
|
|
except Exception:
|
|
pass
|
|
self.view.after(0, _refresh_results_tab)
|
|
|
|
# Mostrar resumen rápido
|
|
# Usar tabla results (1 fila = 1 documento), no el join con matches
|
|
docs = self.db_manager.search_results(run_id=run_id, status="Todos", text=None)
|
|
match_count = sum(1 for r in docs if r.get("status") == "MATCH")
|
|
self.view.append_preclass_output_safe(f"Total procesados: {len(docs)}")
|
|
self.view.append_preclass_output_safe(f"Coincidencias (MATCH): {match_count}")
|
|
|
|
# Detalle opcional
|
|
for res in docs[:10]: # mostrar primeros 10
|
|
status_icon = "✅" if res.get("status") == "MATCH" else "❌"
|
|
self.view.append_preclass_output_safe(f"{status_icon} {res.get('radicado')} ({res.get('status')})")
|
|
|
|
if len(docs) > 10:
|
|
self.view.append_preclass_output_safe("... ver base de datos para más detalles.")
|
|
|
|
self.view.after(0, lambda: mb.showinfo("Completado", f"Preclasificación finalizada.\nCoincidencias: {match_count}/{len(docs)}"))
|
|
|
|
except Exception as e:
|
|
logger.exception("Error en preclasificación")
|
|
self.view.append_preclass_output_safe(f"ERROR CRÍTICO: {str(e)}")
|
|
self.view.after(0, lambda: mb.showerror("Error", f"Ocurrió un error: {e}"))
|
|
finally:
|
|
self.view.set_processing(False)
|
|
|
|
threading.Thread(target=worker, daemon=True).start()
|
|
|
|
def on_export_excel(self):
|
|
"""Exporta los resultados de la última preclasificación a Excel."""
|
|
if not self._last_run_id:
|
|
mb.showwarning("Aviso", "No hay resultados de preclasificación recientes para exportar.")
|
|
return
|
|
|
|
path = fd.asksaveasfilename(
|
|
defaultextension=".xlsx",
|
|
filetypes=[("Excel", "*.xlsx"), ("CSV", "*.csv"), ("Todos", "*.*")]
|
|
)
|
|
if not path:
|
|
return
|
|
|
|
try:
|
|
pre_service = PreclassificationService(self.service.extractor)
|
|
ok = pre_service.export_to_excel(self._last_run_id, path)
|
|
if ok:
|
|
mb.showinfo("Exportar", "Archivo exportado correctamente.")
|
|
else:
|
|
mb.showerror("Exportar", "No se encontraron datos para exportar.")
|
|
except Exception as e:
|
|
logger.exception("Error exportando excel")
|
|
mb.showerror("Error", f"Error al exportar: {e}")
|
|
|
|
def on_organize_files(self):
|
|
"""Organiza los archivos MATCH en carpetas (Run actual)."""
|
|
if not self._last_run_id:
|
|
mb.showwarning("Aviso", "No hay resultados para organizar.")
|
|
return
|
|
self._organize_run_generic(self._last_run_id)
|
|
|
|
def _organize_run_generic(self, run_id: int):
|
|
target_dir = fd.askdirectory(title="Seleccionar carpeta de destino para organizar")
|
|
if not target_dir:
|
|
return
|
|
|
|
if not mb.askyesno("Confirmar", "¿Está seguro de copiar y organizar los archivos en la carpeta seleccionada?\nEsta acción creará una estructura de carpetas basada en los datos extraídos."):
|
|
return
|
|
|
|
try:
|
|
# Usamos append_preclass_output si estamos en esa pestaña, pero desde Resultados?
|
|
# El metodo de view inserta en el widget de log. Si estamos en tab Resultados, no se ve.
|
|
# Pero no importa, mostramos mb.showinfo al final.
|
|
pre_service = PreclassificationService(self.service.extractor)
|
|
ok_count, err_count = pre_service.organize_files(run_id, target_dir)
|
|
|
|
msg = f"Organización finalizada.\nCopiados exitosos: {ok_count}\nErrores: {err_count}"
|
|
mb.showinfo("Organizar", msg)
|
|
except Exception as e:
|
|
logger.exception("Error organizando archivos")
|
|
mb.showerror("Error", f"Error al organizar: {e}")
|
|
|
|
# --- Pestaña Resultados ---
|
|
def on_res_select(self, event):
|
|
"""Muestra detalles del resultado seleccionado."""
|
|
selection = self.view.tree_results.selection()
|
|
if not selection:
|
|
return
|
|
|
|
# values: (ID, Run, Fecha, Radicado, Estado, Archivo)
|
|
item_vals = self.view.tree_results.item(selection[0], "values")
|
|
if not item_vals:
|
|
return
|
|
|
|
try:
|
|
res_id = int(item_vals[0])
|
|
matches = self.db_manager.get_matches_for_result(res_id)
|
|
self.view.update_res_matches_table(matches)
|
|
except Exception as e:
|
|
logger.error(f"Error cargando detalle: {e}")
|
|
|
|
def on_res_organize_run(self):
|
|
"""Organiza archivos del Run seleccionado en el filtro."""
|
|
filters = self.view.get_res_filters()
|
|
run_val = filters["run_id"]
|
|
|
|
if run_val == "Todos" or not run_val:
|
|
mb.showwarning("Organizar", "Por favor seleccione un 'Run ID' específico en el filtro para organizar.")
|
|
return
|
|
|
|
try:
|
|
run_id = int(run_val)
|
|
self._organize_run_generic(run_id)
|
|
except ValueError:
|
|
mb.showerror("Error", "ID de Run inválido.")
|
|
|
|
def on_res_refresh(self):
|
|
"""Refresca la lista de Runs en el combo."""
|
|
try:
|
|
runs = self.db_manager.get_all_runs()
|
|
ids = [r["id"] for r in runs]
|
|
self.view.update_res_runs_combo(ids)
|
|
# Tambien refrescar tabla
|
|
self.on_res_search()
|
|
except Exception as e:
|
|
logger.error(f"Error refrescando runs: {e}")
|
|
|
|
def on_res_search(self):
|
|
"""Busca resultados con los filtros actuales."""
|
|
filters = self.view.get_res_filters()
|
|
run_id = None
|
|
if filters["run_id"] != "Todos":
|
|
try:
|
|
run_id = int(filters["run_id"])
|
|
except: pass
|
|
|
|
try:
|
|
rows = self.db_manager.search_results(
|
|
run_id=run_id,
|
|
status=filters["status"],
|
|
text=filters["text"]
|
|
)
|
|
self.view.update_res_table(rows)
|
|
except Exception as e:
|
|
logger.error(f"Error buscando resultados: {e}")
|
|
mb.showerror("Error", f"Error en búsqueda: {e}")
|
|
|
|
def on_res_export(self):
|
|
"""Exporta la vista actual de resultados a Excel."""
|
|
path = fd.asksaveasfilename(
|
|
defaultextension=".xlsx",
|
|
filetypes=[("Excel", "*.xlsx"), ("CSV", "*.csv")]
|
|
)
|
|
if not path:
|
|
return
|
|
|
|
# Obtener los datos actuales (repite la busqueda)
|
|
# Podriamos optimizar guardando el ultimo result set
|
|
try:
|
|
filters = self.view.get_res_filters()
|
|
run_id = None
|
|
if filters["run_id"] != "Todos":
|
|
try: run_id = int(filters["run_id"])
|
|
except: pass
|
|
|
|
rows = self.db_manager.search_results(
|
|
run_id=run_id,
|
|
status=filters["status"],
|
|
text=filters["text"]
|
|
)
|
|
|
|
if not rows:
|
|
mb.showwarning("Exportar", "No hay datos para exportar.")
|
|
return
|
|
|
|
df = pd.DataFrame(rows)
|
|
if path.endswith(".csv"):
|
|
df.to_csv(path, index=False, sep=";")
|
|
else:
|
|
df.to_excel(path, index=False)
|
|
|
|
mb.showinfo("Exportar", "Datos exportados correctamente.")
|
|
except Exception as e:
|
|
logger.exception("Error exportando vista")
|
|
mb.showerror("Error", f"Error al exportar: {e}")
|
|
|
|
def on_select_csv(self):
|
|
"""Abre un diálogo de guardado y establece la ruta del CSV de salida."""
|
|
selected = fd.asksaveasfilename(defaultextension=".csv", filetypes=[("CSV", "*.csv"), ("Todos", "*.*")])
|
|
if selected:
|
|
self.view.csv_entry.delete(0, 'end')
|
|
self.view.csv_entry.insert(0, selected)
|
|
|
|
def on_process(self):
|
|
"""Inicia el flujo de procesamiento por radicado (solo facturas) en un hilo de fondo.
|
|
|
|
- Si el servicio requiere selección de campos, abre `FieldSelectorDialog` y relanza.
|
|
- Emite resultados en el área de salida y opcionalmente escribe CSV incremental.
|
|
"""
|
|
root = self.view.get_root_path()
|
|
# Tipos de Documento (multi-doc)
|
|
doc_types = []
|
|
try:
|
|
doc_types = self.view.get_doc_types()
|
|
except Exception:
|
|
doc_types = []
|
|
if doc_types:
|
|
try:
|
|
ok, msg = self.view.validate_doc_types()
|
|
except Exception:
|
|
ok, msg = True, None
|
|
if not ok:
|
|
self.view.append_output_safe(msg or "Tipos de Documento inválidos.")
|
|
return
|
|
keywords_str = self.view.get_keywords()
|
|
fields_str = self.view.get_fields()
|
|
structure_mode = self.view.get_process_mode()
|
|
|
|
if structure_mode == "general":
|
|
# En modo general, si está vacío o es el default, procesar todo (None)
|
|
# Si el usuario escribió algo específico, usarlo.
|
|
kws = parse_comma_separated(keywords_str)
|
|
# Check if it matches default legacy keywords to avoid accidental filtering
|
|
if kws == ["Factura_O_Documento_Equivalente", "factura"]:
|
|
keywords = None
|
|
else:
|
|
keywords = kws if kws else None
|
|
else:
|
|
keywords = parse_comma_separated(keywords_str) or ["Factura_O_Documento_Equivalente", "factura"]
|
|
|
|
fields = parse_comma_separated(fields_str) if fields_str else None
|
|
|
|
# Validar ruta antes de deshabilitar la UI
|
|
if not root or not os.path.isdir(root):
|
|
self.view.append_output_safe(f"Error: La ruta raíz no es un directorio: {root or ''}")
|
|
return
|
|
|
|
# Si es modo general, ignorar doc_types
|
|
if structure_mode == "general":
|
|
doc_types = []
|
|
|
|
# Determinar si el primer documento a procesar es TIFF; si no lo es, ocultar
|
|
# los controles de rango TIFF y no aplicar ningún recorte.
|
|
tiff_page_range = None
|
|
has_tiff_first = False
|
|
try:
|
|
if structure_mode == "general":
|
|
preview_entries = collect_documents_flat(root, keywords)
|
|
else:
|
|
preview_entries = collect_radicado_docs(root, keywords)
|
|
|
|
if preview_entries:
|
|
first_path = str(preview_entries[0].get("path") or "").lower()
|
|
has_tiff_first = first_path.endswith((".tif", ".tiff"))
|
|
except Exception:
|
|
has_tiff_first = False
|
|
try:
|
|
# Mostrar/ocultar controles en función de si el primer doc es TIFF
|
|
self.view.show_tiff_range_controls(has_tiff_first)
|
|
except Exception:
|
|
pass
|
|
|
|
if has_tiff_first:
|
|
# Rango opcional de páginas TIFF (1-based; 0 = sin límite)
|
|
try:
|
|
tiff_start_raw = int(getattr(self.view, "get_tiff_start_page", lambda: 0)() or 0)
|
|
except Exception:
|
|
tiff_start_raw = 0
|
|
try:
|
|
tiff_end_raw = int(getattr(self.view, "get_tiff_end_page", lambda: 0)() or 0)
|
|
except Exception:
|
|
tiff_end_raw = 0
|
|
|
|
if tiff_start_raw < 0 or tiff_end_raw < 0:
|
|
self.view.append_output_safe("Error: Las páginas TIFF deben ser mayores o iguales a 0.")
|
|
return
|
|
|
|
tiff_start = tiff_start_raw if tiff_start_raw > 0 else None
|
|
tiff_end = tiff_end_raw if tiff_end_raw > 0 else None
|
|
if tiff_start is not None and tiff_end is not None and tiff_start > tiff_end:
|
|
self.view.append_output_safe("Error: La página de inicio TIFF no puede ser mayor que la página de fin.")
|
|
return
|
|
# Si ambos son 0 (None/None), el usuario quiere "todas las páginas":
|
|
# usamos el centinela (None, None) para indicar "no recortar TIFF".
|
|
if tiff_start is None and tiff_end is None:
|
|
tiff_page_range = (None, None)
|
|
else:
|
|
tiff_page_range = (tiff_start, tiff_end)
|
|
|
|
def worker():
|
|
resultados = None
|
|
if doc_types:
|
|
try:
|
|
self.view.append_output_safe(f"Procesando raíz: {root} | tipos: {[(dt.get('name'), dt.get('keyword')) for dt in doc_types]}")
|
|
except Exception:
|
|
self.view.append_output_safe(f"Procesando raíz: {root} | tipos: {len(doc_types)}")
|
|
else:
|
|
self.view.append_output_safe(f"Procesando raíz: {root} | keywords: {keywords}")
|
|
# limpiar pestaña Uso
|
|
try:
|
|
self._total_prompt_tokens = 0
|
|
self._total_cached_tokens = 0
|
|
self._total_completion_tokens = 0
|
|
self._total_usd = 0.0
|
|
self._hist_requests.clear()
|
|
self._per_doc_costs.clear()
|
|
self._completed_doc_count = 0
|
|
self._phase_tokens = {
|
|
"extract": {"input": 0, "cached": 0, "output": 0, "usd": 0.0},
|
|
"normalize": {"input": 0, "cached": 0, "output": 0, "usd": 0.0},
|
|
}
|
|
self.view.clear_usage()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
def on_progress(radicado, idx, total):
|
|
pct = int((idx / total) * 100)
|
|
self.view.append_output_safe(f"Progreso: {idx}/{total} ({pct}%) - radicado {radicado}")
|
|
|
|
csv_path = self.view.get_csv_path() or None
|
|
|
|
# Primera ejecución: si fields es None, el servicio devolverá _needs_fields
|
|
if doc_types:
|
|
resultados = self.service.process_root_with_field_selection(
|
|
root_path=root,
|
|
doc_types=doc_types,
|
|
fields=fields,
|
|
on_progress=on_progress,
|
|
csv_output_path=csv_path,
|
|
tiff_page_range=tiff_page_range,
|
|
structure_mode=structure_mode,
|
|
)
|
|
else:
|
|
resultados = self.service.process_root_with_field_selection(
|
|
root_path=root,
|
|
keywords=keywords,
|
|
fields=fields,
|
|
on_progress=on_progress,
|
|
csv_output_path=csv_path,
|
|
tiff_page_range=tiff_page_range,
|
|
structure_mode=structure_mode,
|
|
)
|
|
|
|
# Si requiere selección de campos abrir diálogo; de lo contrario, mostrar resultados
|
|
if resultados and isinstance(resultados, list) and isinstance(resultados[0], dict) and resultados[0].get('_needs_fields'):
|
|
schema = resultados[0].get('schema')
|
|
sample = resultados[0].get('first_full')
|
|
first_pdf_path = resultados[0].get('_source_pdf') or ""
|
|
schema_by_type = resultados[0].get('schema_by_type')
|
|
first_full_by_type = resultados[0].get('first_full_by_type')
|
|
first_radicado = resultados[0].get('first_radicado') or ""
|
|
combined_schema = None
|
|
combined_sample = None
|
|
if isinstance(schema_by_type, dict) and schema_by_type:
|
|
combined_schema = {}
|
|
try:
|
|
for tname, sch in schema_by_type.items():
|
|
if not isinstance(sch, dict):
|
|
continue
|
|
for path, typ in sch.items():
|
|
combined_schema[f"{tname}/{path}"] = typ
|
|
except Exception:
|
|
combined_schema = {}
|
|
# Construir muestra combinada para el botón "Ver muestra"
|
|
try:
|
|
combined_sample = {"radicado": first_radicado}
|
|
if isinstance(first_full_by_type, dict):
|
|
for tname, s in first_full_by_type.items():
|
|
combined_sample[tname] = s or {}
|
|
except Exception:
|
|
combined_sample = None
|
|
|
|
def on_submit(selected_fields, alias_map):
|
|
if not selected_fields:
|
|
self.view.append_output_safe("No se seleccionaron campos.")
|
|
self.view.set_processing(False)
|
|
return
|
|
# Estimar tokens/costos para toda la carpeta en hilo de fondo y pedir confirmación en hilo principal
|
|
def do_estimate_and_prompt():
|
|
try:
|
|
if doc_types and schema_by_type:
|
|
est = self.service.estimate_tokens_and_cost(
|
|
root_path=root,
|
|
doc_types=doc_types,
|
|
schema_by_type=schema_by_type,
|
|
selected_paths=selected_fields,
|
|
structure_mode=structure_mode,
|
|
)
|
|
else:
|
|
est = self.service.estimate_tokens_and_cost(
|
|
root_path=root,
|
|
keywords=keywords,
|
|
schema=schema,
|
|
selected_fields=selected_fields,
|
|
structure_mode=structure_mode,
|
|
)
|
|
total_docs = est.get("total_docs", 0)
|
|
per_doc_cost = est.get("per_doc_cost", {}) or {}
|
|
estimated_total = est.get("estimated_total_cost", {}) or {}
|
|
msg = (
|
|
f"Se procesarán {total_docs} documentos.\n\n"
|
|
f"Costo estimado por documento (USD): {per_doc_cost.get('total_usd', 'N/D')}\n"
|
|
f"Costo estimado total (USD): {estimated_total.get('total_usd', 'N/D')}\n\n"
|
|
f"¿Desea continuar?"
|
|
)
|
|
except Exception as e_est:
|
|
msg = f"No se pudo calcular la estimación ({e_est}). ¿Desea continuar sin estimación?"
|
|
|
|
def prompt_and_continue():
|
|
ok = mb.askyesno("Confirmar estimación", msg)
|
|
if not ok:
|
|
self.view.set_processing(False)
|
|
return
|
|
# Evitar doble arranque
|
|
if self._worker2_running:
|
|
return
|
|
self._worker2_running = True
|
|
# Si acepta, lanzar procesamiento en otro hilo
|
|
threading.Thread(target=worker2, daemon=True).start()
|
|
|
|
# Volver al hilo principal para mostrar el cuadro de diálogo
|
|
self.view.after(0, prompt_and_continue)
|
|
|
|
threading.Thread(target=do_estimate_and_prompt, daemon=True).start()
|
|
def worker2():
|
|
try:
|
|
loop = asyncio.new_event_loop()
|
|
asyncio.set_event_loop(loop)
|
|
|
|
async def on_error_retry(error):
|
|
# Este callback corre en el loop de asyncio (worker thread)
|
|
future = loop.create_future()
|
|
|
|
def ask():
|
|
msg = (
|
|
f"⚠️ Error de Cuota o Límite (429):\n\n{error}\n\n"
|
|
"¿Desea PAUSAR el proceso, añadir fondos/esperar y REINTENTAR este radicado?\n\n"
|
|
"Si selecciona 'No', el proceso se detendrá y se guardará lo avanzado hasta ahora."
|
|
)
|
|
res = mb.askyesno("Límite de Cuota Excedido", msg)
|
|
# Regresar el resultado al loop de asyncio
|
|
loop.call_soon_threadsafe(future.set_result, res)
|
|
|
|
self.view.after(0, ask)
|
|
return await future
|
|
|
|
# callback para uso por petición
|
|
def on_usage(phase, radicado, usage, cost):
|
|
try:
|
|
cached = 0
|
|
if isinstance(usage, dict):
|
|
pt = int(usage.get("prompt_tokens") or 0)
|
|
ct = int(usage.get("completion_tokens") or 0)
|
|
details = usage.get("prompt_tokens_details") or {}
|
|
cached = int(details.get("cached_tokens") or 0)
|
|
self._total_prompt_tokens += pt
|
|
self._total_cached_tokens += cached
|
|
self._total_completion_tokens += ct
|
|
if isinstance(cost, dict):
|
|
self._total_usd += float(cost.get("total_usd") or 0.0)
|
|
# Registrar histórico
|
|
per_cost = float((cost or {}).get("total_usd") or 0.0)
|
|
self._hist_requests.append({
|
|
"radicado": radicado, "phase": phase,
|
|
"prompt": int((usage or {}).get("prompt_tokens") or 0),
|
|
"cached": cached,
|
|
"completion": int((usage or {}).get("completion_tokens") or 0),
|
|
"cost_usd": per_cost,
|
|
})
|
|
# Actualizar per-doc y graficar cuando tengamos ambas fases
|
|
bucket = self._per_doc_costs.get(radicado) or {"extract": None, "normalize": None}
|
|
bucket[phase] = per_cost
|
|
self._per_doc_costs[radicado] = bucket
|
|
if bucket.get("extract") is not None and bucket.get("normalize") is not None:
|
|
self._completed_doc_count += 1
|
|
total_cost_doc = float(bucket["extract"] or 0.0) + float(bucket["normalize"] or 0.0)
|
|
# actualizar línea en hilo principal
|
|
self.view.after(0, lambda idx=self._completed_doc_count, val=total_cost_doc: self.view.update_cost_line(idx, val))
|
|
# Actualizar totales por fase
|
|
if phase in self._phase_tokens:
|
|
self._phase_tokens[phase]["input"] += int((usage or {}).get("prompt_tokens") or 0)
|
|
self._phase_tokens[phase]["cached"] += cached
|
|
self._phase_tokens[phase]["output"] += int((usage or {}).get("completion_tokens") or 0)
|
|
self._phase_tokens[phase]["usd"] += per_cost
|
|
# Actualizar barras por fase
|
|
ex = self._phase_tokens["extract"]
|
|
nz = self._phase_tokens["normalize"]
|
|
self.view.after(0, lambda: self.view.update_token_bars(
|
|
ex["input"], ex["cached"], ex["output"], nz["input"], nz["cached"], nz["output"]
|
|
))
|
|
# Actualizar KPIs
|
|
overall = {
|
|
"input": self._total_prompt_tokens,
|
|
"cached": self._total_cached_tokens,
|
|
"output": self._total_completion_tokens,
|
|
"usd": self._total_usd,
|
|
}
|
|
self.view.after(0, lambda ov=overall, exd=ex.copy(), nzd=nz.copy(): self.view.update_kpis(ov, exd, nzd))
|
|
# Añadir fila textual
|
|
self.view.after(0, lambda p=phase, r=radicado, u=usage, c=cost: self.view.append_usage_row(p, r, u, c))
|
|
# actualizar totales
|
|
self.view.set_usage_totals(
|
|
prompt_tokens=self._total_prompt_tokens,
|
|
cached_tokens=self._total_cached_tokens,
|
|
completion_tokens=self._total_completion_tokens,
|
|
total_usd=self._total_usd,
|
|
)
|
|
except Exception:
|
|
pass
|
|
if doc_types:
|
|
res = self.service.process_root_with_field_selection(
|
|
root_path=root,
|
|
doc_types=doc_types,
|
|
fields=selected_fields,
|
|
on_progress=on_progress,
|
|
csv_output_path=csv_path,
|
|
on_usage=on_usage,
|
|
on_error_retry=on_error_retry,
|
|
alias_map=alias_map,
|
|
tiff_page_range=tiff_page_range,
|
|
structure_mode=structure_mode,
|
|
)
|
|
else:
|
|
res = self.service.process_root_with_field_selection(
|
|
root_path=root,
|
|
keywords=keywords,
|
|
fields=selected_fields,
|
|
on_progress=on_progress,
|
|
csv_output_path=csv_path,
|
|
on_usage=on_usage,
|
|
on_error_retry=on_error_retry,
|
|
alias_map=alias_map,
|
|
tiff_page_range=tiff_page_range,
|
|
structure_mode=structure_mode,
|
|
)
|
|
self.view.append_output_safe(json.dumps(res, ensure_ascii=False, indent=2))
|
|
except Exception as e2:
|
|
logging.exception("Error en reproceso")
|
|
self.view.append_output_safe(f"Error: {e2}")
|
|
finally:
|
|
# Liberar guard al finalizar
|
|
self._worker2_running = False
|
|
self.view.set_processing(False)
|
|
self.view.after(0, lambda: mb.showinfo("Proceso Finalizado", "El procesamiento ha finalizado."))
|
|
|
|
# Preparar callback de reprocesamiento
|
|
reprocess_cb = None
|
|
if doc_types:
|
|
def _reprocess_multi():
|
|
s_by_type, f_full_by_type, f_rad = self.service.build_schema_from_first_for_types(
|
|
root, doc_types, tiff_page_range=tiff_page_range, structure_mode=structure_mode
|
|
)
|
|
c_schema = {}
|
|
try:
|
|
for tname, sch in s_by_type.items():
|
|
if not isinstance(sch, dict): continue
|
|
for p, typ in sch.items():
|
|
c_schema[f"{tname}/{p}"] = typ
|
|
except Exception: pass
|
|
|
|
c_sample = {"radicado": f_rad}
|
|
try:
|
|
for tname, s in f_full_by_type.items():
|
|
c_sample[tname] = s or {}
|
|
except Exception: pass
|
|
return c_schema, c_sample
|
|
reprocess_cb = _reprocess_multi
|
|
elif first_pdf_path:
|
|
def _reprocess_single():
|
|
return self.service.build_schema_from_first(first_pdf_path, tiff_page_range=tiff_page_range)
|
|
reprocess_cb = _reprocess_single
|
|
|
|
# Asegurar que siempre pasamos un schema dict (aunque sea vacío) al selector
|
|
effective_schema = combined_schema if combined_schema is not None else (schema or {})
|
|
effective_sample = combined_sample if combined_sample is not None else sample
|
|
|
|
self.view.after(0, lambda: self._open_field_selector(effective_schema, effective_sample, on_submit, reprocess_cb))
|
|
else:
|
|
self.view.append_output_safe(json.dumps(resultados, ensure_ascii=False, indent=2))
|
|
except Exception as e:
|
|
logger.exception("Error en procesamiento")
|
|
self.view.append_output_safe(f"Error: {e}")
|
|
finally:
|
|
# Si se abrió selector, se reactivará al terminar worker2
|
|
if not (isinstance(resultados, list) and resultados and isinstance(resultados[0], dict) and resultados[0].get('_needs_fields')):
|
|
self.view.set_processing(False)
|
|
self.view.after(0, lambda: mb.showinfo("Proceso Finalizado", "El procesamiento ha finalizado."))
|
|
|
|
self.view.set_processing(True)
|
|
threading.Thread(target=worker, daemon=True).start()
|
|
|
|
def _open_field_selector(self, schema, sample, on_submit, on_reprocess=None):
|
|
"""Abre el diálogo de selección de campos.
|
|
|
|
Args:
|
|
schema: Mapa path->tipo que ofrece el servicio para ayudar a seleccionar.
|
|
sample: JSON de muestra del primer documento.
|
|
on_submit: Callback `(selected_fields, alias_map)` a invocar al aceptar.
|
|
on_reprocess: Callback opcional para reprocesar muestra y schema.
|
|
"""
|
|
dlg = FieldSelectorDialog(self.view, schema=schema, sample=sample)
|
|
dlg.set_on_submit(on_submit)
|
|
if on_reprocess:
|
|
dlg.set_on_reprocess(on_reprocess)
|