1343 lines
53 KiB
Python
1343 lines
53 KiB
Python
"""Controlador de la aplicación GUI: coordina vista y servicio.
|
|
|
|
Gestiona selección de carpetas, ejecución en hilo de procesamiento, selector
|
|
interactivo de campos y actualización de configuración del extractor.
|
|
"""
|
|
import json
|
|
import logging
|
|
import threading
|
|
import asyncio
|
|
import tkinter.filedialog as fd
|
|
import tkinter.messagebox as mb
|
|
import os
|
|
|
|
from core.utils import parse_comma_separated, collect_radicado_docs, collect_documents_flat
|
|
from application.logging_config import configure_logging
|
|
from application.app_prefs import load_app_prefs, save_app_prefs, normalize_document_types
|
|
from gui.field_selector import FieldSelectorDialog
|
|
from application.preclassification import PreclassificationService
|
|
from application.strategies import OllamaExtractor
|
|
from application.database import DatabaseManager
|
|
from application.config import settings
|
|
from application.costs import MODEL_PRICING
|
|
from application import postprocess_agent
|
|
from application import csv_cleaning
|
|
from openai import OpenAI
|
|
import pandas as pd
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class AppController:
|
|
"""Controlador principal de la GUI que orquesta la vista y el servicio.
|
|
|
|
Se encarga de:
|
|
- Vincular callbacks de la vista.
|
|
- Disparar el flujo de procesamiento de facturas.
|
|
- Abrir el diálogo de selección de campos cuando el servicio lo solicite.
|
|
"""
|
|
def __init__(self, view, service):
|
|
"""Inicializa el controlador y vincula handlers de la vista.
|
|
|
|
Args:
|
|
view: Instancia de vista (por ejemplo, `InvoiceAppView`).
|
|
service: Servicio de facturas compatible con `process_root_with_field_selection`.
|
|
"""
|
|
# asegurar logging configurado
|
|
try:
|
|
configure_logging()
|
|
except Exception:
|
|
pass
|
|
self.view = view
|
|
self.service = service
|
|
# Acumuladores de uso
|
|
self._total_prompt_tokens = 0
|
|
self._total_cached_tokens = 0
|
|
self._total_completion_tokens = 0
|
|
self._total_usd = 0.0
|
|
# Estado para dashboard
|
|
self._hist_requests = [] # list[dict]: {radicado, phase, prompt, cached, completion, cost_usd}
|
|
self._per_doc_costs = {} # radicado -> {"extract": float|None, "normalize": float|None}
|
|
self._completed_doc_count = 0
|
|
self._phase_tokens = {
|
|
"extract": {"input": 0, "cached": 0, "output": 0, "usd": 0.0},
|
|
"normalize": {"input": 0, "cached": 0, "output": 0, "usd": 0.0},
|
|
}
|
|
# Guard para evitar doble lanzamiento de worker2 (por doble click o condiciones de carrera)
|
|
self._worker2_running = False
|
|
|
|
# Estado Preclasificación
|
|
self._last_run_id = None
|
|
self.db_manager = DatabaseManager() # Para pestaña resultados
|
|
|
|
self.view.set_on_select_root(self.on_select_root)
|
|
try:
|
|
self.view.set_on_select_preclass_root(self.on_select_preclass_root)
|
|
self.view.set_on_run_preclassification(self.on_run_preclassification)
|
|
self.view.set_on_export_excel(self.on_export_excel)
|
|
self.view.set_on_organize_files(self.on_organize_files)
|
|
|
|
# Resultados
|
|
self.view.set_on_res_search(self.on_res_search)
|
|
self.view.set_on_res_export(self.on_res_export)
|
|
self.view.set_on_res_refresh(self.on_res_refresh)
|
|
self.view.set_on_res_organize(self.on_res_organize_run)
|
|
self.view.set_on_res_select(self.on_res_select)
|
|
|
|
# Carga inicial
|
|
self.on_res_refresh()
|
|
except Exception:
|
|
pass
|
|
self.view.set_on_process(self.on_process)
|
|
self.view.set_on_select_csv(self.on_select_csv)
|
|
self.view.set_on_save_settings(self.on_save_settings)
|
|
|
|
# Post-procesamiento
|
|
self._pp_df: pd.DataFrame | None = None
|
|
self._pp_result_df: pd.DataFrame | None = None
|
|
self._pp_csv_sample: str = ""
|
|
self._pp_client = OpenAI(api_key=settings.api.api_key, base_url=settings.api.base_url)
|
|
# Métricas acumuladas de post-procesamiento
|
|
self._pp_total_usd = 0.0
|
|
self._pp_total_calls = 0
|
|
self._pp_total_retries = 0
|
|
self._pp_total_input_tokens = 0
|
|
self._pp_total_output_tokens = 0
|
|
try:
|
|
self.view.set_on_postprocess_select_csv(self.on_pp_select_csv)
|
|
self.view.set_on_postprocess_run(self.on_pp_run)
|
|
self.view.set_on_postprocess_save(self.on_pp_save)
|
|
except Exception:
|
|
pass
|
|
|
|
# Proceso de limpieza (sin LLM)
|
|
self._cleaning_df: pd.DataFrame | None = None
|
|
self._cleaning_result_df: pd.DataFrame | None = None
|
|
try:
|
|
self.view.set_on_cleaning_select_csv(self.on_cleaning_select_csv)
|
|
self.view.set_on_cleaning_apply(self.on_cleaning_apply)
|
|
self.view.set_on_cleaning_save(self.on_cleaning_save)
|
|
except Exception:
|
|
pass
|
|
|
|
# Mapeo de nombres de UI a IDs de modelo (y viceversa)
|
|
self.ui_to_model = {
|
|
"5.4": "gpt-5.4",
|
|
"5.4-mini": "gpt-5.4-mini",
|
|
"5.4-nano": "gpt-5.4-nano",
|
|
"5.2": "gpt-5.2",
|
|
"4.1-mini": "gpt-4.1-mini",
|
|
}
|
|
self.model_to_ui = {v: k for k, v in self.ui_to_model.items()}
|
|
|
|
# Inicializar vista con configuración actual
|
|
try:
|
|
# Llenar tabla de precios de modelos
|
|
pricing_data = self._get_model_pricing_data()
|
|
self.view.update_comparison_table(pricing_data)
|
|
|
|
current_model = getattr(self.service.extractor, "model", "")
|
|
# Intentar mapear modelo backend -> nombre UI, si no existe usar raw
|
|
ui_name = self.model_to_ui.get(current_model, current_model)
|
|
if ui_name:
|
|
self.view.set_selected_model(ui_name)
|
|
|
|
# Cargar concurrencia
|
|
self.view.set_concurrency_limit(settings.processing.concurrency_limit)
|
|
except Exception:
|
|
pass
|
|
|
|
self._load_persisted_preferences()
|
|
try:
|
|
self.view.protocol("WM_DELETE_WINDOW", self.on_app_close)
|
|
except Exception:
|
|
pass
|
|
|
|
def _get_model_pricing_data(self):
|
|
"""Obtiene las tarifas por 1M de tokens para cada modelo."""
|
|
results = []
|
|
for model_id, rates in MODEL_PRICING.items():
|
|
results.append({
|
|
"model": self.model_to_ui.get(model_id, model_id),
|
|
"input": rates["input"],
|
|
"cached_input": rates["cached_input"],
|
|
"output": rates["output"]
|
|
})
|
|
# Ordenar por precio de input descendente
|
|
results.sort(key=lambda x: x["input"], reverse=True)
|
|
return results
|
|
|
|
def on_save_settings(self):
|
|
"""Actualiza la configuración del servicio (modelo LLM y concurrencia)."""
|
|
selected_name = self.view.get_selected_model()
|
|
concurrency = self.view.get_concurrency_limit()
|
|
|
|
if not selected_name:
|
|
mb.showwarning("Configuración", "El modelo no puede estar vacío.")
|
|
return
|
|
|
|
if concurrency <= 0:
|
|
mb.showwarning("Configuración", "El límite de concurrencia debe ser mayor a 0.")
|
|
return
|
|
|
|
# Traducir nombre UI -> ID modelo real
|
|
real_model_id = self.ui_to_model.get(selected_name, selected_name)
|
|
|
|
try:
|
|
# Actualizar extractor
|
|
if hasattr(self.service.extractor, "update_config"):
|
|
self.service.extractor.update_config(model=real_model_id)
|
|
|
|
# Actualizar settings globales
|
|
settings.processing.concurrency_limit = concurrency
|
|
self.view.set_allowed_document_types_hint(self.view.get_allowed_document_types())
|
|
self._save_preferences()
|
|
|
|
mb.showinfo("Configuración", f"Configuración actualizada.\nModelo: {selected_name}\nConcurrencia: {concurrency}")
|
|
self.view.append_output_safe(f"Configuración actualizada: Modelo={real_model_id}, Concurrencia={concurrency}")
|
|
except Exception as e:
|
|
logger.exception("Error actualizando configuración")
|
|
mb.showerror("Error", f"No se pudo actualizar la configuración: {e}")
|
|
|
|
def on_select_root(self):
|
|
"""Abre un selector de carpeta y establece la ruta raíz en la vista.
|
|
|
|
Además, si el primer documento detectado es TIFF, muestra los controles
|
|
de rango TIFF para que el usuario pueda ajustar el índice antes de procesar.
|
|
"""
|
|
selected = fd.askdirectory()
|
|
if not selected:
|
|
return
|
|
self.view.set_root_path(selected)
|
|
self._save_preferences()
|
|
|
|
# Detectar si el primer documento a procesar es TIFF para esta carpeta
|
|
try:
|
|
keywords_str = self.view.get_keywords()
|
|
keywords = parse_comma_separated(keywords_str) or ["Factura_O_Documento_Equivalente", "factura"]
|
|
except Exception:
|
|
keywords = ["Factura_O_Documento_Equivalente", "factura"]
|
|
has_tiff_first = False
|
|
try:
|
|
preview_entries = collect_radicado_docs(selected, keywords)
|
|
if preview_entries:
|
|
first_path = str(preview_entries[0].get("path") or "").lower()
|
|
has_tiff_first = first_path.endswith((".tif", ".tiff"))
|
|
except Exception:
|
|
has_tiff_first = False
|
|
try:
|
|
self.view.show_tiff_range_controls(has_tiff_first)
|
|
except Exception:
|
|
pass
|
|
|
|
def on_select_preclass_root(self):
|
|
"""Abre un selector de carpeta y establece la ruta raíz para preclasificación."""
|
|
selected = fd.askdirectory()
|
|
if not selected:
|
|
return
|
|
self.view.set_preclass_root_path(selected)
|
|
|
|
def on_run_preclassification(self):
|
|
"""Ejecuta la preclasificación en un hilo separado."""
|
|
# 1. Leer inputs de la UI
|
|
root = self.view.get_preclass_root_path()
|
|
patterns = self.view.get_preclass_patterns()
|
|
target_info = self.view.get_preclass_target_info()
|
|
page_range = self.view.get_preclass_range()
|
|
mode = self.view.get_preclass_mode()
|
|
|
|
# Validaciones básicas
|
|
if not root or not os.path.isdir(root):
|
|
mb.showwarning("Error", "Seleccione una carpeta raíz válida.")
|
|
return
|
|
|
|
if not patterns:
|
|
mb.showwarning("Error", "Debe agregar al menos un patrón.")
|
|
return
|
|
|
|
if target_info["enabled"] and not target_info["pattern"]:
|
|
mb.showwarning("Error", "Si habilita archivo objetivo, debe especificar un patrón de archivo.")
|
|
return
|
|
|
|
# Bloquear UI
|
|
self.view.set_processing(True)
|
|
self.view.enable_preclass_actions(False)
|
|
self.view.append_preclass_output("Iniciando preclasificación...")
|
|
self.view.append_preclass_output(f"Raíz: {root}")
|
|
self.view.append_preclass_output(f"Modo: {mode}")
|
|
self.view.append_preclass_output(f"Patrones: {len(patterns)}")
|
|
if target_info["enabled"]:
|
|
self.view.append_preclass_output(f"Filtro: {target_info['pattern']}")
|
|
|
|
# Instanciar servicio (usando el extractor actual)
|
|
# Nota: PreclassificationService maneja su propia instancia de DB
|
|
pre_service = PreclassificationService(self.service.extractor)
|
|
|
|
def worker():
|
|
try:
|
|
def on_progress(radicado, idx, total):
|
|
pct = int((idx / total) * 100)
|
|
msg = f"[{idx}/{total}] Procesando radicado: {radicado} ({pct}%)"
|
|
self.view.append_preclass_output_safe(msg)
|
|
|
|
run_id = pre_service.run_preclassification(
|
|
root_path=root,
|
|
mode=mode,
|
|
patterns=patterns,
|
|
target_info=target_info,
|
|
page_range=page_range,
|
|
on_progress=on_progress
|
|
)
|
|
|
|
self.view.append_preclass_output_safe(f"\n--- Ejecución finalizada (ID: {run_id}) ---")
|
|
|
|
# Guardar ID
|
|
self._last_run_id = run_id
|
|
def _refresh_results_tab():
|
|
# Recargar runs, seleccionar el run recién ejecutado y listar sus resultados
|
|
try:
|
|
self.on_res_refresh()
|
|
self.view.set_res_run(run_id)
|
|
self.on_res_search()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
self.view.enable_preclass_actions(True)
|
|
except Exception:
|
|
pass
|
|
self.view.after(0, _refresh_results_tab)
|
|
|
|
# Mostrar resumen rápido
|
|
# Usar tabla results (1 fila = 1 documento), no el join con matches
|
|
docs, _ = self.db_manager.search_results(
|
|
run_id=run_id, status="Todos", text=None, relax_if_empty=False
|
|
)
|
|
match_count = sum(1 for r in docs if r.get("status") == "MATCH")
|
|
self.view.append_preclass_output_safe(f"Total procesados: {len(docs)}")
|
|
self.view.append_preclass_output_safe(f"Coincidencias (MATCH): {match_count}")
|
|
|
|
# Detalle opcional
|
|
for res in docs[:10]: # mostrar primeros 10
|
|
status_icon = "✅" if res.get("status") == "MATCH" else "❌"
|
|
self.view.append_preclass_output_safe(f"{status_icon} {res.get('radicado')} ({res.get('status')})")
|
|
|
|
if len(docs) > 10:
|
|
self.view.append_preclass_output_safe("... ver base de datos para más detalles.")
|
|
|
|
self.view.after(0, lambda: mb.showinfo("Completado", f"Preclasificación finalizada.\nCoincidencias: {match_count}/{len(docs)}"))
|
|
|
|
except Exception as e:
|
|
logger.exception("Error en preclasificación")
|
|
self.view.append_preclass_output_safe(f"ERROR CRÍTICO: {str(e)}")
|
|
self.view.after(0, lambda: mb.showerror("Error", f"Ocurrió un error: {e}"))
|
|
finally:
|
|
self.view.set_processing(False)
|
|
|
|
threading.Thread(target=worker, daemon=True).start()
|
|
|
|
def on_export_excel(self):
|
|
"""Exporta los resultados de la última preclasificación a Excel."""
|
|
if not self._last_run_id:
|
|
mb.showwarning("Aviso", "No hay resultados de preclasificación recientes para exportar.")
|
|
return
|
|
|
|
path = fd.asksaveasfilename(
|
|
defaultextension=".xlsx",
|
|
filetypes=[("Excel", "*.xlsx"), ("CSV", "*.csv"), ("Todos", "*.*")]
|
|
)
|
|
if not path:
|
|
return
|
|
|
|
try:
|
|
pre_service = PreclassificationService(self.service.extractor)
|
|
ok = pre_service.export_to_excel(self._last_run_id, path)
|
|
if ok:
|
|
mb.showinfo("Exportar", "Archivo exportado correctamente.")
|
|
else:
|
|
mb.showerror("Exportar", "No se encontraron datos para exportar.")
|
|
except Exception as e:
|
|
logger.exception("Error exportando excel")
|
|
mb.showerror("Error", f"Error al exportar: {e}")
|
|
|
|
def on_organize_files(self):
|
|
"""Organiza los archivos MATCH en carpetas (Run actual)."""
|
|
if not self._last_run_id:
|
|
mb.showwarning("Aviso", "No hay resultados para organizar.")
|
|
return
|
|
self._organize_run_generic(self._last_run_id)
|
|
|
|
def _organize_run_generic(self, run_id: int):
|
|
target_dir = fd.askdirectory(title="Seleccionar carpeta de destino para organizar")
|
|
if not target_dir:
|
|
return
|
|
|
|
if not mb.askyesno("Confirmar", "¿Está seguro de copiar y organizar los archivos en la carpeta seleccionada?\nEsta acción creará una estructura de carpetas basada en los datos extraídos."):
|
|
return
|
|
|
|
try:
|
|
# Usamos append_preclass_output si estamos en esa pestaña, pero desde Resultados?
|
|
# El metodo de view inserta en el widget de log. Si estamos en tab Resultados, no se ve.
|
|
# Pero no importa, mostramos mb.showinfo al final.
|
|
pre_service = PreclassificationService(self.service.extractor)
|
|
ok_count, err_count = pre_service.organize_files(run_id, target_dir)
|
|
|
|
msg = f"Organización finalizada.\nCopiados exitosos: {ok_count}\nErrores: {err_count}"
|
|
mb.showinfo("Organizar", msg)
|
|
except Exception as e:
|
|
logger.exception("Error organizando archivos")
|
|
mb.showerror("Error", f"Error al organizar: {e}")
|
|
|
|
# --- Pestaña Resultados ---
|
|
def on_res_select(self, event):
|
|
"""Muestra detalles del resultado seleccionado."""
|
|
item_vals = self.view.get_selected_result_values()
|
|
if not item_vals:
|
|
return
|
|
try:
|
|
res_id = int(item_vals[0])
|
|
matches = self.db_manager.get_matches_for_result(res_id)
|
|
self.view.update_res_matches_table(matches)
|
|
except Exception as e:
|
|
logger.error(f"Error cargando detalle: {e}")
|
|
|
|
def on_res_organize_run(self):
|
|
"""Organiza archivos del Run seleccionado en el filtro."""
|
|
filters = self.view.get_res_filters()
|
|
run_val = filters["run_id"]
|
|
|
|
if run_val == "Todos" or not run_val:
|
|
mb.showwarning("Organizar", "Por favor seleccione un 'Run ID' específico en el filtro para organizar.")
|
|
return
|
|
|
|
try:
|
|
run_id = int(run_val)
|
|
self._organize_run_generic(run_id)
|
|
except ValueError:
|
|
mb.showerror("Error", "ID de Run inválido.")
|
|
|
|
def on_res_refresh(self):
|
|
"""Refresca la lista de Runs en el combo."""
|
|
try:
|
|
runs = self.db_manager.get_all_runs()
|
|
ids = [r["id"] for r in runs]
|
|
self.view.update_res_runs_combo(ids)
|
|
# Tambien refrescar tabla
|
|
self.on_res_search()
|
|
except Exception as e:
|
|
logger.error(f"Error refrescando runs: {e}")
|
|
|
|
def on_res_search(self):
|
|
"""Busca resultados con los filtros actuales."""
|
|
self.view.clear_res_filter_note()
|
|
filters = self.view.get_res_filters()
|
|
run_id = None
|
|
if filters["run_id"] != "Todos":
|
|
try:
|
|
run_id = int(filters["run_id"])
|
|
except: pass
|
|
|
|
try:
|
|
rows, relax_note = self.db_manager.search_results(
|
|
run_id=run_id,
|
|
status=filters["status"],
|
|
text=filters["text"],
|
|
relax_if_empty=bool(filters.get("relax_on_empty", False)),
|
|
)
|
|
self.view.update_res_table(rows)
|
|
if relax_note:
|
|
self.view.set_res_filter_note(relax_note)
|
|
except Exception as e:
|
|
logger.error(f"Error buscando resultados: {e}")
|
|
mb.showerror("Error", f"Error en búsqueda: {e}")
|
|
|
|
def on_res_export(self):
|
|
"""Exporta la vista actual de resultados a Excel."""
|
|
path = fd.asksaveasfilename(
|
|
defaultextension=".xlsx",
|
|
filetypes=[("Excel", "*.xlsx"), ("CSV", "*.csv")]
|
|
)
|
|
if not path:
|
|
return
|
|
|
|
# Obtener los datos actuales (repite la busqueda)
|
|
# Podriamos optimizar guardando el ultimo result set
|
|
try:
|
|
filters = self.view.get_res_filters()
|
|
run_id = None
|
|
if filters["run_id"] != "Todos":
|
|
try: run_id = int(filters["run_id"])
|
|
except: pass
|
|
|
|
rows, _ = self.db_manager.search_results(
|
|
run_id=run_id,
|
|
status=filters["status"],
|
|
text=filters["text"],
|
|
relax_if_empty=bool(filters.get("relax_on_empty", False)),
|
|
)
|
|
|
|
if not rows:
|
|
mb.showwarning("Exportar", "No hay datos para exportar.")
|
|
return
|
|
|
|
df = pd.DataFrame(rows)
|
|
if path.endswith(".csv"):
|
|
df.to_csv(path, index=False, sep=";")
|
|
else:
|
|
df.to_excel(path, index=False)
|
|
|
|
mb.showinfo("Exportar", "Datos exportados correctamente.")
|
|
except Exception as e:
|
|
logger.exception("Error exportando vista")
|
|
mb.showerror("Error", f"Error al exportar: {e}")
|
|
|
|
def on_select_csv(self):
|
|
"""Abre un diálogo de guardado y establece la ruta del CSV de salida."""
|
|
selected = fd.asksaveasfilename(defaultextension=".csv", filetypes=[("CSV", "*.csv"), ("Todos", "*.*")])
|
|
if selected:
|
|
self.view.set_csv_path(selected)
|
|
self._save_preferences()
|
|
|
|
def _collect_preferences(self) -> dict:
|
|
"""Toma una foto de la configuración visible de la app."""
|
|
try:
|
|
doc_types = self.view.get_doc_types()
|
|
except Exception:
|
|
doc_types = []
|
|
|
|
return {
|
|
"selected_model": self.view.get_selected_model(),
|
|
"concurrency_limit": self.view.get_concurrency_limit(),
|
|
"allowed_document_types": self.view.get_allowed_document_types(),
|
|
"retry_on_empty": self.view.get_retry_empty_enabled(),
|
|
"process_mode": self.view.get_process_mode(),
|
|
"root_path": self.view.get_root_path(),
|
|
"csv_path": self.view.get_csv_path(),
|
|
"doc_types": doc_types,
|
|
}
|
|
|
|
def _save_preferences(self) -> None:
|
|
"""Guarda preferencias de la GUI en disco."""
|
|
save_app_prefs(self._collect_preferences())
|
|
|
|
def _load_persisted_preferences(self) -> None:
|
|
"""Carga preferencias persistidas y las aplica a la vista/servicio."""
|
|
prefs = load_app_prefs()
|
|
if not prefs:
|
|
return
|
|
|
|
try:
|
|
selected_model = str(prefs.get("selected_model") or "").strip()
|
|
if selected_model:
|
|
self.view.set_selected_model(selected_model)
|
|
real_model_id = self.ui_to_model.get(selected_model, selected_model)
|
|
if hasattr(self.service.extractor, "update_config"):
|
|
self.service.extractor.update_config(model=real_model_id)
|
|
except Exception:
|
|
logger.exception("No se pudo restaurar el modelo persistido")
|
|
|
|
try:
|
|
concurrency = int(prefs.get("concurrency_limit") or settings.processing.concurrency_limit)
|
|
self.view.set_concurrency_limit(concurrency)
|
|
settings.processing.concurrency_limit = concurrency
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
allowed_document_types = normalize_document_types(prefs.get("allowed_document_types"))
|
|
self.view.set_allowed_document_types(allowed_document_types)
|
|
self.view.set_allowed_document_types_hint(allowed_document_types)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
self.view.set_retry_empty_enabled(bool(prefs.get("retry_on_empty", False)))
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
self.view.set_process_mode(str(prefs.get("process_mode") or "radicados"))
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
root_path = str(prefs.get("root_path") or "").strip()
|
|
if root_path:
|
|
self.view.set_root_path(root_path)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
csv_path = str(prefs.get("csv_path") or "").strip()
|
|
if csv_path:
|
|
self.view.set_csv_path(csv_path)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
doc_types = prefs.get("doc_types")
|
|
if isinstance(doc_types, list):
|
|
self.view.set_doc_types(doc_types)
|
|
except Exception:
|
|
pass
|
|
|
|
def on_app_close(self):
|
|
"""Guarda preferencias antes de cerrar la ventana."""
|
|
try:
|
|
self._save_preferences()
|
|
finally:
|
|
self.view.destroy()
|
|
|
|
def on_process(self):
|
|
"""Inicia el flujo de procesamiento por radicado (solo facturas) en un hilo de fondo.
|
|
|
|
- Si el servicio requiere selección de campos, abre `FieldSelectorDialog` y relanza.
|
|
- Emite resultados en el área de salida y opcionalmente escribe CSV incremental.
|
|
"""
|
|
self._save_preferences()
|
|
root = self.view.get_root_path()
|
|
# Tipos de Documento (multi-doc)
|
|
doc_types = []
|
|
try:
|
|
doc_types = self.view.get_doc_types()
|
|
except Exception:
|
|
doc_types = []
|
|
if doc_types:
|
|
try:
|
|
ok, msg = self.view.validate_doc_types()
|
|
except Exception:
|
|
ok, msg = True, None
|
|
if not ok:
|
|
self.view.append_output_safe(msg or "Tipos de Documento inválidos.")
|
|
return
|
|
keywords_str = self.view.get_keywords()
|
|
fields_str = self.view.get_fields()
|
|
structure_mode = self.view.get_process_mode()
|
|
retry_on_empty = self.view.get_retry_empty_enabled()
|
|
|
|
# Filtro de radicados
|
|
radicado_filter = None
|
|
try:
|
|
if self.view.get_radicado_filter_enabled():
|
|
radicado_filter = self.view.get_radicado_filter_list() or None
|
|
except Exception:
|
|
radicado_filter = None
|
|
|
|
if structure_mode == "general":
|
|
# En modo general, si está vacío o es el default, procesar todo (None)
|
|
# Si el usuario escribió algo específico, usarlo.
|
|
kws = parse_comma_separated(keywords_str)
|
|
# Check if it matches default legacy keywords to avoid accidental filtering
|
|
if kws == ["Factura_O_Documento_Equivalente", "factura"]:
|
|
keywords = None
|
|
else:
|
|
keywords = kws if kws else None
|
|
else:
|
|
keywords = parse_comma_separated(keywords_str) or ["Factura_O_Documento_Equivalente", "factura"]
|
|
|
|
fields = parse_comma_separated(fields_str) if fields_str else None
|
|
|
|
# Validar ruta antes de deshabilitar la UI
|
|
if not root or not os.path.isdir(root):
|
|
self.view.append_output_safe(f"Error: La ruta raíz no es un directorio: {root or ''}")
|
|
return
|
|
|
|
# Si es modo general, ignorar doc_types
|
|
if structure_mode == "general":
|
|
doc_types = []
|
|
|
|
# Determinar si el primer documento a procesar es TIFF; si no lo es, ocultar
|
|
# los controles de rango TIFF y no aplicar ningún recorte.
|
|
tiff_page_range = None
|
|
has_tiff_first = False
|
|
try:
|
|
if structure_mode == "general":
|
|
preview_entries = collect_documents_flat(root, keywords)
|
|
else:
|
|
preview_entries = collect_radicado_docs(root, keywords)
|
|
|
|
if preview_entries:
|
|
first_path = str(preview_entries[0].get("path") or "").lower()
|
|
has_tiff_first = first_path.endswith((".tif", ".tiff"))
|
|
except Exception:
|
|
has_tiff_first = False
|
|
try:
|
|
# Mostrar/ocultar controles en función de si el primer doc es TIFF
|
|
self.view.show_tiff_range_controls(has_tiff_first)
|
|
except Exception:
|
|
pass
|
|
|
|
if has_tiff_first:
|
|
# Rango opcional de páginas TIFF (1-based; 0 = sin límite)
|
|
try:
|
|
tiff_start_raw = int(getattr(self.view, "get_tiff_start_page", lambda: 0)() or 0)
|
|
except Exception:
|
|
tiff_start_raw = 0
|
|
try:
|
|
tiff_end_raw = int(getattr(self.view, "get_tiff_end_page", lambda: 0)() or 0)
|
|
except Exception:
|
|
tiff_end_raw = 0
|
|
|
|
if tiff_start_raw < 0 or tiff_end_raw < 0:
|
|
self.view.append_output_safe("Error: Las páginas TIFF deben ser mayores o iguales a 0.")
|
|
return
|
|
|
|
tiff_start = tiff_start_raw if tiff_start_raw > 0 else None
|
|
tiff_end = tiff_end_raw if tiff_end_raw > 0 else None
|
|
if tiff_start is not None and tiff_end is not None and tiff_start > tiff_end:
|
|
self.view.append_output_safe("Error: La página de inicio TIFF no puede ser mayor que la página de fin.")
|
|
return
|
|
# Si ambos son 0 (None/None), el usuario quiere "todas las páginas":
|
|
# usamos el centinela (None, None) para indicar "no recortar TIFF".
|
|
if tiff_start is None and tiff_end is None:
|
|
tiff_page_range = (None, None)
|
|
else:
|
|
tiff_page_range = (tiff_start, tiff_end)
|
|
|
|
def worker():
|
|
resultados = None
|
|
if doc_types:
|
|
try:
|
|
self.view.append_output_safe(f"Procesando raíz: {root} | tipos: {[(dt.get('name'), dt.get('keyword')) for dt in doc_types]}")
|
|
except Exception:
|
|
self.view.append_output_safe(f"Procesando raíz: {root} | tipos: {len(doc_types)}")
|
|
else:
|
|
self.view.append_output_safe(f"Procesando raíz: {root} | keywords: {keywords}")
|
|
# limpiar pestaña Uso
|
|
try:
|
|
self._total_prompt_tokens = 0
|
|
self._total_cached_tokens = 0
|
|
self._total_completion_tokens = 0
|
|
self._total_usd = 0.0
|
|
self._hist_requests.clear()
|
|
self._per_doc_costs.clear()
|
|
self._completed_doc_count = 0
|
|
self._phase_tokens = {
|
|
"extract": {"input": 0, "cached": 0, "output": 0, "usd": 0.0},
|
|
"normalize": {"input": 0, "cached": 0, "output": 0, "usd": 0.0},
|
|
}
|
|
self.view.clear_usage()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
def on_progress(radicado, idx, total):
|
|
pct = int((idx / total) * 100)
|
|
self.view.append_output_safe(f"Progreso: {idx}/{total} ({pct}%) - radicado {radicado}")
|
|
|
|
csv_path = self.view.get_csv_path() or None
|
|
|
|
# Primera ejecución: si fields es None, el servicio devolverá _needs_fields
|
|
if doc_types:
|
|
resultados = self.service.process_root_with_field_selection(
|
|
root_path=root,
|
|
doc_types=doc_types,
|
|
fields=fields,
|
|
on_progress=on_progress,
|
|
csv_output_path=csv_path,
|
|
tiff_page_range=tiff_page_range,
|
|
structure_mode=structure_mode,
|
|
retry_on_empty=retry_on_empty,
|
|
radicado_filter=radicado_filter,
|
|
)
|
|
else:
|
|
resultados = self.service.process_root_with_field_selection(
|
|
root_path=root,
|
|
keywords=keywords,
|
|
fields=fields,
|
|
on_progress=on_progress,
|
|
csv_output_path=csv_path,
|
|
tiff_page_range=tiff_page_range,
|
|
structure_mode=structure_mode,
|
|
retry_on_empty=retry_on_empty,
|
|
radicado_filter=radicado_filter,
|
|
)
|
|
|
|
# Si requiere selección de campos abrir diálogo; de lo contrario, mostrar resultados
|
|
if resultados and isinstance(resultados, list) and isinstance(resultados[0], dict) and resultados[0].get('_needs_fields'):
|
|
schema = resultados[0].get('schema')
|
|
sample = resultados[0].get('first_full')
|
|
first_pdf_path = resultados[0].get('_source_pdf') or ""
|
|
schema_by_type = resultados[0].get('schema_by_type')
|
|
first_full_by_type = resultados[0].get('first_full_by_type')
|
|
first_radicado = resultados[0].get('first_radicado') or ""
|
|
combined_schema = None
|
|
combined_sample = None
|
|
if isinstance(schema_by_type, dict) and schema_by_type:
|
|
combined_schema = {}
|
|
try:
|
|
for tname, sch in schema_by_type.items():
|
|
if not isinstance(sch, dict):
|
|
continue
|
|
for path, typ in sch.items():
|
|
combined_schema[f"{tname}/{path}"] = typ
|
|
except Exception:
|
|
combined_schema = {}
|
|
# Construir muestra combinada para el botón "Ver muestra"
|
|
try:
|
|
combined_sample = {"radicado": first_radicado}
|
|
if isinstance(first_full_by_type, dict):
|
|
for tname, s in first_full_by_type.items():
|
|
combined_sample[tname] = s or {}
|
|
except Exception:
|
|
combined_sample = None
|
|
|
|
def on_submit(selected_fields, alias_map):
|
|
if not selected_fields:
|
|
self.view.append_output_safe("No se seleccionaron campos.")
|
|
self.view.set_processing(False)
|
|
return
|
|
# Estimar tokens/costos para toda la carpeta en hilo de fondo y pedir confirmación en hilo principal
|
|
def do_estimate_and_prompt():
|
|
try:
|
|
if doc_types and schema_by_type:
|
|
est = self.service.estimate_tokens_and_cost(
|
|
root_path=root,
|
|
doc_types=doc_types,
|
|
schema_by_type=schema_by_type,
|
|
selected_paths=selected_fields,
|
|
structure_mode=structure_mode,
|
|
)
|
|
else:
|
|
est = self.service.estimate_tokens_and_cost(
|
|
root_path=root,
|
|
keywords=keywords,
|
|
schema=schema,
|
|
selected_fields=selected_fields,
|
|
structure_mode=structure_mode,
|
|
)
|
|
total_docs = est.get("total_docs", 0)
|
|
per_doc_cost = est.get("per_doc_cost", {}) or {}
|
|
estimated_total = est.get("estimated_total_cost", {}) or {}
|
|
msg = (
|
|
f"Se procesarán {total_docs} documentos.\n\n"
|
|
f"Costo estimado por documento (USD): {per_doc_cost.get('total_usd', 'N/D')}\n"
|
|
f"Costo estimado total (USD): {estimated_total.get('total_usd', 'N/D')}\n\n"
|
|
f"¿Desea continuar?"
|
|
)
|
|
except Exception as e_est:
|
|
msg = f"No se pudo calcular la estimación ({e_est}). ¿Desea continuar sin estimación?"
|
|
|
|
def prompt_and_continue():
|
|
ok = mb.askyesno("Confirmar estimación", msg)
|
|
if not ok:
|
|
self.view.set_processing(False)
|
|
return
|
|
# Evitar doble arranque
|
|
if self._worker2_running:
|
|
return
|
|
self._worker2_running = True
|
|
# Si acepta, lanzar procesamiento en otro hilo
|
|
threading.Thread(target=worker2, daemon=True).start()
|
|
|
|
# Volver al hilo principal para mostrar el cuadro de diálogo
|
|
self.view.after(0, prompt_and_continue)
|
|
|
|
threading.Thread(target=do_estimate_and_prompt, daemon=True).start()
|
|
def worker2():
|
|
try:
|
|
loop = asyncio.new_event_loop()
|
|
asyncio.set_event_loop(loop)
|
|
|
|
async def on_error_retry(error):
|
|
# Este callback corre en el loop de asyncio (worker thread)
|
|
future = loop.create_future()
|
|
|
|
def ask():
|
|
msg = (
|
|
f"⚠️ Error de Cuota o Límite (429):\n\n{error}\n\n"
|
|
"¿Desea PAUSAR el proceso, añadir fondos/esperar y REINTENTAR este radicado?\n\n"
|
|
"Si selecciona 'No', el proceso se detendrá y se guardará lo avanzado hasta ahora."
|
|
)
|
|
res = mb.askyesno("Límite de Cuota Excedido", msg)
|
|
# Regresar el resultado al loop de asyncio
|
|
loop.call_soon_threadsafe(future.set_result, res)
|
|
|
|
self.view.after(0, ask)
|
|
return await future
|
|
|
|
# callback para uso por petición
|
|
def on_usage(phase, radicado, usage, cost):
|
|
try:
|
|
cached = 0
|
|
if isinstance(usage, dict):
|
|
pt = int(usage.get("prompt_tokens") or 0)
|
|
ct = int(usage.get("completion_tokens") or 0)
|
|
details = usage.get("prompt_tokens_details") or {}
|
|
cached = int(details.get("cached_tokens") or 0)
|
|
self._total_prompt_tokens += pt
|
|
self._total_cached_tokens += cached
|
|
self._total_completion_tokens += ct
|
|
if isinstance(cost, dict):
|
|
self._total_usd += float(cost.get("total_usd") or 0.0)
|
|
# Registrar histórico
|
|
per_cost = float((cost or {}).get("total_usd") or 0.0)
|
|
self._hist_requests.append({
|
|
"radicado": radicado, "phase": phase,
|
|
"prompt": int((usage or {}).get("prompt_tokens") or 0),
|
|
"cached": cached,
|
|
"completion": int((usage or {}).get("completion_tokens") or 0),
|
|
"cost_usd": per_cost,
|
|
})
|
|
# Actualizar per-doc y graficar cuando tengamos ambas fases
|
|
bucket = self._per_doc_costs.get(radicado) or {"extract": None, "normalize": None}
|
|
bucket[phase] = per_cost
|
|
self._per_doc_costs[radicado] = bucket
|
|
if bucket.get("extract") is not None and bucket.get("normalize") is not None:
|
|
self._completed_doc_count += 1
|
|
total_cost_doc = float(bucket["extract"] or 0.0) + float(bucket["normalize"] or 0.0)
|
|
self.view.after(0, lambda idx=self._completed_doc_count, val=total_cost_doc: self.view.update_cost_line(idx, val))
|
|
avg_cost = self._total_usd / max(self._completed_doc_count, 1)
|
|
self.view.after(0, lambda p=self._completed_doc_count, a=avg_cost: self.view.update_doc_kpi(p, a))
|
|
# Actualizar totales por fase
|
|
if phase in self._phase_tokens:
|
|
self._phase_tokens[phase]["input"] += int((usage or {}).get("prompt_tokens") or 0)
|
|
self._phase_tokens[phase]["cached"] += cached
|
|
self._phase_tokens[phase]["output"] += int((usage or {}).get("completion_tokens") or 0)
|
|
self._phase_tokens[phase]["usd"] += per_cost
|
|
ex = self._phase_tokens["extract"]
|
|
nz = self._phase_tokens["normalize"]
|
|
pp_in = self._pp_total_input_tokens
|
|
pp_out = self._pp_total_output_tokens
|
|
self.view.after(0, lambda: self.view.update_token_bars(
|
|
ex["input"], ex["cached"], ex["output"],
|
|
nz["input"], nz["cached"], nz["output"],
|
|
pp_in, pp_out,
|
|
))
|
|
# Actualizar KPIs
|
|
overall = {
|
|
"input": self._total_prompt_tokens,
|
|
"cached": self._total_cached_tokens,
|
|
"output": self._total_completion_tokens,
|
|
"usd": self._total_usd,
|
|
}
|
|
self.view.after(0, lambda ov=overall, exd=ex.copy(), nzd=nz.copy(): self.view.update_kpis(ov, exd, nzd))
|
|
# Añadir fila textual
|
|
self.view.after(0, lambda p=phase, r=radicado, u=usage, c=cost: self.view.append_usage_row(p, r, u, c))
|
|
# actualizar totales
|
|
self.view.set_usage_totals(
|
|
prompt_tokens=self._total_prompt_tokens,
|
|
cached_tokens=self._total_cached_tokens,
|
|
completion_tokens=self._total_completion_tokens,
|
|
total_usd=self._total_usd,
|
|
)
|
|
except Exception:
|
|
pass
|
|
if doc_types:
|
|
res = self.service.process_root_with_field_selection(
|
|
root_path=root,
|
|
doc_types=doc_types,
|
|
fields=selected_fields,
|
|
on_progress=on_progress,
|
|
csv_output_path=csv_path,
|
|
on_usage=on_usage,
|
|
on_error_retry=on_error_retry,
|
|
alias_map=alias_map,
|
|
tiff_page_range=tiff_page_range,
|
|
structure_mode=structure_mode,
|
|
retry_on_empty=retry_on_empty,
|
|
radicado_filter=radicado_filter,
|
|
)
|
|
else:
|
|
res = self.service.process_root_with_field_selection(
|
|
root_path=root,
|
|
keywords=keywords,
|
|
fields=selected_fields,
|
|
on_progress=on_progress,
|
|
csv_output_path=csv_path,
|
|
on_usage=on_usage,
|
|
on_error_retry=on_error_retry,
|
|
alias_map=alias_map,
|
|
tiff_page_range=tiff_page_range,
|
|
structure_mode=structure_mode,
|
|
retry_on_empty=retry_on_empty,
|
|
radicado_filter=radicado_filter,
|
|
)
|
|
self.view.append_output_safe(json.dumps(res, ensure_ascii=False, indent=2))
|
|
except Exception as e2:
|
|
logging.exception("Error en reproceso")
|
|
self.view.append_output_safe(f"Error: {e2}")
|
|
finally:
|
|
# Liberar guard al finalizar
|
|
self._worker2_running = False
|
|
self.view.set_processing(False)
|
|
self.view.after(0, lambda: mb.showinfo("Proceso Finalizado", "El procesamiento ha finalizado."))
|
|
|
|
# Preparar callback de reprocesamiento
|
|
reprocess_cb = None
|
|
if doc_types:
|
|
def _reprocess_multi():
|
|
s_by_type, f_full_by_type, f_rad = self.service.build_schema_from_first_for_types(
|
|
root, doc_types, tiff_page_range=tiff_page_range, structure_mode=structure_mode
|
|
)
|
|
c_schema = {}
|
|
try:
|
|
for tname, sch in s_by_type.items():
|
|
if not isinstance(sch, dict): continue
|
|
for p, typ in sch.items():
|
|
c_schema[f"{tname}/{p}"] = typ
|
|
except Exception: pass
|
|
|
|
c_sample = {"radicado": f_rad}
|
|
try:
|
|
for tname, s in f_full_by_type.items():
|
|
c_sample[tname] = s or {}
|
|
except Exception: pass
|
|
return c_schema, c_sample
|
|
reprocess_cb = _reprocess_multi
|
|
elif first_pdf_path:
|
|
def _reprocess_single():
|
|
return self.service.build_schema_from_first(first_pdf_path, tiff_page_range=tiff_page_range)
|
|
reprocess_cb = _reprocess_single
|
|
|
|
# Asegurar que siempre pasamos un schema dict (aunque sea vacío) al selector
|
|
effective_schema = combined_schema if combined_schema is not None else (schema or {})
|
|
effective_sample = combined_sample if combined_sample is not None else sample
|
|
|
|
self.view.after(0, lambda: self._open_field_selector(effective_schema, effective_sample, on_submit, reprocess_cb))
|
|
else:
|
|
self.view.append_output_safe(json.dumps(resultados, ensure_ascii=False, indent=2))
|
|
except Exception as e:
|
|
logger.exception("Error en procesamiento")
|
|
self.view.append_output_safe(f"Error: {e}")
|
|
finally:
|
|
# Si se abrió selector, se reactivará al terminar worker2
|
|
if not (isinstance(resultados, list) and resultados and isinstance(resultados[0], dict) and resultados[0].get('_needs_fields')):
|
|
self.view.set_processing(False)
|
|
self.view.after(0, lambda: mb.showinfo("Proceso Finalizado", "El procesamiento ha finalizado."))
|
|
|
|
self.view.set_processing(True)
|
|
threading.Thread(target=worker, daemon=True).start()
|
|
|
|
def _open_field_selector(self, schema, sample, on_submit, on_reprocess=None):
|
|
"""Abre el diálogo de selección de campos.
|
|
|
|
Args:
|
|
schema: Mapa path->tipo que ofrece el servicio para ayudar a seleccionar.
|
|
sample: JSON de muestra del primer documento.
|
|
on_submit: Callback `(selected_fields, alias_map)` a invocar al aceptar.
|
|
on_reprocess: Callback opcional para reprocesar muestra y schema.
|
|
"""
|
|
dlg = FieldSelectorDialog(self.view, schema=schema, sample=sample)
|
|
dlg.set_on_submit(on_submit)
|
|
if on_reprocess:
|
|
dlg.set_on_reprocess(on_reprocess)
|
|
|
|
# =========================================================================
|
|
# --- POST-PROCESAMIENTO ---
|
|
# =========================================================================
|
|
|
|
def on_pp_select_csv(self):
|
|
"""Abre un selector de archivo CSV y muestra la vista previa."""
|
|
filepath = fd.askopenfilename(
|
|
filetypes=[("CSV", "*.csv"), ("Todos", "*.*")],
|
|
title="Seleccionar CSV para post-procesamiento",
|
|
)
|
|
if not filepath:
|
|
return
|
|
|
|
try:
|
|
delimiter = postprocess_agent.detect_delimiter(filepath)
|
|
df = pd.read_csv(filepath, delimiter=delimiter, dtype=str)
|
|
except Exception as e:
|
|
mb.showerror("Error", f"No se pudo leer el CSV:\n{e}")
|
|
return
|
|
|
|
self._pp_df = df
|
|
self._pp_result_df = None
|
|
self.view.set_postprocess_csv_path(filepath)
|
|
self.view.enable_postprocess_save(False)
|
|
self.view.set_postprocess_code("")
|
|
self.view.set_postprocess_result("")
|
|
self.view.clear_postprocess_log()
|
|
|
|
sample_text, _ = postprocess_agent.sample_dataframe(df)
|
|
self._pp_csv_sample = sample_text
|
|
self.view.set_postprocess_preview(sample_text)
|
|
|
|
def on_pp_run(self):
|
|
"""Genera y ejecuta código de transformación con el agente LLM."""
|
|
if self._pp_df is None:
|
|
mb.showwarning("Aviso", "Primero seleccione un archivo CSV.")
|
|
return
|
|
|
|
instructions = self.view.get_postprocess_instructions()
|
|
if not instructions:
|
|
mb.showwarning("Aviso", "Escriba las instrucciones de transformación.")
|
|
return
|
|
|
|
self.view.enable_postprocess_controls(False)
|
|
self.view.enable_postprocess_save(False)
|
|
self.view.clear_postprocess_log()
|
|
self.view.set_postprocess_code("")
|
|
self.view.set_postprocess_result("")
|
|
self.view.set_postprocess_status("Procesando...")
|
|
|
|
csv_sample = self._pp_csv_sample
|
|
df = self._pp_df
|
|
|
|
def track_usage(usage, is_retry=False):
|
|
self._pp_total_calls += 1
|
|
if is_retry:
|
|
self._pp_total_retries += 1
|
|
if isinstance(usage, dict):
|
|
inp = int(usage.get("input_tokens") or 0)
|
|
out = int(usage.get("output_tokens") or 0)
|
|
self._pp_total_input_tokens += inp
|
|
self._pp_total_output_tokens += out
|
|
cost = (inp / 1_000_000.0) * 0.25 + (out / 1_000_000.0) * 2.00
|
|
self._pp_total_usd += cost
|
|
self.view.append_postprocess_log_safe(
|
|
f" Tokens: in={inp:,} out={out:,} | Costo: ${cost:.4f}"
|
|
)
|
|
|
|
def worker():
|
|
max_retries = 3
|
|
code = None
|
|
error_msg = None
|
|
|
|
try:
|
|
self.view.append_postprocess_log_safe("Generando código con el agente...")
|
|
code, usage = postprocess_agent.generate_code(
|
|
self._pp_client, csv_sample, instructions,
|
|
)
|
|
track_usage(usage)
|
|
self.view.after(0, lambda c=code: self.view.set_postprocess_code(c))
|
|
self.view.append_postprocess_log_safe("Código generado. Ejecutando...")
|
|
|
|
result_df, error_msg = postprocess_agent.execute_code(code, df)
|
|
|
|
attempt = 1
|
|
while error_msg and attempt < max_retries:
|
|
attempt += 1
|
|
self.view.append_postprocess_log_safe(
|
|
f"Error en ejecución: {error_msg}\nReintentando ({attempt}/{max_retries})..."
|
|
)
|
|
code, usage = postprocess_agent.retry_with_error(
|
|
self._pp_client, csv_sample, instructions, code, error_msg,
|
|
)
|
|
track_usage(usage, is_retry=True)
|
|
self.view.after(0, lambda c=code: self.view.set_postprocess_code(c))
|
|
self.view.append_postprocess_log_safe("Código corregido. Ejecutando...")
|
|
result_df, error_msg = postprocess_agent.execute_code(code, df)
|
|
|
|
# Actualizar dashboard de PP
|
|
short_instr = instructions[:60] + ("..." if len(instructions) > 60 else "")
|
|
self.view.after(0, lambda: self.view.update_pp_kpis(
|
|
self._pp_total_usd, self._pp_total_calls, self._pp_total_retries,
|
|
self._pp_total_input_tokens, self._pp_total_output_tokens,
|
|
short_instr,
|
|
))
|
|
self.view.after(0, lambda: self.view.append_usage_row(
|
|
"post-proc", short_instr,
|
|
{"prompt_tokens": self._pp_total_input_tokens, "completion_tokens": self._pp_total_output_tokens},
|
|
{"total_usd": self._pp_total_usd},
|
|
))
|
|
|
|
if error_msg:
|
|
self.view.append_postprocess_log_safe(
|
|
f"Error final tras {max_retries} intentos: {error_msg}"
|
|
)
|
|
self.view.after(0, lambda: self.view.set_postprocess_status("Error"))
|
|
final_err = error_msg
|
|
self.view.after(0, lambda: mb.showerror(
|
|
"Error", f"No se pudo aplicar la transformación tras {max_retries} intentos:\n{final_err}",
|
|
))
|
|
return
|
|
|
|
self._pp_result_df = result_df
|
|
result_sample, _ = postprocess_agent.sample_dataframe(result_df)
|
|
self.view.after(0, lambda s=result_sample: self.view.set_postprocess_result(s))
|
|
self.view.append_postprocess_log_safe(
|
|
f"Ejecución exitosa. Filas: {len(result_df):,} | Columnas: {len(result_df.columns)}"
|
|
)
|
|
self.view.after(0, lambda: self.view.enable_postprocess_save(True))
|
|
self.view.after(0, lambda: self.view.set_postprocess_status("Completado"))
|
|
|
|
except Exception as exc:
|
|
logger.exception("Error en post-procesamiento")
|
|
err_str = str(exc)
|
|
self.view.append_postprocess_log_safe(f"Error inesperado: {err_str}")
|
|
self.view.after(0, lambda: self.view.set_postprocess_status("Error"))
|
|
self.view.after(0, lambda m=err_str: mb.showerror("Error", f"Error inesperado:\n{m}"))
|
|
finally:
|
|
self.view.after(0, lambda: self.view.enable_postprocess_controls(True))
|
|
|
|
threading.Thread(target=worker, daemon=True).start()
|
|
|
|
def on_pp_save(self):
|
|
"""Guarda el DataFrame resultante del post-procesamiento."""
|
|
if self._pp_result_df is None:
|
|
mb.showwarning("Aviso", "No hay resultado para guardar.")
|
|
return
|
|
|
|
path = fd.asksaveasfilename(
|
|
defaultextension=".csv",
|
|
filetypes=[("CSV", "*.csv"), ("Excel", "*.xlsx"), ("Todos", "*.*")],
|
|
title="Guardar CSV post-procesado",
|
|
)
|
|
if not path:
|
|
return
|
|
|
|
try:
|
|
if path.endswith(".xlsx"):
|
|
self._pp_result_df.to_excel(path, index=False)
|
|
else:
|
|
self._pp_result_df.to_csv(path, index=False)
|
|
mb.showinfo("Guardado", f"Archivo guardado correctamente:\n{path}")
|
|
except Exception as e:
|
|
logger.exception("Error guardando CSV post-procesado")
|
|
mb.showerror("Error", f"No se pudo guardar el archivo:\n{e}")
|
|
|
|
# =========================================================================
|
|
# --- PROCESO DE LIMPIEZA ---
|
|
# =========================================================================
|
|
|
|
def on_cleaning_select_csv(self):
|
|
"""Abre un selector de CSV y prepara el resumen de columnas."""
|
|
filepath = fd.askopenfilename(
|
|
filetypes=[("CSV", "*.csv"), ("Todos", "*.*")],
|
|
title="Seleccionar CSV para limpieza",
|
|
)
|
|
if not filepath:
|
|
return
|
|
|
|
self._cleaning_result_df = None
|
|
self.view.set_cleaning_csv_path(filepath)
|
|
self.view.enable_cleaning_controls(False)
|
|
self.view.enable_cleaning_save(False)
|
|
self.view.set_cleaning_result(None)
|
|
self.view.set_cleaning_preview(None)
|
|
self.view.set_cleaning_status("Cargando CSV...")
|
|
try:
|
|
self.view.set_cleaning_loading(True, "Leyendo archivo CSV...")
|
|
except Exception:
|
|
pass
|
|
|
|
def worker():
|
|
try:
|
|
df, delimiter = csv_cleaning.load_csv(filepath)
|
|
self.view.after(
|
|
0,
|
|
lambda d=df, delim=delimiter: self._on_cleaning_csv_loaded(
|
|
d, delim
|
|
),
|
|
)
|
|
summaries = csv_cleaning.analyze_dataframe(df)
|
|
self._cleaning_df = df
|
|
self.view.after(
|
|
0,
|
|
lambda d=df, s=summaries: self._finish_cleaning_csv_load(d, s),
|
|
)
|
|
except Exception as e:
|
|
logger.exception("Error leyendo o analizando CSV para limpieza")
|
|
err = str(e)
|
|
self.view.after(0, lambda m=err: self._fail_cleaning_csv_load(m))
|
|
|
|
threading.Thread(target=worker, daemon=True).start()
|
|
|
|
def _on_cleaning_csv_loaded(self, df: pd.DataFrame, delimiter: str):
|
|
self.view.set_cleaning_status(
|
|
f"CSV leído: {len(df):,} filas x {len(df.columns)} columnas. Analizando columnas..."
|
|
)
|
|
try:
|
|
self.view.set_cleaning_loading(True, "Analizando columnas...")
|
|
self.view.set_cleaning_delimiter_chip(delimiter)
|
|
except Exception:
|
|
pass
|
|
|
|
def _finish_cleaning_csv_load(self, df: pd.DataFrame, summaries):
|
|
self.view.populate_cleaning_columns(summaries)
|
|
self.view.set_cleaning_preview(df)
|
|
self.view.set_cleaning_status(
|
|
f"CSV cargado: {len(df):,} filas x {len(df.columns)} columnas."
|
|
)
|
|
try:
|
|
self.view.set_cleaning_loading(False)
|
|
except Exception:
|
|
pass
|
|
self.view.enable_cleaning_controls(True)
|
|
|
|
def _fail_cleaning_csv_load(self, message: str):
|
|
self._cleaning_df = None
|
|
self.view.set_cleaning_status("Error al cargar CSV")
|
|
try:
|
|
self.view.set_cleaning_loading(False)
|
|
except Exception:
|
|
pass
|
|
self.view.enable_cleaning_controls(True)
|
|
mb.showerror("Error", f"No se pudo cargar o analizar el CSV:\n{message}")
|
|
|
|
def on_cleaning_apply(self):
|
|
"""Aplica las reglas seleccionadas en un hilo y actualiza el resultado."""
|
|
if self._cleaning_df is None:
|
|
mb.showwarning("Aviso", "Primero seleccione un archivo CSV.")
|
|
return
|
|
|
|
rules = self.view.get_cleaning_selected_rules()
|
|
if not rules:
|
|
mb.showwarning(
|
|
"Aviso",
|
|
"No hay reglas seleccionadas. Marque al menos una regla para alguna columna.",
|
|
)
|
|
return
|
|
|
|
self.view.enable_cleaning_controls(False)
|
|
self.view.enable_cleaning_save(False)
|
|
self.view.set_cleaning_status("Aplicando reglas...")
|
|
self.view.set_cleaning_result(None)
|
|
|
|
df = self._cleaning_df
|
|
|
|
def worker():
|
|
try:
|
|
result_df, log = csv_cleaning.apply_rules(df, rules)
|
|
self._cleaning_result_df = result_df
|
|
ok_count = sum(1 for line in log if line.startswith("[OK]"))
|
|
err_count = sum(1 for line in log if line.startswith("[ERROR]"))
|
|
warn_count = sum(1 for line in log if line.startswith("[WARN]"))
|
|
info = f"{ok_count} reglas aplicadas"
|
|
if err_count or warn_count:
|
|
info += f" · {err_count} errores · {warn_count} avisos"
|
|
self.view.after(0, lambda d=result_df, t=info: self.view.set_cleaning_result(d, t))
|
|
self.view.after(0, lambda: self.view.enable_cleaning_save(True))
|
|
status = (
|
|
f"Completado: {ok_count} reglas aplicadas correctamente"
|
|
+ (f", {err_count} errores" if err_count else "")
|
|
+ (f", {warn_count} avisos" if warn_count else "")
|
|
+ "."
|
|
)
|
|
self.view.after(0, lambda s=status: self.view.set_cleaning_status(s))
|
|
except Exception as exc:
|
|
logger.exception("Error aplicando reglas de limpieza")
|
|
err = str(exc)
|
|
self.view.after(0, lambda: self.view.set_cleaning_status("Error"))
|
|
self.view.after(0, lambda m=err: mb.showerror("Error", f"Error aplicando reglas:\n{m}"))
|
|
finally:
|
|
self.view.after(0, lambda: self.view.enable_cleaning_controls(True))
|
|
|
|
threading.Thread(target=worker, daemon=True).start()
|
|
|
|
def on_cleaning_save(self):
|
|
"""Guarda el DataFrame limpio en disco."""
|
|
if self._cleaning_result_df is None:
|
|
mb.showwarning("Aviso", "No hay resultado para guardar.")
|
|
return
|
|
|
|
path = fd.asksaveasfilename(
|
|
defaultextension=".csv",
|
|
filetypes=[("CSV", "*.csv"), ("Excel", "*.xlsx"), ("Todos", "*.*")],
|
|
title="Guardar CSV limpio",
|
|
)
|
|
if not path:
|
|
return
|
|
|
|
try:
|
|
if path.lower().endswith(".xlsx"):
|
|
self._cleaning_result_df.to_excel(path, index=False)
|
|
else:
|
|
self._cleaning_result_df.to_csv(path, index=False, sep="|")
|
|
mb.showinfo("Guardado", f"Archivo guardado correctamente:\n{path}")
|
|
except Exception as e:
|
|
logger.exception("Error guardando CSV limpio")
|
|
mb.showerror("Error", f"No se pudo guardar el archivo:\n{e}")
|