"""Controlador de la aplicación GUI: coordina vista y servicio. Gestiona selección de carpetas, ejecución en hilo de procesamiento, selector interactivo de campos y actualización de configuración del extractor. """ import json import logging import threading import asyncio import tkinter.filedialog as fd import tkinter.messagebox as mb import os from core.utils import parse_comma_separated, collect_radicado_docs, collect_documents_flat from application.logging_config import configure_logging from application.app_prefs import load_app_prefs, save_app_prefs, normalize_document_types from gui.field_selector import FieldSelectorDialog from application.preclassification import PreclassificationService from application.strategies import OllamaExtractor from application.database import DatabaseManager from application.config import settings from application.costs import MODEL_PRICING from application import postprocess_agent from application import csv_cleaning from openai import OpenAI import pandas as pd logger = logging.getLogger(__name__) class AppController: """Controlador principal de la GUI que orquesta la vista y el servicio. Se encarga de: - Vincular callbacks de la vista. - Disparar el flujo de procesamiento de facturas. - Abrir el diálogo de selección de campos cuando el servicio lo solicite. """ def __init__(self, view, service): """Inicializa el controlador y vincula handlers de la vista. Args: view: Instancia de vista (por ejemplo, `InvoiceAppView`). service: Servicio de facturas compatible con `process_root_with_field_selection`. """ # asegurar logging configurado try: configure_logging() except Exception: pass self.view = view self.service = service # Acumuladores de uso self._total_prompt_tokens = 0 self._total_cached_tokens = 0 self._total_completion_tokens = 0 self._total_usd = 0.0 # Estado para dashboard self._hist_requests = [] # list[dict]: {radicado, phase, prompt, cached, completion, cost_usd} self._per_doc_costs = {} # radicado -> {"extract": float|None, "normalize": float|None} self._completed_doc_count = 0 self._phase_tokens = { "extract": {"input": 0, "cached": 0, "output": 0, "usd": 0.0}, "normalize": {"input": 0, "cached": 0, "output": 0, "usd": 0.0}, } # Guard para evitar doble lanzamiento de worker2 (por doble click o condiciones de carrera) self._worker2_running = False # Estado Preclasificación self._last_run_id = None self.db_manager = DatabaseManager() # Para pestaña resultados self.view.set_on_select_root(self.on_select_root) try: self.view.set_on_select_preclass_root(self.on_select_preclass_root) self.view.set_on_run_preclassification(self.on_run_preclassification) self.view.set_on_export_excel(self.on_export_excel) self.view.set_on_organize_files(self.on_organize_files) # Resultados self.view.set_on_res_search(self.on_res_search) self.view.set_on_res_export(self.on_res_export) self.view.set_on_res_refresh(self.on_res_refresh) self.view.set_on_res_organize(self.on_res_organize_run) self.view.set_on_res_select(self.on_res_select) # Carga inicial self.on_res_refresh() except Exception: pass self.view.set_on_process(self.on_process) self.view.set_on_select_csv(self.on_select_csv) self.view.set_on_save_settings(self.on_save_settings) # Post-procesamiento self._pp_df: pd.DataFrame | None = None self._pp_result_df: pd.DataFrame | None = None self._pp_csv_sample: str = "" self._pp_client = OpenAI(api_key=settings.api.api_key, base_url=settings.api.base_url) # Métricas acumuladas de post-procesamiento self._pp_total_usd = 0.0 self._pp_total_calls = 0 self._pp_total_retries = 0 self._pp_total_input_tokens = 0 self._pp_total_output_tokens = 0 try: self.view.set_on_postprocess_select_csv(self.on_pp_select_csv) self.view.set_on_postprocess_run(self.on_pp_run) self.view.set_on_postprocess_save(self.on_pp_save) except Exception: pass # Proceso de limpieza (sin LLM) self._cleaning_df: pd.DataFrame | None = None self._cleaning_result_df: pd.DataFrame | None = None try: self.view.set_on_cleaning_select_csv(self.on_cleaning_select_csv) self.view.set_on_cleaning_apply(self.on_cleaning_apply) self.view.set_on_cleaning_save(self.on_cleaning_save) except Exception: pass # Mapeo de nombres de UI a IDs de modelo (y viceversa) self.ui_to_model = { "5.4": "gpt-5.4", "5.4-mini": "gpt-5.4-mini", "5.4-nano": "gpt-5.4-nano", "5.2": "gpt-5.2", "4.1-mini": "gpt-4.1-mini", } self.model_to_ui = {v: k for k, v in self.ui_to_model.items()} # Inicializar vista con configuración actual try: # Llenar tabla de precios de modelos pricing_data = self._get_model_pricing_data() self.view.update_comparison_table(pricing_data) current_model = getattr(self.service.extractor, "model", "") # Intentar mapear modelo backend -> nombre UI, si no existe usar raw ui_name = self.model_to_ui.get(current_model, current_model) if ui_name: self.view.set_selected_model(ui_name) # Cargar concurrencia self.view.set_concurrency_limit(settings.processing.concurrency_limit) except Exception: pass self._load_persisted_preferences() try: self.view.protocol("WM_DELETE_WINDOW", self.on_app_close) except Exception: pass def _get_model_pricing_data(self): """Obtiene las tarifas por 1M de tokens para cada modelo.""" results = [] for model_id, rates in MODEL_PRICING.items(): results.append({ "model": self.model_to_ui.get(model_id, model_id), "input": rates["input"], "cached_input": rates["cached_input"], "output": rates["output"] }) # Ordenar por precio de input descendente results.sort(key=lambda x: x["input"], reverse=True) return results def on_save_settings(self): """Actualiza la configuración del servicio (modelo LLM y concurrencia).""" selected_name = self.view.get_selected_model() concurrency = self.view.get_concurrency_limit() if not selected_name: mb.showwarning("Configuración", "El modelo no puede estar vacío.") return if concurrency <= 0: mb.showwarning("Configuración", "El límite de concurrencia debe ser mayor a 0.") return # Traducir nombre UI -> ID modelo real real_model_id = self.ui_to_model.get(selected_name, selected_name) try: # Actualizar extractor if hasattr(self.service.extractor, "update_config"): self.service.extractor.update_config(model=real_model_id) # Actualizar settings globales settings.processing.concurrency_limit = concurrency self.view.set_allowed_document_types_hint(self.view.get_allowed_document_types()) self._save_preferences() mb.showinfo("Configuración", f"Configuración actualizada.\nModelo: {selected_name}\nConcurrencia: {concurrency}") self.view.append_output_safe(f"Configuración actualizada: Modelo={real_model_id}, Concurrencia={concurrency}") except Exception as e: logger.exception("Error actualizando configuración") mb.showerror("Error", f"No se pudo actualizar la configuración: {e}") def on_select_root(self): """Abre un selector de carpeta y establece la ruta raíz en la vista. Además, si el primer documento detectado es TIFF, muestra los controles de rango TIFF para que el usuario pueda ajustar el índice antes de procesar. """ selected = fd.askdirectory() if not selected: return self.view.set_root_path(selected) self._save_preferences() # Detectar si el primer documento a procesar es TIFF para esta carpeta try: keywords_str = self.view.get_keywords() keywords = parse_comma_separated(keywords_str) or ["Factura_O_Documento_Equivalente", "factura"] except Exception: keywords = ["Factura_O_Documento_Equivalente", "factura"] has_tiff_first = False try: preview_entries = collect_radicado_docs(selected, keywords) if preview_entries: first_path = str(preview_entries[0].get("path") or "").lower() has_tiff_first = first_path.endswith((".tif", ".tiff")) except Exception: has_tiff_first = False try: self.view.show_tiff_range_controls(has_tiff_first) except Exception: pass def on_select_preclass_root(self): """Abre un selector de carpeta y establece la ruta raíz para preclasificación.""" selected = fd.askdirectory() if not selected: return self.view.set_preclass_root_path(selected) def on_run_preclassification(self): """Ejecuta la preclasificación en un hilo separado.""" # 1. Leer inputs de la UI root = self.view.get_preclass_root_path() patterns = self.view.get_preclass_patterns() target_info = self.view.get_preclass_target_info() page_range = self.view.get_preclass_range() mode = self.view.get_preclass_mode() # Validaciones básicas if not root or not os.path.isdir(root): mb.showwarning("Error", "Seleccione una carpeta raíz válida.") return if not patterns: mb.showwarning("Error", "Debe agregar al menos un patrón.") return if target_info["enabled"] and not target_info["pattern"]: mb.showwarning("Error", "Si habilita archivo objetivo, debe especificar un patrón de archivo.") return # Bloquear UI self.view.set_processing(True) self.view.enable_preclass_actions(False) self.view.append_preclass_output("Iniciando preclasificación...") self.view.append_preclass_output(f"Raíz: {root}") self.view.append_preclass_output(f"Modo: {mode}") self.view.append_preclass_output(f"Patrones: {len(patterns)}") if target_info["enabled"]: self.view.append_preclass_output(f"Filtro: {target_info['pattern']}") # Instanciar servicio (usando el extractor actual) # Nota: PreclassificationService maneja su propia instancia de DB pre_service = PreclassificationService(self.service.extractor) def worker(): try: def on_progress(radicado, idx, total): pct = int((idx / total) * 100) msg = f"[{idx}/{total}] Procesando radicado: {radicado} ({pct}%)" self.view.append_preclass_output_safe(msg) run_id = pre_service.run_preclassification( root_path=root, mode=mode, patterns=patterns, target_info=target_info, page_range=page_range, on_progress=on_progress ) self.view.append_preclass_output_safe(f"\n--- Ejecución finalizada (ID: {run_id}) ---") # Guardar ID self._last_run_id = run_id def _refresh_results_tab(): # Recargar runs, seleccionar el run recién ejecutado y listar sus resultados try: self.on_res_refresh() self.view.set_res_run(run_id) self.on_res_search() except Exception: pass try: self.view.enable_preclass_actions(True) except Exception: pass self.view.after(0, _refresh_results_tab) # Mostrar resumen rápido # Usar tabla results (1 fila = 1 documento), no el join con matches docs, _ = self.db_manager.search_results( run_id=run_id, status="Todos", text=None, relax_if_empty=False ) match_count = sum(1 for r in docs if r.get("status") == "MATCH") self.view.append_preclass_output_safe(f"Total procesados: {len(docs)}") self.view.append_preclass_output_safe(f"Coincidencias (MATCH): {match_count}") # Detalle opcional for res in docs[:10]: # mostrar primeros 10 status_icon = "✅" if res.get("status") == "MATCH" else "❌" self.view.append_preclass_output_safe(f"{status_icon} {res.get('radicado')} ({res.get('status')})") if len(docs) > 10: self.view.append_preclass_output_safe("... ver base de datos para más detalles.") self.view.after(0, lambda: mb.showinfo("Completado", f"Preclasificación finalizada.\nCoincidencias: {match_count}/{len(docs)}")) except Exception as e: logger.exception("Error en preclasificación") self.view.append_preclass_output_safe(f"ERROR CRÍTICO: {str(e)}") self.view.after(0, lambda: mb.showerror("Error", f"Ocurrió un error: {e}")) finally: self.view.set_processing(False) threading.Thread(target=worker, daemon=True).start() def on_export_excel(self): """Exporta los resultados de la última preclasificación a Excel.""" if not self._last_run_id: mb.showwarning("Aviso", "No hay resultados de preclasificación recientes para exportar.") return path = fd.asksaveasfilename( defaultextension=".xlsx", filetypes=[("Excel", "*.xlsx"), ("CSV", "*.csv"), ("Todos", "*.*")] ) if not path: return try: pre_service = PreclassificationService(self.service.extractor) ok = pre_service.export_to_excel(self._last_run_id, path) if ok: mb.showinfo("Exportar", "Archivo exportado correctamente.") else: mb.showerror("Exportar", "No se encontraron datos para exportar.") except Exception as e: logger.exception("Error exportando excel") mb.showerror("Error", f"Error al exportar: {e}") def on_organize_files(self): """Organiza los archivos MATCH en carpetas (Run actual).""" if not self._last_run_id: mb.showwarning("Aviso", "No hay resultados para organizar.") return self._organize_run_generic(self._last_run_id) def _organize_run_generic(self, run_id: int): target_dir = fd.askdirectory(title="Seleccionar carpeta de destino para organizar") if not target_dir: return if not mb.askyesno("Confirmar", "¿Está seguro de copiar y organizar los archivos en la carpeta seleccionada?\nEsta acción creará una estructura de carpetas basada en los datos extraídos."): return try: # Usamos append_preclass_output si estamos en esa pestaña, pero desde Resultados? # El metodo de view inserta en el widget de log. Si estamos en tab Resultados, no se ve. # Pero no importa, mostramos mb.showinfo al final. pre_service = PreclassificationService(self.service.extractor) ok_count, err_count = pre_service.organize_files(run_id, target_dir) msg = f"Organización finalizada.\nCopiados exitosos: {ok_count}\nErrores: {err_count}" mb.showinfo("Organizar", msg) except Exception as e: logger.exception("Error organizando archivos") mb.showerror("Error", f"Error al organizar: {e}") # --- Pestaña Resultados --- def on_res_select(self, event): """Muestra detalles del resultado seleccionado.""" item_vals = self.view.get_selected_result_values() if not item_vals: return try: res_id = int(item_vals[0]) matches = self.db_manager.get_matches_for_result(res_id) self.view.update_res_matches_table(matches) except Exception as e: logger.error(f"Error cargando detalle: {e}") def on_res_organize_run(self): """Organiza archivos del Run seleccionado en el filtro.""" filters = self.view.get_res_filters() run_val = filters["run_id"] if run_val == "Todos" or not run_val: mb.showwarning("Organizar", "Por favor seleccione un 'Run ID' específico en el filtro para organizar.") return try: run_id = int(run_val) self._organize_run_generic(run_id) except ValueError: mb.showerror("Error", "ID de Run inválido.") def on_res_refresh(self): """Refresca la lista de Runs en el combo.""" try: runs = self.db_manager.get_all_runs() ids = [r["id"] for r in runs] self.view.update_res_runs_combo(ids) # Tambien refrescar tabla self.on_res_search() except Exception as e: logger.error(f"Error refrescando runs: {e}") def on_res_search(self): """Busca resultados con los filtros actuales.""" self.view.clear_res_filter_note() filters = self.view.get_res_filters() run_id = None if filters["run_id"] != "Todos": try: run_id = int(filters["run_id"]) except: pass try: rows, relax_note = self.db_manager.search_results( run_id=run_id, status=filters["status"], text=filters["text"], relax_if_empty=bool(filters.get("relax_on_empty", False)), ) self.view.update_res_table(rows) if relax_note: self.view.set_res_filter_note(relax_note) except Exception as e: logger.error(f"Error buscando resultados: {e}") mb.showerror("Error", f"Error en búsqueda: {e}") def on_res_export(self): """Exporta la vista actual de resultados a Excel.""" path = fd.asksaveasfilename( defaultextension=".xlsx", filetypes=[("Excel", "*.xlsx"), ("CSV", "*.csv")] ) if not path: return # Obtener los datos actuales (repite la busqueda) # Podriamos optimizar guardando el ultimo result set try: filters = self.view.get_res_filters() run_id = None if filters["run_id"] != "Todos": try: run_id = int(filters["run_id"]) except: pass rows, _ = self.db_manager.search_results( run_id=run_id, status=filters["status"], text=filters["text"], relax_if_empty=bool(filters.get("relax_on_empty", False)), ) if not rows: mb.showwarning("Exportar", "No hay datos para exportar.") return df = pd.DataFrame(rows) if path.endswith(".csv"): df.to_csv(path, index=False, sep=";") else: df.to_excel(path, index=False) mb.showinfo("Exportar", "Datos exportados correctamente.") except Exception as e: logger.exception("Error exportando vista") mb.showerror("Error", f"Error al exportar: {e}") def on_select_csv(self): """Abre un diálogo de guardado y establece la ruta del CSV de salida.""" selected = fd.asksaveasfilename(defaultextension=".csv", filetypes=[("CSV", "*.csv"), ("Todos", "*.*")]) if selected: self.view.set_csv_path(selected) self._save_preferences() def _collect_preferences(self) -> dict: """Toma una foto de la configuración visible de la app.""" try: doc_types = self.view.get_doc_types() except Exception: doc_types = [] return { "selected_model": self.view.get_selected_model(), "concurrency_limit": self.view.get_concurrency_limit(), "allowed_document_types": self.view.get_allowed_document_types(), "retry_on_empty": self.view.get_retry_empty_enabled(), "process_mode": self.view.get_process_mode(), "root_path": self.view.get_root_path(), "csv_path": self.view.get_csv_path(), "doc_types": doc_types, } def _save_preferences(self) -> None: """Guarda preferencias de la GUI en disco.""" save_app_prefs(self._collect_preferences()) def _load_persisted_preferences(self) -> None: """Carga preferencias persistidas y las aplica a la vista/servicio.""" prefs = load_app_prefs() if not prefs: return try: selected_model = str(prefs.get("selected_model") or "").strip() if selected_model: self.view.set_selected_model(selected_model) real_model_id = self.ui_to_model.get(selected_model, selected_model) if hasattr(self.service.extractor, "update_config"): self.service.extractor.update_config(model=real_model_id) except Exception: logger.exception("No se pudo restaurar el modelo persistido") try: concurrency = int(prefs.get("concurrency_limit") or settings.processing.concurrency_limit) self.view.set_concurrency_limit(concurrency) settings.processing.concurrency_limit = concurrency except Exception: pass try: allowed_document_types = normalize_document_types(prefs.get("allowed_document_types")) self.view.set_allowed_document_types(allowed_document_types) self.view.set_allowed_document_types_hint(allowed_document_types) except Exception: pass try: self.view.set_retry_empty_enabled(bool(prefs.get("retry_on_empty", False))) except Exception: pass try: self.view.set_process_mode(str(prefs.get("process_mode") or "radicados")) except Exception: pass try: root_path = str(prefs.get("root_path") or "").strip() if root_path: self.view.set_root_path(root_path) except Exception: pass try: csv_path = str(prefs.get("csv_path") or "").strip() if csv_path: self.view.set_csv_path(csv_path) except Exception: pass try: doc_types = prefs.get("doc_types") if isinstance(doc_types, list): self.view.set_doc_types(doc_types) except Exception: pass def on_app_close(self): """Guarda preferencias antes de cerrar la ventana.""" try: self._save_preferences() finally: self.view.destroy() def on_process(self): """Inicia el flujo de procesamiento por radicado (solo facturas) en un hilo de fondo. - Si el servicio requiere selección de campos, abre `FieldSelectorDialog` y relanza. - Emite resultados en el área de salida y opcionalmente escribe CSV incremental. """ self._save_preferences() root = self.view.get_root_path() # Tipos de Documento (multi-doc) doc_types = [] try: doc_types = self.view.get_doc_types() except Exception: doc_types = [] if doc_types: try: ok, msg = self.view.validate_doc_types() except Exception: ok, msg = True, None if not ok: self.view.append_output_safe(msg or "Tipos de Documento inválidos.") return keywords_str = self.view.get_keywords() fields_str = self.view.get_fields() structure_mode = self.view.get_process_mode() retry_on_empty = self.view.get_retry_empty_enabled() # Filtro de radicados radicado_filter = None try: if self.view.get_radicado_filter_enabled(): radicado_filter = self.view.get_radicado_filter_list() or None except Exception: radicado_filter = None if structure_mode == "general": # En modo general, si está vacío o es el default, procesar todo (None) # Si el usuario escribió algo específico, usarlo. kws = parse_comma_separated(keywords_str) # Check if it matches default legacy keywords to avoid accidental filtering if kws == ["Factura_O_Documento_Equivalente", "factura"]: keywords = None else: keywords = kws if kws else None else: keywords = parse_comma_separated(keywords_str) or ["Factura_O_Documento_Equivalente", "factura"] fields = parse_comma_separated(fields_str) if fields_str else None # Validar ruta antes de deshabilitar la UI if not root or not os.path.isdir(root): self.view.append_output_safe(f"Error: La ruta raíz no es un directorio: {root or ''}") return # Si es modo general, ignorar doc_types if structure_mode == "general": doc_types = [] # Determinar si el primer documento a procesar es TIFF; si no lo es, ocultar # los controles de rango TIFF y no aplicar ningún recorte. tiff_page_range = None has_tiff_first = False try: if structure_mode == "general": preview_entries = collect_documents_flat(root, keywords) else: preview_entries = collect_radicado_docs(root, keywords) if preview_entries: first_path = str(preview_entries[0].get("path") or "").lower() has_tiff_first = first_path.endswith((".tif", ".tiff")) except Exception: has_tiff_first = False try: # Mostrar/ocultar controles en función de si el primer doc es TIFF self.view.show_tiff_range_controls(has_tiff_first) except Exception: pass if has_tiff_first: # Rango opcional de páginas TIFF (1-based; 0 = sin límite) try: tiff_start_raw = int(getattr(self.view, "get_tiff_start_page", lambda: 0)() or 0) except Exception: tiff_start_raw = 0 try: tiff_end_raw = int(getattr(self.view, "get_tiff_end_page", lambda: 0)() or 0) except Exception: tiff_end_raw = 0 if tiff_start_raw < 0 or tiff_end_raw < 0: self.view.append_output_safe("Error: Las páginas TIFF deben ser mayores o iguales a 0.") return tiff_start = tiff_start_raw if tiff_start_raw > 0 else None tiff_end = tiff_end_raw if tiff_end_raw > 0 else None if tiff_start is not None and tiff_end is not None and tiff_start > tiff_end: self.view.append_output_safe("Error: La página de inicio TIFF no puede ser mayor que la página de fin.") return # Si ambos son 0 (None/None), el usuario quiere "todas las páginas": # usamos el centinela (None, None) para indicar "no recortar TIFF". if tiff_start is None and tiff_end is None: tiff_page_range = (None, None) else: tiff_page_range = (tiff_start, tiff_end) def worker(): resultados = None if doc_types: try: self.view.append_output_safe(f"Procesando raíz: {root} | tipos: {[(dt.get('name'), dt.get('keyword')) for dt in doc_types]}") except Exception: self.view.append_output_safe(f"Procesando raíz: {root} | tipos: {len(doc_types)}") else: self.view.append_output_safe(f"Procesando raíz: {root} | keywords: {keywords}") # limpiar pestaña Uso try: self._total_prompt_tokens = 0 self._total_cached_tokens = 0 self._total_completion_tokens = 0 self._total_usd = 0.0 self._hist_requests.clear() self._per_doc_costs.clear() self._completed_doc_count = 0 self._phase_tokens = { "extract": {"input": 0, "cached": 0, "output": 0, "usd": 0.0}, "normalize": {"input": 0, "cached": 0, "output": 0, "usd": 0.0}, } self.view.clear_usage() except Exception: pass try: def on_progress(radicado, idx, total): pct = int((idx / total) * 100) self.view.append_output_safe(f"Progreso: {idx}/{total} ({pct}%) - radicado {radicado}") csv_path = self.view.get_csv_path() or None # Primera ejecución: si fields es None, el servicio devolverá _needs_fields if doc_types: resultados = self.service.process_root_with_field_selection( root_path=root, doc_types=doc_types, fields=fields, on_progress=on_progress, csv_output_path=csv_path, tiff_page_range=tiff_page_range, structure_mode=structure_mode, retry_on_empty=retry_on_empty, radicado_filter=radicado_filter, ) else: resultados = self.service.process_root_with_field_selection( root_path=root, keywords=keywords, fields=fields, on_progress=on_progress, csv_output_path=csv_path, tiff_page_range=tiff_page_range, structure_mode=structure_mode, retry_on_empty=retry_on_empty, radicado_filter=radicado_filter, ) # Si requiere selección de campos abrir diálogo; de lo contrario, mostrar resultados if resultados and isinstance(resultados, list) and isinstance(resultados[0], dict) and resultados[0].get('_needs_fields'): schema = resultados[0].get('schema') sample = resultados[0].get('first_full') first_pdf_path = resultados[0].get('_source_pdf') or "" schema_by_type = resultados[0].get('schema_by_type') first_full_by_type = resultados[0].get('first_full_by_type') first_radicado = resultados[0].get('first_radicado') or "" combined_schema = None combined_sample = None if isinstance(schema_by_type, dict) and schema_by_type: combined_schema = {} try: for tname, sch in schema_by_type.items(): if not isinstance(sch, dict): continue for path, typ in sch.items(): combined_schema[f"{tname}/{path}"] = typ except Exception: combined_schema = {} # Construir muestra combinada para el botón "Ver muestra" try: combined_sample = {"radicado": first_radicado} if isinstance(first_full_by_type, dict): for tname, s in first_full_by_type.items(): combined_sample[tname] = s or {} except Exception: combined_sample = None def on_submit(selected_fields, alias_map): if not selected_fields: self.view.append_output_safe("No se seleccionaron campos.") self.view.set_processing(False) return # Estimar tokens/costos para toda la carpeta en hilo de fondo y pedir confirmación en hilo principal def do_estimate_and_prompt(): try: if doc_types and schema_by_type: est = self.service.estimate_tokens_and_cost( root_path=root, doc_types=doc_types, schema_by_type=schema_by_type, selected_paths=selected_fields, structure_mode=structure_mode, ) else: est = self.service.estimate_tokens_and_cost( root_path=root, keywords=keywords, schema=schema, selected_fields=selected_fields, structure_mode=structure_mode, ) total_docs = est.get("total_docs", 0) per_doc_cost = est.get("per_doc_cost", {}) or {} estimated_total = est.get("estimated_total_cost", {}) or {} msg = ( f"Se procesarán {total_docs} documentos.\n\n" f"Costo estimado por documento (USD): {per_doc_cost.get('total_usd', 'N/D')}\n" f"Costo estimado total (USD): {estimated_total.get('total_usd', 'N/D')}\n\n" f"¿Desea continuar?" ) except Exception as e_est: msg = f"No se pudo calcular la estimación ({e_est}). ¿Desea continuar sin estimación?" def prompt_and_continue(): ok = mb.askyesno("Confirmar estimación", msg) if not ok: self.view.set_processing(False) return # Evitar doble arranque if self._worker2_running: return self._worker2_running = True # Si acepta, lanzar procesamiento en otro hilo threading.Thread(target=worker2, daemon=True).start() # Volver al hilo principal para mostrar el cuadro de diálogo self.view.after(0, prompt_and_continue) threading.Thread(target=do_estimate_and_prompt, daemon=True).start() def worker2(): try: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) async def on_error_retry(error): # Este callback corre en el loop de asyncio (worker thread) future = loop.create_future() def ask(): msg = ( f"⚠️ Error de Cuota o Límite (429):\n\n{error}\n\n" "¿Desea PAUSAR el proceso, añadir fondos/esperar y REINTENTAR este radicado?\n\n" "Si selecciona 'No', el proceso se detendrá y se guardará lo avanzado hasta ahora." ) res = mb.askyesno("Límite de Cuota Excedido", msg) # Regresar el resultado al loop de asyncio loop.call_soon_threadsafe(future.set_result, res) self.view.after(0, ask) return await future # callback para uso por petición def on_usage(phase, radicado, usage, cost): try: cached = 0 if isinstance(usage, dict): pt = int(usage.get("prompt_tokens") or 0) ct = int(usage.get("completion_tokens") or 0) details = usage.get("prompt_tokens_details") or {} cached = int(details.get("cached_tokens") or 0) self._total_prompt_tokens += pt self._total_cached_tokens += cached self._total_completion_tokens += ct if isinstance(cost, dict): self._total_usd += float(cost.get("total_usd") or 0.0) # Registrar histórico per_cost = float((cost or {}).get("total_usd") or 0.0) self._hist_requests.append({ "radicado": radicado, "phase": phase, "prompt": int((usage or {}).get("prompt_tokens") or 0), "cached": cached, "completion": int((usage or {}).get("completion_tokens") or 0), "cost_usd": per_cost, }) # Actualizar per-doc y graficar cuando tengamos ambas fases bucket = self._per_doc_costs.get(radicado) or {"extract": None, "normalize": None} bucket[phase] = per_cost self._per_doc_costs[radicado] = bucket if bucket.get("extract") is not None and bucket.get("normalize") is not None: self._completed_doc_count += 1 total_cost_doc = float(bucket["extract"] or 0.0) + float(bucket["normalize"] or 0.0) self.view.after(0, lambda idx=self._completed_doc_count, val=total_cost_doc: self.view.update_cost_line(idx, val)) avg_cost = self._total_usd / max(self._completed_doc_count, 1) self.view.after(0, lambda p=self._completed_doc_count, a=avg_cost: self.view.update_doc_kpi(p, a)) # Actualizar totales por fase if phase in self._phase_tokens: self._phase_tokens[phase]["input"] += int((usage or {}).get("prompt_tokens") or 0) self._phase_tokens[phase]["cached"] += cached self._phase_tokens[phase]["output"] += int((usage or {}).get("completion_tokens") or 0) self._phase_tokens[phase]["usd"] += per_cost ex = self._phase_tokens["extract"] nz = self._phase_tokens["normalize"] pp_in = self._pp_total_input_tokens pp_out = self._pp_total_output_tokens self.view.after(0, lambda: self.view.update_token_bars( ex["input"], ex["cached"], ex["output"], nz["input"], nz["cached"], nz["output"], pp_in, pp_out, )) # Actualizar KPIs overall = { "input": self._total_prompt_tokens, "cached": self._total_cached_tokens, "output": self._total_completion_tokens, "usd": self._total_usd, } self.view.after(0, lambda ov=overall, exd=ex.copy(), nzd=nz.copy(): self.view.update_kpis(ov, exd, nzd)) # Añadir fila textual self.view.after(0, lambda p=phase, r=radicado, u=usage, c=cost: self.view.append_usage_row(p, r, u, c)) # actualizar totales self.view.set_usage_totals( prompt_tokens=self._total_prompt_tokens, cached_tokens=self._total_cached_tokens, completion_tokens=self._total_completion_tokens, total_usd=self._total_usd, ) except Exception: pass if doc_types: res = self.service.process_root_with_field_selection( root_path=root, doc_types=doc_types, fields=selected_fields, on_progress=on_progress, csv_output_path=csv_path, on_usage=on_usage, on_error_retry=on_error_retry, alias_map=alias_map, tiff_page_range=tiff_page_range, structure_mode=structure_mode, retry_on_empty=retry_on_empty, radicado_filter=radicado_filter, ) else: res = self.service.process_root_with_field_selection( root_path=root, keywords=keywords, fields=selected_fields, on_progress=on_progress, csv_output_path=csv_path, on_usage=on_usage, on_error_retry=on_error_retry, alias_map=alias_map, tiff_page_range=tiff_page_range, structure_mode=structure_mode, retry_on_empty=retry_on_empty, radicado_filter=radicado_filter, ) self.view.append_output_safe(json.dumps(res, ensure_ascii=False, indent=2)) except Exception as e2: logging.exception("Error en reproceso") self.view.append_output_safe(f"Error: {e2}") finally: # Liberar guard al finalizar self._worker2_running = False self.view.set_processing(False) self.view.after(0, lambda: mb.showinfo("Proceso Finalizado", "El procesamiento ha finalizado.")) # Preparar callback de reprocesamiento reprocess_cb = None if doc_types: def _reprocess_multi(): s_by_type, f_full_by_type, f_rad = self.service.build_schema_from_first_for_types( root, doc_types, tiff_page_range=tiff_page_range, structure_mode=structure_mode ) c_schema = {} try: for tname, sch in s_by_type.items(): if not isinstance(sch, dict): continue for p, typ in sch.items(): c_schema[f"{tname}/{p}"] = typ except Exception: pass c_sample = {"radicado": f_rad} try: for tname, s in f_full_by_type.items(): c_sample[tname] = s or {} except Exception: pass return c_schema, c_sample reprocess_cb = _reprocess_multi elif first_pdf_path: def _reprocess_single(): return self.service.build_schema_from_first(first_pdf_path, tiff_page_range=tiff_page_range) reprocess_cb = _reprocess_single # Asegurar que siempre pasamos un schema dict (aunque sea vacío) al selector effective_schema = combined_schema if combined_schema is not None else (schema or {}) effective_sample = combined_sample if combined_sample is not None else sample self.view.after(0, lambda: self._open_field_selector(effective_schema, effective_sample, on_submit, reprocess_cb)) else: self.view.append_output_safe(json.dumps(resultados, ensure_ascii=False, indent=2)) except Exception as e: logger.exception("Error en procesamiento") self.view.append_output_safe(f"Error: {e}") finally: # Si se abrió selector, se reactivará al terminar worker2 if not (isinstance(resultados, list) and resultados and isinstance(resultados[0], dict) and resultados[0].get('_needs_fields')): self.view.set_processing(False) self.view.after(0, lambda: mb.showinfo("Proceso Finalizado", "El procesamiento ha finalizado.")) self.view.set_processing(True) threading.Thread(target=worker, daemon=True).start() def _open_field_selector(self, schema, sample, on_submit, on_reprocess=None): """Abre el diálogo de selección de campos. Args: schema: Mapa path->tipo que ofrece el servicio para ayudar a seleccionar. sample: JSON de muestra del primer documento. on_submit: Callback `(selected_fields, alias_map)` a invocar al aceptar. on_reprocess: Callback opcional para reprocesar muestra y schema. """ dlg = FieldSelectorDialog(self.view, schema=schema, sample=sample) dlg.set_on_submit(on_submit) if on_reprocess: dlg.set_on_reprocess(on_reprocess) # ========================================================================= # --- POST-PROCESAMIENTO --- # ========================================================================= def on_pp_select_csv(self): """Abre un selector de archivo CSV y muestra la vista previa.""" filepath = fd.askopenfilename( filetypes=[("CSV", "*.csv"), ("Todos", "*.*")], title="Seleccionar CSV para post-procesamiento", ) if not filepath: return try: delimiter = postprocess_agent.detect_delimiter(filepath) df = pd.read_csv(filepath, delimiter=delimiter, dtype=str) except Exception as e: mb.showerror("Error", f"No se pudo leer el CSV:\n{e}") return self._pp_df = df self._pp_result_df = None self.view.set_postprocess_csv_path(filepath) self.view.enable_postprocess_save(False) self.view.set_postprocess_code("") self.view.set_postprocess_result("") self.view.clear_postprocess_log() sample_text, _ = postprocess_agent.sample_dataframe(df) self._pp_csv_sample = sample_text self.view.set_postprocess_preview(sample_text) def on_pp_run(self): """Genera y ejecuta código de transformación con el agente LLM.""" if self._pp_df is None: mb.showwarning("Aviso", "Primero seleccione un archivo CSV.") return instructions = self.view.get_postprocess_instructions() if not instructions: mb.showwarning("Aviso", "Escriba las instrucciones de transformación.") return self.view.enable_postprocess_controls(False) self.view.enable_postprocess_save(False) self.view.clear_postprocess_log() self.view.set_postprocess_code("") self.view.set_postprocess_result("") self.view.set_postprocess_status("Procesando...") csv_sample = self._pp_csv_sample df = self._pp_df def track_usage(usage, is_retry=False): self._pp_total_calls += 1 if is_retry: self._pp_total_retries += 1 if isinstance(usage, dict): inp = int(usage.get("input_tokens") or 0) out = int(usage.get("output_tokens") or 0) self._pp_total_input_tokens += inp self._pp_total_output_tokens += out cost = (inp / 1_000_000.0) * 0.25 + (out / 1_000_000.0) * 2.00 self._pp_total_usd += cost self.view.append_postprocess_log_safe( f" Tokens: in={inp:,} out={out:,} | Costo: ${cost:.4f}" ) def worker(): max_retries = 3 code = None error_msg = None try: self.view.append_postprocess_log_safe("Generando código con el agente...") code, usage = postprocess_agent.generate_code( self._pp_client, csv_sample, instructions, ) track_usage(usage) self.view.after(0, lambda c=code: self.view.set_postprocess_code(c)) self.view.append_postprocess_log_safe("Código generado. Ejecutando...") result_df, error_msg = postprocess_agent.execute_code(code, df) attempt = 1 while error_msg and attempt < max_retries: attempt += 1 self.view.append_postprocess_log_safe( f"Error en ejecución: {error_msg}\nReintentando ({attempt}/{max_retries})..." ) code, usage = postprocess_agent.retry_with_error( self._pp_client, csv_sample, instructions, code, error_msg, ) track_usage(usage, is_retry=True) self.view.after(0, lambda c=code: self.view.set_postprocess_code(c)) self.view.append_postprocess_log_safe("Código corregido. Ejecutando...") result_df, error_msg = postprocess_agent.execute_code(code, df) # Actualizar dashboard de PP short_instr = instructions[:60] + ("..." if len(instructions) > 60 else "") self.view.after(0, lambda: self.view.update_pp_kpis( self._pp_total_usd, self._pp_total_calls, self._pp_total_retries, self._pp_total_input_tokens, self._pp_total_output_tokens, short_instr, )) self.view.after(0, lambda: self.view.append_usage_row( "post-proc", short_instr, {"prompt_tokens": self._pp_total_input_tokens, "completion_tokens": self._pp_total_output_tokens}, {"total_usd": self._pp_total_usd}, )) if error_msg: self.view.append_postprocess_log_safe( f"Error final tras {max_retries} intentos: {error_msg}" ) self.view.after(0, lambda: self.view.set_postprocess_status("Error")) final_err = error_msg self.view.after(0, lambda: mb.showerror( "Error", f"No se pudo aplicar la transformación tras {max_retries} intentos:\n{final_err}", )) return self._pp_result_df = result_df result_sample, _ = postprocess_agent.sample_dataframe(result_df) self.view.after(0, lambda s=result_sample: self.view.set_postprocess_result(s)) self.view.append_postprocess_log_safe( f"Ejecución exitosa. Filas: {len(result_df):,} | Columnas: {len(result_df.columns)}" ) self.view.after(0, lambda: self.view.enable_postprocess_save(True)) self.view.after(0, lambda: self.view.set_postprocess_status("Completado")) except Exception as exc: logger.exception("Error en post-procesamiento") err_str = str(exc) self.view.append_postprocess_log_safe(f"Error inesperado: {err_str}") self.view.after(0, lambda: self.view.set_postprocess_status("Error")) self.view.after(0, lambda m=err_str: mb.showerror("Error", f"Error inesperado:\n{m}")) finally: self.view.after(0, lambda: self.view.enable_postprocess_controls(True)) threading.Thread(target=worker, daemon=True).start() def on_pp_save(self): """Guarda el DataFrame resultante del post-procesamiento.""" if self._pp_result_df is None: mb.showwarning("Aviso", "No hay resultado para guardar.") return path = fd.asksaveasfilename( defaultextension=".csv", filetypes=[("CSV", "*.csv"), ("Excel", "*.xlsx"), ("Todos", "*.*")], title="Guardar CSV post-procesado", ) if not path: return try: if path.endswith(".xlsx"): self._pp_result_df.to_excel(path, index=False) else: self._pp_result_df.to_csv(path, index=False) mb.showinfo("Guardado", f"Archivo guardado correctamente:\n{path}") except Exception as e: logger.exception("Error guardando CSV post-procesado") mb.showerror("Error", f"No se pudo guardar el archivo:\n{e}") # ========================================================================= # --- PROCESO DE LIMPIEZA --- # ========================================================================= def on_cleaning_select_csv(self): """Abre un selector de CSV y prepara el resumen de columnas.""" filepath = fd.askopenfilename( filetypes=[("CSV", "*.csv"), ("Todos", "*.*")], title="Seleccionar CSV para limpieza", ) if not filepath: return self._cleaning_result_df = None self.view.set_cleaning_csv_path(filepath) self.view.enable_cleaning_controls(False) self.view.enable_cleaning_save(False) self.view.set_cleaning_result(None) self.view.set_cleaning_preview(None) self.view.set_cleaning_status("Cargando CSV...") try: self.view.set_cleaning_loading(True, "Leyendo archivo CSV...") except Exception: pass def worker(): try: df, delimiter = csv_cleaning.load_csv(filepath) self.view.after( 0, lambda d=df, delim=delimiter: self._on_cleaning_csv_loaded( d, delim ), ) summaries = csv_cleaning.analyze_dataframe(df) self._cleaning_df = df self.view.after( 0, lambda d=df, s=summaries: self._finish_cleaning_csv_load(d, s), ) except Exception as e: logger.exception("Error leyendo o analizando CSV para limpieza") err = str(e) self.view.after(0, lambda m=err: self._fail_cleaning_csv_load(m)) threading.Thread(target=worker, daemon=True).start() def _on_cleaning_csv_loaded(self, df: pd.DataFrame, delimiter: str): self.view.set_cleaning_status( f"CSV leído: {len(df):,} filas x {len(df.columns)} columnas. Analizando columnas..." ) try: self.view.set_cleaning_loading(True, "Analizando columnas...") self.view.set_cleaning_delimiter_chip(delimiter) except Exception: pass def _finish_cleaning_csv_load(self, df: pd.DataFrame, summaries): self.view.populate_cleaning_columns(summaries) self.view.set_cleaning_preview(df) self.view.set_cleaning_status( f"CSV cargado: {len(df):,} filas x {len(df.columns)} columnas." ) try: self.view.set_cleaning_loading(False) except Exception: pass self.view.enable_cleaning_controls(True) def _fail_cleaning_csv_load(self, message: str): self._cleaning_df = None self.view.set_cleaning_status("Error al cargar CSV") try: self.view.set_cleaning_loading(False) except Exception: pass self.view.enable_cleaning_controls(True) mb.showerror("Error", f"No se pudo cargar o analizar el CSV:\n{message}") def on_cleaning_apply(self): """Aplica las reglas seleccionadas en un hilo y actualiza el resultado.""" if self._cleaning_df is None: mb.showwarning("Aviso", "Primero seleccione un archivo CSV.") return rules = self.view.get_cleaning_selected_rules() if not rules: mb.showwarning( "Aviso", "No hay reglas seleccionadas. Marque al menos una regla para alguna columna.", ) return self.view.enable_cleaning_controls(False) self.view.enable_cleaning_save(False) self.view.set_cleaning_status("Aplicando reglas...") self.view.set_cleaning_result(None) df = self._cleaning_df def worker(): try: result_df, log = csv_cleaning.apply_rules(df, rules) self._cleaning_result_df = result_df ok_count = sum(1 for line in log if line.startswith("[OK]")) err_count = sum(1 for line in log if line.startswith("[ERROR]")) warn_count = sum(1 for line in log if line.startswith("[WARN]")) info = f"{ok_count} reglas aplicadas" if err_count or warn_count: info += f" · {err_count} errores · {warn_count} avisos" self.view.after(0, lambda d=result_df, t=info: self.view.set_cleaning_result(d, t)) self.view.after(0, lambda: self.view.enable_cleaning_save(True)) status = ( f"Completado: {ok_count} reglas aplicadas correctamente" + (f", {err_count} errores" if err_count else "") + (f", {warn_count} avisos" if warn_count else "") + "." ) self.view.after(0, lambda s=status: self.view.set_cleaning_status(s)) except Exception as exc: logger.exception("Error aplicando reglas de limpieza") err = str(exc) self.view.after(0, lambda: self.view.set_cleaning_status("Error")) self.view.after(0, lambda m=err: mb.showerror("Error", f"Error aplicando reglas:\n{m}")) finally: self.view.after(0, lambda: self.view.enable_cleaning_controls(True)) threading.Thread(target=worker, daemon=True).start() def on_cleaning_save(self): """Guarda el DataFrame limpio en disco.""" if self._cleaning_result_df is None: mb.showwarning("Aviso", "No hay resultado para guardar.") return path = fd.asksaveasfilename( defaultextension=".csv", filetypes=[("CSV", "*.csv"), ("Excel", "*.xlsx"), ("Todos", "*.*")], title="Guardar CSV limpio", ) if not path: return try: if path.lower().endswith(".xlsx"): self._cleaning_result_df.to_excel(path, index=False) else: self._cleaning_result_df.to_csv(path, index=False, sep="|") mb.showinfo("Guardado", f"Archivo guardado correctamente:\n{path}") except Exception as e: logger.exception("Error guardando CSV limpio") mb.showerror("Error", f"No se pudo guardar el archivo:\n{e}")