feat: Integrate CSV cleaning functionality into the application, adding a new GUI tab for user interaction, enabling CSV loading, analysis, and rule application for data cleaning. Implement backend tracking for batch processing and enhance the BatchProcessor with tracking capabilities.

2026-04-22 00:04:15 -05:00
parent 7a38b0843b
commit 6b3eccdc19
30 changed files with 3590 additions and 19 deletions
@@ -21,6 +21,7 @@ from application.database import DatabaseManager
 from application.config import settings
 from application.costs import MODEL_PRICING
 from application import postprocess_agent
+from application import csv_cleaning
 from openai import OpenAI
 import pandas as pd

@@ -110,6 +111,16 @@ class AppController:
    except Exception:
      pass

+    # Proceso de limpieza (sin LLM)
+    self._cleaning_df: pd.DataFrame | None = None
+    self._cleaning_result_df: pd.DataFrame | None = None
+    try:
+      self.view.set_on_cleaning_select_csv(self.on_cleaning_select_csv)
+      self.view.set_on_cleaning_apply(self.on_cleaning_apply)
+      self.view.set_on_cleaning_save(self.on_cleaning_save)
+    except Exception:
+      pass
+
    # Mapeo de nombres de UI a IDs de modelo (y viceversa)
    self.ui_to_model = {
        "5.4": "gpt-5.4",
@@ -1169,3 +1180,120 @@ class AppController:
    except Exception as e:
      logger.exception("Error guardando CSV post-procesado")
      mb.showerror("Error", f"No se pudo guardar el archivo:\n{e}")
+
+  # =========================================================================
+  # --- PROCESO DE LIMPIEZA ---
+  # =========================================================================
+
+  def on_cleaning_select_csv(self):
+    """Abre un selector de CSV y prepara el resumen de columnas."""
+    filepath = fd.askopenfilename(
+      filetypes=[("CSV", "*.csv"), ("Todos", "*.*")],
+      title="Seleccionar CSV para limpieza",
+    )
+    if not filepath:
+      return
+
+    try:
+      df, delimiter = csv_cleaning.load_csv(filepath)
+    except Exception as e:
+      logger.exception("Error leyendo CSV para limpieza")
+      mb.showerror("Error", f"No se pudo leer el CSV:\n{e}")
+      return
+
+    self._cleaning_df = df
+    self._cleaning_result_df = None
+    self.view.set_cleaning_csv_path(filepath)
+    self.view.enable_cleaning_save(False)
+    self.view.set_cleaning_result(None)
+    self.view.set_cleaning_status(
+      f"CSV cargado: {len(df):,} filas x {len(df.columns)} columnas."
+    )
+    try:
+      self.view.set_cleaning_delimiter_chip(delimiter)
+    except Exception:
+      pass
+
+    try:
+      summaries = csv_cleaning.analyze_dataframe(df)
+    except Exception as e:
+      logger.exception("Error analizando CSV para limpieza")
+      mb.showerror("Error", f"No se pudo analizar el CSV:\n{e}")
+      return
+
+    self.view.populate_cleaning_columns(summaries)
+    self.view.set_cleaning_preview(df)
+
+  def on_cleaning_apply(self):
+    """Aplica las reglas seleccionadas en un hilo y actualiza el resultado."""
+    if self._cleaning_df is None:
+      mb.showwarning("Aviso", "Primero seleccione un archivo CSV.")
+      return
+
+    rules = self.view.get_cleaning_selected_rules()
+    if not rules:
+      mb.showwarning(
+        "Aviso",
+        "No hay reglas seleccionadas. Marque al menos una regla para alguna columna.",
+      )
+      return
+
+    self.view.enable_cleaning_controls(False)
+    self.view.enable_cleaning_save(False)
+    self.view.set_cleaning_status("Aplicando reglas...")
+    self.view.set_cleaning_result(None)
+
+    df = self._cleaning_df
+
+    def worker():
+      try:
+        result_df, log = csv_cleaning.apply_rules(df, rules)
+        self._cleaning_result_df = result_df
+        ok_count = sum(1 for line in log if line.startswith("[OK]"))
+        err_count = sum(1 for line in log if line.startswith("[ERROR]"))
+        warn_count = sum(1 for line in log if line.startswith("[WARN]"))
+        info = f"{ok_count} reglas aplicadas"
+        if err_count or warn_count:
+          info += f" · {err_count} errores · {warn_count} avisos"
+        self.view.after(0, lambda d=result_df, t=info: self.view.set_cleaning_result(d, t))
+        self.view.after(0, lambda: self.view.enable_cleaning_save(True))
+        status = (
+          f"Completado: {ok_count} reglas aplicadas correctamente"
+          + (f", {err_count} errores" if err_count else "")
+          + (f", {warn_count} avisos" if warn_count else "")
+          + "."
+        )
+        self.view.after(0, lambda s=status: self.view.set_cleaning_status(s))
+      except Exception as exc:
+        logger.exception("Error aplicando reglas de limpieza")
+        err = str(exc)
+        self.view.after(0, lambda: self.view.set_cleaning_status("Error"))
+        self.view.after(0, lambda m=err: mb.showerror("Error", f"Error aplicando reglas:\n{m}"))
+      finally:
+        self.view.after(0, lambda: self.view.enable_cleaning_controls(True))
+
+    threading.Thread(target=worker, daemon=True).start()
+
+  def on_cleaning_save(self):
+    """Guarda el DataFrame limpio en disco."""
+    if self._cleaning_result_df is None:
+      mb.showwarning("Aviso", "No hay resultado para guardar.")
+      return
+
+    path = fd.asksaveasfilename(
+      defaultextension=".csv",
+      filetypes=[("CSV", "*.csv"), ("Excel", "*.xlsx"), ("Todos", "*.*")],
+      title="Guardar CSV limpio",
+    )
+    if not path:
+      return
+
+    try:
+      if path.lower().endswith(".xlsx"):
+        self._cleaning_result_df.to_excel(path, index=False)
+      else:
+        self._cleaning_result_df.to_csv(path, index=False)
+      mb.showinfo("Guardado", f"Archivo guardado correctamente:\n{path}")
+    except Exception as e:
+      logger.exception("Error guardando CSV limpio")
+      mb.showerror("Error", f"No se pudo guardar el archivo:\n{e}")
@@ -5,5 +5,14 @@ from gui.tabs.main_tab import MainProcessingTab
 from gui.tabs.preclass_tab import PreclassificationTab
 from gui.tabs.metrics_tab import MetricsTab
 from gui.tabs.postprocess_tab import PostProcessTab
+from gui.tabs.cleaning_tab import CleaningTab

-__all__ = ["SettingsTab", "ResultsTab", "MainProcessingTab", "PreclassificationTab", "MetricsTab", "PostProcessTab"]
+__all__ = [
+  "SettingsTab",
+  "ResultsTab",
+  "MainProcessingTab",
+  "PreclassificationTab",
+  "MetricsTab",
+  "PostProcessTab",
+  "CleaningTab",
+]
@@ -6,7 +6,7 @@ Expone métodos para enlazar callbacks, leer entradas y escribir salida en un
 import os
 import customtkinter as ctk
 from gui.styles import AppStyles
-from gui.tabs import SettingsTab, ResultsTab, MainProcessingTab, PreclassificationTab, MetricsTab, PostProcessTab
+from gui.tabs import SettingsTab, ResultsTab, MainProcessingTab, PreclassificationTab, MetricsTab, PostProcessTab, CleaningTab


 class InvoiceAppView(ctk.CTk):
@@ -44,6 +44,7 @@ class InvoiceAppView(ctk.CTk):
    self.tab_settings = self.tabs.add("Configuración")
    self.tab_usage = self.tabs.add("Dashboard & Métricas")
    self.tab_postprocess = self.tabs.add("Post-Procesamiento")
+    self.tab_cleaning = self.tabs.add("Proceso de Limpieza")

    # =========================================================================
    # --- TAB: PROCESAR DOCUMENTOS ---
@@ -77,6 +78,11 @@ class InvoiceAppView(ctk.CTk):
    # =========================================================================
    self.postprocess_tab = PostProcessTab(self.tab_postprocess)

+    # =========================================================================
+    # --- TAB: PROCESO DE LIMPIEZA ---
+    # =========================================================================
+    self.cleaning_tab = CleaningTab(self.tab_cleaning)
+

  # ---------------------------------------------------------------------------
  # MÉTODOS PÚBLICOS (API COMPATIBLE)
@@ -353,3 +359,41 @@ class InvoiceAppView(ctk.CTk):

  def validate_doc_types(self) -> tuple[bool, str | None]:
    return self.main_tab.validate_doc_types()
+
+  # --- TAB PROCESO DE LIMPIEZA ---
+
+  def set_on_cleaning_select_csv(self, callback):
+    self.cleaning_tab.set_on_select_csv(callback)
+
+  def set_on_cleaning_apply(self, callback):
+    self.cleaning_tab.set_on_apply(callback)
+
+  def set_on_cleaning_save(self, callback):
+    self.cleaning_tab.set_on_save(callback)
+
+  def set_cleaning_csv_path(self, path: str):
+    self.cleaning_tab.set_csv_path(path)
+
+  def set_cleaning_preview(self, df):
+    self.cleaning_tab.set_preview(df)
+
+  def populate_cleaning_columns(self, summaries):
+    self.cleaning_tab.populate_columns(summaries)
+
+  def get_cleaning_selected_rules(self) -> dict[str, list[str]]:
+    return self.cleaning_tab.get_selected_rules()
+
+  def set_cleaning_result(self, df, info_text: str = ""):
+    self.cleaning_tab.set_result(df, info_text)
+
+  def enable_cleaning_save(self, enabled: bool):
+    self.cleaning_tab.enable_save(enabled)
+
+  def enable_cleaning_controls(self, enabled: bool):
+    self.cleaning_tab.enable_controls(enabled)
+
+  def set_cleaning_status(self, text: str):
+    self.cleaning_tab.set_status(text)
+
+  def set_cleaning_delimiter_chip(self, delimiter: str):
+    self.cleaning_tab.set_delimiter_chip(delimiter)