diff --git a/application/batch_processor.py b/application/batch_processor.py index fe84f60..78c6459 100644 --- a/application/batch_processor.py +++ b/application/batch_processor.py @@ -19,6 +19,7 @@ from application.file_discovery import get_sorted_radicados, get_files_as_radica from application.csv_manager import CSVManager from application.config import settings from application.app_prefs import get_allowed_document_types +from application.tracking_manager import TrackingManager from core.utils import collect_radicado_docs, find_first_matching_file, collect_documents_flat from core.csv_utils import rows_from_data @@ -43,9 +44,15 @@ class BatchProcessor: """ self.service = service self.csv_manager = CSVManager() + self.tracking_manager = TrackingManager() self._pause_event = asyncio.Event() self._pause_event.set() # No pausado inicialmente self._stop_requested = False + # Contadores acumulados para tracking web + self._track_tokens: int = 0 + self._track_cost_usd: float = 0.0 + self._track_success: int = 0 + self._track_errors: int = 0 def process_root( self, @@ -70,9 +77,38 @@ class BatchProcessor: Delega en métodos específicos para multi-documento o single-document. """ - # Resetear estado de pausa/parada + # Resetear estado de pausa/parada y contadores de tracking self._pause_event.set() self._stop_requested = False + self._track_tokens = 0 + self._track_cost_usd = 0.0 + self._track_success = 0 + self._track_errors = 0 + + # Notificar inicio a tracking si es procesamiento masivo + if fields: + # Intentar estimar total de radicados + try: + if structure_mode == "general": + temp_rads = get_files_as_radicados(root_path) + else: + temp_rads = get_sorted_radicados(root_path) + + if radicado_filter: + filter_set = {r.strip() for r in radicado_filter if r.strip()} + temp_rads = [r for r in temp_rads if r in filter_set] + + self.tracking_manager.start_batch( + root_path, + len(temp_rads), + config={ + "structure_mode": structure_mode, + "fields_count": len(fields) + }, + csv_output_path=csv_output_path, + ) + except Exception as e: + logger.error(f"Error al iniciar tracking: {e}") # 1. Preparación de Schemas (Fase 1 Multi-doc) if doc_types and not fields: @@ -249,7 +285,9 @@ class BatchProcessor: except Exception: prompt_t = PromptBuilder.build(selected_paths_for_type, {}, profile_name=prof.get('name', 'invoice')) - # Llamada ASYNC + # Llamada ASYNC - Fase: Extracción + self.tracking_manager.report_phase("extract", rad_id) + self.tracking_manager.log(f"Iniciando extracción IA", level="info", radicado=rad_id) data, usage_ext = await self.service.extract_full_from_pdf_async(match, instruction=prompt_t, tiff_page_range=tiff_page_range) self._accumulate_usage(agg_extract_usage, usage_ext) @@ -260,7 +298,9 @@ class BatchProcessor: else: data = {"radicado": rad_id, "data": data} - # Normalización en thread aparte + # Normalización en thread aparte - Fase: Normalización + self.tracking_manager.report_phase("normalize", rad_id) + self.tracking_manager.log(f"Extracción OK — normalizando campos", level="info", radicado=rad_id) norm = await asyncio.to_thread( normalize_to_schema, data, @@ -436,7 +476,9 @@ class BatchProcessor: try: logger.info(f"Procesando radicado {entry['radicado']} [{idx}/{total}]", extra={"radicado": entry['radicado'], "pdf": entry['path']}) - # Async Extract + # Async Extract - Fase: Extracción + self.tracking_manager.report_phase("extract", entry['radicado']) + self.tracking_manager.log(f"Iniciando extracción IA", level="info", radicado=entry['radicado']) data, usage_ext = await self.service.extract_full_from_pdf_async(entry["path"], instruction=prompt, tiff_page_range=tiff_page_range) self._report_usage(on_usage, entry['radicado'], "extract", usage_ext) @@ -447,7 +489,9 @@ class BatchProcessor: data = {"radicado": entry['radicado'], "data": data} data["_source_pdf"] = entry["path"] - # Normalizar (Thread) + # Normalizar (Thread) - Fase: Normalización + self.tracking_manager.report_phase("normalize", entry['radicado']) + self.tracking_manager.log(f"Extracción OK — normalizando campos", level="info", radicado=entry['radicado']) norm = await asyncio.to_thread( normalize_to_schema, data, paths=selected_fields, profile_name=prof.get('name', 'invoice') @@ -667,23 +711,62 @@ class BatchProcessor: callback(rad, idx, total) except Exception: pass + + # Notificar a tracking web con contadores reales acumulados + try: + self._track_success = idx - self._track_errors + # Log de radicado completado exitosamente + if rad and self._track_errors < idx: + self.tracking_manager.log( + f"Radicado completado ({self._track_success}/{total} exitosos)", + level="success", + radicado=str(rad) + ) + if idx >= total: + self.tracking_manager.complete_batch( + success_count=self._track_success, + error_count=self._track_errors, + total_tokens=self._track_tokens, + total_cost_usd=self._track_cost_usd, + ) + else: + self.tracking_manager.update_progress( + current_index=idx, + success_count=self._track_success, + error_count=self._track_errors, + total_tokens=self._track_tokens, + total_cost_usd=self._track_cost_usd, + status="processing" + ) + except Exception as e: + logger.debug(f"Error al notificar progreso a tracking: {e}") def _accumulate_usage(self, agg, new_usage): accumulate_usage(agg, new_usage) def _report_usage(self, callback, rad, phase, usage): - if callback and usage: - try: - model = None - if phase == "extract": - model = getattr(self.service.extractor, "model", None) - elif phase == "normalize": - # El normalizador está hardcodeado a gpt-4.1-mini en normalizer.py - model = "gpt-4.1-mini" - - callback(phase, rad, usage, compute_cost_from_usage(usage, phase, model=model)) - except Exception: - pass + if not usage: + return + try: + model = None + if phase == "extract": + model = getattr(self.service.extractor, "model", None) + elif phase == "normalize": + model = "gpt-4.1-mini" + + cost = compute_cost_from_usage(usage, phase, model=model) + + # Acumular tokens y costos para el tracking web + prompt_tokens = int(usage.get("prompt_tokens") or 0) + completion_tokens = int(usage.get("completion_tokens") or 0) + self._track_tokens += prompt_tokens + completion_tokens + if isinstance(cost, dict): + self._track_cost_usd += float(cost.get("total_usd") or 0.0) + + if callback: + callback(phase, rad, usage, cost) + except Exception: + pass def _write_csv_multidoc(self, path, data, fields, doc_types, prof, header, alias_map, rad): try: @@ -787,6 +870,22 @@ class BatchProcessor: if pdf_path: error_obj["_source_pdf"] = pdf_path logger.exception(f"Error procesando radicado {rad}", extra={"radicado": rad}) + # Acumular error en el contador de tracking + self._track_errors += 1 + error_message = str(e) + try: + self.tracking_manager.report_error(f"[{rad}] {error_message}") + except Exception: + pass + try: + self.tracking_manager.log( + message=f"Error procesando radicado: {error_message}", + level="error", + radicado=str(rad) + ) + except Exception: + pass + if csv_path: header_err = header if header else ["radicado", "error"] try: diff --git a/application/csv_cleaning.py b/application/csv_cleaning.py new file mode 100644 index 0000000..7dd04c4 --- /dev/null +++ b/application/csv_cleaning.py @@ -0,0 +1,777 @@ +"""Limpieza de CSV con análisis inteligente local y reglas configurables. + +Este módulo es pura lógica (sin GUI, sin LLM). Provee: +- `analyze_dataframe` para obtener un resumen por columna (tipo inferido, + ejemplo aleatorio, nulos, únicos, estadísticas específicas por tipo). +- Un registro `CLEANING_RULES` con reglas aplicables a columnas. +- `apply_rules` que aplica las reglas seleccionadas por columna y devuelve + el DataFrame resultante junto con un log de operaciones/advertencias. +""" +from __future__ import annotations + +import csv as _csv +import logging +import random +import re +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +import numpy as np +import pandas as pd + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# MODELO DE DATOS +# --------------------------------------------------------------------------- + +TYPE_INTEGER = "integer" +TYPE_DECIMAL = "decimal" +TYPE_DATE = "date" +TYPE_BOOLEAN = "boolean" +TYPE_CATEGORICAL = "categorical" +TYPE_TEXT = "text" +TYPE_MIXED = "mixed" +TYPE_EMPTY = "empty" + + +@dataclass +class ColumnSummary: + """Resumen inteligente para una columna del DataFrame.""" + name: str + inferred_type: str + example: str = "" + null_count: int = 0 + null_pct: float = 0.0 + unique_count: int = 0 + unique_pct: float = 0.0 + min: Optional[float] = None + max: Optional[float] = None + mean: Optional[float] = None + avg_length: Optional[float] = None + top_values: list[tuple[str, int]] = field(default_factory=list) + detected_date_format: Optional[str] = None + applicable_rules: list[str] = field(default_factory=list) + + def info_text(self) -> str: + """Texto humano con información adicional (para subtítulo de la UI).""" + parts: list[str] = [] + parts.append(f"nulos: {self.null_count} ({self.null_pct:.0f}%)") + parts.append(f"únicos: {self.unique_count}") + if self.inferred_type in (TYPE_INTEGER, TYPE_DECIMAL): + if self.min is not None: + parts.append(f"min: {_fmt_num(self.min)}") + if self.max is not None: + parts.append(f"max: {_fmt_num(self.max)}") + if self.mean is not None: + parts.append(f"media: {_fmt_num(self.mean)}") + elif self.inferred_type == TYPE_DATE: + if self.detected_date_format: + parts.append(f"formato: {self.detected_date_format}") + elif self.inferred_type in (TYPE_TEXT, TYPE_CATEGORICAL): + if self.avg_length is not None: + parts.append(f"long. prom: {self.avg_length:.1f}") + if self.top_values: + tops = ", ".join(f'"{v}" ({c})' for v, c in self.top_values[:3]) + parts.append(f"top: {tops}") + return " · ".join(parts) + + +def _fmt_num(x: float) -> str: + if x is None: + return "" + if isinstance(x, float) and x.is_integer(): + return str(int(x)) + if isinstance(x, float): + return f"{x:.2f}" + return str(x) + + +# --------------------------------------------------------------------------- +# DETECCIÓN DE FECHAS +# --------------------------------------------------------------------------- + +_DATE_NUM_RE = re.compile(r"^\s*(\d{1,4})[\s\/\-\.](\d{1,2})[\s\/\-\.](\d{1,4})\s*$") +_DATE_COMPACT_RE = re.compile(r"^\s*(\d{6}|\d{8})\s*$") +# Hora opcional: HH[:MM[:SS]][ AM|PM] al final de la cadena +_TIME_TAIL_RE = re.compile( + r"[\sT]+(\d{1,2})(?::(\d{1,2})(?::(\d{1,2}))?)?\s*([APap]\.?\s*[Mm]\.?)?\s*$" +) + + +def _split_date_time(value: str) -> tuple[str, Optional[str]]: + """Separa la parte de fecha de la parte de hora si está presente. + + Devuelve `(date_part, time_part_normalized)`. `time_part_normalized` es + None si no hay hora. La hora se normaliza a `HH:MM[:SS][ AM|PM]`. + """ + if value is None: + return "", None + s = str(value).strip() + if not s: + return "", None + + m = _TIME_TAIL_RE.search(s) + if not m: + return s, None + + hh = int(m.group(1)) + mm = int(m.group(2)) if m.group(2) else 0 + ss = m.group(3) + ampm = m.group(4) + if not (0 <= hh <= 23 and 0 <= mm <= 59): + return s, None + if ss is not None: + try: + ssi = int(ss) + if not (0 <= ssi <= 59): + return s, None + except ValueError: + return s, None + # Validar contra AM/PM (con AM/PM, hora 1-12) + if ampm and not (1 <= hh <= 12): + return s, None + + date_part = s[: m.start()].rstrip(" T") + if not date_part: + return s, None + + parts: list[str] = [f"{hh:02d}:{mm:02d}"] + if ss is not None: + parts[0] += f":{int(ss):02d}" + if ampm: + norm_ampm = ampm.replace(".", "").replace(" ", "").upper() + if norm_ampm in ("AM", "PM"): + parts.append(norm_ampm) + time_norm = " ".join(parts) + return date_part, time_norm + + +def _extract_date_parts(value: str) -> Optional[tuple[int, int, int, str]]: + """Intenta extraer (a, b, c, layout) de una fecha en cualquier formato común. + + `layout` es uno de: 'sep' (con separadores), 'compact8' (YYYYMMDD o + DDMMYYYY), 'compact6' (DDMMYY o YYMMDD). No interpreta todavía qué + componente es día/mes/año; eso lo decide `detect_date_format` viendo la + columna completa. Acepta valores con hora al final (ignorada para los + componentes de fecha). + """ + if value is None: + return None + s = str(value).strip() + if not s or s.lower() in ("nan", "none", "nat"): + return None + + date_part, _ = _split_date_time(s) + if date_part: + s = date_part + + m = _DATE_NUM_RE.match(s) + if m: + try: + a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)) + return (a, b, c, "sep") + except ValueError: + return None + + m2 = _DATE_COMPACT_RE.match(s) + if m2: + digits = m2.group(1) + if len(digits) == 8: + return (int(digits[0:2]), int(digits[2:4]), int(digits[4:8]), "compact8") + if len(digits) == 6: + return (int(digits[0:2]), int(digits[2:4]), int(digits[4:6]), "compact6") + + return None + + +def detect_date_format(series: pd.Series, sample_size: int = 500) -> Optional[str]: + """Analiza la serie y determina el formato de fecha dominante. + + Devuelve un string con el layout detectado (p. ej. "DD/MM/YYYY", + "MM/DD/YYYY", "YYYYMMDD", "DDMMYYYY") o None si no parece fechas. + """ + if series is None or len(series) == 0: + return None + + non_null = series.dropna() + if non_null.empty: + return None + + if len(non_null) > sample_size: + non_null = non_null.sample(sample_size, random_state=42) + + layouts = {"sep": 0, "compact8": 0, "compact6": 0} + sep_first_gt12 = 0 # primer componente > 12 → probable día + sep_second_gt12 = 0 # segundo componente > 12 → probable mes solo posible si MM/DD + sep_third_4digits = 0 # tercer componente de 4 dígitos → año al final + sep_first_4digits = 0 # primer componente de 4 dígitos → año al inicio + compact8_year_first = 0 # primeros 4 dígitos > 1900 → YYYYMMDD + compact8_year_last = 0 # últimos 4 dígitos > 1900 → DDMMYYYY + total_parsed = 0 + + for val in non_null: + parsed = _extract_date_parts(val) + if parsed is None: + continue + total_parsed += 1 + a, b, c, layout = parsed + layouts[layout] = layouts.get(layout, 0) + 1 + + if layout == "sep": + # año puede estar al inicio o al final según cantidad de dígitos + if _digit_count(a) == 4: + sep_first_4digits += 1 + if _digit_count(c) == 4: + sep_third_4digits += 1 + if a > 12: + sep_first_gt12 += 1 + if b > 12: + sep_second_gt12 += 1 + elif layout == "compact8": + # a son los primeros 2 dígitos, c son los últimos 4 + if c >= 1900: + compact8_year_last += 1 + # a concatenado con b son los primeros 4 dígitos + first4 = a * 100 + b + if first4 >= 1900: + compact8_year_first += 1 + + if total_parsed == 0: + return None + + # Requerimos que >=70% de los no nulos parseen como fecha para considerarla + # columna de fechas. + if total_parsed / len(non_null) < 0.7: + return None + + dominant = max(layouts, key=lambda k: layouts[k]) + + if dominant == "compact8": + if compact8_year_first > compact8_year_last: + return "YYYYMMDD" + return "DDMMYYYY" + if dominant == "compact6": + return "DDMMYY" + + # Caso con separadores + if sep_first_4digits > sep_third_4digits: + return "YYYY/MM/DD" + # año al final + if sep_first_gt12 > 0 and sep_second_gt12 == 0: + return "DD/MM/YYYY" + if sep_second_gt12 > 0 and sep_first_gt12 == 0: + return "MM/DD/YYYY" + # ambiguo → preferencia hispana + return "DD/MM/YYYY" + + +def _digit_count(n: int) -> int: + return len(str(abs(n))) + + +def _parse_date_with_format(value: str, fmt: str) -> Optional[tuple[int, int, int]]: + """Convierte un valor de fecha a (día, mes, año) según el layout detectado.""" + parsed = _extract_date_parts(value) + if parsed is None: + return None + a, b, c, layout = parsed + + if fmt == "DD/MM/YYYY": + d, m, y = a, b, c + elif fmt == "MM/DD/YYYY": + m, d, y = a, b, c + elif fmt == "YYYY/MM/DD": + y, m, d = a, b, c + elif fmt == "YYYYMMDD" and layout == "compact8": + # reconstruir: primer4 = a*100 + b, luego mes y día + y = a * 100 + b + m = c // 100 + d = c % 100 + elif fmt == "DDMMYYYY" and layout == "compact8": + d = a + m = b + y = c + elif fmt == "DDMMYY" and layout == "compact6": + d = a + m = b + y = 2000 + c if c < 50 else 1900 + c + else: + # intentar como sep genérico + d, m, y = a, b, c + + if not (1 <= m <= 12 and 1 <= d <= 31 and 1000 <= y <= 9999): + # año en 2 dígitos + if y < 100: + y = 2000 + y if y < 50 else 1900 + y + if not (1 <= m <= 12 and 1 <= d <= 31 and 1000 <= y <= 9999): + return None + return (d, m, y) + + +# --------------------------------------------------------------------------- +# ANÁLISIS DE COLUMNAS +# --------------------------------------------------------------------------- + +_BOOLEAN_VALUES = {"true", "false", "si", "sí", "no", "1", "0", "yes"} + + +def _infer_column_type(series: pd.Series) -> tuple[str, Optional[str]]: + """Devuelve (tipo, formato_fecha_detectado).""" + non_null = series.dropna().astype(str).str.strip() + non_null = non_null[non_null != ""] + if non_null.empty: + return TYPE_EMPTY, None + + # Booleano + lowered = non_null.str.lower() + if lowered.isin(_BOOLEAN_VALUES).all() and lowered.nunique() <= 3: + return TYPE_BOOLEAN, None + + # Numérico + # Aceptamos separador de miles con "." o "," y decimal con "," o ".". + # Estrategia: intentar to_numeric tras normalizar. + numeric_candidate = _to_numeric_lenient(non_null) + numeric_ratio = numeric_candidate.notna().mean() + if numeric_ratio >= 0.9: + is_all_integer = (numeric_candidate.dropna() % 1 == 0).all() + return (TYPE_INTEGER if is_all_integer else TYPE_DECIMAL), None + + # Fecha + date_fmt = detect_date_format(non_null) + if date_fmt: + return TYPE_DATE, date_fmt + + # Categórico vs texto + unique_ratio = non_null.nunique() / len(non_null) + if non_null.nunique() <= 15 and unique_ratio < 0.3: + return TYPE_CATEGORICAL, None + + # Mixto si mezcla números con texto y nada es dominante + if 0.1 < numeric_ratio < 0.9: + return TYPE_MIXED, None + + return TYPE_TEXT, None + + +def _to_numeric_lenient(series: pd.Series) -> pd.Series: + """Intenta convertir a numérico aceptando separadores comunes.""" + s = series.astype(str).str.strip() + s = s.str.replace(r"\s+", "", regex=True) + + # Si hay ambos "." y ",", asumir "." como miles y "," como decimal + def _norm(x: str) -> str: + if "." in x and "," in x: + return x.replace(".", "").replace(",", ".") + if "," in x and "." not in x: + # coma decimal + return x.replace(",", ".") + return x + + s = s.map(_norm) + return pd.to_numeric(s, errors="coerce") + + +def _applicable_rules_for_type(inferred_type: str) -> list[str]: + """Reglas sugeridas según tipo inferido.""" + base = ["strip_special_spaces", "strip_special_keep_date_seps", "strip_spaces"] + if inferred_type in (TYPE_INTEGER, TYPE_DECIMAL): + return base + ["to_integer", "round_decimals_custom"] + if inferred_type == TYPE_DATE: + return base + ["date_numeric_to_ddmmyyyy", "date_alpha_to_ddmmyyyy"] + if inferred_type == TYPE_MIXED: + return base + [ + "to_integer", + "round_decimals_custom", + "date_numeric_to_ddmmyyyy", + "date_alpha_to_ddmmyyyy", + ] + # Texto / categórico / boolean / empty → todas las de limpieza de texto + las + # conversoras (el usuario sabrá si aplica). + return base + [ + "to_integer", + "round_decimals_custom", + "date_numeric_to_ddmmyyyy", + "date_alpha_to_ddmmyyyy", + ] + + +def analyze_dataframe(df: pd.DataFrame) -> list[ColumnSummary]: + """Genera un resumen por columna del DataFrame.""" + summaries: list[ColumnSummary] = [] + total_rows = len(df) + rng = random.Random(42) + + for col in df.columns: + series = df[col] + null_count = int(series.isna().sum() + (series.astype(str).str.strip() == "").sum()) + # Evitar doble conteo: considerar vacío solo si no-NaN + null_count = int((series.isna() | (series.astype(str).str.strip() == "")).sum()) + null_pct = (null_count / total_rows * 100) if total_rows else 0.0 + + non_null = series.dropna().astype(str) + non_null = non_null[non_null.str.strip() != ""] + + example = "" + if not non_null.empty: + example = str(rng.choice(non_null.tolist())) + if len(example) > 80: + example = example[:77] + "..." + + unique_count = int(non_null.nunique()) + unique_pct = (unique_count / total_rows * 100) if total_rows else 0.0 + + inferred_type, date_fmt = _infer_column_type(series) + + summary = ColumnSummary( + name=str(col), + inferred_type=inferred_type, + example=example, + null_count=null_count, + null_pct=null_pct, + unique_count=unique_count, + unique_pct=unique_pct, + detected_date_format=date_fmt, + ) + + if inferred_type in (TYPE_INTEGER, TYPE_DECIMAL): + numeric = _to_numeric_lenient(non_null) + if numeric.notna().any(): + summary.min = float(numeric.min()) + summary.max = float(numeric.max()) + summary.mean = float(numeric.mean()) + elif inferred_type in (TYPE_TEXT, TYPE_CATEGORICAL): + if not non_null.empty: + summary.avg_length = float(non_null.str.len().mean()) + counts = non_null.value_counts().head(3) + summary.top_values = [(str(v), int(c)) for v, c in counts.items()] + + summary.applicable_rules = _applicable_rules_for_type(inferred_type) + summaries.append(summary) + + return summaries + + +# --------------------------------------------------------------------------- +# REGLAS DE LIMPIEZA +# --------------------------------------------------------------------------- + + +def _rule_strip_special_spaces(series: pd.Series) -> pd.Series: + """Elimina todo lo que no sea alfanumérico (incluye espacios).""" + pattern = re.compile(r"[^A-Za-z0-9]", flags=re.UNICODE) + + def _fn(x): + if pd.isna(x): + return x + return pattern.sub("", str(x)) + + return series.map(_fn) + + +def _rule_strip_special_keep_date_seps(series: pd.Series) -> pd.Series: + """Conserva alfanuméricos y separadores de fecha: '/', '-', espacio.""" + pattern = re.compile(r"[^A-Za-z0-9\/\- ]", flags=re.UNICODE) + + def _fn(x): + if pd.isna(x): + return x + return pattern.sub("", str(x)) + + return series.map(_fn) + + +def _rule_strip_spaces(series: pd.Series) -> pd.Series: + """Elimina únicamente espacios (incluidos tabs y saltos de línea).""" + def _fn(x): + if pd.isna(x): + return x + return re.sub(r"\s+", "", str(x)) + + return series.map(_fn) + + +def _rule_to_integer(series: pd.Series) -> pd.Series: + """Convierte a entero (Int64 nullable) con parser tolerante.""" + numeric = _to_numeric_lenient(series.astype(str)) + return numeric.round().astype("Int64") + + +def _rule_round_decimals_custom(series: pd.Series) -> pd.Series: + """Aproximación custom: fracción > 0.5 → entero superior, resto → inferior. + + Funciona con positivos y negativos (usa parte entera hacia 0 y suma o + resta 1 según el signo y la fracción). + """ + numeric = _to_numeric_lenient(series.astype(str)) + + def _fn(x): + if pd.isna(x): + return pd.NA + sign = -1 if x < 0 else 1 + ax = abs(float(x)) + integer_part = int(ax) + frac = ax - integer_part + rounded = integer_part + (1 if frac > 0.5 else 0) + return sign * rounded + + return numeric.map(_fn).astype("Int64") + + +def _rule_date_numeric_to_ddmmyyyy(series: pd.Series) -> pd.Series: + """Entrada: fechas en formato numérico (ej. 20240315 o 15032024). + + Salida: "DD/MM/AAAA" tras detectar el formato predominante. + """ + # Para esta regla consideramos los valores "numéricos" (quitamos separadores + # comunes si los hay para detectar un YYYYMMDD/DDMMYYYY confiable). + stripped = series.astype(str).str.replace(r"[\s\/\-\.]", "", regex=True) + stripped = stripped.where(series.notna(), other=pd.NA) + fmt = detect_date_format(stripped) + if fmt is None: + # fallback: intentar con detector normal + fmt = detect_date_format(series) + return _format_dates_to_ddmmyyyy(series, fmt) + + +def _rule_date_alpha_to_ddmmyyyy(series: pd.Series) -> pd.Series: + """Entrada: fechas en formato alfanumérico con separadores. + + Salida: "DD/MM/AAAA". + """ + fmt = detect_date_format(series) + return _format_dates_to_ddmmyyyy(series, fmt) + + +def _format_dates_to_ddmmyyyy(series: pd.Series, fmt: Optional[str]) -> pd.Series: + """Convierte cada valor al formato DD/MM/AAAA usando `fmt` como layout de + entrada detectado. La parte de hora (si existe) se descarta. Mantiene el + valor original si no puede parsear la fecha. + """ + if fmt is None: + return series.copy() + + def _fn(x): + if pd.isna(x) or str(x).strip() == "": + return x + raw = str(x) + date_part, _time_part = _split_date_time(raw) + target = date_part if date_part else raw + parsed = _parse_date_with_format(target, fmt) + if parsed is None: + return x + d, m, y = parsed + return f"{d:02d}/{m:02d}/{y:04d}" + + return series.map(_fn) + + +# Registro de reglas +CLEANING_RULES: list[dict[str, Any]] = [ + { + "id": "strip_special_spaces", + "name": "Eliminar caracteres especiales y espacios", + "description": "Deja solo caracteres alfanuméricos (A-Z, a-z, 0-9).", + "fn": _rule_strip_special_spaces, + }, + { + "id": "strip_special_keep_date_seps", + "name": "Eliminar caracteres especiales y espacios (excepto / - y espacio, p. ej. Fecha Servicio)", + "description": "Conserva alfanuméricos y los separadores '/', '-' y espacios.", + "fn": _rule_strip_special_keep_date_seps, + }, + { + "id": "strip_spaces", + "name": "Eliminar únicamente espacios", + "description": "Elimina todos los caracteres de espacio en blanco.", + "fn": _rule_strip_spaces, + }, + { + "id": "to_integer", + "name": "Convertir a Entero", + "description": "Convierte la columna a enteros (los decimales se redondean).", + "fn": _rule_to_integer, + }, + { + "id": "date_numeric_to_ddmmyyyy", + "name": "Conversión de formato numérico (Fechas) → DD/MM/AAAA", + "description": "Detecta el formato numérico predominante (YYYYMMDD, DDMMYYYY, etc.) y normaliza a DD/MM/AAAA.", + "fn": _rule_date_numeric_to_ddmmyyyy, + }, + { + "id": "date_alpha_to_ddmmyyyy", + "name": "Conversión de formato alfanumérico (Fechas) → DD/MM/AAAA", + "description": "Detecta el formato con separadores predominante (DD/MM/YYYY, MM/DD/YYYY, YYYY/MM/DD, etc.) y normaliza a DD/MM/AAAA.", + "fn": _rule_date_alpha_to_ddmmyyyy, + }, + { + "id": "round_decimals_custom", + "name": "Aproximación Decimales (>0.5 sube, ≤0.5 baja)", + "description": "Si la fracción decimal es mayor a 0.5 se aproxima al entero superior; de lo contrario, al inferior.", + "fn": _rule_round_decimals_custom, + }, +] + +_RULE_BY_ID: dict[str, Callable[[pd.Series], pd.Series]] = { + r["id"]: r["fn"] for r in CLEANING_RULES +} + +_RULE_NAME_BY_ID: dict[str, str] = {r["id"]: r["name"] for r in CLEANING_RULES} + + +def get_rule(rule_id: str) -> Optional[dict[str, Any]]: + """Devuelve el registro de una regla por id.""" + for r in CLEANING_RULES: + if r["id"] == rule_id: + return r + return None + + +# --------------------------------------------------------------------------- +# APLICACIÓN DE REGLAS +# --------------------------------------------------------------------------- + + +def apply_rules( + df: pd.DataFrame, + rules_per_column: dict[str, list[str]], +) -> tuple[pd.DataFrame, list[str]]: + """Aplica las reglas seleccionadas a cada columna. + + `rules_per_column` es un dict de `columna -> lista de IDs de reglas en el + orden a aplicar`. Devuelve el DataFrame resultante y un log. + """ + result = df.copy() + log: list[str] = [] + + for col, rule_ids in rules_per_column.items(): + if not rule_ids: + continue + if col not in result.columns: + log.append(f"[WARN] Columna no encontrada: {col}") + continue + for rule_id in rule_ids: + fn = _RULE_BY_ID.get(rule_id) + if fn is None: + log.append(f"[WARN] Regla desconocida '{rule_id}' en columna '{col}'") + continue + try: + result[col] = fn(result[col]) + log.append(f"[OK] '{col}' <- {_RULE_NAME_BY_ID.get(rule_id, rule_id)}") + except Exception as exc: # noqa: BLE001 + logger.exception("Error aplicando regla %s a %s", rule_id, col) + log.append(f"[ERROR] '{col}' regla '{rule_id}': {exc}") + + return result, log + + +# --------------------------------------------------------------------------- +# CARGA ROBUSTA DE CSV +# --------------------------------------------------------------------------- + +_CANDIDATE_DELIMITERS = ["|", ";", "\t", ",", ":"] + + +def detect_delimiter(filepath: str) -> str: + """Detecta el delimitador del CSV de forma robusta. + + Estrategia: + 1. Intentar `csv.Sniffer` con un sample grande. + 2. Si falla o el resultado parece incorrecto, contar ocurrencias por línea + de cada delimitador candidato y elegir el que produce el mayor número + de columnas estable entre líneas. + """ + encodings = ("utf-8", "utf-8-sig", "latin-1", "cp1252") + sample = "" + for enc in encodings: + try: + with open(filepath, "r", encoding=enc, errors="replace") as f: + sample = f.read(65536) + break + except Exception: + continue + + if not sample: + return "," + + try: + dialect = _csv.Sniffer().sniff(sample, delimiters="".join(_CANDIDATE_DELIMITERS)) + sniffed = dialect.delimiter + except _csv.Error: + sniffed = None + + lines = [ln for ln in sample.splitlines() if ln.strip()] + if not lines: + return sniffed or "," + + # Para cada candidato, calcular el número promedio de columnas y la + # consistencia (cuántas líneas tienen el mismo conteo dominante). + scores: dict[str, tuple[int, int]] = {} + for delim in _CANDIDATE_DELIMITERS: + counts = [ln.count(delim) for ln in lines[:50]] + if max(counts) == 0: + continue + # Modo (valor más frecuente) + from collections import Counter + common = Counter(counts).most_common(1)[0] + mode_count, mode_freq = common + if mode_count == 0: + continue + scores[delim] = (mode_count, mode_freq) + + if scores: + # Elegir el delimitador con mayor número de columnas (mode_count) y, + # como desempate, mayor consistencia (mode_freq). + best = max(scores.items(), key=lambda kv: (kv[1][0], kv[1][1])) + best_delim = best[0] + # Si el sniffer dio algo razonable y coincide, prevalece; si difiere y + # nuestro candidato es claramente mejor, usamos el nuestro. + if sniffed and sniffed in scores: + sniffed_score = scores[sniffed] + if sniffed_score[0] >= best[1][0] - 1: + return sniffed + return best_delim + + return sniffed or "," + + +def load_csv(filepath: str) -> tuple[pd.DataFrame, str]: + """Carga un CSV con detección de delimitador y tolerancia a filas malas. + + Devuelve `(df, delimiter)`. Lanza excepción si todos los intentos fallan. + """ + delimiter = detect_delimiter(filepath) + + encodings_to_try = ["utf-8", "utf-8-sig", "latin-1", "cp1252"] + last_error: Optional[Exception] = None + + for enc in encodings_to_try: + try: + df = pd.read_csv( + filepath, + delimiter=delimiter, + dtype=str, + encoding=enc, + engine="python", + on_bad_lines="skip", + ) + return df, delimiter + except Exception as exc: # noqa: BLE001 + last_error = exc + continue + + # Último intento con quoting permisivo y sin separador detectado + try: + df = pd.read_csv( + filepath, + sep=None, + dtype=str, + engine="python", + on_bad_lines="skip", + ) + return df, "(auto)" + except Exception as exc: # noqa: BLE001 + last_error = exc + + raise RuntimeError(f"No se pudo leer el CSV: {last_error}") diff --git a/application/tracking_manager.py b/application/tracking_manager.py new file mode 100644 index 0000000..ae1a5b0 --- /dev/null +++ b/application/tracking_manager.py @@ -0,0 +1,107 @@ +import requests +import logging +import os +import socket +import getpass +from typing import Optional, Dict, Any +from datetime import datetime + +logger = logging.getLogger(__name__) + +class TrackingManager: + """Gestiona la comunicación con el servidor de seguimiento web.""" + + def __init__(self, base_url: str = "http://localhost:8000"): + self.base_url = base_url + self.current_batch_id: Optional[str] = None + self.is_enabled = True + self._session = requests.Session() # Usar sesión para reutilizar conexiones + + def start_batch(self, root_path: str, total_radicados: int, config: Optional[Dict] = None, + csv_output_path: Optional[str] = None): + """Notifica el inicio de un nuevo lote al servidor.""" + if not self.is_enabled: + return + + try: + payload = { + "root_path": root_path, + "total_radicados": total_radicados, + "csv_output_path": csv_output_path, + "hostname": socket.gethostname(), + "username": getpass.getuser(), + "config": config or {} + } + response = self._session.post(f"{self.base_url}/batches", json=payload, timeout=5) + if response.status_code == 200: + self.current_batch_id = response.json().get("id") + logger.info(f"Tracking iniciado: Batch ID {self.current_batch_id}") + else: + logger.warning(f"Error al iniciar tracking: {response.status_code}") + except Exception as e: + logger.error(f"Error de conexión con el servidor de tracking: {e}") + + def update_progress(self, current_index: int, success_count: int, error_count: int, + total_tokens: int, total_cost_usd: float, status: str = "processing"): + """Actualiza el progreso del lote actual.""" + if not self.is_enabled or not self.current_batch_id: + return + + try: + payload = { + "current_radicado_index": current_index, + "success_count": success_count, + "error_count": error_count, + "total_tokens": total_tokens, + "total_cost_usd": total_cost_usd, + "status": status + } + self._session.patch(f"{self.base_url}/batches/{self.current_batch_id}", json=payload, timeout=1) + except Exception as e: + logger.debug(f"Error al actualizar tracking: {e}") + + def log(self, message: str, level: str = "info", radicado: str = None): + """Envía una entrada de log al servidor de tracking.""" + if not self.is_enabled or not self.current_batch_id: + return + try: + payload = {"message": message, "level": level, "radicado": radicado} + self._session.post( + f"{self.base_url}/batches/{self.current_batch_id}/logs", + json=payload, timeout=1 + ) + except Exception as e: + logger.debug(f"Error al enviar log a tracking: {e}") + + def report_phase(self, phase: str, radicado: str = None): + """Reporta la fase actual de procesamiento (extract, normalize, csv, completed).""" + if not self.is_enabled or not self.current_batch_id: + return + try: + payload = {"current_phase": phase, "current_radicado": radicado} + self._session.patch(f"{self.base_url}/batches/{self.current_batch_id}", json=payload, timeout=1) + except Exception as e: + logger.debug(f"Error al reportar fase a tracking: {e}") + + def report_error(self, error_message: str): + """Reporta un error crítico al servidor.""" + if not self.is_enabled or not self.current_batch_id: + return + + try: + payload = {"last_error": error_message} + self._session.patch(f"{self.base_url}/batches/{self.current_batch_id}", json=payload, timeout=1) + except Exception as e: + logger.debug(f"Error al reportar error de tracking: {e}") + + def complete_batch(self, success_count: int, error_count: int, total_tokens: int, total_cost_usd: float): + """Notifica la finalización exitosa del lote.""" + self.update_progress( + current_index=-1, # Indica finalización si se desea + success_count=success_count, + error_count=error_count, + total_tokens=total_tokens, + total_cost_usd=total_cost_usd, + status="completed" + ) + self.current_batch_id = None diff --git a/gui/controller.py b/gui/controller.py index 8ffc4d8..2cdbda7 100644 --- a/gui/controller.py +++ b/gui/controller.py @@ -21,6 +21,7 @@ from application.database import DatabaseManager from application.config import settings from application.costs import MODEL_PRICING from application import postprocess_agent +from application import csv_cleaning from openai import OpenAI import pandas as pd @@ -110,6 +111,16 @@ class AppController: except Exception: pass + # Proceso de limpieza (sin LLM) + self._cleaning_df: pd.DataFrame | None = None + self._cleaning_result_df: pd.DataFrame | None = None + try: + self.view.set_on_cleaning_select_csv(self.on_cleaning_select_csv) + self.view.set_on_cleaning_apply(self.on_cleaning_apply) + self.view.set_on_cleaning_save(self.on_cleaning_save) + except Exception: + pass + # Mapeo de nombres de UI a IDs de modelo (y viceversa) self.ui_to_model = { "5.4": "gpt-5.4", @@ -1169,3 +1180,120 @@ class AppController: except Exception as e: logger.exception("Error guardando CSV post-procesado") mb.showerror("Error", f"No se pudo guardar el archivo:\n{e}") + + # ========================================================================= + # --- PROCESO DE LIMPIEZA --- + # ========================================================================= + + def on_cleaning_select_csv(self): + """Abre un selector de CSV y prepara el resumen de columnas.""" + filepath = fd.askopenfilename( + filetypes=[("CSV", "*.csv"), ("Todos", "*.*")], + title="Seleccionar CSV para limpieza", + ) + if not filepath: + return + + try: + df, delimiter = csv_cleaning.load_csv(filepath) + except Exception as e: + logger.exception("Error leyendo CSV para limpieza") + mb.showerror("Error", f"No se pudo leer el CSV:\n{e}") + return + + self._cleaning_df = df + self._cleaning_result_df = None + self.view.set_cleaning_csv_path(filepath) + self.view.enable_cleaning_save(False) + self.view.set_cleaning_result(None) + self.view.set_cleaning_status( + f"CSV cargado: {len(df):,} filas x {len(df.columns)} columnas." + ) + try: + self.view.set_cleaning_delimiter_chip(delimiter) + except Exception: + pass + + try: + summaries = csv_cleaning.analyze_dataframe(df) + except Exception as e: + logger.exception("Error analizando CSV para limpieza") + mb.showerror("Error", f"No se pudo analizar el CSV:\n{e}") + return + + self.view.populate_cleaning_columns(summaries) + self.view.set_cleaning_preview(df) + + def on_cleaning_apply(self): + """Aplica las reglas seleccionadas en un hilo y actualiza el resultado.""" + if self._cleaning_df is None: + mb.showwarning("Aviso", "Primero seleccione un archivo CSV.") + return + + rules = self.view.get_cleaning_selected_rules() + if not rules: + mb.showwarning( + "Aviso", + "No hay reglas seleccionadas. Marque al menos una regla para alguna columna.", + ) + return + + self.view.enable_cleaning_controls(False) + self.view.enable_cleaning_save(False) + self.view.set_cleaning_status("Aplicando reglas...") + self.view.set_cleaning_result(None) + + df = self._cleaning_df + + def worker(): + try: + result_df, log = csv_cleaning.apply_rules(df, rules) + self._cleaning_result_df = result_df + ok_count = sum(1 for line in log if line.startswith("[OK]")) + err_count = sum(1 for line in log if line.startswith("[ERROR]")) + warn_count = sum(1 for line in log if line.startswith("[WARN]")) + info = f"{ok_count} reglas aplicadas" + if err_count or warn_count: + info += f" · {err_count} errores · {warn_count} avisos" + self.view.after(0, lambda d=result_df, t=info: self.view.set_cleaning_result(d, t)) + self.view.after(0, lambda: self.view.enable_cleaning_save(True)) + status = ( + f"Completado: {ok_count} reglas aplicadas correctamente" + + (f", {err_count} errores" if err_count else "") + + (f", {warn_count} avisos" if warn_count else "") + + "." + ) + self.view.after(0, lambda s=status: self.view.set_cleaning_status(s)) + except Exception as exc: + logger.exception("Error aplicando reglas de limpieza") + err = str(exc) + self.view.after(0, lambda: self.view.set_cleaning_status("Error")) + self.view.after(0, lambda m=err: mb.showerror("Error", f"Error aplicando reglas:\n{m}")) + finally: + self.view.after(0, lambda: self.view.enable_cleaning_controls(True)) + + threading.Thread(target=worker, daemon=True).start() + + def on_cleaning_save(self): + """Guarda el DataFrame limpio en disco.""" + if self._cleaning_result_df is None: + mb.showwarning("Aviso", "No hay resultado para guardar.") + return + + path = fd.asksaveasfilename( + defaultextension=".csv", + filetypes=[("CSV", "*.csv"), ("Excel", "*.xlsx"), ("Todos", "*.*")], + title="Guardar CSV limpio", + ) + if not path: + return + + try: + if path.lower().endswith(".xlsx"): + self._cleaning_result_df.to_excel(path, index=False) + else: + self._cleaning_result_df.to_csv(path, index=False) + mb.showinfo("Guardado", f"Archivo guardado correctamente:\n{path}") + except Exception as e: + logger.exception("Error guardando CSV limpio") + mb.showerror("Error", f"No se pudo guardar el archivo:\n{e}") diff --git a/gui/tabs/__init__.py b/gui/tabs/__init__.py index 535d8bc..9c80cab 100644 --- a/gui/tabs/__init__.py +++ b/gui/tabs/__init__.py @@ -5,5 +5,14 @@ from gui.tabs.main_tab import MainProcessingTab from gui.tabs.preclass_tab import PreclassificationTab from gui.tabs.metrics_tab import MetricsTab from gui.tabs.postprocess_tab import PostProcessTab +from gui.tabs.cleaning_tab import CleaningTab -__all__ = ["SettingsTab", "ResultsTab", "MainProcessingTab", "PreclassificationTab", "MetricsTab", "PostProcessTab"] +__all__ = [ + "SettingsTab", + "ResultsTab", + "MainProcessingTab", + "PreclassificationTab", + "MetricsTab", + "PostProcessTab", + "CleaningTab", +] diff --git a/gui/tabs/cleaning_tab.py b/gui/tabs/cleaning_tab.py new file mode 100644 index 0000000..a5b608c --- /dev/null +++ b/gui/tabs/cleaning_tab.py @@ -0,0 +1,1014 @@ +"""Pestaña de Proceso de Limpieza de CSV. + +Permite al usuario cargar un CSV, ver un resumen inteligente por columna y +aplicar reglas de limpieza seleccionables antes de exportar el resultado. +""" +from __future__ import annotations + +from dataclasses import dataclass +from tkinter import ttk +from typing import Optional + +import customtkinter as ctk +import pandas as pd + +from application.csv_cleaning import CLEANING_RULES, ColumnSummary +from gui.styles import AppStyles + + +_PREVIEW_MAX_ROWS = 50 +_PREVIEW_MAX_CELL_LEN = 80 + + +_TYPE_LABELS = { + "integer": "Entero", + "decimal": "Decimal", + "date": "Fecha", + "boolean": "Booleano", + "categorical": "Categórico", + "text": "Texto", + "mixed": "Mixto", + "empty": "Vacío", +} + +# Color por tipo (badge). Tuplas (light, dark) compatibles con CTk. +_TYPE_COLORS: dict[str, tuple[str, str]] = { + "integer": ("#1F6FEB", "#1F6FEB"), + "decimal": ("#0EA5E9", "#0EA5E9"), + "date": ("#D97706", "#D97706"), + "boolean": ("#7C3AED", "#7C3AED"), + "categorical": ("#16A34A", "#16A34A"), + "text": ("#475569", "#64748B"), + "mixed": ("#DC2626", "#DC2626"), + "empty": ("#6B7280", "#6B7280"), +} + +_RULE_SHORT_LABELS: dict[str, str] = { + "strip_special_spaces": "Sin especiales/espacios", + "strip_special_keep_date_seps": "Sin especiales (mantiene / - espacio)", + "strip_spaces": "Sin espacios", + "to_integer": "A entero", + "date_numeric_to_ddmmyyyy": "Fecha numérica → DD/MM/AAAA", + "date_alpha_to_ddmmyyyy": "Fecha alfanumérica → DD/MM/AAAA", + "round_decimals_custom": "Aprox. decimales (>0.5 sube)", +} + + +def _fmt_number(x) -> str: + """Formatea un número para mostrar en chip (entero limpio o 2 decimales).""" + if x is None: + return "—" + try: + f = float(x) + except (TypeError, ValueError): + return str(x) + if f.is_integer(): + return f"{int(f):,}" + return f"{f:,.2f}" + + +@dataclass +class _ColumnUI: + """Referencias a los widgets de una tarjeta de columna.""" + card: ctk.CTkFrame + body: ctk.CTkFrame + toggle_btn: ctk.CTkButton + count_label: ctk.CTkLabel + rule_vars: list[tuple[str, ctk.CTkCheckBox, ctk.BooleanVar]] + summary: ColumnSummary + expanded: bool = True + + +class CleaningTab: + """UI de la pestaña Proceso de Limpieza.""" + + def __init__(self, parent_tab): + self.tab = parent_tab + self.fonts = AppStyles.get_fonts() + self.colors = AppStyles.get_colors() + self.spacing = AppStyles.get_spacing() + + self._column_uis: dict[str, _ColumnUI] = {} + self._search_var: Optional[ctk.StringVar] = None + + self._setup_layout() + self._create_file_section() + self._create_preview_section() + self._create_bulk_actions_section() + self._create_columns_section() + self._create_action_section() + self._create_result_section() + + # -- Layout ------------------------------------------------------------------ + + def _setup_layout(self): + self.tab.grid_columnconfigure(0, weight=1) + self.tab.grid_rowconfigure(0, weight=1) + + self.scroll = ctk.CTkScrollableFrame(self.tab, fg_color="transparent") + self.scroll.grid(row=0, column=0, sticky="nsew", padx=10, pady=10) + self.scroll.grid_columnconfigure(0, weight=1) + + # -- Sección archivo --------------------------------------------------------- + + def _create_file_section(self): + frame = ctk.CTkFrame(self.scroll, corner_radius=8) + frame.grid(row=0, column=0, sticky="ew", padx=5, pady=(5, 4)) + frame.grid_columnconfigure(2, weight=1) + + lbl_title = ctk.CTkLabel( + frame, text="Archivo CSV", font=self.fonts["section"], + ) + lbl_title.grid(row=0, column=0, columnspan=4, sticky="w", padx=15, pady=(10, 4)) + + self.btn_select_csv = ctk.CTkButton( + frame, text="📂 Seleccionar CSV", font=self.fonts["ui"], width=170, + fg_color=self.colors["primary_button"], hover_color=self.colors["primary_hover"], + ) + self.btn_select_csv.grid(row=1, column=0, padx=(15, 8), pady=(0, 12), sticky="w") + + self.lbl_csv_path = ctk.CTkLabel( + frame, text="Ningún archivo seleccionado", + font=self.fonts["italic_small"], text_color=self.colors["gray_light"], + anchor="w", + ) + self.lbl_csv_path.grid(row=1, column=1, columnspan=2, padx=4, pady=(0, 12), sticky="ew") + + # Chips informativos: filas, columnas, delimitador + chips_frame = ctk.CTkFrame(frame, fg_color="transparent") + chips_frame.grid(row=1, column=3, padx=(8, 15), pady=(0, 12), sticky="e") + self.chip_rows = self._make_chip(chips_frame, "Filas: —") + self.chip_rows.pack(side="left", padx=(0, 6)) + self.chip_cols = self._make_chip(chips_frame, "Columnas: —") + self.chip_cols.pack(side="left", padx=(0, 6)) + self.chip_delim = self._make_chip(chips_frame, "Delim: —") + self.chip_delim.pack(side="left") + + def _make_chip(self, parent, text: str) -> ctk.CTkLabel: + return ctk.CTkLabel( + parent, text=text, font=self.fonts["italic_small"], + fg_color=("gray85", "gray25"), corner_radius=12, + padx=10, pady=2, + ) + + # -- Vista previa ------------------------------------------------------------ + + def _create_preview_section(self): + frame = ctk.CTkFrame(self.scroll, corner_radius=8) + frame.grid(row=1, column=0, sticky="ew", padx=5, pady=4) + frame.grid_columnconfigure(0, weight=1) + + header = ctk.CTkFrame(frame, fg_color="transparent") + header.grid(row=0, column=0, sticky="ew", padx=15, pady=(10, 4)) + header.grid_columnconfigure(0, weight=1) + + lbl = ctk.CTkLabel(header, text="Vista previa", font=self.fonts["section"]) + lbl.pack(side="left") + + self.lbl_preview_info = ctk.CTkLabel( + header, text="(sin datos)", font=self.fonts["italic_small"], + text_color=self.colors["gray_light"], + ) + self.lbl_preview_info.pack(side="left", padx=(12, 0)) + + self.btn_toggle_preview = ctk.CTkButton( + header, text="Ocultar", width=80, height=24, + font=self.fonts["italic_small"], + fg_color="transparent", border_width=1, + command=self._on_toggle_preview, + ) + self.btn_toggle_preview.pack(side="right") + + self.preview_table_holder = ctk.CTkFrame( + frame, fg_color=("gray92", "gray18"), corner_radius=6, height=220, + ) + self.preview_table_holder.grid(row=1, column=0, sticky="ew", padx=15, pady=(0, 10)) + self.preview_table_holder.grid_propagate(False) + self.preview_table_holder.grid_columnconfigure(0, weight=1) + self.preview_table_holder.grid_rowconfigure(0, weight=1) + + self._preview_tree: Optional[ttk.Treeview] = None + self._preview_visible = True + self._render_table_placeholder( + self.preview_table_holder, "Selecciona un CSV para ver la vista previa.", + ) + + def _on_toggle_preview(self): + if self._preview_visible: + self.preview_table_holder.grid_remove() + self.btn_toggle_preview.configure(text="Mostrar") + else: + self.preview_table_holder.grid() + self.btn_toggle_preview.configure(text="Ocultar") + self._preview_visible = not self._preview_visible + + # -- Acciones masivas -------------------------------------------------------- + + def _create_bulk_actions_section(self): + frame = ctk.CTkFrame(self.scroll, corner_radius=8) + frame.grid(row=2, column=0, sticky="ew", padx=5, pady=4) + frame.grid_columnconfigure(0, weight=1) + + header = ctk.CTkFrame(frame, fg_color="transparent") + header.grid(row=0, column=0, sticky="ew", padx=15, pady=(10, 4)) + header.grid_columnconfigure(0, weight=1) + + lbl = ctk.CTkLabel( + header, text="Acciones rápidas", font=self.fonts["section"], + ) + lbl.pack(side="left") + + body = ctk.CTkFrame(frame, fg_color="transparent") + body.grid(row=1, column=0, sticky="ew", padx=15, pady=(0, 10)) + body.grid_columnconfigure(2, weight=1) + + # Buscador + lbl_search = ctk.CTkLabel(body, text="🔍 Buscar columna:", font=self.fonts["ui"]) + lbl_search.grid(row=0, column=0, padx=(0, 6), pady=4, sticky="w") + self._search_var = ctk.StringVar(value="") + self._search_var.trace_add("write", lambda *_: self._apply_search_filter()) + self.entry_search = ctk.CTkEntry( + body, textvariable=self._search_var, width=240, + placeholder_text="nombre de columna...", + ) + self.entry_search.grid(row=0, column=1, padx=(0, 12), pady=4, sticky="w") + + # Botones colapsar/expandir todas + btns_right = ctk.CTkFrame(body, fg_color="transparent") + btns_right.grid(row=0, column=2, sticky="e") + + self.btn_expand_all = ctk.CTkButton( + btns_right, text="Expandir todas", width=130, height=28, + font=self.fonts["ui"], fg_color="transparent", border_width=1, + command=lambda: self._set_all_expanded(True), + ) + self.btn_expand_all.pack(side="left", padx=4) + self.btn_collapse_all = ctk.CTkButton( + btns_right, text="Colapsar todas", width=130, height=28, + font=self.fonts["ui"], fg_color="transparent", border_width=1, + command=lambda: self._set_all_expanded(False), + ) + self.btn_collapse_all.pack(side="left", padx=4) + self.btn_clear_rules = ctk.CTkButton( + btns_right, text="Limpiar selección", width=140, height=28, + font=self.fonts["ui"], + fg_color=self.colors["danger"], hover_color=self.colors["danger_hover"], + command=self._clear_all_rules, + ) + self.btn_clear_rules.pack(side="left", padx=4) + + # Aplicar regla a todas las de un tipo + bulk_row = ctk.CTkFrame(body, fg_color="transparent") + bulk_row.grid(row=1, column=0, columnspan=3, sticky="ew", pady=(8, 0)) + + lbl_bulk = ctk.CTkLabel( + bulk_row, text="Aplicar regla a todas las columnas de tipo:", + font=self.fonts["ui"], + ) + lbl_bulk.pack(side="left", padx=(0, 6)) + + type_options = list(_TYPE_LABELS.values()) + self._bulk_type_var = ctk.StringVar(value=type_options[0]) + self.bulk_type_menu = ctk.CTkOptionMenu( + bulk_row, values=type_options, variable=self._bulk_type_var, + width=130, font=self.fonts["ui"], + ) + self.bulk_type_menu.pack(side="left", padx=(0, 6)) + + rule_options = [r["name"] for r in CLEANING_RULES] + self._bulk_rule_var = ctk.StringVar(value=rule_options[0]) + self.bulk_rule_menu = ctk.CTkOptionMenu( + bulk_row, values=rule_options, variable=self._bulk_rule_var, + width=380, font=self.fonts["ui"], + ) + self.bulk_rule_menu.pack(side="left", padx=(0, 6)) + + self.btn_bulk_apply = ctk.CTkButton( + bulk_row, text="Aplicar a todas", width=140, height=28, + font=self.fonts["ui"], + fg_color=self.colors["secondary_button"], + hover_color=self.colors["secondary_hover"], + command=self._on_bulk_apply, + ) + self.btn_bulk_apply.pack(side="left", padx=4) + + # -- Columnas y reglas ------------------------------------------------------- + + def _create_columns_section(self): + frame = ctk.CTkFrame(self.scroll, corner_radius=8) + frame.grid(row=3, column=0, sticky="ew", padx=5, pady=4) + frame.grid_columnconfigure(0, weight=1) + + header = ctk.CTkFrame(frame, fg_color="transparent") + header.grid(row=0, column=0, sticky="ew", padx=15, pady=(10, 4)) + header.grid_columnconfigure(0, weight=1) + + lbl = ctk.CTkLabel( + header, text="Resumen y reglas por columna", font=self.fonts["section"], + ) + lbl.pack(side="left") + + self.lbl_global_count = ctk.CTkLabel( + header, text="0 reglas seleccionadas", + font=self.fonts["italic_small"], text_color=self.colors["gray_light"], + ) + self.lbl_global_count.pack(side="right") + + self.columns_container = ctk.CTkScrollableFrame( + frame, fg_color="transparent", height=420, + ) + self.columns_container.grid(row=1, column=0, sticky="ew", padx=10, pady=(0, 10)) + self.columns_container.grid_columnconfigure(0, weight=1) + + self._empty_label = ctk.CTkLabel( + self.columns_container, + text="Selecciona un CSV para ver el resumen de columnas.", + font=self.fonts["italic_small"], text_color=self.colors["gray_light"], + ) + self._empty_label.grid(row=0, column=0, sticky="w", padx=10, pady=20) + + # -- Acción ------------------------------------------------------------------ + + def _create_action_section(self): + frame = ctk.CTkFrame(self.scroll, corner_radius=8) + frame.grid(row=4, column=0, sticky="ew", padx=5, pady=4) + frame.grid_columnconfigure(1, weight=1) + + self.btn_apply = ctk.CTkButton( + frame, text="✨ Aplicar limpieza", font=self.fonts["button_large"], + fg_color=self.colors["primary_button"], hover_color=self.colors["primary_hover"], + height=42, width=220, + ) + self.btn_apply.grid(row=0, column=0, padx=15, pady=12, sticky="w") + + self.lbl_status = ctk.CTkLabel( + frame, text="", font=self.fonts["italic_small"], + text_color=self.colors["gray_light"], anchor="w", + ) + self.lbl_status.grid(row=0, column=1, padx=10, pady=12, sticky="ew") + + # -- Resultado --------------------------------------------------------------- + + def _create_result_section(self): + frame = ctk.CTkFrame(self.scroll, corner_radius=8) + frame.grid(row=5, column=0, sticky="ew", padx=5, pady=(4, 6)) + frame.grid_columnconfigure(0, weight=1) + + top = ctk.CTkFrame(frame, fg_color="transparent") + top.grid(row=0, column=0, sticky="ew", padx=15, pady=(10, 4)) + top.grid_columnconfigure(0, weight=1) + + lbl = ctk.CTkLabel( + top, text="Vista previa del resultado", font=self.fonts["section"], + ) + lbl.pack(side="left") + + self.lbl_result_info = ctk.CTkLabel( + top, text="", font=self.fonts["italic_small"], + text_color=self.colors["gray_light"], + ) + self.lbl_result_info.pack(side="left", padx=(12, 0)) + + self.btn_save = ctk.CTkButton( + top, text="💾 Guardar CSV", font=self.fonts["ui"], width=160, height=32, + fg_color=self.colors["success"], hover_color=self.colors["success_hover"], + state="disabled", + ) + self.btn_save.pack(side="right") + + self.result_table_holder = ctk.CTkFrame( + frame, fg_color=("gray92", "gray18"), corner_radius=6, height=260, + ) + self.result_table_holder.grid(row=1, column=0, sticky="ew", padx=15, pady=(0, 10)) + self.result_table_holder.grid_propagate(False) + self.result_table_holder.grid_columnconfigure(0, weight=1) + self.result_table_holder.grid_rowconfigure(0, weight=1) + + self._result_tree: Optional[ttk.Treeview] = None + self._render_table_placeholder( + self.result_table_holder, "Aplica reglas para ver el resultado.", + ) + + # =========================================================================== + # API pública + # =========================================================================== + + def set_on_select_csv(self, callback): + self.btn_select_csv.configure(command=callback) + + def set_on_apply(self, callback): + self.btn_apply.configure(command=callback) + + def set_on_save(self, callback): + self.btn_save.configure(command=callback) + + def set_csv_path(self, path: str): + self.lbl_csv_path.configure(text=path or "Ningún archivo seleccionado") + + def set_status(self, text: str): + self.lbl_status.configure(text=text) + + def set_preview(self, df: Optional[pd.DataFrame]): + """Renderiza el DataFrame original en la tabla de vista previa.""" + if df is None or len(df) == 0: + self._render_table_placeholder( + self.preview_table_holder, "(sin datos)", + ) + self.lbl_preview_info.configure(text="(sin datos)") + self._preview_tree = None + return + info = ( + f"{len(df):,} filas · {len(df.columns)} columnas · " + f"mostrando primeras {min(_PREVIEW_MAX_ROWS, len(df))}" + ) + self.lbl_preview_info.configure(text=info) + self._preview_tree = self._render_dataframe_table( + self.preview_table_holder, df, + ) + self.chip_rows.configure(text=f"Filas: {len(df):,}") + self.chip_cols.configure(text=f"Columnas: {len(df.columns)}") + + def set_result(self, df: Optional[pd.DataFrame], info_text: str = ""): + """Renderiza el DataFrame resultante en la tabla de resultado.""" + if df is None or len(df) == 0: + self._render_table_placeholder( + self.result_table_holder, "(sin datos)", + ) + self.lbl_result_info.configure(text=info_text) + self._result_tree = None + return + summary = ( + f"{len(df):,} filas · {len(df.columns)} columnas · " + f"mostrando primeras {min(_PREVIEW_MAX_ROWS, len(df))}" + ) + if info_text: + summary = f"{summary} · {info_text}" + self.lbl_result_info.configure(text=summary) + self._result_tree = self._render_dataframe_table( + self.result_table_holder, df, + ) + + def enable_save(self, enabled: bool): + self.btn_save.configure(state="normal" if enabled else "disabled") + + def enable_controls(self, enabled: bool): + state = "normal" if enabled else "disabled" + self.btn_select_csv.configure(state=state) + self.btn_apply.configure(state=state) + self.btn_bulk_apply.configure(state=state) + self.btn_clear_rules.configure(state=state) + self.btn_expand_all.configure(state=state) + self.btn_collapse_all.configure(state=state) + + def populate_columns(self, summaries: list[ColumnSummary]): + """Renderiza la lista de columnas con sus reglas aplicables.""" + for child in list(self.columns_container.winfo_children()): + child.destroy() + self._column_uis.clear() + if self._search_var is not None: + try: + self._search_var.set("") + except Exception: + pass + + if not summaries: + empty = ctk.CTkLabel( + self.columns_container, + text="El CSV no tiene columnas.", + font=self.fonts["italic_small"], text_color=self.colors["gray_light"], + ) + empty.grid(row=0, column=0, sticky="w", padx=10, pady=20) + self._update_global_count() + self.chip_cols.configure(text=f"Columnas: 0") + return + + rules_by_id = {r["id"]: r for r in CLEANING_RULES} + + for row_idx, summary in enumerate(summaries): + ui = self._build_column_card(row_idx, summary, rules_by_id) + self._column_uis[summary.name] = ui + + self.chip_cols.configure(text=f"Columnas: {len(summaries)}") + self._update_global_count() + + def get_selected_rules(self) -> dict[str, list[str]]: + """Devuelve un mapa columna -> lista de rule_ids en el orden visual.""" + result: dict[str, list[str]] = {} + for col, ui in self._column_uis.items(): + selected = [rid for rid, _chk, var in ui.rule_vars if bool(var.get())] + if selected: + result[col] = selected + return result + + # =========================================================================== + # Construcción de tarjeta de columna + # =========================================================================== + + def _build_column_card( + self, row_idx: int, summary: ColumnSummary, rules_by_id: dict, + ) -> _ColumnUI: + card = ctk.CTkFrame(self.columns_container, corner_radius=8, border_width=1) + card.grid(row=row_idx, column=0, sticky="ew", padx=2, pady=4) + card.grid_columnconfigure(0, weight=1) + + # ---- Header ---- + header = ctk.CTkFrame(card, fg_color="transparent") + header.grid(row=0, column=0, sticky="ew", padx=10, pady=(8, 4)) + header.grid_columnconfigure(2, weight=1) + + toggle_btn = ctk.CTkButton( + header, text="▼", width=28, height=28, + font=self.fonts["ui"], fg_color="transparent", + border_width=0, hover=True, + ) + toggle_btn.grid(row=0, column=0, padx=(0, 6), sticky="w") + + type_color = _TYPE_COLORS.get(summary.inferred_type, ("gray60", "gray40")) + type_label = _TYPE_LABELS.get(summary.inferred_type, summary.inferred_type) + badge = ctk.CTkLabel( + header, text=type_label.upper(), + font=ctk.CTkFont(size=10, weight="bold"), + fg_color=type_color, text_color="white", + corner_radius=10, padx=10, pady=2, + ) + badge.grid(row=0, column=1, padx=(0, 10), sticky="w") + + name_lbl = ctk.CTkLabel( + header, text=summary.name, font=self.fonts["bold_large"], anchor="w", + ) + name_lbl.grid(row=0, column=2, sticky="w") + + count_lbl = ctk.CTkLabel( + header, text="0 reglas", + font=self.fonts["italic_small"], text_color=self.colors["gray_light"], + ) + count_lbl.grid(row=0, column=3, padx=(8, 0), sticky="e") + + # ---- Body (info + reglas) ---- + body = ctk.CTkFrame(card, fg_color="transparent") + body.grid(row=1, column=0, sticky="ew", padx=10, pady=(0, 10)) + body.grid_columnconfigure(0, weight=1) + + example = summary.example or "(sin datos)" + if len(example) > 100: + example = example[:97] + "..." + lbl_example = ctk.CTkLabel( + body, text=f'Ejemplo: "{example}"', + font=self.fonts["mono_small"], anchor="w", + text_color=self.colors["gray_dark"], justify="left", + ) + lbl_example.grid(row=0, column=0, sticky="ew", padx=4, pady=(0, 4)) + + self._build_stats_chips(body, summary).grid( + row=1, column=0, sticky="ew", padx=4, pady=(0, 4), + ) + + if summary.top_values: + self._build_top_values_panel(body, summary.top_values).grid( + row=2, column=0, sticky="ew", padx=4, pady=(0, 6), + ) + + rules_frame = ctk.CTkFrame(body, fg_color=("gray92", "gray18"), corner_radius=6) + rules_frame.grid(row=3, column=0, sticky="ew", padx=0, pady=(2, 0)) + rules_frame.grid_columnconfigure((0, 1), weight=1, uniform="rules") + + applicable_ids = summary.applicable_rules or [r["id"] for r in CLEANING_RULES] + ordered_ids = [r["id"] for r in CLEANING_RULES if r["id"] in applicable_ids] + + vars_list: list[tuple[str, ctk.CTkCheckBox, ctk.BooleanVar]] = [] + col_in_row = 0 + row_in_rules = 0 + for rid in ordered_ids: + rule = rules_by_id.get(rid) + if rule is None: + continue + var = ctk.BooleanVar(value=False) + var.trace_add("write", lambda *_a, c=summary.name: self._on_rule_toggle(c)) + label = _RULE_SHORT_LABELS.get(rid, rule["name"]) + chk = ctk.CTkCheckBox( + rules_frame, text=label, variable=var, + font=self.fonts["ui"], checkbox_width=18, checkbox_height=18, + ) + chk.grid(row=row_in_rules, column=col_in_row, sticky="w", padx=10, pady=4) + vars_list.append((rid, chk, var)) + col_in_row += 1 + if col_in_row >= 2: + col_in_row = 0 + row_in_rules += 1 + + ui = _ColumnUI( + card=card, body=body, toggle_btn=toggle_btn, + count_label=count_lbl, rule_vars=vars_list, summary=summary, + ) + toggle_btn.configure(command=lambda c=summary.name: self._toggle_column(c)) + return ui + + # =========================================================================== + # Stats visuales por columna + # =========================================================================== + + def _build_stats_chips(self, parent, summary: ColumnSummary) -> ctk.CTkFrame: + """Construye una fila de chips con la información clave de la columna.""" + row = ctk.CTkFrame(parent, fg_color="transparent") + + # Sin datos + nulls_color = ( + ("#FEE2E2", "#7F1D1D") if summary.null_count > 0 + else ("gray88", "gray22") + ) + nulls_text_color = ( + ("#991B1B", "#FECACA") if summary.null_count > 0 + else ("gray40", "gray70") + ) + self._make_stat_chip( + row, + label="Sin datos", + value=f"{summary.null_count} ({summary.null_pct:.0f}%)", + bg=nulls_color, + text_color=nulls_text_color, + ).pack(side="left", padx=(0, 6), pady=2) + + # Únicos + self._make_stat_chip( + row, + label="Únicos", + value=f"{summary.unique_count}", + bg=("#DBEAFE", "#1E3A8A"), + text_color=("#1E40AF", "#BFDBFE"), + ).pack(side="left", padx=(0, 6), pady=2) + + # Stats por tipo + if summary.inferred_type in ("integer", "decimal"): + if summary.min is not None: + self._make_stat_chip( + row, label="Mín", value=_fmt_number(summary.min), + bg=("#DCFCE7", "#14532D"), text_color=("#166534", "#BBF7D0"), + ).pack(side="left", padx=(0, 6), pady=2) + if summary.max is not None: + self._make_stat_chip( + row, label="Máx", value=_fmt_number(summary.max), + bg=("#DCFCE7", "#14532D"), text_color=("#166534", "#BBF7D0"), + ).pack(side="left", padx=(0, 6), pady=2) + if summary.mean is not None: + self._make_stat_chip( + row, label="Promedio", value=_fmt_number(summary.mean), + bg=("#DCFCE7", "#14532D"), text_color=("#166534", "#BBF7D0"), + ).pack(side="left", padx=(0, 6), pady=2) + elif summary.inferred_type == "date": + if summary.detected_date_format: + self._make_stat_chip( + row, label="Formato detectado", + value=summary.detected_date_format, + bg=("#FFEDD5", "#7C2D12"), + text_color=("#9A3412", "#FED7AA"), + ).pack(side="left", padx=(0, 6), pady=2) + elif summary.inferred_type in ("text", "categorical"): + if summary.avg_length is not None: + self._make_stat_chip( + row, label="Longitud promedio", + value=f"{summary.avg_length:.0f} caracteres", + bg=("gray88", "gray22"), text_color=("gray35", "gray75"), + ).pack(side="left", padx=(0, 6), pady=2) + + return row + + def _make_stat_chip( + self, + parent, + *, + label: str, + value: str, + bg, + text_color, + ) -> ctk.CTkFrame: + """Chip con etiqueta pequeña + valor en negrita.""" + chip = ctk.CTkFrame(parent, fg_color=bg, corner_radius=10) + inner = ctk.CTkFrame(chip, fg_color="transparent") + inner.pack(padx=8, pady=2) + ctk.CTkLabel( + inner, text=label, font=ctk.CTkFont(size=10), + text_color=text_color, + ).pack(side="left", padx=(0, 4)) + ctk.CTkLabel( + inner, text=value, font=ctk.CTkFont(size=11, weight="bold"), + text_color=text_color, + ).pack(side="left") + return chip + + def _build_top_values_panel( + self, parent, top_values: list[tuple[str, int]], + ) -> ctk.CTkFrame: + """Panel con los valores más frecuentes en formato chip.""" + container = ctk.CTkFrame(parent, fg_color="transparent") + container.grid_columnconfigure(1, weight=1) + + ctk.CTkLabel( + container, text="Más frecuentes:", + font=ctk.CTkFont(size=11, weight="bold"), + text_color=self.colors["text_muted"], anchor="w", + ).grid(row=0, column=0, padx=(0, 8), sticky="nw") + + chips_row = ctk.CTkFrame(container, fg_color="transparent") + chips_row.grid(row=0, column=1, sticky="ew") + + for value, count in top_values[:5]: + display = str(value) + if len(display) > 60: + display = display[:57] + "..." + chip = ctk.CTkFrame( + chips_row, fg_color=("gray90", "gray20"), corner_radius=10, + ) + chip.pack(side="left", padx=(0, 6), pady=2) + ctk.CTkLabel( + chip, text=f'"{display}"', + font=ctk.CTkFont(size=11), + text_color=("gray25", "gray85"), + ).pack(side="left", padx=(8, 4), pady=2) + ctk.CTkLabel( + chip, text=f"x{count}", + font=ctk.CTkFont(size=10, weight="bold"), + text_color=self.colors["primary_button"], + ).pack(side="left", padx=(0, 8), pady=2) + + return container + + # =========================================================================== + # Lógica interna + # =========================================================================== + + def _toggle_column(self, col_name: str): + ui = self._column_uis.get(col_name) + if ui is None: + return + if ui.expanded: + ui.body.grid_remove() + ui.toggle_btn.configure(text="▶") + else: + ui.body.grid() + ui.toggle_btn.configure(text="▼") + ui.expanded = not ui.expanded + + def _set_all_expanded(self, expanded: bool): + for col_name, ui in self._column_uis.items(): + if ui.expanded != expanded: + self._toggle_column(col_name) + + def _on_rule_toggle(self, col_name: str): + ui = self._column_uis.get(col_name) + if ui is None: + return + selected = sum(1 for _rid, _chk, var in ui.rule_vars if bool(var.get())) + text = f"{selected} regla{'s' if selected != 1 else ''}" + color = self.colors["success"] if selected else self.colors["gray_light"] + ui.count_label.configure(text=text, text_color=color) + self._update_global_count() + + def _update_global_count(self): + cols_with_rules = 0 + total_rules = 0 + for ui in self._column_uis.values(): + sel = sum(1 for _rid, _chk, var in ui.rule_vars if bool(var.get())) + if sel: + cols_with_rules += 1 + total_rules += sel + if total_rules == 0: + self.lbl_global_count.configure( + text="0 reglas seleccionadas", + text_color=self.colors["gray_light"], + ) + else: + self.lbl_global_count.configure( + text=f"{total_rules} reglas en {cols_with_rules} columnas", + text_color=self.colors["success"], + ) + + def _clear_all_rules(self): + for ui in self._column_uis.values(): + for _rid, _chk, var in ui.rule_vars: + if bool(var.get()): + var.set(False) + + def _on_bulk_apply(self): + """Aplica la regla seleccionada a todas las columnas del tipo elegido.""" + type_label = self._bulk_type_var.get() + rule_name = self._bulk_rule_var.get() + + type_id = next( + (k for k, v in _TYPE_LABELS.items() if v == type_label), None, + ) + rule_id = next( + (r["id"] for r in CLEANING_RULES if r["name"] == rule_name), None, + ) + if not type_id or not rule_id: + return + + matched = 0 + for ui in self._column_uis.values(): + if ui.summary.inferred_type != type_id: + continue + for rid, _chk, var in ui.rule_vars: + if rid == rule_id: + var.set(True) + matched += 1 + break + self.set_status( + f"Regla aplicada a {matched} columna(s) de tipo {type_label}." + ) + + def _apply_search_filter(self): + if self._search_var is None: + return + query = (self._search_var.get() or "").strip().lower() + for col_name, ui in self._column_uis.items(): + if not query or query in col_name.lower(): + ui.card.grid() + else: + ui.card.grid_remove() + + def set_delimiter_chip(self, delimiter: str): + """Actualiza el chip del delimitador detectado.""" + label = {"\t": "TAB", " ": "SPACE"}.get(delimiter, delimiter) + self.chip_delim.configure(text=f"Delim: {label}") + + # =========================================================================== + # Renderizado de tablas + # =========================================================================== + + def _render_table_placeholder(self, holder: ctk.CTkFrame, text: str): + """Limpia el holder y muestra un mensaje centrado.""" + for child in list(holder.winfo_children()): + child.destroy() + placeholder = ctk.CTkLabel( + holder, text=text, font=self.fonts["italic_small"], + text_color=self.colors["gray_light"], + ) + placeholder.grid(row=0, column=0, padx=20, pady=20, sticky="nsew") + + def _render_dataframe_table( + self, holder: ctk.CTkFrame, df: pd.DataFrame, + ) -> ttk.Treeview: + """Renderiza un DataFrame como ttk.Treeview con scroll y estilo pulido.""" + for child in list(holder.winfo_children()): + child.destroy() + + palette = self._table_palette() + style = ttk.Style() + try: + style.theme_use("clam") + except Exception: + pass + style.configure( + "Cleaning.Treeview", + rowheight=30, + font=("Consolas", 10), + background=palette["row_even"], + fieldbackground=palette["row_even"], + foreground=palette["text"], + borderwidth=0, + relief="flat", + ) + style.configure( + "Cleaning.Treeview.Heading", + font=("Roboto", 10, "bold"), + background=palette["header_bg"], + foreground=palette["header_fg"], + relief="flat", + padding=(12, 10), + borderwidth=0, + ) + # Header con separadores verticales (el borde aparece al poner relief="solid") + style.map( + "Cleaning.Treeview.Heading", + background=[("active", palette["header_hover"])], + relief=[("active", "flat"), ("!active", "flat")], + ) + style.map( + "Cleaning.Treeview", + background=[("selected", palette["selected_bg"])], + foreground=[("selected", palette["selected_fg"])], + ) + # Truco: el layout de la celda le añade un separador visible ajustando + # el padding lateral para que la zebra se "vea" entre columnas. + try: + style.layout( + "Cleaning.Treeview.Heading", + [ + ("Treeheading.cell", {"sticky": "nswe"}), + ("Treeheading.border", { + "sticky": "nswe", "children": [ + ("Treeheading.padding", { + "sticky": "nswe", "children": [ + ("Treeheading.image", {"side": "right", "sticky": ""}), + ("Treeheading.text", {"sticky": "we"}), + ], + }), + ], + }), + ], + ) + except Exception: + pass + + # Wrapper con borde sutil para resaltar la tabla + outer = ctk.CTkFrame( + holder, fg_color=palette["outer_bg"], + border_width=1, border_color=palette["border"], + corner_radius=6, + ) + outer.grid(row=0, column=0, sticky="nsew", padx=4, pady=4) + outer.grid_columnconfigure(0, weight=1) + outer.grid_rowconfigure(0, weight=1) + + cols = [str(c) for c in df.columns] + tree = ttk.Treeview( + outer, columns=cols, show="headings", style="Cleaning.Treeview", + selectmode="browse", + ) + + for i, col in enumerate(cols): + tree.heading(col, text=f" {col} ", anchor="w") + width = self._estimate_column_width(df[col], col) + anchor = "w" + tree.column(col, width=width, minwidth=80, anchor=anchor, stretch=False) + + # Zebra striping + primera columna con acento sutil + tree.tag_configure( + "odd", background=palette["row_odd"], foreground=palette["text"], + ) + tree.tag_configure( + "even", background=palette["row_even"], foreground=palette["text"], + ) + + visible = df.head(_PREVIEW_MAX_ROWS) + for idx, (_, row) in enumerate(visible.iterrows()): + # Espacios laterales en cada celda para separar visualmente las columnas + values = [f" {self._truncate_cell(v)}" for v in row.tolist()] + tag = "odd" if idx % 2 else "even" + tree.insert("", "end", values=values, tags=(tag,)) + + sb_y = ttk.Scrollbar(outer, orient="vertical", command=tree.yview) + sb_x = ttk.Scrollbar(outer, orient="horizontal", command=tree.xview) + tree.configure(yscrollcommand=sb_y.set, xscrollcommand=sb_x.set) + + tree.grid(row=0, column=0, sticky="nsew") + sb_y.grid(row=0, column=1, sticky="ns") + sb_x.grid(row=1, column=0, sticky="ew") + + holder.grid_columnconfigure(0, weight=1) + holder.grid_rowconfigure(0, weight=1) + return tree + + def _table_palette(self) -> dict[str, str]: + """Paleta de colores adaptada al tema claro/oscuro activo.""" + try: + mode = ctk.get_appearance_mode() + except Exception: + mode = "Light" + if mode == "Dark": + return { + "outer_bg": "#1F2937", + "border": "#374151", + "header_bg": "#111827", + "header_fg": "#F9FAFB", + "header_hover": "#1F2937", + "row_even": "#1F2937", + "row_odd": "#273142", + "text": "#E5E7EB", + "selected_bg": "#2563EB", + "selected_fg": "#FFFFFF", + } + return { + "outer_bg": "#FFFFFF", + "border": "#D1D5DB", + "header_bg": "#1F2937", + "header_fg": "#FFFFFF", + "header_hover": "#374151", + "row_even": "#FFFFFF", + "row_odd": "#F3F4F6", + "text": "#111827", + "selected_bg": "#2563EB", + "selected_fg": "#FFFFFF", + } + + @staticmethod + def _truncate_cell(value) -> str: + if value is None: + return "" + if isinstance(value, float) and pd.isna(value): + return "" + s = str(value) + if len(s) > _PREVIEW_MAX_CELL_LEN: + return s[: _PREVIEW_MAX_CELL_LEN - 3] + "..." + return s + + @staticmethod + def _estimate_column_width(series: pd.Series, col_name: str) -> int: + """Estima el ancho de una columna en pixeles a partir del contenido.""" + char_w = 8 + # Muestreo más amplio para evitar cortar valores largos. + sample = series.head(40).astype(str).tolist() + max_len = max([len(col_name)] + [len(s) for s in sample] + [6]) + max_len = min(max_len, _PREVIEW_MAX_CELL_LEN) + # Padding extra para los espacios laterales agregados en cada celda + # (formato " valor ") que mejoran la separación visual entre columnas. + return max(110, min(460, max_len * char_w + 40)) diff --git a/gui/view.py b/gui/view.py index 9c8eaac..e0901a9 100644 --- a/gui/view.py +++ b/gui/view.py @@ -6,7 +6,7 @@ Expone métodos para enlazar callbacks, leer entradas y escribir salida en un import os import customtkinter as ctk from gui.styles import AppStyles -from gui.tabs import SettingsTab, ResultsTab, MainProcessingTab, PreclassificationTab, MetricsTab, PostProcessTab +from gui.tabs import SettingsTab, ResultsTab, MainProcessingTab, PreclassificationTab, MetricsTab, PostProcessTab, CleaningTab class InvoiceAppView(ctk.CTk): @@ -44,6 +44,7 @@ class InvoiceAppView(ctk.CTk): self.tab_settings = self.tabs.add("Configuración") self.tab_usage = self.tabs.add("Dashboard & Métricas") self.tab_postprocess = self.tabs.add("Post-Procesamiento") + self.tab_cleaning = self.tabs.add("Proceso de Limpieza") # ========================================================================= # --- TAB: PROCESAR DOCUMENTOS --- @@ -77,6 +78,11 @@ class InvoiceAppView(ctk.CTk): # ========================================================================= self.postprocess_tab = PostProcessTab(self.tab_postprocess) + # ========================================================================= + # --- TAB: PROCESO DE LIMPIEZA --- + # ========================================================================= + self.cleaning_tab = CleaningTab(self.tab_cleaning) + # --------------------------------------------------------------------------- # MÉTODOS PÚBLICOS (API COMPATIBLE) @@ -353,3 +359,41 @@ class InvoiceAppView(ctk.CTk): def validate_doc_types(self) -> tuple[bool, str | None]: return self.main_tab.validate_doc_types() + + # --- TAB PROCESO DE LIMPIEZA --- + + def set_on_cleaning_select_csv(self, callback): + self.cleaning_tab.set_on_select_csv(callback) + + def set_on_cleaning_apply(self, callback): + self.cleaning_tab.set_on_apply(callback) + + def set_on_cleaning_save(self, callback): + self.cleaning_tab.set_on_save(callback) + + def set_cleaning_csv_path(self, path: str): + self.cleaning_tab.set_csv_path(path) + + def set_cleaning_preview(self, df): + self.cleaning_tab.set_preview(df) + + def populate_cleaning_columns(self, summaries): + self.cleaning_tab.populate_columns(summaries) + + def get_cleaning_selected_rules(self) -> dict[str, list[str]]: + return self.cleaning_tab.get_selected_rules() + + def set_cleaning_result(self, df, info_text: str = ""): + self.cleaning_tab.set_result(df, info_text) + + def enable_cleaning_save(self, enabled: bool): + self.cleaning_tab.enable_save(enabled) + + def enable_cleaning_controls(self, enabled: bool): + self.cleaning_tab.enable_controls(enabled) + + def set_cleaning_status(self, text: str): + self.cleaning_tab.set_status(text) + + def set_cleaning_delimiter_chip(self, delimiter: str): + self.cleaning_tab.set_delimiter_chip(delimiter) diff --git a/server/app/__init__.py b/server/app/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/server/app/__init__.py @@ -0,0 +1 @@ + diff --git a/server/app/database.py b/server/app/database.py new file mode 100644 index 0000000..2cc751a --- /dev/null +++ b/server/app/database.py @@ -0,0 +1,159 @@ +"""Capa de persistencia SQLite para el servidor de tracking. + +Guarda el historial completo de lotes de extracción, incluyendo +métricas de progreso, costos y la ruta del CSV generado. +""" +import sqlite3 +import json +import os +from typing import Dict, Any, List, Optional +from datetime import datetime + +DB_PATH = os.path.join(os.path.dirname(__file__), "..", "tracking.db") + + +def get_connection() -> sqlite3.Connection: + conn = sqlite3.connect(DB_PATH) + conn.row_factory = sqlite3.Row + return conn + + +def init_db(): + """Crea las tablas si no existen.""" + with get_connection() as conn: + conn.execute(""" + CREATE TABLE IF NOT EXISTS batch_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + batch_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + level TEXT NOT NULL DEFAULT 'info', + radicado TEXT, + message TEXT NOT NULL, + FOREIGN KEY (batch_id) REFERENCES batches(id) + ) + """) + conn.execute("CREATE INDEX IF NOT EXISTS idx_batch_logs_batch_id ON batch_logs(batch_id)") + conn.execute(""" + CREATE TABLE IF NOT EXISTS batches ( + id TEXT PRIMARY KEY, + root_path TEXT NOT NULL, + csv_output_path TEXT, + hostname TEXT, + username TEXT, + total_radicados INTEGER NOT NULL DEFAULT 0, + config TEXT, + status TEXT NOT NULL DEFAULT 'processing', + current_radicado_index INTEGER NOT NULL DEFAULT 0, + success_count INTEGER NOT NULL DEFAULT 0, + error_count INTEGER NOT NULL DEFAULT 0, + total_tokens INTEGER NOT NULL DEFAULT 0, + total_cost_usd REAL NOT NULL DEFAULT 0.0, + last_error TEXT, + current_phase TEXT, + current_radicado TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + """) + conn.commit() + + +def insert_batch(batch: Dict[str, Any]): + """Inserta un nuevo lote en la base de datos.""" + with get_connection() as conn: + conn.execute(""" + INSERT INTO batches ( + id, root_path, csv_output_path, hostname, username, total_radicados, config, + status, current_radicado_index, success_count, error_count, + total_tokens, total_cost_usd, last_error, current_phase, current_radicado, + created_at, updated_at + ) VALUES ( + :id, :root_path, :csv_output_path, :hostname, :username, :total_radicados, :config, + :status, :current_radicado_index, :success_count, :error_count, + :total_tokens, :total_cost_usd, :last_error, :current_phase, :current_radicado, + :created_at, :updated_at + ) + """, { + **batch, + "config": json.dumps(batch.get("config") or {}), + "created_at": _to_iso(batch.get("created_at")), + "updated_at": _to_iso(batch.get("updated_at")), + "current_phase": batch.get("current_phase"), + "current_radicado": batch.get("current_radicado"), + }) + conn.commit() + + +def update_batch(batch_id: str, fields: Dict[str, Any]): + """Actualiza campos de un lote existente.""" + if not fields: + return + fields["updated_at"] = _to_iso(datetime.now()) + set_clause = ", ".join(f"{k} = :{k}" for k in fields) + fields["batch_id"] = batch_id + with get_connection() as conn: + conn.execute( + f"UPDATE batches SET {set_clause} WHERE id = :batch_id", + fields + ) + conn.commit() + + +def get_all_batches() -> List[Dict[str, Any]]: + """Retorna todos los lotes ordenados por fecha de creación descendente.""" + with get_connection() as conn: + rows = conn.execute( + "SELECT * FROM batches ORDER BY created_at DESC" + ).fetchall() + return [_row_to_dict(r) for r in rows] + + +def insert_log(batch_id: str, message: str, level: str = "info", radicado: str = None): + """Inserta una entrada de log para un lote.""" + with get_connection() as conn: + conn.execute( + "INSERT INTO batch_logs (batch_id, timestamp, level, radicado, message) VALUES (?, ?, ?, ?, ?)", + (batch_id, datetime.now().isoformat(), level, radicado, message) + ) + conn.commit() + + +def get_logs(batch_id: str, limit: int = 200) -> List[Dict[str, Any]]: + """Retorna los logs de un lote, del más reciente al más antiguo.""" + with get_connection() as conn: + rows = conn.execute( + "SELECT * FROM batch_logs WHERE batch_id = ? ORDER BY id DESC LIMIT ?", + (batch_id, limit) + ).fetchall() + return [dict(r) for r in rows] + + +def get_batch(batch_id: str) -> Optional[Dict[str, Any]]: + """Retorna un lote por su ID.""" + with get_connection() as conn: + row = conn.execute( + "SELECT * FROM batches WHERE id = ?", (batch_id,) + ).fetchone() + return _row_to_dict(row) if row else None + + +def _row_to_dict(row: sqlite3.Row) -> Dict[str, Any]: + d = dict(row) + try: + d["config"] = json.loads(d.get("config") or "{}") + except Exception: + d["config"] = {} + + # Asegurar que hostname y username estén presentes + if "hostname" not in d: d["hostname"] = None + if "username" not in d: d["username"] = None + + return d + + +def _to_iso(value) -> str: + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, str): + return value + return datetime.now().isoformat() diff --git a/server/app/main.py b/server/app/main.py new file mode 100644 index 0000000..b24114a --- /dev/null +++ b/server/app/main.py @@ -0,0 +1,187 @@ +"""Servidor de tracking de IPSExtractor. + +Persiste el historial de lotes en SQLite y distribuye actualizaciones +en tiempo real vía WebSockets al dashboard web. +""" +from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from fastapi.encoders import jsonable_encoder +from fastapi.responses import FileResponse +from contextlib import asynccontextmanager +from typing import Dict, List, Any, Optional +from pydantic import BaseModel +import uuid +import os +from datetime import datetime + +from .schemas.batch import BatchCreate, BatchUpdate, BatchRead, BatchStatus +from .database import init_db, insert_batch, update_batch, get_all_batches, get_batch, insert_log, get_logs + + +@asynccontextmanager +async def lifespan(app: FastAPI): + init_db() + yield + +app = FastAPI(title="IPSExtractor Tracking API", lifespan=lifespan) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Cache en memoria para lotes activos (para broadcast rápido) +active_batches: Dict[str, Dict[str, Any]] = {} +active_connections: List[WebSocket] = [] + + +async def broadcast(message: dict): + json_compatible_message = jsonable_encoder(message) + disconnected = [] + for connection in active_connections: + try: + await connection.send_json(json_compatible_message) + except Exception: + disconnected.append(connection) + for conn in disconnected: + active_connections.remove(conn) + + +@app.post("/batches", response_model=BatchRead) +async def create_batch(batch: BatchCreate): + batch_id = str(uuid.uuid4()) + now = datetime.now() + new_batch = { + "id": batch_id, + **batch.model_dump(), + "status": BatchStatus.PROCESSING, + "current_radicado_index": 0, + "success_count": 0, + "error_count": 0, + "total_tokens": 0, + "total_cost_usd": 0.0, + "last_error": None, + "created_at": now, + "updated_at": now, + } + active_batches[batch_id] = new_batch + insert_batch(new_batch) + await broadcast({"event": "batch_created", "data": new_batch}) + return new_batch + + +@app.get("/batches", response_model=List[BatchRead]) +async def list_batches(): + # Retorna historial completo desde SQLite + db_batches = get_all_batches() + # Sobreescribe con datos en memoria si el lote está activo (más fresco) + result = [] + for b in db_batches: + if b["id"] in active_batches: + result.append(active_batches[b["id"]]) + else: + result.append(b) + return result + + +@app.get("/batches/{batch_id}", response_model=BatchRead) +async def get_batch_endpoint(batch_id: str): + if batch_id in active_batches: + return active_batches[batch_id] + batch = get_batch(batch_id) + if not batch: + raise HTTPException(status_code=404, detail="Lote no encontrado") + return batch + + +@app.patch("/batches/{batch_id}", response_model=BatchRead) +async def update_batch_endpoint(batch_id: str, update: BatchUpdate): + if batch_id in active_batches: + batch = active_batches[batch_id] + update_data = update.model_dump(exclude_unset=True) + for key, value in update_data.items(): + batch[key] = value + batch["updated_at"] = datetime.now() + + # Si el lote se completó, sacarlo del cache activo + if batch.get("status") in (BatchStatus.COMPLETED, BatchStatus.FAILED): + active_batches.pop(batch_id, None) + + update_batch(batch_id, update_data) + await broadcast({"event": "batch_updated", "data": batch}) + return batch + + # Lote no está en memoria (completado), actualizar solo en DB + batch = get_batch(batch_id) + if not batch: + raise HTTPException(status_code=404, detail="Lote no encontrado") + update_data = update.model_dump(exclude_unset=True) + update_batch(batch_id, update_data) + batch.update(update_data) + return batch + + +class LogEntry(BaseModel): + message: str + level: str = "info" + radicado: Optional[str] = None + +class LogRead(BaseModel): + id: int + batch_id: str + timestamp: str + level: str + radicado: Optional[str] + message: str + +@app.post("/batches/{batch_id}/logs", status_code=201) +async def add_log(batch_id: str, entry: LogEntry): + """Añade una entrada de log para un lote.""" + insert_log(batch_id, entry.message, entry.level, entry.radicado) + # Broadcast log también por WebSocket para actualizaciones en vivo + await broadcast({"event": "log_added", "batch_id": batch_id, "data": { + "level": entry.level, "radicado": entry.radicado, "message": entry.message + }}) + return {"ok": True} + + +@app.get("/batches/{batch_id}/logs", response_model=List[LogRead]) +async def list_logs(batch_id: str, limit: int = 200): + """Retorna el historial de logs de un lote.""" + return get_logs(batch_id, limit) + + +@app.get("/download/{batch_id}") +async def download_csv(batch_id: str): + """Sirve el archivo CSV generado por el lote de extracción.""" + batch = get_batch(batch_id) + if not batch: + raise HTTPException(status_code=404, detail="Lote no encontrado") + + csv_path = batch.get("csv_output_path") + if not csv_path: + raise HTTPException(status_code=404, detail="Este lote no tiene un archivo CSV asociado") + if not os.path.isfile(csv_path): + raise HTTPException(status_code=404, detail=f"Archivo CSV no encontrado en el servidor: {csv_path}") + + filename = os.path.basename(csv_path) + return FileResponse( + path=csv_path, + media_type="text/csv", + filename=filename, + ) + + +@app.websocket("/ws/tracking") +async def websocket_endpoint(websocket: WebSocket): + await websocket.accept() + active_connections.append(websocket) + try: + while True: + await websocket.receive_text() + except WebSocketDisconnect: + if websocket in active_connections: + active_connections.remove(websocket) diff --git a/server/app/schemas/__init__.py b/server/app/schemas/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/server/app/schemas/__init__.py @@ -0,0 +1 @@ + diff --git a/server/app/schemas/batch.py b/server/app/schemas/batch.py new file mode 100644 index 0000000..0b846ff --- /dev/null +++ b/server/app/schemas/batch.py @@ -0,0 +1,54 @@ +from pydantic import BaseModel +from typing import Optional, List, Dict +from datetime import datetime +from enum import Enum + + +class BatchStatus(str, Enum): + PENDING = "pending" + PROCESSING = "processing" + COMPLETED = "completed" + FAILED = "failed" + + +class BatchBase(BaseModel): + root_path: str + total_radicados: int + csv_output_path: Optional[str] = None + hostname: Optional[str] = None + username: Optional[str] = None + config: Optional[Dict] = None + + +class BatchCreate(BatchBase): + pass + + +class BatchUpdate(BaseModel): + status: Optional[BatchStatus] = None + current_radicado_index: Optional[int] = None + success_count: Optional[int] = None + error_count: Optional[int] = None + total_tokens: Optional[int] = None + total_cost_usd: Optional[float] = None + last_error: Optional[str] = None + current_phase: Optional[str] = None + current_radicado: Optional[str] = None + + +class BatchRead(BatchBase): + id: str + status: BatchStatus + current_radicado_index: int = 0 + success_count: int = 0 + error_count: int = 0 + total_tokens: int = 0 + total_cost_usd: float = 0.0 + last_error: Optional[str] = None + current_phase: Optional[str] = None + current_radicado: Optional[str] = None + created_at: datetime + updated_at: datetime + + class Config: + from_attributes = True diff --git a/web-app/.gitignore b/web-app/.gitignore new file mode 100644 index 0000000..5ef6a52 --- /dev/null +++ b/web-app/.gitignore @@ -0,0 +1,41 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# env files (can opt-in for committing if needed) +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/web-app/AGENTS.md b/web-app/AGENTS.md new file mode 100644 index 0000000..8bd0e39 --- /dev/null +++ b/web-app/AGENTS.md @@ -0,0 +1,5 @@ + +# This is NOT the Next.js you know + +This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` before writing any code. Heed deprecation notices. + diff --git a/web-app/CLAUDE.md b/web-app/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/web-app/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/web-app/README.md b/web-app/README.md new file mode 100644 index 0000000..66bb426 --- /dev/null +++ b/web-app/README.md @@ -0,0 +1,36 @@ +This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app). + +## Getting Started + +First, run the development server: + +```bash +npm run dev +# or +yarn dev +# or +pnpm dev +# or +bun dev +``` + +Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. + +You can start editing the page by modifying `app/page.js`. The page auto-updates as you edit the file. + +This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel. + +## Learn More + +To learn more about Next.js, take a look at the following resources: + +- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. +- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. + +You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome! + +## Deploy on Vercel + +The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. + +Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details. diff --git a/web-app/eslint.config.mjs b/web-app/eslint.config.mjs new file mode 100644 index 0000000..f443835 --- /dev/null +++ b/web-app/eslint.config.mjs @@ -0,0 +1,16 @@ +import { defineConfig, globalIgnores } from "eslint/config"; +import nextVitals from "eslint-config-next/core-web-vitals"; + +const eslintConfig = defineConfig([ + ...nextVitals, + // Override default ignores of eslint-config-next. + globalIgnores([ + // Default ignores of eslint-config-next: + ".next/**", + "out/**", + "build/**", + "next-env.d.ts", + ]), +]); + +export default eslintConfig; diff --git a/web-app/next.config.mjs b/web-app/next.config.mjs new file mode 100644 index 0000000..b108e1a --- /dev/null +++ b/web-app/next.config.mjs @@ -0,0 +1,6 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = { + /* config options here */ +}; + +export default nextConfig; diff --git a/web-app/postcss.config.mjs b/web-app/postcss.config.mjs new file mode 100644 index 0000000..61e3684 --- /dev/null +++ b/web-app/postcss.config.mjs @@ -0,0 +1,7 @@ +const config = { + plugins: { + "@tailwindcss/postcss": {}, + }, +}; + +export default config; diff --git a/web-app/public/file.svg b/web-app/public/file.svg new file mode 100644 index 0000000..004145c --- /dev/null +++ b/web-app/public/file.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web-app/public/globe.svg b/web-app/public/globe.svg new file mode 100644 index 0000000..567f17b --- /dev/null +++ b/web-app/public/globe.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web-app/public/next.svg b/web-app/public/next.svg new file mode 100644 index 0000000..5174b28 --- /dev/null +++ b/web-app/public/next.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web-app/public/vercel.svg b/web-app/public/vercel.svg new file mode 100644 index 0000000..7705396 --- /dev/null +++ b/web-app/public/vercel.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web-app/public/window.svg b/web-app/public/window.svg new file mode 100644 index 0000000..b2b2a44 --- /dev/null +++ b/web-app/public/window.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web-app/src/app/components/AppShell.js b/web-app/src/app/components/AppShell.js new file mode 100644 index 0000000..89c1b74 --- /dev/null +++ b/web-app/src/app/components/AppShell.js @@ -0,0 +1,128 @@ +"use client"; +import { useState } from "react"; +import Link from "next/link"; +import { usePathname } from "next/navigation"; +import { + Database, LayoutDashboard, History, TrendingUp, Settings, + ChevronLeft, ChevronRight, LogOut, User, Bell +} from "lucide-react"; +import { clsx } from "clsx"; +import { twMerge } from "tailwind-merge"; + +function cn(...inputs) { + return twMerge(clsx(inputs)); +} + +const NAV_ITEMS = [ + { href: "/", icon: LayoutDashboard, label: "Dashboard" }, + { href: "/processes", icon: History, label: "Procesos" }, + { href: "/analytics", icon: TrendingUp, label: "Analíticas" }, + { href: "/settings", icon: Settings, label: "Configuración" }, +]; + +export default function AppShell({ children }) { + const [collapsed, setCollapsed] = useState(false); + const pathname = usePathname(); + + return ( +
+ {/* Sidebar */} + + + {/* Main area */} +
+ {/* Topbar */} +
+
+ + {NAV_ITEMS.find(n => n.href === "/" ? pathname === "/" : pathname.startsWith(n.href))?.label || ""} + +
+
+ +
+ AD +
+
+
+ + {/* Page content */} +
+ {children} +
+
+
+ ); +} diff --git a/web-app/src/app/favicon.ico b/web-app/src/app/favicon.ico new file mode 100644 index 0000000..718d6fe Binary files /dev/null and b/web-app/src/app/favicon.ico differ diff --git a/web-app/src/app/globals.css b/web-app/src/app/globals.css new file mode 100644 index 0000000..291882a --- /dev/null +++ b/web-app/src/app/globals.css @@ -0,0 +1,22 @@ +@import "tailwindcss"; + +:root { + --background: #F8FAFC; + --foreground: #0f172a; +} + +body { + background: var(--background); + color: var(--foreground); +} + +::-webkit-scrollbar { width: 5px; height: 5px; } +::-webkit-scrollbar-track { background: transparent; } +::-webkit-scrollbar-thumb { background: #E2E8F0; border-radius: 10px; } +::-webkit-scrollbar-thumb:hover { background: #CBD5E1; } + +@keyframes spin-slow { + from { transform: rotate(0deg); } + to { transform: rotate(360deg); } +} +.animate-spin-slow { animation: spin-slow 3s linear infinite; } diff --git a/web-app/src/app/layout.js b/web-app/src/app/layout.js new file mode 100644 index 0000000..3f6445f --- /dev/null +++ b/web-app/src/app/layout.js @@ -0,0 +1,31 @@ +import { Geist, Geist_Mono } from "next/font/google"; +import "./globals.css"; +import AppShell from "./components/AppShell"; + +const geistSans = Geist({ + variable: "--font-geist-sans", + subsets: ["latin"], +}); + +const geistMono = Geist_Mono({ + variable: "--font-geist-mono", + subsets: ["latin"], +}); + +export const metadata = { + title: "IPSExtractor Enterprise", + description: "Enterprise Document Processing Monitor", +}; + +export default function RootLayout({ children }) { + return ( + + + {children} + + + ); +} diff --git a/web-app/src/app/page.js b/web-app/src/app/page.js new file mode 100644 index 0000000..0c35d00 --- /dev/null +++ b/web-app/src/app/page.js @@ -0,0 +1,372 @@ +"use client"; +import { useEffect, useState, useCallback, useMemo } from "react"; +import Link from "next/link"; +import { + Activity, CheckCircle2, AlertCircle, Download, History, + TrendingUp, DollarSign, FileSearch, GitMerge, FileSpreadsheet, + ArrowRight, Cpu, Wifi, WifiOff, RefreshCw, FolderOpen, Clock, + ChevronRight +} from "lucide-react"; +import { clsx } from "clsx"; +import { twMerge } from "tailwind-merge"; + +function cn(...inputs) { return twMerge(clsx(inputs)); } + +const API_URL = "http://localhost:8000"; + +// --- Definición de fases del pipeline --- +const PHASES = [ + { key: "prepare", label: "Preparación", icon: FolderOpen, desc: "Escaneando documentos..." }, + { key: "extract", label: "Extracción IA", icon: Cpu, desc: "Analizando con LLM..." }, + { key: "normalize", label: "Normalización", icon: GitMerge, desc: "Estructurando datos..." }, + { key: "csv", label: "Exportación", icon: FileSpreadsheet, desc: "Generando CSV..." }, +]; + +function getPhaseIndex(phase) { + if (!phase) return 0; + const idx = PHASES.findIndex(p => p.key === phase); + return idx >= 0 ? idx : 0; +} + +// --- Componentes --- +function Card({ children, className }) { + return ( +
+ {children} +
+ ); +} + +function KpiCard({ label, value, sub, icon: Icon, color }) { + const colors = { + blue: { bg: "bg-blue-50", text: "text-blue-600", glow: "shadow-blue-100" }, + emerald: { bg: "bg-emerald-50", text: "text-emerald-600", glow: "shadow-emerald-100" }, + amber: { bg: "bg-amber-50", text: "text-amber-600", glow: "shadow-amber-100" }, + violet: { bg: "bg-violet-50", text: "text-violet-600", glow: "shadow-violet-100" }, + }; + const c = colors[color] || colors.blue; + return ( + +
+
+ +
+
+

{value}

+

{label}

+ {sub &&

{sub}

} + + ); +} + +function PipelineVisual({ batch }) { + const phaseIdx = getPhaseIndex(batch?.current_phase); + const progress = batch?.current_radicado_index || 0; + const total = batch?.total_radicados || 1; + const pct = Math.min((progress / total) * 100, 100); + + return ( +
+ {/* Radicado actual */} + {batch?.current_radicado && ( +
+
+
+

Procesando ahora

+

{batch.current_radicado}

+
+
+ )} + + {/* Pipeline steps */} +
+ {/* Connecting line */} +
+
+ +
+ {PHASES.map((phase, i) => { + const done = i < phaseIdx; + const active = i === phaseIdx && batch?.status === "processing"; + const pending = i > phaseIdx; + const Icon = phase.icon; + + return ( +
+
+ {done ? ( + + ) : active ? ( + + ) : ( + + )} +
+
+

{phase.label}

+ {active && ( +

{phase.desc}

+ )} +
+
+ ); + })} +
+
+ + {/* Progress bar */} +
+
+ Progreso Global + {progress} / {total} +
+
+
+
+
+ {Math.round(pct)}% completado + {total - progress} restantes +
+
+ + {/* Live stats */} +
+ {[ + { label: "Exitosos", value: batch?.success_count || 0, color: "text-emerald-600 bg-emerald-50" }, + { label: "Errores", value: batch?.error_count || 0, color: "text-red-500 bg-red-50" }, + { label: "Tokens", value: (batch?.total_tokens || 0).toLocaleString(), color: "text-blue-600 bg-blue-50" }, + ].map(s => ( +
+

{s.value}

+

{s.label}

+
+ ))} +
+
+ ); +} + +function EmptyProcess() { + return ( +
+
+ +
+

Sin procesos activos

+

Inicia un proceso desde la aplicación desktop para verlo aquí en tiempo real.

+
+ ); +} + +function RecentBatchRow({ batch }) { + const pct = Math.min(((batch.current_radicado_index || 0) / (batch.total_radicados || 1)) * 100, 100); + return ( + +
+ {batch.status === "completed" + ? + : + } +
+
+

{batch.root_path.split(/[\\/]/).pop()}

+
+
+
+
+ {batch.success_count || 0} ok +
+
+
+

${(batch.total_cost_usd || 0).toFixed(3)}

+

{(batch.total_tokens || 0).toLocaleString()} tok.

+
+ + + ); +} + +export default function HomePage() { + const [batches, setBatches] = useState([]); + const [wsStatus, setWsStatus] = useState("connecting"); + + const fetchBatches = useCallback(() => { + fetch(`${API_URL}/batches`).then(r => r.json()).then(setBatches).catch(console.error); + }, []); + + useEffect(() => { + fetchBatches(); + const ws = new WebSocket("ws://localhost:8000/ws/tracking"); + ws.onopen = () => setWsStatus("connected"); + ws.onclose = () => { setWsStatus("disconnected"); setTimeout(fetchBatches, 5000); }; + ws.onmessage = (event) => { + const { event: evt, data } = JSON.parse(event.data); + if (evt === "batch_created") setBatches(prev => [data, ...prev]); + else if (evt === "batch_updated") setBatches(prev => prev.map(b => b.id === data.id ? data : b)); + }; + return () => ws.close(); + }, [fetchBatches]); + + const activeProcess = useMemo(() => batches.find(b => b.status === "processing"), [batches]); + const recentDone = useMemo(() => batches.filter(b => b.status !== "processing").slice(0, 5), [batches]); + const completed = useMemo(() => batches.filter(b => b.status === "completed"), [batches]); + const stats = useMemo(() => ({ + total: batches.length, + processed: completed.reduce((a, b) => a + (b.success_count || 0), 0), + cost: completed.reduce((a, b) => a + (b.total_cost_usd || 0), 0), + tokens: completed.reduce((a, b) => a + (b.total_tokens || 0), 0), + }), [batches, completed]); + + return ( +
+ {/* Status pill */} +
+
+

Dashboard

+

Monitoreo en tiempo real del procesamiento

+
+
+ {wsStatus === "connected" + ? + : + } + {wsStatus === "connected" ? "En Vivo" : "Sin conexión"} +
+
+ + {/* KPIs */} +
+ + + + +
+ + {/* Main content: Process live + Recents */} +
+ {/* Live process */} +
+ +
+
+
+

+ {activeProcess ? "Proceso en Ejecución" : "Sin proceso activo"} +

+
+ {activeProcess && ( +
+ + EN VIVO +
+ )} +
+
+ {activeProcess + ? + : + } +
+ {activeProcess && ( +
+
+ +
+

Ruta de origen

+

{activeProcess.root_path}

+
+
+
+
+ Equipo: + {activeProcess.hostname || "—"} +
+
+ Usuario: + {activeProcess.username || "—"} +
+
+
+ )} + +
+ + {/* Right column */} +
+ {/* Cost live if active */} + {activeProcess && ( + +

Costo en Vivo

+
+

+ ${(activeProcess.total_cost_usd || 0).toFixed(4)} +

+

USD hasta ahora

+
+
+

{(activeProcess.total_tokens || 0).toLocaleString()}

+

Tokens

+
+
+

{activeProcess.success_count || 0}

+

Exitosos

+
+
+

{activeProcess.error_count || 0}

+

Errores

+
+
+
+ + )} + + {/* Recent */} + +
+

Recientes

+ + Ver todos + +
+
+ {recentDone.length > 0 + ? recentDone.map(b => ) + : ( +

+ Aún no hay procesos completados. +

+ ) + } +
+ +
+
+
+ ); +} diff --git a/web-app/src/app/processes/page.js b/web-app/src/app/processes/page.js new file mode 100644 index 0000000..c72fb62 --- /dev/null +++ b/web-app/src/app/processes/page.js @@ -0,0 +1,321 @@ +"use client"; +import { useEffect, useState, useCallback, useMemo } from "react"; +import { + CheckCircle2, AlertCircle, RefreshCw, Download, + Search, Clock, Monitor, User, FolderOpen, + Copy, Check, ChevronDown, ChevronUp, List, TriangleAlert +} from "lucide-react"; +import { clsx } from "clsx"; +import { twMerge } from "tailwind-merge"; + +function cn(...inputs) { return twMerge(clsx(inputs)); } + +const API_URL = "http://localhost:8000"; + +function formatDate(iso) { + if (!iso) return "-"; + return new Date(iso).toLocaleString("es-CO", { + year: "numeric", month: "short", day: "numeric", + hour: "2-digit", minute: "2-digit" + }); +} + +function StatusBadge({ status }) { + const cfg = { + processing: { label: "En Vivo", cls: "bg-blue-50 text-blue-600 border-blue-100", dot: "bg-blue-500 animate-pulse" }, + completed: { label: "Completado", cls: "bg-emerald-50 text-emerald-700 border-emerald-100", dot: "bg-emerald-500" }, + failed: { label: "Fallido", cls: "bg-red-50 text-red-700 border-red-100", dot: "bg-red-500" }, + }; + const c = cfg[status] || cfg.failed; + return ( + + + {c.label} + + ); +} + +function CopyButton({ text }) { + const [copied, setCopied] = useState(false); + const copy = () => { + navigator.clipboard.writeText(text); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + }; + return ( + + ); +} + +const LOG_LEVEL_STYLE = { + info: { cls: "text-slate-400", dot: "bg-slate-300", label: "INFO" }, + success: { cls: "text-emerald-600", dot: "bg-emerald-400", label: "OK" }, + error: { cls: "text-red-500", dot: "bg-red-500", label: "ERR" }, + warning: { cls: "text-amber-500", dot: "bg-amber-400", label: "WARN" }, +}; + +function LogPanel({ batchId, open }) { + const [logs, setLogs] = useState(null); + const [loading, setLoading] = useState(false); + + useEffect(() => { + if (!open || logs !== null) return; + setLoading(true); + fetch(`${API_URL}/batches/${batchId}/logs?limit=200`) + .then(r => r.json()) + .then(data => { setLogs(data); setLoading(false); }) + .catch(() => { setLogs([]); setLoading(false); }); + }, [open, batchId, logs]); + + if (!open) return null; + + return ( +
+ {/* Terminal header */} +
+ + + + Historial de logs + {logs && {logs.length} entradas} +
+ +
+ {loading && ( +

Cargando logs…

+ )} + {!loading && logs && logs.length === 0 && ( +

Sin registros para este proceso.

+ )} + {!loading && logs && [...logs].reverse().map((log) => { + const style = LOG_LEVEL_STYLE[log.level] || LOG_LEVEL_STYLE.info; + const time = log.timestamp ? log.timestamp.split("T")[1]?.slice(0, 8) : ""; + return ( +
+ {time} + {style.label} + {log.radicado && ( + + {log.radicado} + + )} + {log.message} +
+ ); + })} +
+
+ ); +} + +function BatchCard({ batch }) { + const progress = batch.current_radicado_index || 0; + const total = batch.total_radicados || 1; + const pct = Math.min((progress / total) * 100, 100); + const isActive = batch.status === "processing"; + const [showLogs, setShowLogs] = useState(false); + + return ( +
+ {/* Header */} +
+
+ {batch.status === "completed" ? + : batch.status === "failed" ? + : } +
+ +
+
+

+ {batch.root_path.split(/[\\/]/).pop()} +

+ +
+ + {/* Path */} +
+ +

{batch.root_path}

+ +
+ + {/* Meta */} +
+ {formatDate(batch.created_at)} + {batch.hostname || "—"} + {batch.username || "—"} + #{batch.id.slice(0, 8)} +
+
+
+ + {/* Error banner */} + {batch.last_error && ( +
+ +
+

Último error

+

{batch.last_error}

+
+
+ )} + + {/* Progress bar */} +
+
+ {isActive ? `Fase: ${batch.current_phase || "preparando"}` : "Completado"} + {progress} / {total} radicados +
+
+
+
+
+ + {/* Stats footer */} +
+ {[ + { label: "Exitosos", value: batch.success_count || 0, color: "text-emerald-600" }, + { label: "Errores", value: batch.error_count || 0, color: "text-red-500" }, + { label: "Tokens", value: (batch.total_tokens || 0).toLocaleString(), color: "text-blue-600" }, + { label: "Costo USD", value: `$${(batch.total_cost_usd || 0).toFixed(4)}`, color: "text-slate-700" }, + ].map(s => ( +
+

{s.value}

+

{s.label}

+
+ ))} +
+ + {/* Actions bar */} +
+ {batch.status === "completed" && batch.csv_output_path && ( + + Descargar CSV + + )} + +
+ + {/* Log panel */} + +
+ ); +} + +export default function ProcessesPage() { + const [batches, setBatches] = useState([]); + const [search, setSearch] = useState(""); + const [filter, setFilter] = useState("all"); + + const fetchBatches = useCallback(() => { + fetch(`${API_URL}/batches`).then(r => r.json()).then(setBatches).catch(console.error); + }, []); + + useEffect(() => { + fetchBatches(); + const ws = new WebSocket("ws://localhost:8000/ws/tracking"); + ws.onopen = fetchBatches; + ws.onmessage = (event) => { + const { event: evt, data } = JSON.parse(event.data); + if (evt === "batch_created") setBatches(prev => [data, ...prev]); + else if (evt === "batch_updated") setBatches(prev => prev.map(b => b.id === data.id ? data : b)); + }; + return () => ws.close(); + }, [fetchBatches]); + + const filtered = useMemo(() => { + return batches + .filter(b => filter === "all" || b.status === filter) + .filter(b => b.root_path.toLowerCase().includes(search.toLowerCase())); + }, [batches, search, filter]); + + const counts = useMemo(() => ({ + all: batches.length, + processing: batches.filter(b => b.status === "processing").length, + completed: batches.filter(b => b.status === "completed").length, + failed: batches.filter(b => b.status === "failed").length, + }), [batches]); + + return ( +
+ {/* Header */} +
+

Procesos

+

Historial completo de lotes de extracción

+
+ + {/* Filters */} +
+
+ + setSearch(e.target.value)} + /> +
+
+ {[ + { key: "all", label: `Todos (${counts.all})` }, + { key: "processing", label: `En vivo (${counts.processing})` }, + { key: "completed", label: `Completados (${counts.completed})` }, + { key: "failed", label: `Fallidos (${counts.failed})` }, + ].map(f => ( + + ))} +
+
+ + {/* List */} +
+ {filtered.length > 0 ? ( + filtered.map(b => ) + ) : ( +
+ +

No se encontraron procesos

+

Prueba con otro filtro o búsqueda

+
+ )} +
+
+ ); +}