feat: Integrate CSV cleaning functionality into the application, adding a new GUI tab for user interaction, enabling CSV loading, analysis, and rule application for data cleaning. Implement backend tracking for batch processing and enhance the BatchProcessor with tracking capabilities.
This commit is contained in:
erik
+116
-17
@@ -19,6 +19,7 @@ from application.file_discovery import get_sorted_radicados, get_files_as_radica
|
||||
from application.csv_manager import CSVManager
|
||||
from application.config import settings
|
||||
from application.app_prefs import get_allowed_document_types
|
||||
from application.tracking_manager import TrackingManager
|
||||
from core.utils import collect_radicado_docs, find_first_matching_file, collect_documents_flat
|
||||
from core.csv_utils import rows_from_data
|
||||
|
||||
@@ -43,9 +44,15 @@ class BatchProcessor:
|
||||
"""
|
||||
self.service = service
|
||||
self.csv_manager = CSVManager()
|
||||
self.tracking_manager = TrackingManager()
|
||||
self._pause_event = asyncio.Event()
|
||||
self._pause_event.set() # No pausado inicialmente
|
||||
self._stop_requested = False
|
||||
# Contadores acumulados para tracking web
|
||||
self._track_tokens: int = 0
|
||||
self._track_cost_usd: float = 0.0
|
||||
self._track_success: int = 0
|
||||
self._track_errors: int = 0
|
||||
|
||||
def process_root(
|
||||
self,
|
||||
@@ -70,9 +77,38 @@ class BatchProcessor:
|
||||
|
||||
Delega en métodos específicos para multi-documento o single-document.
|
||||
"""
|
||||
# Resetear estado de pausa/parada
|
||||
# Resetear estado de pausa/parada y contadores de tracking
|
||||
self._pause_event.set()
|
||||
self._stop_requested = False
|
||||
self._track_tokens = 0
|
||||
self._track_cost_usd = 0.0
|
||||
self._track_success = 0
|
||||
self._track_errors = 0
|
||||
|
||||
# Notificar inicio a tracking si es procesamiento masivo
|
||||
if fields:
|
||||
# Intentar estimar total de radicados
|
||||
try:
|
||||
if structure_mode == "general":
|
||||
temp_rads = get_files_as_radicados(root_path)
|
||||
else:
|
||||
temp_rads = get_sorted_radicados(root_path)
|
||||
|
||||
if radicado_filter:
|
||||
filter_set = {r.strip() for r in radicado_filter if r.strip()}
|
||||
temp_rads = [r for r in temp_rads if r in filter_set]
|
||||
|
||||
self.tracking_manager.start_batch(
|
||||
root_path,
|
||||
len(temp_rads),
|
||||
config={
|
||||
"structure_mode": structure_mode,
|
||||
"fields_count": len(fields)
|
||||
},
|
||||
csv_output_path=csv_output_path,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error al iniciar tracking: {e}")
|
||||
|
||||
# 1. Preparación de Schemas (Fase 1 Multi-doc)
|
||||
if doc_types and not fields:
|
||||
@@ -249,7 +285,9 @@ class BatchProcessor:
|
||||
except Exception:
|
||||
prompt_t = PromptBuilder.build(selected_paths_for_type, {}, profile_name=prof.get('name', 'invoice'))
|
||||
|
||||
# Llamada ASYNC
|
||||
# Llamada ASYNC - Fase: Extracción
|
||||
self.tracking_manager.report_phase("extract", rad_id)
|
||||
self.tracking_manager.log(f"Iniciando extracción IA", level="info", radicado=rad_id)
|
||||
data, usage_ext = await self.service.extract_full_from_pdf_async(match, instruction=prompt_t, tiff_page_range=tiff_page_range)
|
||||
self._accumulate_usage(agg_extract_usage, usage_ext)
|
||||
|
||||
@@ -260,7 +298,9 @@ class BatchProcessor:
|
||||
else:
|
||||
data = {"radicado": rad_id, "data": data}
|
||||
|
||||
# Normalización en thread aparte
|
||||
# Normalización en thread aparte - Fase: Normalización
|
||||
self.tracking_manager.report_phase("normalize", rad_id)
|
||||
self.tracking_manager.log(f"Extracción OK — normalizando campos", level="info", radicado=rad_id)
|
||||
norm = await asyncio.to_thread(
|
||||
normalize_to_schema,
|
||||
data,
|
||||
@@ -436,7 +476,9 @@ class BatchProcessor:
|
||||
try:
|
||||
logger.info(f"Procesando radicado {entry['radicado']} [{idx}/{total}]", extra={"radicado": entry['radicado'], "pdf": entry['path']})
|
||||
|
||||
# Async Extract
|
||||
# Async Extract - Fase: Extracción
|
||||
self.tracking_manager.report_phase("extract", entry['radicado'])
|
||||
self.tracking_manager.log(f"Iniciando extracción IA", level="info", radicado=entry['radicado'])
|
||||
data, usage_ext = await self.service.extract_full_from_pdf_async(entry["path"], instruction=prompt, tiff_page_range=tiff_page_range)
|
||||
self._report_usage(on_usage, entry['radicado'], "extract", usage_ext)
|
||||
|
||||
@@ -447,7 +489,9 @@ class BatchProcessor:
|
||||
data = {"radicado": entry['radicado'], "data": data}
|
||||
data["_source_pdf"] = entry["path"]
|
||||
|
||||
# Normalizar (Thread)
|
||||
# Normalizar (Thread) - Fase: Normalización
|
||||
self.tracking_manager.report_phase("normalize", entry['radicado'])
|
||||
self.tracking_manager.log(f"Extracción OK — normalizando campos", level="info", radicado=entry['radicado'])
|
||||
norm = await asyncio.to_thread(
|
||||
normalize_to_schema,
|
||||
data, paths=selected_fields, profile_name=prof.get('name', 'invoice')
|
||||
@@ -667,23 +711,62 @@ class BatchProcessor:
|
||||
callback(rad, idx, total)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Notificar a tracking web con contadores reales acumulados
|
||||
try:
|
||||
self._track_success = idx - self._track_errors
|
||||
# Log de radicado completado exitosamente
|
||||
if rad and self._track_errors < idx:
|
||||
self.tracking_manager.log(
|
||||
f"Radicado completado ({self._track_success}/{total} exitosos)",
|
||||
level="success",
|
||||
radicado=str(rad)
|
||||
)
|
||||
if idx >= total:
|
||||
self.tracking_manager.complete_batch(
|
||||
success_count=self._track_success,
|
||||
error_count=self._track_errors,
|
||||
total_tokens=self._track_tokens,
|
||||
total_cost_usd=self._track_cost_usd,
|
||||
)
|
||||
else:
|
||||
self.tracking_manager.update_progress(
|
||||
current_index=idx,
|
||||
success_count=self._track_success,
|
||||
error_count=self._track_errors,
|
||||
total_tokens=self._track_tokens,
|
||||
total_cost_usd=self._track_cost_usd,
|
||||
status="processing"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error al notificar progreso a tracking: {e}")
|
||||
|
||||
def _accumulate_usage(self, agg, new_usage):
|
||||
accumulate_usage(agg, new_usage)
|
||||
|
||||
def _report_usage(self, callback, rad, phase, usage):
|
||||
if callback and usage:
|
||||
try:
|
||||
model = None
|
||||
if phase == "extract":
|
||||
model = getattr(self.service.extractor, "model", None)
|
||||
elif phase == "normalize":
|
||||
# El normalizador está hardcodeado a gpt-4.1-mini en normalizer.py
|
||||
model = "gpt-4.1-mini"
|
||||
|
||||
callback(phase, rad, usage, compute_cost_from_usage(usage, phase, model=model))
|
||||
except Exception:
|
||||
pass
|
||||
if not usage:
|
||||
return
|
||||
try:
|
||||
model = None
|
||||
if phase == "extract":
|
||||
model = getattr(self.service.extractor, "model", None)
|
||||
elif phase == "normalize":
|
||||
model = "gpt-4.1-mini"
|
||||
|
||||
cost = compute_cost_from_usage(usage, phase, model=model)
|
||||
|
||||
# Acumular tokens y costos para el tracking web
|
||||
prompt_tokens = int(usage.get("prompt_tokens") or 0)
|
||||
completion_tokens = int(usage.get("completion_tokens") or 0)
|
||||
self._track_tokens += prompt_tokens + completion_tokens
|
||||
if isinstance(cost, dict):
|
||||
self._track_cost_usd += float(cost.get("total_usd") or 0.0)
|
||||
|
||||
if callback:
|
||||
callback(phase, rad, usage, cost)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _write_csv_multidoc(self, path, data, fields, doc_types, prof, header, alias_map, rad):
|
||||
try:
|
||||
@@ -787,6 +870,22 @@ class BatchProcessor:
|
||||
if pdf_path: error_obj["_source_pdf"] = pdf_path
|
||||
logger.exception(f"Error procesando radicado {rad}", extra={"radicado": rad})
|
||||
|
||||
# Acumular error en el contador de tracking
|
||||
self._track_errors += 1
|
||||
error_message = str(e)
|
||||
try:
|
||||
self.tracking_manager.report_error(f"[{rad}] {error_message}")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.tracking_manager.log(
|
||||
message=f"Error procesando radicado: {error_message}",
|
||||
level="error",
|
||||
radicado=str(rad)
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if csv_path:
|
||||
header_err = header if header else ["radicado", "error"]
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,777 @@
|
||||
"""Limpieza de CSV con análisis inteligente local y reglas configurables.
|
||||
|
||||
Este módulo es pura lógica (sin GUI, sin LLM). Provee:
|
||||
- `analyze_dataframe` para obtener un resumen por columna (tipo inferido,
|
||||
ejemplo aleatorio, nulos, únicos, estadísticas específicas por tipo).
|
||||
- Un registro `CLEANING_RULES` con reglas aplicables a columnas.
|
||||
- `apply_rules` que aplica las reglas seleccionadas por columna y devuelve
|
||||
el DataFrame resultante junto con un log de operaciones/advertencias.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv as _csv
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MODELO DE DATOS
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TYPE_INTEGER = "integer"
|
||||
TYPE_DECIMAL = "decimal"
|
||||
TYPE_DATE = "date"
|
||||
TYPE_BOOLEAN = "boolean"
|
||||
TYPE_CATEGORICAL = "categorical"
|
||||
TYPE_TEXT = "text"
|
||||
TYPE_MIXED = "mixed"
|
||||
TYPE_EMPTY = "empty"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ColumnSummary:
|
||||
"""Resumen inteligente para una columna del DataFrame."""
|
||||
name: str
|
||||
inferred_type: str
|
||||
example: str = ""
|
||||
null_count: int = 0
|
||||
null_pct: float = 0.0
|
||||
unique_count: int = 0
|
||||
unique_pct: float = 0.0
|
||||
min: Optional[float] = None
|
||||
max: Optional[float] = None
|
||||
mean: Optional[float] = None
|
||||
avg_length: Optional[float] = None
|
||||
top_values: list[tuple[str, int]] = field(default_factory=list)
|
||||
detected_date_format: Optional[str] = None
|
||||
applicable_rules: list[str] = field(default_factory=list)
|
||||
|
||||
def info_text(self) -> str:
|
||||
"""Texto humano con información adicional (para subtítulo de la UI)."""
|
||||
parts: list[str] = []
|
||||
parts.append(f"nulos: {self.null_count} ({self.null_pct:.0f}%)")
|
||||
parts.append(f"únicos: {self.unique_count}")
|
||||
if self.inferred_type in (TYPE_INTEGER, TYPE_DECIMAL):
|
||||
if self.min is not None:
|
||||
parts.append(f"min: {_fmt_num(self.min)}")
|
||||
if self.max is not None:
|
||||
parts.append(f"max: {_fmt_num(self.max)}")
|
||||
if self.mean is not None:
|
||||
parts.append(f"media: {_fmt_num(self.mean)}")
|
||||
elif self.inferred_type == TYPE_DATE:
|
||||
if self.detected_date_format:
|
||||
parts.append(f"formato: {self.detected_date_format}")
|
||||
elif self.inferred_type in (TYPE_TEXT, TYPE_CATEGORICAL):
|
||||
if self.avg_length is not None:
|
||||
parts.append(f"long. prom: {self.avg_length:.1f}")
|
||||
if self.top_values:
|
||||
tops = ", ".join(f'"{v}" ({c})' for v, c in self.top_values[:3])
|
||||
parts.append(f"top: {tops}")
|
||||
return " · ".join(parts)
|
||||
|
||||
|
||||
def _fmt_num(x: float) -> str:
|
||||
if x is None:
|
||||
return ""
|
||||
if isinstance(x, float) and x.is_integer():
|
||||
return str(int(x))
|
||||
if isinstance(x, float):
|
||||
return f"{x:.2f}"
|
||||
return str(x)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DETECCIÓN DE FECHAS
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DATE_NUM_RE = re.compile(r"^\s*(\d{1,4})[\s\/\-\.](\d{1,2})[\s\/\-\.](\d{1,4})\s*$")
|
||||
_DATE_COMPACT_RE = re.compile(r"^\s*(\d{6}|\d{8})\s*$")
|
||||
# Hora opcional: HH[:MM[:SS]][ AM|PM] al final de la cadena
|
||||
_TIME_TAIL_RE = re.compile(
|
||||
r"[\sT]+(\d{1,2})(?::(\d{1,2})(?::(\d{1,2}))?)?\s*([APap]\.?\s*[Mm]\.?)?\s*$"
|
||||
)
|
||||
|
||||
|
||||
def _split_date_time(value: str) -> tuple[str, Optional[str]]:
|
||||
"""Separa la parte de fecha de la parte de hora si está presente.
|
||||
|
||||
Devuelve `(date_part, time_part_normalized)`. `time_part_normalized` es
|
||||
None si no hay hora. La hora se normaliza a `HH:MM[:SS][ AM|PM]`.
|
||||
"""
|
||||
if value is None:
|
||||
return "", None
|
||||
s = str(value).strip()
|
||||
if not s:
|
||||
return "", None
|
||||
|
||||
m = _TIME_TAIL_RE.search(s)
|
||||
if not m:
|
||||
return s, None
|
||||
|
||||
hh = int(m.group(1))
|
||||
mm = int(m.group(2)) if m.group(2) else 0
|
||||
ss = m.group(3)
|
||||
ampm = m.group(4)
|
||||
if not (0 <= hh <= 23 and 0 <= mm <= 59):
|
||||
return s, None
|
||||
if ss is not None:
|
||||
try:
|
||||
ssi = int(ss)
|
||||
if not (0 <= ssi <= 59):
|
||||
return s, None
|
||||
except ValueError:
|
||||
return s, None
|
||||
# Validar contra AM/PM (con AM/PM, hora 1-12)
|
||||
if ampm and not (1 <= hh <= 12):
|
||||
return s, None
|
||||
|
||||
date_part = s[: m.start()].rstrip(" T")
|
||||
if not date_part:
|
||||
return s, None
|
||||
|
||||
parts: list[str] = [f"{hh:02d}:{mm:02d}"]
|
||||
if ss is not None:
|
||||
parts[0] += f":{int(ss):02d}"
|
||||
if ampm:
|
||||
norm_ampm = ampm.replace(".", "").replace(" ", "").upper()
|
||||
if norm_ampm in ("AM", "PM"):
|
||||
parts.append(norm_ampm)
|
||||
time_norm = " ".join(parts)
|
||||
return date_part, time_norm
|
||||
|
||||
|
||||
def _extract_date_parts(value: str) -> Optional[tuple[int, int, int, str]]:
|
||||
"""Intenta extraer (a, b, c, layout) de una fecha en cualquier formato común.
|
||||
|
||||
`layout` es uno de: 'sep' (con separadores), 'compact8' (YYYYMMDD o
|
||||
DDMMYYYY), 'compact6' (DDMMYY o YYMMDD). No interpreta todavía qué
|
||||
componente es día/mes/año; eso lo decide `detect_date_format` viendo la
|
||||
columna completa. Acepta valores con hora al final (ignorada para los
|
||||
componentes de fecha).
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
s = str(value).strip()
|
||||
if not s or s.lower() in ("nan", "none", "nat"):
|
||||
return None
|
||||
|
||||
date_part, _ = _split_date_time(s)
|
||||
if date_part:
|
||||
s = date_part
|
||||
|
||||
m = _DATE_NUM_RE.match(s)
|
||||
if m:
|
||||
try:
|
||||
a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||||
return (a, b, c, "sep")
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
m2 = _DATE_COMPACT_RE.match(s)
|
||||
if m2:
|
||||
digits = m2.group(1)
|
||||
if len(digits) == 8:
|
||||
return (int(digits[0:2]), int(digits[2:4]), int(digits[4:8]), "compact8")
|
||||
if len(digits) == 6:
|
||||
return (int(digits[0:2]), int(digits[2:4]), int(digits[4:6]), "compact6")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def detect_date_format(series: pd.Series, sample_size: int = 500) -> Optional[str]:
|
||||
"""Analiza la serie y determina el formato de fecha dominante.
|
||||
|
||||
Devuelve un string con el layout detectado (p. ej. "DD/MM/YYYY",
|
||||
"MM/DD/YYYY", "YYYYMMDD", "DDMMYYYY") o None si no parece fechas.
|
||||
"""
|
||||
if series is None or len(series) == 0:
|
||||
return None
|
||||
|
||||
non_null = series.dropna()
|
||||
if non_null.empty:
|
||||
return None
|
||||
|
||||
if len(non_null) > sample_size:
|
||||
non_null = non_null.sample(sample_size, random_state=42)
|
||||
|
||||
layouts = {"sep": 0, "compact8": 0, "compact6": 0}
|
||||
sep_first_gt12 = 0 # primer componente > 12 → probable día
|
||||
sep_second_gt12 = 0 # segundo componente > 12 → probable mes solo posible si MM/DD
|
||||
sep_third_4digits = 0 # tercer componente de 4 dígitos → año al final
|
||||
sep_first_4digits = 0 # primer componente de 4 dígitos → año al inicio
|
||||
compact8_year_first = 0 # primeros 4 dígitos > 1900 → YYYYMMDD
|
||||
compact8_year_last = 0 # últimos 4 dígitos > 1900 → DDMMYYYY
|
||||
total_parsed = 0
|
||||
|
||||
for val in non_null:
|
||||
parsed = _extract_date_parts(val)
|
||||
if parsed is None:
|
||||
continue
|
||||
total_parsed += 1
|
||||
a, b, c, layout = parsed
|
||||
layouts[layout] = layouts.get(layout, 0) + 1
|
||||
|
||||
if layout == "sep":
|
||||
# año puede estar al inicio o al final según cantidad de dígitos
|
||||
if _digit_count(a) == 4:
|
||||
sep_first_4digits += 1
|
||||
if _digit_count(c) == 4:
|
||||
sep_third_4digits += 1
|
||||
if a > 12:
|
||||
sep_first_gt12 += 1
|
||||
if b > 12:
|
||||
sep_second_gt12 += 1
|
||||
elif layout == "compact8":
|
||||
# a son los primeros 2 dígitos, c son los últimos 4
|
||||
if c >= 1900:
|
||||
compact8_year_last += 1
|
||||
# a concatenado con b son los primeros 4 dígitos
|
||||
first4 = a * 100 + b
|
||||
if first4 >= 1900:
|
||||
compact8_year_first += 1
|
||||
|
||||
if total_parsed == 0:
|
||||
return None
|
||||
|
||||
# Requerimos que >=70% de los no nulos parseen como fecha para considerarla
|
||||
# columna de fechas.
|
||||
if total_parsed / len(non_null) < 0.7:
|
||||
return None
|
||||
|
||||
dominant = max(layouts, key=lambda k: layouts[k])
|
||||
|
||||
if dominant == "compact8":
|
||||
if compact8_year_first > compact8_year_last:
|
||||
return "YYYYMMDD"
|
||||
return "DDMMYYYY"
|
||||
if dominant == "compact6":
|
||||
return "DDMMYY"
|
||||
|
||||
# Caso con separadores
|
||||
if sep_first_4digits > sep_third_4digits:
|
||||
return "YYYY/MM/DD"
|
||||
# año al final
|
||||
if sep_first_gt12 > 0 and sep_second_gt12 == 0:
|
||||
return "DD/MM/YYYY"
|
||||
if sep_second_gt12 > 0 and sep_first_gt12 == 0:
|
||||
return "MM/DD/YYYY"
|
||||
# ambiguo → preferencia hispana
|
||||
return "DD/MM/YYYY"
|
||||
|
||||
|
||||
def _digit_count(n: int) -> int:
|
||||
return len(str(abs(n)))
|
||||
|
||||
|
||||
def _parse_date_with_format(value: str, fmt: str) -> Optional[tuple[int, int, int]]:
|
||||
"""Convierte un valor de fecha a (día, mes, año) según el layout detectado."""
|
||||
parsed = _extract_date_parts(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
a, b, c, layout = parsed
|
||||
|
||||
if fmt == "DD/MM/YYYY":
|
||||
d, m, y = a, b, c
|
||||
elif fmt == "MM/DD/YYYY":
|
||||
m, d, y = a, b, c
|
||||
elif fmt == "YYYY/MM/DD":
|
||||
y, m, d = a, b, c
|
||||
elif fmt == "YYYYMMDD" and layout == "compact8":
|
||||
# reconstruir: primer4 = a*100 + b, luego mes y día
|
||||
y = a * 100 + b
|
||||
m = c // 100
|
||||
d = c % 100
|
||||
elif fmt == "DDMMYYYY" and layout == "compact8":
|
||||
d = a
|
||||
m = b
|
||||
y = c
|
||||
elif fmt == "DDMMYY" and layout == "compact6":
|
||||
d = a
|
||||
m = b
|
||||
y = 2000 + c if c < 50 else 1900 + c
|
||||
else:
|
||||
# intentar como sep genérico
|
||||
d, m, y = a, b, c
|
||||
|
||||
if not (1 <= m <= 12 and 1 <= d <= 31 and 1000 <= y <= 9999):
|
||||
# año en 2 dígitos
|
||||
if y < 100:
|
||||
y = 2000 + y if y < 50 else 1900 + y
|
||||
if not (1 <= m <= 12 and 1 <= d <= 31 and 1000 <= y <= 9999):
|
||||
return None
|
||||
return (d, m, y)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ANÁLISIS DE COLUMNAS
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_BOOLEAN_VALUES = {"true", "false", "si", "sí", "no", "1", "0", "yes"}
|
||||
|
||||
|
||||
def _infer_column_type(series: pd.Series) -> tuple[str, Optional[str]]:
|
||||
"""Devuelve (tipo, formato_fecha_detectado)."""
|
||||
non_null = series.dropna().astype(str).str.strip()
|
||||
non_null = non_null[non_null != ""]
|
||||
if non_null.empty:
|
||||
return TYPE_EMPTY, None
|
||||
|
||||
# Booleano
|
||||
lowered = non_null.str.lower()
|
||||
if lowered.isin(_BOOLEAN_VALUES).all() and lowered.nunique() <= 3:
|
||||
return TYPE_BOOLEAN, None
|
||||
|
||||
# Numérico
|
||||
# Aceptamos separador de miles con "." o "," y decimal con "," o ".".
|
||||
# Estrategia: intentar to_numeric tras normalizar.
|
||||
numeric_candidate = _to_numeric_lenient(non_null)
|
||||
numeric_ratio = numeric_candidate.notna().mean()
|
||||
if numeric_ratio >= 0.9:
|
||||
is_all_integer = (numeric_candidate.dropna() % 1 == 0).all()
|
||||
return (TYPE_INTEGER if is_all_integer else TYPE_DECIMAL), None
|
||||
|
||||
# Fecha
|
||||
date_fmt = detect_date_format(non_null)
|
||||
if date_fmt:
|
||||
return TYPE_DATE, date_fmt
|
||||
|
||||
# Categórico vs texto
|
||||
unique_ratio = non_null.nunique() / len(non_null)
|
||||
if non_null.nunique() <= 15 and unique_ratio < 0.3:
|
||||
return TYPE_CATEGORICAL, None
|
||||
|
||||
# Mixto si mezcla números con texto y nada es dominante
|
||||
if 0.1 < numeric_ratio < 0.9:
|
||||
return TYPE_MIXED, None
|
||||
|
||||
return TYPE_TEXT, None
|
||||
|
||||
|
||||
def _to_numeric_lenient(series: pd.Series) -> pd.Series:
|
||||
"""Intenta convertir a numérico aceptando separadores comunes."""
|
||||
s = series.astype(str).str.strip()
|
||||
s = s.str.replace(r"\s+", "", regex=True)
|
||||
|
||||
# Si hay ambos "." y ",", asumir "." como miles y "," como decimal
|
||||
def _norm(x: str) -> str:
|
||||
if "." in x and "," in x:
|
||||
return x.replace(".", "").replace(",", ".")
|
||||
if "," in x and "." not in x:
|
||||
# coma decimal
|
||||
return x.replace(",", ".")
|
||||
return x
|
||||
|
||||
s = s.map(_norm)
|
||||
return pd.to_numeric(s, errors="coerce")
|
||||
|
||||
|
||||
def _applicable_rules_for_type(inferred_type: str) -> list[str]:
|
||||
"""Reglas sugeridas según tipo inferido."""
|
||||
base = ["strip_special_spaces", "strip_special_keep_date_seps", "strip_spaces"]
|
||||
if inferred_type in (TYPE_INTEGER, TYPE_DECIMAL):
|
||||
return base + ["to_integer", "round_decimals_custom"]
|
||||
if inferred_type == TYPE_DATE:
|
||||
return base + ["date_numeric_to_ddmmyyyy", "date_alpha_to_ddmmyyyy"]
|
||||
if inferred_type == TYPE_MIXED:
|
||||
return base + [
|
||||
"to_integer",
|
||||
"round_decimals_custom",
|
||||
"date_numeric_to_ddmmyyyy",
|
||||
"date_alpha_to_ddmmyyyy",
|
||||
]
|
||||
# Texto / categórico / boolean / empty → todas las de limpieza de texto + las
|
||||
# conversoras (el usuario sabrá si aplica).
|
||||
return base + [
|
||||
"to_integer",
|
||||
"round_decimals_custom",
|
||||
"date_numeric_to_ddmmyyyy",
|
||||
"date_alpha_to_ddmmyyyy",
|
||||
]
|
||||
|
||||
|
||||
def analyze_dataframe(df: pd.DataFrame) -> list[ColumnSummary]:
|
||||
"""Genera un resumen por columna del DataFrame."""
|
||||
summaries: list[ColumnSummary] = []
|
||||
total_rows = len(df)
|
||||
rng = random.Random(42)
|
||||
|
||||
for col in df.columns:
|
||||
series = df[col]
|
||||
null_count = int(series.isna().sum() + (series.astype(str).str.strip() == "").sum())
|
||||
# Evitar doble conteo: considerar vacío solo si no-NaN
|
||||
null_count = int((series.isna() | (series.astype(str).str.strip() == "")).sum())
|
||||
null_pct = (null_count / total_rows * 100) if total_rows else 0.0
|
||||
|
||||
non_null = series.dropna().astype(str)
|
||||
non_null = non_null[non_null.str.strip() != ""]
|
||||
|
||||
example = ""
|
||||
if not non_null.empty:
|
||||
example = str(rng.choice(non_null.tolist()))
|
||||
if len(example) > 80:
|
||||
example = example[:77] + "..."
|
||||
|
||||
unique_count = int(non_null.nunique())
|
||||
unique_pct = (unique_count / total_rows * 100) if total_rows else 0.0
|
||||
|
||||
inferred_type, date_fmt = _infer_column_type(series)
|
||||
|
||||
summary = ColumnSummary(
|
||||
name=str(col),
|
||||
inferred_type=inferred_type,
|
||||
example=example,
|
||||
null_count=null_count,
|
||||
null_pct=null_pct,
|
||||
unique_count=unique_count,
|
||||
unique_pct=unique_pct,
|
||||
detected_date_format=date_fmt,
|
||||
)
|
||||
|
||||
if inferred_type in (TYPE_INTEGER, TYPE_DECIMAL):
|
||||
numeric = _to_numeric_lenient(non_null)
|
||||
if numeric.notna().any():
|
||||
summary.min = float(numeric.min())
|
||||
summary.max = float(numeric.max())
|
||||
summary.mean = float(numeric.mean())
|
||||
elif inferred_type in (TYPE_TEXT, TYPE_CATEGORICAL):
|
||||
if not non_null.empty:
|
||||
summary.avg_length = float(non_null.str.len().mean())
|
||||
counts = non_null.value_counts().head(3)
|
||||
summary.top_values = [(str(v), int(c)) for v, c in counts.items()]
|
||||
|
||||
summary.applicable_rules = _applicable_rules_for_type(inferred_type)
|
||||
summaries.append(summary)
|
||||
|
||||
return summaries
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# REGLAS DE LIMPIEZA
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _rule_strip_special_spaces(series: pd.Series) -> pd.Series:
|
||||
"""Elimina todo lo que no sea alfanumérico (incluye espacios)."""
|
||||
pattern = re.compile(r"[^A-Za-z0-9]", flags=re.UNICODE)
|
||||
|
||||
def _fn(x):
|
||||
if pd.isna(x):
|
||||
return x
|
||||
return pattern.sub("", str(x))
|
||||
|
||||
return series.map(_fn)
|
||||
|
||||
|
||||
def _rule_strip_special_keep_date_seps(series: pd.Series) -> pd.Series:
|
||||
"""Conserva alfanuméricos y separadores de fecha: '/', '-', espacio."""
|
||||
pattern = re.compile(r"[^A-Za-z0-9\/\- ]", flags=re.UNICODE)
|
||||
|
||||
def _fn(x):
|
||||
if pd.isna(x):
|
||||
return x
|
||||
return pattern.sub("", str(x))
|
||||
|
||||
return series.map(_fn)
|
||||
|
||||
|
||||
def _rule_strip_spaces(series: pd.Series) -> pd.Series:
|
||||
"""Elimina únicamente espacios (incluidos tabs y saltos de línea)."""
|
||||
def _fn(x):
|
||||
if pd.isna(x):
|
||||
return x
|
||||
return re.sub(r"\s+", "", str(x))
|
||||
|
||||
return series.map(_fn)
|
||||
|
||||
|
||||
def _rule_to_integer(series: pd.Series) -> pd.Series:
|
||||
"""Convierte a entero (Int64 nullable) con parser tolerante."""
|
||||
numeric = _to_numeric_lenient(series.astype(str))
|
||||
return numeric.round().astype("Int64")
|
||||
|
||||
|
||||
def _rule_round_decimals_custom(series: pd.Series) -> pd.Series:
|
||||
"""Aproximación custom: fracción > 0.5 → entero superior, resto → inferior.
|
||||
|
||||
Funciona con positivos y negativos (usa parte entera hacia 0 y suma o
|
||||
resta 1 según el signo y la fracción).
|
||||
"""
|
||||
numeric = _to_numeric_lenient(series.astype(str))
|
||||
|
||||
def _fn(x):
|
||||
if pd.isna(x):
|
||||
return pd.NA
|
||||
sign = -1 if x < 0 else 1
|
||||
ax = abs(float(x))
|
||||
integer_part = int(ax)
|
||||
frac = ax - integer_part
|
||||
rounded = integer_part + (1 if frac > 0.5 else 0)
|
||||
return sign * rounded
|
||||
|
||||
return numeric.map(_fn).astype("Int64")
|
||||
|
||||
|
||||
def _rule_date_numeric_to_ddmmyyyy(series: pd.Series) -> pd.Series:
|
||||
"""Entrada: fechas en formato numérico (ej. 20240315 o 15032024).
|
||||
|
||||
Salida: "DD/MM/AAAA" tras detectar el formato predominante.
|
||||
"""
|
||||
# Para esta regla consideramos los valores "numéricos" (quitamos separadores
|
||||
# comunes si los hay para detectar un YYYYMMDD/DDMMYYYY confiable).
|
||||
stripped = series.astype(str).str.replace(r"[\s\/\-\.]", "", regex=True)
|
||||
stripped = stripped.where(series.notna(), other=pd.NA)
|
||||
fmt = detect_date_format(stripped)
|
||||
if fmt is None:
|
||||
# fallback: intentar con detector normal
|
||||
fmt = detect_date_format(series)
|
||||
return _format_dates_to_ddmmyyyy(series, fmt)
|
||||
|
||||
|
||||
def _rule_date_alpha_to_ddmmyyyy(series: pd.Series) -> pd.Series:
|
||||
"""Entrada: fechas en formato alfanumérico con separadores.
|
||||
|
||||
Salida: "DD/MM/AAAA".
|
||||
"""
|
||||
fmt = detect_date_format(series)
|
||||
return _format_dates_to_ddmmyyyy(series, fmt)
|
||||
|
||||
|
||||
def _format_dates_to_ddmmyyyy(series: pd.Series, fmt: Optional[str]) -> pd.Series:
|
||||
"""Convierte cada valor al formato DD/MM/AAAA usando `fmt` como layout de
|
||||
entrada detectado. La parte de hora (si existe) se descarta. Mantiene el
|
||||
valor original si no puede parsear la fecha.
|
||||
"""
|
||||
if fmt is None:
|
||||
return series.copy()
|
||||
|
||||
def _fn(x):
|
||||
if pd.isna(x) or str(x).strip() == "":
|
||||
return x
|
||||
raw = str(x)
|
||||
date_part, _time_part = _split_date_time(raw)
|
||||
target = date_part if date_part else raw
|
||||
parsed = _parse_date_with_format(target, fmt)
|
||||
if parsed is None:
|
||||
return x
|
||||
d, m, y = parsed
|
||||
return f"{d:02d}/{m:02d}/{y:04d}"
|
||||
|
||||
return series.map(_fn)
|
||||
|
||||
|
||||
# Registro de reglas
|
||||
CLEANING_RULES: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": "strip_special_spaces",
|
||||
"name": "Eliminar caracteres especiales y espacios",
|
||||
"description": "Deja solo caracteres alfanuméricos (A-Z, a-z, 0-9).",
|
||||
"fn": _rule_strip_special_spaces,
|
||||
},
|
||||
{
|
||||
"id": "strip_special_keep_date_seps",
|
||||
"name": "Eliminar caracteres especiales y espacios (excepto / - y espacio, p. ej. Fecha Servicio)",
|
||||
"description": "Conserva alfanuméricos y los separadores '/', '-' y espacios.",
|
||||
"fn": _rule_strip_special_keep_date_seps,
|
||||
},
|
||||
{
|
||||
"id": "strip_spaces",
|
||||
"name": "Eliminar únicamente espacios",
|
||||
"description": "Elimina todos los caracteres de espacio en blanco.",
|
||||
"fn": _rule_strip_spaces,
|
||||
},
|
||||
{
|
||||
"id": "to_integer",
|
||||
"name": "Convertir a Entero",
|
||||
"description": "Convierte la columna a enteros (los decimales se redondean).",
|
||||
"fn": _rule_to_integer,
|
||||
},
|
||||
{
|
||||
"id": "date_numeric_to_ddmmyyyy",
|
||||
"name": "Conversión de formato numérico (Fechas) → DD/MM/AAAA",
|
||||
"description": "Detecta el formato numérico predominante (YYYYMMDD, DDMMYYYY, etc.) y normaliza a DD/MM/AAAA.",
|
||||
"fn": _rule_date_numeric_to_ddmmyyyy,
|
||||
},
|
||||
{
|
||||
"id": "date_alpha_to_ddmmyyyy",
|
||||
"name": "Conversión de formato alfanumérico (Fechas) → DD/MM/AAAA",
|
||||
"description": "Detecta el formato con separadores predominante (DD/MM/YYYY, MM/DD/YYYY, YYYY/MM/DD, etc.) y normaliza a DD/MM/AAAA.",
|
||||
"fn": _rule_date_alpha_to_ddmmyyyy,
|
||||
},
|
||||
{
|
||||
"id": "round_decimals_custom",
|
||||
"name": "Aproximación Decimales (>0.5 sube, ≤0.5 baja)",
|
||||
"description": "Si la fracción decimal es mayor a 0.5 se aproxima al entero superior; de lo contrario, al inferior.",
|
||||
"fn": _rule_round_decimals_custom,
|
||||
},
|
||||
]
|
||||
|
||||
_RULE_BY_ID: dict[str, Callable[[pd.Series], pd.Series]] = {
|
||||
r["id"]: r["fn"] for r in CLEANING_RULES
|
||||
}
|
||||
|
||||
_RULE_NAME_BY_ID: dict[str, str] = {r["id"]: r["name"] for r in CLEANING_RULES}
|
||||
|
||||
|
||||
def get_rule(rule_id: str) -> Optional[dict[str, Any]]:
|
||||
"""Devuelve el registro de una regla por id."""
|
||||
for r in CLEANING_RULES:
|
||||
if r["id"] == rule_id:
|
||||
return r
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# APLICACIÓN DE REGLAS
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def apply_rules(
|
||||
df: pd.DataFrame,
|
||||
rules_per_column: dict[str, list[str]],
|
||||
) -> tuple[pd.DataFrame, list[str]]:
|
||||
"""Aplica las reglas seleccionadas a cada columna.
|
||||
|
||||
`rules_per_column` es un dict de `columna -> lista de IDs de reglas en el
|
||||
orden a aplicar`. Devuelve el DataFrame resultante y un log.
|
||||
"""
|
||||
result = df.copy()
|
||||
log: list[str] = []
|
||||
|
||||
for col, rule_ids in rules_per_column.items():
|
||||
if not rule_ids:
|
||||
continue
|
||||
if col not in result.columns:
|
||||
log.append(f"[WARN] Columna no encontrada: {col}")
|
||||
continue
|
||||
for rule_id in rule_ids:
|
||||
fn = _RULE_BY_ID.get(rule_id)
|
||||
if fn is None:
|
||||
log.append(f"[WARN] Regla desconocida '{rule_id}' en columna '{col}'")
|
||||
continue
|
||||
try:
|
||||
result[col] = fn(result[col])
|
||||
log.append(f"[OK] '{col}' <- {_RULE_NAME_BY_ID.get(rule_id, rule_id)}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Error aplicando regla %s a %s", rule_id, col)
|
||||
log.append(f"[ERROR] '{col}' regla '{rule_id}': {exc}")
|
||||
|
||||
return result, log
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CARGA ROBUSTA DE CSV
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_CANDIDATE_DELIMITERS = ["|", ";", "\t", ",", ":"]
|
||||
|
||||
|
||||
def detect_delimiter(filepath: str) -> str:
|
||||
"""Detecta el delimitador del CSV de forma robusta.
|
||||
|
||||
Estrategia:
|
||||
1. Intentar `csv.Sniffer` con un sample grande.
|
||||
2. Si falla o el resultado parece incorrecto, contar ocurrencias por línea
|
||||
de cada delimitador candidato y elegir el que produce el mayor número
|
||||
de columnas estable entre líneas.
|
||||
"""
|
||||
encodings = ("utf-8", "utf-8-sig", "latin-1", "cp1252")
|
||||
sample = ""
|
||||
for enc in encodings:
|
||||
try:
|
||||
with open(filepath, "r", encoding=enc, errors="replace") as f:
|
||||
sample = f.read(65536)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not sample:
|
||||
return ","
|
||||
|
||||
try:
|
||||
dialect = _csv.Sniffer().sniff(sample, delimiters="".join(_CANDIDATE_DELIMITERS))
|
||||
sniffed = dialect.delimiter
|
||||
except _csv.Error:
|
||||
sniffed = None
|
||||
|
||||
lines = [ln for ln in sample.splitlines() if ln.strip()]
|
||||
if not lines:
|
||||
return sniffed or ","
|
||||
|
||||
# Para cada candidato, calcular el número promedio de columnas y la
|
||||
# consistencia (cuántas líneas tienen el mismo conteo dominante).
|
||||
scores: dict[str, tuple[int, int]] = {}
|
||||
for delim in _CANDIDATE_DELIMITERS:
|
||||
counts = [ln.count(delim) for ln in lines[:50]]
|
||||
if max(counts) == 0:
|
||||
continue
|
||||
# Modo (valor más frecuente)
|
||||
from collections import Counter
|
||||
common = Counter(counts).most_common(1)[0]
|
||||
mode_count, mode_freq = common
|
||||
if mode_count == 0:
|
||||
continue
|
||||
scores[delim] = (mode_count, mode_freq)
|
||||
|
||||
if scores:
|
||||
# Elegir el delimitador con mayor número de columnas (mode_count) y,
|
||||
# como desempate, mayor consistencia (mode_freq).
|
||||
best = max(scores.items(), key=lambda kv: (kv[1][0], kv[1][1]))
|
||||
best_delim = best[0]
|
||||
# Si el sniffer dio algo razonable y coincide, prevalece; si difiere y
|
||||
# nuestro candidato es claramente mejor, usamos el nuestro.
|
||||
if sniffed and sniffed in scores:
|
||||
sniffed_score = scores[sniffed]
|
||||
if sniffed_score[0] >= best[1][0] - 1:
|
||||
return sniffed
|
||||
return best_delim
|
||||
|
||||
return sniffed or ","
|
||||
|
||||
|
||||
def load_csv(filepath: str) -> tuple[pd.DataFrame, str]:
|
||||
"""Carga un CSV con detección de delimitador y tolerancia a filas malas.
|
||||
|
||||
Devuelve `(df, delimiter)`. Lanza excepción si todos los intentos fallan.
|
||||
"""
|
||||
delimiter = detect_delimiter(filepath)
|
||||
|
||||
encodings_to_try = ["utf-8", "utf-8-sig", "latin-1", "cp1252"]
|
||||
last_error: Optional[Exception] = None
|
||||
|
||||
for enc in encodings_to_try:
|
||||
try:
|
||||
df = pd.read_csv(
|
||||
filepath,
|
||||
delimiter=delimiter,
|
||||
dtype=str,
|
||||
encoding=enc,
|
||||
engine="python",
|
||||
on_bad_lines="skip",
|
||||
)
|
||||
return df, delimiter
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_error = exc
|
||||
continue
|
||||
|
||||
# Último intento con quoting permisivo y sin separador detectado
|
||||
try:
|
||||
df = pd.read_csv(
|
||||
filepath,
|
||||
sep=None,
|
||||
dtype=str,
|
||||
engine="python",
|
||||
on_bad_lines="skip",
|
||||
)
|
||||
return df, "(auto)"
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_error = exc
|
||||
|
||||
raise RuntimeError(f"No se pudo leer el CSV: {last_error}")
|
||||
@@ -0,0 +1,107 @@
|
||||
import requests
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import getpass
|
||||
from typing import Optional, Dict, Any
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class TrackingManager:
|
||||
"""Gestiona la comunicación con el servidor de seguimiento web."""
|
||||
|
||||
def __init__(self, base_url: str = "http://localhost:8000"):
|
||||
self.base_url = base_url
|
||||
self.current_batch_id: Optional[str] = None
|
||||
self.is_enabled = True
|
||||
self._session = requests.Session() # Usar sesión para reutilizar conexiones
|
||||
|
||||
def start_batch(self, root_path: str, total_radicados: int, config: Optional[Dict] = None,
|
||||
csv_output_path: Optional[str] = None):
|
||||
"""Notifica el inicio de un nuevo lote al servidor."""
|
||||
if not self.is_enabled:
|
||||
return
|
||||
|
||||
try:
|
||||
payload = {
|
||||
"root_path": root_path,
|
||||
"total_radicados": total_radicados,
|
||||
"csv_output_path": csv_output_path,
|
||||
"hostname": socket.gethostname(),
|
||||
"username": getpass.getuser(),
|
||||
"config": config or {}
|
||||
}
|
||||
response = self._session.post(f"{self.base_url}/batches", json=payload, timeout=5)
|
||||
if response.status_code == 200:
|
||||
self.current_batch_id = response.json().get("id")
|
||||
logger.info(f"Tracking iniciado: Batch ID {self.current_batch_id}")
|
||||
else:
|
||||
logger.warning(f"Error al iniciar tracking: {response.status_code}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error de conexión con el servidor de tracking: {e}")
|
||||
|
||||
def update_progress(self, current_index: int, success_count: int, error_count: int,
|
||||
total_tokens: int, total_cost_usd: float, status: str = "processing"):
|
||||
"""Actualiza el progreso del lote actual."""
|
||||
if not self.is_enabled or not self.current_batch_id:
|
||||
return
|
||||
|
||||
try:
|
||||
payload = {
|
||||
"current_radicado_index": current_index,
|
||||
"success_count": success_count,
|
||||
"error_count": error_count,
|
||||
"total_tokens": total_tokens,
|
||||
"total_cost_usd": total_cost_usd,
|
||||
"status": status
|
||||
}
|
||||
self._session.patch(f"{self.base_url}/batches/{self.current_batch_id}", json=payload, timeout=1)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error al actualizar tracking: {e}")
|
||||
|
||||
def log(self, message: str, level: str = "info", radicado: str = None):
|
||||
"""Envía una entrada de log al servidor de tracking."""
|
||||
if not self.is_enabled or not self.current_batch_id:
|
||||
return
|
||||
try:
|
||||
payload = {"message": message, "level": level, "radicado": radicado}
|
||||
self._session.post(
|
||||
f"{self.base_url}/batches/{self.current_batch_id}/logs",
|
||||
json=payload, timeout=1
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error al enviar log a tracking: {e}")
|
||||
|
||||
def report_phase(self, phase: str, radicado: str = None):
|
||||
"""Reporta la fase actual de procesamiento (extract, normalize, csv, completed)."""
|
||||
if not self.is_enabled or not self.current_batch_id:
|
||||
return
|
||||
try:
|
||||
payload = {"current_phase": phase, "current_radicado": radicado}
|
||||
self._session.patch(f"{self.base_url}/batches/{self.current_batch_id}", json=payload, timeout=1)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error al reportar fase a tracking: {e}")
|
||||
|
||||
def report_error(self, error_message: str):
|
||||
"""Reporta un error crítico al servidor."""
|
||||
if not self.is_enabled or not self.current_batch_id:
|
||||
return
|
||||
|
||||
try:
|
||||
payload = {"last_error": error_message}
|
||||
self._session.patch(f"{self.base_url}/batches/{self.current_batch_id}", json=payload, timeout=1)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error al reportar error de tracking: {e}")
|
||||
|
||||
def complete_batch(self, success_count: int, error_count: int, total_tokens: int, total_cost_usd: float):
|
||||
"""Notifica la finalización exitosa del lote."""
|
||||
self.update_progress(
|
||||
current_index=-1, # Indica finalización si se desea
|
||||
success_count=success_count,
|
||||
error_count=error_count,
|
||||
total_tokens=total_tokens,
|
||||
total_cost_usd=total_cost_usd,
|
||||
status="completed"
|
||||
)
|
||||
self.current_batch_id = None
|
||||
@@ -21,6 +21,7 @@ from application.database import DatabaseManager
|
||||
from application.config import settings
|
||||
from application.costs import MODEL_PRICING
|
||||
from application import postprocess_agent
|
||||
from application import csv_cleaning
|
||||
from openai import OpenAI
|
||||
import pandas as pd
|
||||
|
||||
@@ -110,6 +111,16 @@ class AppController:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Proceso de limpieza (sin LLM)
|
||||
self._cleaning_df: pd.DataFrame | None = None
|
||||
self._cleaning_result_df: pd.DataFrame | None = None
|
||||
try:
|
||||
self.view.set_on_cleaning_select_csv(self.on_cleaning_select_csv)
|
||||
self.view.set_on_cleaning_apply(self.on_cleaning_apply)
|
||||
self.view.set_on_cleaning_save(self.on_cleaning_save)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Mapeo de nombres de UI a IDs de modelo (y viceversa)
|
||||
self.ui_to_model = {
|
||||
"5.4": "gpt-5.4",
|
||||
@@ -1169,3 +1180,120 @@ class AppController:
|
||||
except Exception as e:
|
||||
logger.exception("Error guardando CSV post-procesado")
|
||||
mb.showerror("Error", f"No se pudo guardar el archivo:\n{e}")
|
||||
|
||||
# =========================================================================
|
||||
# --- PROCESO DE LIMPIEZA ---
|
||||
# =========================================================================
|
||||
|
||||
def on_cleaning_select_csv(self):
|
||||
"""Abre un selector de CSV y prepara el resumen de columnas."""
|
||||
filepath = fd.askopenfilename(
|
||||
filetypes=[("CSV", "*.csv"), ("Todos", "*.*")],
|
||||
title="Seleccionar CSV para limpieza",
|
||||
)
|
||||
if not filepath:
|
||||
return
|
||||
|
||||
try:
|
||||
df, delimiter = csv_cleaning.load_csv(filepath)
|
||||
except Exception as e:
|
||||
logger.exception("Error leyendo CSV para limpieza")
|
||||
mb.showerror("Error", f"No se pudo leer el CSV:\n{e}")
|
||||
return
|
||||
|
||||
self._cleaning_df = df
|
||||
self._cleaning_result_df = None
|
||||
self.view.set_cleaning_csv_path(filepath)
|
||||
self.view.enable_cleaning_save(False)
|
||||
self.view.set_cleaning_result(None)
|
||||
self.view.set_cleaning_status(
|
||||
f"CSV cargado: {len(df):,} filas x {len(df.columns)} columnas."
|
||||
)
|
||||
try:
|
||||
self.view.set_cleaning_delimiter_chip(delimiter)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
summaries = csv_cleaning.analyze_dataframe(df)
|
||||
except Exception as e:
|
||||
logger.exception("Error analizando CSV para limpieza")
|
||||
mb.showerror("Error", f"No se pudo analizar el CSV:\n{e}")
|
||||
return
|
||||
|
||||
self.view.populate_cleaning_columns(summaries)
|
||||
self.view.set_cleaning_preview(df)
|
||||
|
||||
def on_cleaning_apply(self):
|
||||
"""Aplica las reglas seleccionadas en un hilo y actualiza el resultado."""
|
||||
if self._cleaning_df is None:
|
||||
mb.showwarning("Aviso", "Primero seleccione un archivo CSV.")
|
||||
return
|
||||
|
||||
rules = self.view.get_cleaning_selected_rules()
|
||||
if not rules:
|
||||
mb.showwarning(
|
||||
"Aviso",
|
||||
"No hay reglas seleccionadas. Marque al menos una regla para alguna columna.",
|
||||
)
|
||||
return
|
||||
|
||||
self.view.enable_cleaning_controls(False)
|
||||
self.view.enable_cleaning_save(False)
|
||||
self.view.set_cleaning_status("Aplicando reglas...")
|
||||
self.view.set_cleaning_result(None)
|
||||
|
||||
df = self._cleaning_df
|
||||
|
||||
def worker():
|
||||
try:
|
||||
result_df, log = csv_cleaning.apply_rules(df, rules)
|
||||
self._cleaning_result_df = result_df
|
||||
ok_count = sum(1 for line in log if line.startswith("[OK]"))
|
||||
err_count = sum(1 for line in log if line.startswith("[ERROR]"))
|
||||
warn_count = sum(1 for line in log if line.startswith("[WARN]"))
|
||||
info = f"{ok_count} reglas aplicadas"
|
||||
if err_count or warn_count:
|
||||
info += f" · {err_count} errores · {warn_count} avisos"
|
||||
self.view.after(0, lambda d=result_df, t=info: self.view.set_cleaning_result(d, t))
|
||||
self.view.after(0, lambda: self.view.enable_cleaning_save(True))
|
||||
status = (
|
||||
f"Completado: {ok_count} reglas aplicadas correctamente"
|
||||
+ (f", {err_count} errores" if err_count else "")
|
||||
+ (f", {warn_count} avisos" if warn_count else "")
|
||||
+ "."
|
||||
)
|
||||
self.view.after(0, lambda s=status: self.view.set_cleaning_status(s))
|
||||
except Exception as exc:
|
||||
logger.exception("Error aplicando reglas de limpieza")
|
||||
err = str(exc)
|
||||
self.view.after(0, lambda: self.view.set_cleaning_status("Error"))
|
||||
self.view.after(0, lambda m=err: mb.showerror("Error", f"Error aplicando reglas:\n{m}"))
|
||||
finally:
|
||||
self.view.after(0, lambda: self.view.enable_cleaning_controls(True))
|
||||
|
||||
threading.Thread(target=worker, daemon=True).start()
|
||||
|
||||
def on_cleaning_save(self):
|
||||
"""Guarda el DataFrame limpio en disco."""
|
||||
if self._cleaning_result_df is None:
|
||||
mb.showwarning("Aviso", "No hay resultado para guardar.")
|
||||
return
|
||||
|
||||
path = fd.asksaveasfilename(
|
||||
defaultextension=".csv",
|
||||
filetypes=[("CSV", "*.csv"), ("Excel", "*.xlsx"), ("Todos", "*.*")],
|
||||
title="Guardar CSV limpio",
|
||||
)
|
||||
if not path:
|
||||
return
|
||||
|
||||
try:
|
||||
if path.lower().endswith(".xlsx"):
|
||||
self._cleaning_result_df.to_excel(path, index=False)
|
||||
else:
|
||||
self._cleaning_result_df.to_csv(path, index=False)
|
||||
mb.showinfo("Guardado", f"Archivo guardado correctamente:\n{path}")
|
||||
except Exception as e:
|
||||
logger.exception("Error guardando CSV limpio")
|
||||
mb.showerror("Error", f"No se pudo guardar el archivo:\n{e}")
|
||||
|
||||
+10
-1
@@ -5,5 +5,14 @@ from gui.tabs.main_tab import MainProcessingTab
|
||||
from gui.tabs.preclass_tab import PreclassificationTab
|
||||
from gui.tabs.metrics_tab import MetricsTab
|
||||
from gui.tabs.postprocess_tab import PostProcessTab
|
||||
from gui.tabs.cleaning_tab import CleaningTab
|
||||
|
||||
__all__ = ["SettingsTab", "ResultsTab", "MainProcessingTab", "PreclassificationTab", "MetricsTab", "PostProcessTab"]
|
||||
__all__ = [
|
||||
"SettingsTab",
|
||||
"ResultsTab",
|
||||
"MainProcessingTab",
|
||||
"PreclassificationTab",
|
||||
"MetricsTab",
|
||||
"PostProcessTab",
|
||||
"CleaningTab",
|
||||
]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+45
-1
@@ -6,7 +6,7 @@ Expone métodos para enlazar callbacks, leer entradas y escribir salida en un
|
||||
import os
|
||||
import customtkinter as ctk
|
||||
from gui.styles import AppStyles
|
||||
from gui.tabs import SettingsTab, ResultsTab, MainProcessingTab, PreclassificationTab, MetricsTab, PostProcessTab
|
||||
from gui.tabs import SettingsTab, ResultsTab, MainProcessingTab, PreclassificationTab, MetricsTab, PostProcessTab, CleaningTab
|
||||
|
||||
|
||||
class InvoiceAppView(ctk.CTk):
|
||||
@@ -44,6 +44,7 @@ class InvoiceAppView(ctk.CTk):
|
||||
self.tab_settings = self.tabs.add("Configuración")
|
||||
self.tab_usage = self.tabs.add("Dashboard & Métricas")
|
||||
self.tab_postprocess = self.tabs.add("Post-Procesamiento")
|
||||
self.tab_cleaning = self.tabs.add("Proceso de Limpieza")
|
||||
|
||||
# =========================================================================
|
||||
# --- TAB: PROCESAR DOCUMENTOS ---
|
||||
@@ -77,6 +78,11 @@ class InvoiceAppView(ctk.CTk):
|
||||
# =========================================================================
|
||||
self.postprocess_tab = PostProcessTab(self.tab_postprocess)
|
||||
|
||||
# =========================================================================
|
||||
# --- TAB: PROCESO DE LIMPIEZA ---
|
||||
# =========================================================================
|
||||
self.cleaning_tab = CleaningTab(self.tab_cleaning)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MÉTODOS PÚBLICOS (API COMPATIBLE)
|
||||
@@ -353,3 +359,41 @@ class InvoiceAppView(ctk.CTk):
|
||||
|
||||
def validate_doc_types(self) -> tuple[bool, str | None]:
|
||||
return self.main_tab.validate_doc_types()
|
||||
|
||||
# --- TAB PROCESO DE LIMPIEZA ---
|
||||
|
||||
def set_on_cleaning_select_csv(self, callback):
|
||||
self.cleaning_tab.set_on_select_csv(callback)
|
||||
|
||||
def set_on_cleaning_apply(self, callback):
|
||||
self.cleaning_tab.set_on_apply(callback)
|
||||
|
||||
def set_on_cleaning_save(self, callback):
|
||||
self.cleaning_tab.set_on_save(callback)
|
||||
|
||||
def set_cleaning_csv_path(self, path: str):
|
||||
self.cleaning_tab.set_csv_path(path)
|
||||
|
||||
def set_cleaning_preview(self, df):
|
||||
self.cleaning_tab.set_preview(df)
|
||||
|
||||
def populate_cleaning_columns(self, summaries):
|
||||
self.cleaning_tab.populate_columns(summaries)
|
||||
|
||||
def get_cleaning_selected_rules(self) -> dict[str, list[str]]:
|
||||
return self.cleaning_tab.get_selected_rules()
|
||||
|
||||
def set_cleaning_result(self, df, info_text: str = ""):
|
||||
self.cleaning_tab.set_result(df, info_text)
|
||||
|
||||
def enable_cleaning_save(self, enabled: bool):
|
||||
self.cleaning_tab.enable_save(enabled)
|
||||
|
||||
def enable_cleaning_controls(self, enabled: bool):
|
||||
self.cleaning_tab.enable_controls(enabled)
|
||||
|
||||
def set_cleaning_status(self, text: str):
|
||||
self.cleaning_tab.set_status(text)
|
||||
|
||||
def set_cleaning_delimiter_chip(self, delimiter: str):
|
||||
self.cleaning_tab.set_delimiter_chip(delimiter)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Capa de persistencia SQLite para el servidor de tracking.
|
||||
|
||||
Guarda el historial completo de lotes de extracción, incluyendo
|
||||
métricas de progreso, costos y la ruta del CSV generado.
|
||||
"""
|
||||
import sqlite3
|
||||
import json
|
||||
import os
|
||||
from typing import Dict, Any, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
DB_PATH = os.path.join(os.path.dirname(__file__), "..", "tracking.db")
|
||||
|
||||
|
||||
def get_connection() -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
def init_db():
|
||||
"""Crea las tablas si no existen."""
|
||||
with get_connection() as conn:
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS batch_logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
batch_id TEXT NOT NULL,
|
||||
timestamp TEXT NOT NULL,
|
||||
level TEXT NOT NULL DEFAULT 'info',
|
||||
radicado TEXT,
|
||||
message TEXT NOT NULL,
|
||||
FOREIGN KEY (batch_id) REFERENCES batches(id)
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_batch_logs_batch_id ON batch_logs(batch_id)")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS batches (
|
||||
id TEXT PRIMARY KEY,
|
||||
root_path TEXT NOT NULL,
|
||||
csv_output_path TEXT,
|
||||
hostname TEXT,
|
||||
username TEXT,
|
||||
total_radicados INTEGER NOT NULL DEFAULT 0,
|
||||
config TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'processing',
|
||||
current_radicado_index INTEGER NOT NULL DEFAULT 0,
|
||||
success_count INTEGER NOT NULL DEFAULT 0,
|
||||
error_count INTEGER NOT NULL DEFAULT 0,
|
||||
total_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
total_cost_usd REAL NOT NULL DEFAULT 0.0,
|
||||
last_error TEXT,
|
||||
current_phase TEXT,
|
||||
current_radicado TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
|
||||
def insert_batch(batch: Dict[str, Any]):
|
||||
"""Inserta un nuevo lote en la base de datos."""
|
||||
with get_connection() as conn:
|
||||
conn.execute("""
|
||||
INSERT INTO batches (
|
||||
id, root_path, csv_output_path, hostname, username, total_radicados, config,
|
||||
status, current_radicado_index, success_count, error_count,
|
||||
total_tokens, total_cost_usd, last_error, current_phase, current_radicado,
|
||||
created_at, updated_at
|
||||
) VALUES (
|
||||
:id, :root_path, :csv_output_path, :hostname, :username, :total_radicados, :config,
|
||||
:status, :current_radicado_index, :success_count, :error_count,
|
||||
:total_tokens, :total_cost_usd, :last_error, :current_phase, :current_radicado,
|
||||
:created_at, :updated_at
|
||||
)
|
||||
""", {
|
||||
**batch,
|
||||
"config": json.dumps(batch.get("config") or {}),
|
||||
"created_at": _to_iso(batch.get("created_at")),
|
||||
"updated_at": _to_iso(batch.get("updated_at")),
|
||||
"current_phase": batch.get("current_phase"),
|
||||
"current_radicado": batch.get("current_radicado"),
|
||||
})
|
||||
conn.commit()
|
||||
|
||||
|
||||
def update_batch(batch_id: str, fields: Dict[str, Any]):
|
||||
"""Actualiza campos de un lote existente."""
|
||||
if not fields:
|
||||
return
|
||||
fields["updated_at"] = _to_iso(datetime.now())
|
||||
set_clause = ", ".join(f"{k} = :{k}" for k in fields)
|
||||
fields["batch_id"] = batch_id
|
||||
with get_connection() as conn:
|
||||
conn.execute(
|
||||
f"UPDATE batches SET {set_clause} WHERE id = :batch_id",
|
||||
fields
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_all_batches() -> List[Dict[str, Any]]:
|
||||
"""Retorna todos los lotes ordenados por fecha de creación descendente."""
|
||||
with get_connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM batches ORDER BY created_at DESC"
|
||||
).fetchall()
|
||||
return [_row_to_dict(r) for r in rows]
|
||||
|
||||
|
||||
def insert_log(batch_id: str, message: str, level: str = "info", radicado: str = None):
|
||||
"""Inserta una entrada de log para un lote."""
|
||||
with get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO batch_logs (batch_id, timestamp, level, radicado, message) VALUES (?, ?, ?, ?, ?)",
|
||||
(batch_id, datetime.now().isoformat(), level, radicado, message)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_logs(batch_id: str, limit: int = 200) -> List[Dict[str, Any]]:
|
||||
"""Retorna los logs de un lote, del más reciente al más antiguo."""
|
||||
with get_connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM batch_logs WHERE batch_id = ? ORDER BY id DESC LIMIT ?",
|
||||
(batch_id, limit)
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def get_batch(batch_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Retorna un lote por su ID."""
|
||||
with get_connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM batches WHERE id = ?", (batch_id,)
|
||||
).fetchone()
|
||||
return _row_to_dict(row) if row else None
|
||||
|
||||
|
||||
def _row_to_dict(row: sqlite3.Row) -> Dict[str, Any]:
|
||||
d = dict(row)
|
||||
try:
|
||||
d["config"] = json.loads(d.get("config") or "{}")
|
||||
except Exception:
|
||||
d["config"] = {}
|
||||
|
||||
# Asegurar que hostname y username estén presentes
|
||||
if "hostname" not in d: d["hostname"] = None
|
||||
if "username" not in d: d["username"] = None
|
||||
|
||||
return d
|
||||
|
||||
|
||||
def _to_iso(value) -> str:
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return datetime.now().isoformat()
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Servidor de tracking de IPSExtractor.
|
||||
|
||||
Persiste el historial de lotes en SQLite y distribuye actualizaciones
|
||||
en tiempo real vía WebSockets al dashboard web.
|
||||
"""
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.responses import FileResponse
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Dict, List, Any, Optional
|
||||
from pydantic import BaseModel
|
||||
import uuid
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
from .schemas.batch import BatchCreate, BatchUpdate, BatchRead, BatchStatus
|
||||
from .database import init_db, insert_batch, update_batch, get_all_batches, get_batch, insert_log, get_logs
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
init_db()
|
||||
yield
|
||||
|
||||
app = FastAPI(title="IPSExtractor Tracking API", lifespan=lifespan)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Cache en memoria para lotes activos (para broadcast rápido)
|
||||
active_batches: Dict[str, Dict[str, Any]] = {}
|
||||
active_connections: List[WebSocket] = []
|
||||
|
||||
|
||||
async def broadcast(message: dict):
|
||||
json_compatible_message = jsonable_encoder(message)
|
||||
disconnected = []
|
||||
for connection in active_connections:
|
||||
try:
|
||||
await connection.send_json(json_compatible_message)
|
||||
except Exception:
|
||||
disconnected.append(connection)
|
||||
for conn in disconnected:
|
||||
active_connections.remove(conn)
|
||||
|
||||
|
||||
@app.post("/batches", response_model=BatchRead)
|
||||
async def create_batch(batch: BatchCreate):
|
||||
batch_id = str(uuid.uuid4())
|
||||
now = datetime.now()
|
||||
new_batch = {
|
||||
"id": batch_id,
|
||||
**batch.model_dump(),
|
||||
"status": BatchStatus.PROCESSING,
|
||||
"current_radicado_index": 0,
|
||||
"success_count": 0,
|
||||
"error_count": 0,
|
||||
"total_tokens": 0,
|
||||
"total_cost_usd": 0.0,
|
||||
"last_error": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
active_batches[batch_id] = new_batch
|
||||
insert_batch(new_batch)
|
||||
await broadcast({"event": "batch_created", "data": new_batch})
|
||||
return new_batch
|
||||
|
||||
|
||||
@app.get("/batches", response_model=List[BatchRead])
|
||||
async def list_batches():
|
||||
# Retorna historial completo desde SQLite
|
||||
db_batches = get_all_batches()
|
||||
# Sobreescribe con datos en memoria si el lote está activo (más fresco)
|
||||
result = []
|
||||
for b in db_batches:
|
||||
if b["id"] in active_batches:
|
||||
result.append(active_batches[b["id"]])
|
||||
else:
|
||||
result.append(b)
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/batches/{batch_id}", response_model=BatchRead)
|
||||
async def get_batch_endpoint(batch_id: str):
|
||||
if batch_id in active_batches:
|
||||
return active_batches[batch_id]
|
||||
batch = get_batch(batch_id)
|
||||
if not batch:
|
||||
raise HTTPException(status_code=404, detail="Lote no encontrado")
|
||||
return batch
|
||||
|
||||
|
||||
@app.patch("/batches/{batch_id}", response_model=BatchRead)
|
||||
async def update_batch_endpoint(batch_id: str, update: BatchUpdate):
|
||||
if batch_id in active_batches:
|
||||
batch = active_batches[batch_id]
|
||||
update_data = update.model_dump(exclude_unset=True)
|
||||
for key, value in update_data.items():
|
||||
batch[key] = value
|
||||
batch["updated_at"] = datetime.now()
|
||||
|
||||
# Si el lote se completó, sacarlo del cache activo
|
||||
if batch.get("status") in (BatchStatus.COMPLETED, BatchStatus.FAILED):
|
||||
active_batches.pop(batch_id, None)
|
||||
|
||||
update_batch(batch_id, update_data)
|
||||
await broadcast({"event": "batch_updated", "data": batch})
|
||||
return batch
|
||||
|
||||
# Lote no está en memoria (completado), actualizar solo en DB
|
||||
batch = get_batch(batch_id)
|
||||
if not batch:
|
||||
raise HTTPException(status_code=404, detail="Lote no encontrado")
|
||||
update_data = update.model_dump(exclude_unset=True)
|
||||
update_batch(batch_id, update_data)
|
||||
batch.update(update_data)
|
||||
return batch
|
||||
|
||||
|
||||
class LogEntry(BaseModel):
|
||||
message: str
|
||||
level: str = "info"
|
||||
radicado: Optional[str] = None
|
||||
|
||||
class LogRead(BaseModel):
|
||||
id: int
|
||||
batch_id: str
|
||||
timestamp: str
|
||||
level: str
|
||||
radicado: Optional[str]
|
||||
message: str
|
||||
|
||||
@app.post("/batches/{batch_id}/logs", status_code=201)
|
||||
async def add_log(batch_id: str, entry: LogEntry):
|
||||
"""Añade una entrada de log para un lote."""
|
||||
insert_log(batch_id, entry.message, entry.level, entry.radicado)
|
||||
# Broadcast log también por WebSocket para actualizaciones en vivo
|
||||
await broadcast({"event": "log_added", "batch_id": batch_id, "data": {
|
||||
"level": entry.level, "radicado": entry.radicado, "message": entry.message
|
||||
}})
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@app.get("/batches/{batch_id}/logs", response_model=List[LogRead])
|
||||
async def list_logs(batch_id: str, limit: int = 200):
|
||||
"""Retorna el historial de logs de un lote."""
|
||||
return get_logs(batch_id, limit)
|
||||
|
||||
|
||||
@app.get("/download/{batch_id}")
|
||||
async def download_csv(batch_id: str):
|
||||
"""Sirve el archivo CSV generado por el lote de extracción."""
|
||||
batch = get_batch(batch_id)
|
||||
if not batch:
|
||||
raise HTTPException(status_code=404, detail="Lote no encontrado")
|
||||
|
||||
csv_path = batch.get("csv_output_path")
|
||||
if not csv_path:
|
||||
raise HTTPException(status_code=404, detail="Este lote no tiene un archivo CSV asociado")
|
||||
if not os.path.isfile(csv_path):
|
||||
raise HTTPException(status_code=404, detail=f"Archivo CSV no encontrado en el servidor: {csv_path}")
|
||||
|
||||
filename = os.path.basename(csv_path)
|
||||
return FileResponse(
|
||||
path=csv_path,
|
||||
media_type="text/csv",
|
||||
filename=filename,
|
||||
)
|
||||
|
||||
|
||||
@app.websocket("/ws/tracking")
|
||||
async def websocket_endpoint(websocket: WebSocket):
|
||||
await websocket.accept()
|
||||
active_connections.append(websocket)
|
||||
try:
|
||||
while True:
|
||||
await websocket.receive_text()
|
||||
except WebSocketDisconnect:
|
||||
if websocket in active_connections:
|
||||
active_connections.remove(websocket)
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, List, Dict
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class BatchStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
PROCESSING = "processing"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class BatchBase(BaseModel):
|
||||
root_path: str
|
||||
total_radicados: int
|
||||
csv_output_path: Optional[str] = None
|
||||
hostname: Optional[str] = None
|
||||
username: Optional[str] = None
|
||||
config: Optional[Dict] = None
|
||||
|
||||
|
||||
class BatchCreate(BatchBase):
|
||||
pass
|
||||
|
||||
|
||||
class BatchUpdate(BaseModel):
|
||||
status: Optional[BatchStatus] = None
|
||||
current_radicado_index: Optional[int] = None
|
||||
success_count: Optional[int] = None
|
||||
error_count: Optional[int] = None
|
||||
total_tokens: Optional[int] = None
|
||||
total_cost_usd: Optional[float] = None
|
||||
last_error: Optional[str] = None
|
||||
current_phase: Optional[str] = None
|
||||
current_radicado: Optional[str] = None
|
||||
|
||||
|
||||
class BatchRead(BatchBase):
|
||||
id: str
|
||||
status: BatchStatus
|
||||
current_radicado_index: int = 0
|
||||
success_count: int = 0
|
||||
error_count: int = 0
|
||||
total_tokens: int = 0
|
||||
total_cost_usd: float = 0.0
|
||||
last_error: Optional[str] = None
|
||||
current_phase: Optional[str] = None
|
||||
current_radicado: Optional[str] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
@@ -0,0 +1,41 @@
|
||||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
/.pnp
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/versions
|
||||
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# next.js
|
||||
/.next/
|
||||
/out/
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
*.pem
|
||||
|
||||
# debug
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env*
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
# typescript
|
||||
*.tsbuildinfo
|
||||
next-env.d.ts
|
||||
@@ -0,0 +1,5 @@
|
||||
<!-- BEGIN:nextjs-agent-rules -->
|
||||
# This is NOT the Next.js you know
|
||||
|
||||
This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` before writing any code. Heed deprecation notices.
|
||||
<!-- END:nextjs-agent-rules -->
|
||||
@@ -0,0 +1 @@
|
||||
@AGENTS.md
|
||||
@@ -0,0 +1,36 @@
|
||||
This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
|
||||
|
||||
## Getting Started
|
||||
|
||||
First, run the development server:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
# or
|
||||
yarn dev
|
||||
# or
|
||||
pnpm dev
|
||||
# or
|
||||
bun dev
|
||||
```
|
||||
|
||||
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
|
||||
|
||||
You can start editing the page by modifying `app/page.js`. The page auto-updates as you edit the file.
|
||||
|
||||
This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel.
|
||||
|
||||
## Learn More
|
||||
|
||||
To learn more about Next.js, take a look at the following resources:
|
||||
|
||||
- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
|
||||
- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
|
||||
|
||||
You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome!
|
||||
|
||||
## Deploy on Vercel
|
||||
|
||||
The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
|
||||
|
||||
Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details.
|
||||
@@ -0,0 +1,16 @@
|
||||
import { defineConfig, globalIgnores } from "eslint/config";
|
||||
import nextVitals from "eslint-config-next/core-web-vitals";
|
||||
|
||||
const eslintConfig = defineConfig([
|
||||
...nextVitals,
|
||||
// Override default ignores of eslint-config-next.
|
||||
globalIgnores([
|
||||
// Default ignores of eslint-config-next:
|
||||
".next/**",
|
||||
"out/**",
|
||||
"build/**",
|
||||
"next-env.d.ts",
|
||||
]),
|
||||
]);
|
||||
|
||||
export default eslintConfig;
|
||||
@@ -0,0 +1,6 @@
|
||||
/** @type {import('next').NextConfig} */
|
||||
const nextConfig = {
|
||||
/* config options here */
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
@@ -0,0 +1,7 @@
|
||||
const config = {
|
||||
plugins: {
|
||||
"@tailwindcss/postcss": {},
|
||||
},
|
||||
};
|
||||
|
||||
export default config;
|
||||
@@ -0,0 +1 @@
|
||||
<svg fill="none" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path d="M14.5 13.5V5.41a1 1 0 0 0-.3-.7L9.8.29A1 1 0 0 0 9.08 0H1.5v13.5A2.5 2.5 0 0 0 4 16h8a2.5 2.5 0 0 0 2.5-2.5m-1.5 0v-7H8v-5H3v12a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1M9.5 5V2.12L12.38 5zM5.13 5h-.62v1.25h2.12V5zm-.62 3h7.12v1.25H4.5zm.62 3h-.62v1.25h7.12V11z" clip-rule="evenodd" fill="#666" fill-rule="evenodd"/></svg>
|
||||
|
After Width: | Height: | Size: 391 B |
@@ -0,0 +1 @@
|
||||
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><g clip-path="url(#a)"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.27 14.1a6.5 6.5 0 0 0 3.67-3.45q-1.24.21-2.7.34-.31 1.83-.97 3.1M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16m.48-1.52a7 7 0 0 1-.96 0H7.5a4 4 0 0 1-.84-1.32q-.38-.89-.63-2.08a40 40 0 0 0 3.92 0q-.25 1.2-.63 2.08a4 4 0 0 1-.84 1.31zm2.94-4.76q1.66-.15 2.95-.43a7 7 0 0 0 0-2.58q-1.3-.27-2.95-.43a18 18 0 0 1 0 3.44m-1.27-3.54a17 17 0 0 1 0 3.64 39 39 0 0 1-4.3 0 17 17 0 0 1 0-3.64 39 39 0 0 1 4.3 0m1.1-1.17q1.45.13 2.69.34a6.5 6.5 0 0 0-3.67-3.44q.65 1.26.98 3.1M8.48 1.5l.01.02q.41.37.84 1.31.38.89.63 2.08a40 40 0 0 0-3.92 0q.25-1.2.63-2.08a4 4 0 0 1 .85-1.32 7 7 0 0 1 .96 0m-2.75.4a6.5 6.5 0 0 0-3.67 3.44 29 29 0 0 1 2.7-.34q.31-1.83.97-3.1M4.58 6.28q-1.66.16-2.95.43a7 7 0 0 0 0 2.58q1.3.27 2.95.43a18 18 0 0 1 0-3.44m.17 4.71q-1.45-.12-2.69-.34a6.5 6.5 0 0 0 3.67 3.44q-.65-1.27-.98-3.1" fill="#666"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h16v16H0z"/></clipPath></defs></svg>
|
||||
|
After Width: | Height: | Size: 1.0 KiB |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
@@ -0,0 +1 @@
|
||||
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1155 1000"><path d="m577.3 0 577.4 1000H0z" fill="#fff"/></svg>
|
||||
|
After Width: | Height: | Size: 128 B |
@@ -0,0 +1 @@
|
||||
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M1.5 2.5h13v10a1 1 0 0 1-1 1h-11a1 1 0 0 1-1-1zM0 1h16v11.5a2.5 2.5 0 0 1-2.5 2.5h-11A2.5 2.5 0 0 1 0 12.5zm3.75 4.5a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5M7 4.75a.75.75 0 1 1-1.5 0 .75.75 0 0 1 1.5 0m1.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5" fill="#666"/></svg>
|
||||
|
After Width: | Height: | Size: 385 B |
@@ -0,0 +1,128 @@
|
||||
"use client";
|
||||
import { useState } from "react";
|
||||
import Link from "next/link";
|
||||
import { usePathname } from "next/navigation";
|
||||
import {
|
||||
Database, LayoutDashboard, History, TrendingUp, Settings,
|
||||
ChevronLeft, ChevronRight, LogOut, User, Bell
|
||||
} from "lucide-react";
|
||||
import { clsx } from "clsx";
|
||||
import { twMerge } from "tailwind-merge";
|
||||
|
||||
function cn(...inputs) {
|
||||
return twMerge(clsx(inputs));
|
||||
}
|
||||
|
||||
const NAV_ITEMS = [
|
||||
{ href: "/", icon: LayoutDashboard, label: "Dashboard" },
|
||||
{ href: "/processes", icon: History, label: "Procesos" },
|
||||
{ href: "/analytics", icon: TrendingUp, label: "Analíticas" },
|
||||
{ href: "/settings", icon: Settings, label: "Configuración" },
|
||||
];
|
||||
|
||||
export default function AppShell({ children }) {
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
const pathname = usePathname();
|
||||
|
||||
return (
|
||||
<div className="flex h-screen overflow-hidden font-sans text-slate-900 selection:bg-blue-100">
|
||||
{/* Sidebar */}
|
||||
<aside className={cn(
|
||||
"bg-white border-r border-slate-200/60 flex flex-col transition-all duration-300 ease-in-out z-40 h-screen",
|
||||
collapsed ? "w-20" : "w-64"
|
||||
)}>
|
||||
{/* Logo */}
|
||||
<div className={cn("p-5 flex items-center gap-3", collapsed && "justify-center")}>
|
||||
<div className="bg-blue-600 p-2.5 rounded-xl shadow-lg shadow-blue-200 flex-shrink-0">
|
||||
<Database className="w-5 h-5 text-white" />
|
||||
</div>
|
||||
{!collapsed && (
|
||||
<div>
|
||||
<h1 className="text-base font-black tracking-tight text-slate-900 leading-none">IPSExtractor</h1>
|
||||
<p className="text-[9px] font-bold text-slate-400 uppercase tracking-widest mt-0.5">Enterprise</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="h-px bg-slate-100 mx-4" />
|
||||
|
||||
{/* Nav */}
|
||||
<nav className="flex-1 px-3 py-4 space-y-1 overflow-y-auto">
|
||||
{!collapsed && (
|
||||
<p className="text-[9px] font-bold text-slate-400 uppercase tracking-widest px-3 pb-2">NAVEGACIÓN</p>
|
||||
)}
|
||||
{NAV_ITEMS.map(({ href, icon: Icon, label }) => {
|
||||
const active = href === "/" ? pathname === "/" : pathname.startsWith(href);
|
||||
return (
|
||||
<Link
|
||||
key={href}
|
||||
href={href}
|
||||
className={cn(
|
||||
"flex items-center gap-3 px-3 py-2.5 rounded-xl transition-all duration-150 group",
|
||||
active
|
||||
? "bg-blue-600 text-white shadow-lg shadow-blue-200"
|
||||
: "text-slate-500 hover:bg-slate-100 hover:text-slate-900"
|
||||
)}
|
||||
>
|
||||
<Icon className={cn("w-5 h-5 flex-shrink-0", !active && "group-hover:text-blue-600")} />
|
||||
{!collapsed && <span className="text-sm font-semibold tracking-tight truncate">{label}</span>}
|
||||
</Link>
|
||||
);
|
||||
})}
|
||||
</nav>
|
||||
|
||||
{/* Footer */}
|
||||
<div className="p-3 space-y-1 border-t border-slate-100">
|
||||
<button
|
||||
onClick={() => setCollapsed(!collapsed)}
|
||||
className="flex items-center gap-3 w-full px-3 py-2 rounded-xl text-slate-400 hover:bg-slate-100 hover:text-slate-700 transition-all"
|
||||
>
|
||||
{collapsed
|
||||
? <ChevronRight className="w-5 h-5 mx-auto" />
|
||||
: <><ChevronLeft className="w-5 h-5" /><span className="text-xs font-semibold">Colapsar</span></>
|
||||
}
|
||||
</button>
|
||||
|
||||
{!collapsed && (
|
||||
<div className="mt-2 p-3 bg-slate-50 rounded-2xl flex items-center gap-3">
|
||||
<div className="w-8 h-8 rounded-full bg-gradient-to-br from-blue-500 to-indigo-600 flex items-center justify-center text-white text-xs font-bold flex-shrink-0">
|
||||
AD
|
||||
</div>
|
||||
<div className="flex-1 min-w-0">
|
||||
<p className="text-xs font-bold text-slate-900 truncate">Admin User</p>
|
||||
<p className="text-[10px] text-slate-400 truncate">admin@ipsextractor.com</p>
|
||||
</div>
|
||||
<LogOut className="w-4 h-4 text-slate-400 hover:text-red-500 cursor-pointer flex-shrink-0" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
{/* Main area */}
|
||||
<div className="flex-1 flex flex-col overflow-hidden">
|
||||
{/* Topbar */}
|
||||
<header className="bg-white/80 backdrop-blur-md border-b border-slate-200/60 px-6 py-3.5 flex items-center justify-between flex-shrink-0">
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-xs font-bold text-slate-400 uppercase tracking-widest">
|
||||
{NAV_ITEMS.find(n => n.href === "/" ? pathname === "/" : pathname.startsWith(n.href))?.label || ""}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-4">
|
||||
<button className="p-2 text-slate-400 hover:text-blue-600 relative transition-colors">
|
||||
<Bell className="w-5 h-5" />
|
||||
<span className="absolute top-1.5 right-1.5 w-1.5 h-1.5 bg-red-500 rounded-full" />
|
||||
</button>
|
||||
<div className="w-8 h-8 rounded-full bg-gradient-to-br from-blue-500 to-indigo-600 flex items-center justify-center text-white text-xs font-bold">
|
||||
AD
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{/* Page content */}
|
||||
<main className="flex-1 overflow-y-auto">
|
||||
{children}
|
||||
</main>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
@@ -0,0 +1,22 @@
|
||||
@import "tailwindcss";
|
||||
|
||||
:root {
|
||||
--background: #F8FAFC;
|
||||
--foreground: #0f172a;
|
||||
}
|
||||
|
||||
body {
|
||||
background: var(--background);
|
||||
color: var(--foreground);
|
||||
}
|
||||
|
||||
::-webkit-scrollbar { width: 5px; height: 5px; }
|
||||
::-webkit-scrollbar-track { background: transparent; }
|
||||
::-webkit-scrollbar-thumb { background: #E2E8F0; border-radius: 10px; }
|
||||
::-webkit-scrollbar-thumb:hover { background: #CBD5E1; }
|
||||
|
||||
@keyframes spin-slow {
|
||||
from { transform: rotate(0deg); }
|
||||
to { transform: rotate(360deg); }
|
||||
}
|
||||
.animate-spin-slow { animation: spin-slow 3s linear infinite; }
|
||||
@@ -0,0 +1,31 @@
|
||||
import { Geist, Geist_Mono } from "next/font/google";
|
||||
import "./globals.css";
|
||||
import AppShell from "./components/AppShell";
|
||||
|
||||
const geistSans = Geist({
|
||||
variable: "--font-geist-sans",
|
||||
subsets: ["latin"],
|
||||
});
|
||||
|
||||
const geistMono = Geist_Mono({
|
||||
variable: "--font-geist-mono",
|
||||
subsets: ["latin"],
|
||||
});
|
||||
|
||||
export const metadata = {
|
||||
title: "IPSExtractor Enterprise",
|
||||
description: "Enterprise Document Processing Monitor",
|
||||
};
|
||||
|
||||
export default function RootLayout({ children }) {
|
||||
return (
|
||||
<html
|
||||
lang="es"
|
||||
className={`${geistSans.variable} ${geistMono.variable} h-full antialiased`}
|
||||
>
|
||||
<body className="min-h-full bg-[#F8FAFC]">
|
||||
<AppShell>{children}</AppShell>
|
||||
</body>
|
||||
</html>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
"use client";
|
||||
import { useEffect, useState, useCallback, useMemo } from "react";
|
||||
import Link from "next/link";
|
||||
import {
|
||||
Activity, CheckCircle2, AlertCircle, Download, History,
|
||||
TrendingUp, DollarSign, FileSearch, GitMerge, FileSpreadsheet,
|
||||
ArrowRight, Cpu, Wifi, WifiOff, RefreshCw, FolderOpen, Clock,
|
||||
ChevronRight
|
||||
} from "lucide-react";
|
||||
import { clsx } from "clsx";
|
||||
import { twMerge } from "tailwind-merge";
|
||||
|
||||
function cn(...inputs) { return twMerge(clsx(inputs)); }
|
||||
|
||||
const API_URL = "http://localhost:8000";
|
||||
|
||||
// --- Definición de fases del pipeline ---
|
||||
const PHASES = [
|
||||
{ key: "prepare", label: "Preparación", icon: FolderOpen, desc: "Escaneando documentos..." },
|
||||
{ key: "extract", label: "Extracción IA", icon: Cpu, desc: "Analizando con LLM..." },
|
||||
{ key: "normalize", label: "Normalización", icon: GitMerge, desc: "Estructurando datos..." },
|
||||
{ key: "csv", label: "Exportación", icon: FileSpreadsheet, desc: "Generando CSV..." },
|
||||
];
|
||||
|
||||
function getPhaseIndex(phase) {
|
||||
if (!phase) return 0;
|
||||
const idx = PHASES.findIndex(p => p.key === phase);
|
||||
return idx >= 0 ? idx : 0;
|
||||
}
|
||||
|
||||
// --- Componentes ---
|
||||
function Card({ children, className }) {
|
||||
return (
|
||||
<div className={cn("bg-white rounded-2xl border border-slate-200/60 shadow-sm", className)}>
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function KpiCard({ label, value, sub, icon: Icon, color }) {
|
||||
const colors = {
|
||||
blue: { bg: "bg-blue-50", text: "text-blue-600", glow: "shadow-blue-100" },
|
||||
emerald: { bg: "bg-emerald-50", text: "text-emerald-600", glow: "shadow-emerald-100" },
|
||||
amber: { bg: "bg-amber-50", text: "text-amber-600", glow: "shadow-amber-100" },
|
||||
violet: { bg: "bg-violet-50", text: "text-violet-600", glow: "shadow-violet-100" },
|
||||
};
|
||||
const c = colors[color] || colors.blue;
|
||||
return (
|
||||
<Card className="p-5">
|
||||
<div className="flex items-start justify-between mb-4">
|
||||
<div className={cn("p-2.5 rounded-xl", c.bg, "shadow-lg", c.glow)}>
|
||||
<Icon className={cn("w-5 h-5", c.text)} />
|
||||
</div>
|
||||
</div>
|
||||
<p className="text-2xl font-black text-slate-900 tracking-tight">{value}</p>
|
||||
<p className="text-xs font-semibold text-slate-500 mt-0.5">{label}</p>
|
||||
{sub && <p className="text-[11px] text-slate-400 mt-1">{sub}</p>}
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
function PipelineVisual({ batch }) {
|
||||
const phaseIdx = getPhaseIndex(batch?.current_phase);
|
||||
const progress = batch?.current_radicado_index || 0;
|
||||
const total = batch?.total_radicados || 1;
|
||||
const pct = Math.min((progress / total) * 100, 100);
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{/* Radicado actual */}
|
||||
{batch?.current_radicado && (
|
||||
<div className="flex items-center gap-3 bg-blue-50 border border-blue-100 rounded-xl px-4 py-2.5">
|
||||
<div className="w-2 h-2 rounded-full bg-blue-500 animate-pulse flex-shrink-0" />
|
||||
<div className="min-w-0">
|
||||
<p className="text-[10px] text-blue-500 font-bold uppercase tracking-wider">Procesando ahora</p>
|
||||
<p className="text-sm font-semibold text-blue-900 truncate">{batch.current_radicado}</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Pipeline steps */}
|
||||
<div className="relative">
|
||||
{/* Connecting line */}
|
||||
<div className="absolute top-6 left-6 right-6 h-0.5 bg-slate-100 z-0" />
|
||||
<div
|
||||
className="absolute top-6 left-6 h-0.5 bg-gradient-to-r from-blue-500 to-blue-400 z-0 transition-all duration-700"
|
||||
style={{ width: `${(phaseIdx / (PHASES.length - 1)) * (100 - 12)}%` }}
|
||||
/>
|
||||
|
||||
<div className="relative z-10 grid grid-cols-4 gap-2">
|
||||
{PHASES.map((phase, i) => {
|
||||
const done = i < phaseIdx;
|
||||
const active = i === phaseIdx && batch?.status === "processing";
|
||||
const pending = i > phaseIdx;
|
||||
const Icon = phase.icon;
|
||||
|
||||
return (
|
||||
<div key={phase.key} className="flex flex-col items-center gap-2">
|
||||
<div className={cn(
|
||||
"w-12 h-12 rounded-2xl flex items-center justify-center transition-all duration-500",
|
||||
done ? "bg-blue-600 shadow-lg shadow-blue-200"
|
||||
: active ? "bg-blue-600 shadow-xl shadow-blue-300 scale-110 ring-4 ring-blue-100"
|
||||
: "bg-slate-100"
|
||||
)}>
|
||||
{done ? (
|
||||
<CheckCircle2 className="w-5 h-5 text-white" />
|
||||
) : active ? (
|
||||
<RefreshCw className="w-5 h-5 text-white animate-spin" />
|
||||
) : (
|
||||
<Icon className="w-5 h-5 text-slate-400" />
|
||||
)}
|
||||
</div>
|
||||
<div className="text-center">
|
||||
<p className={cn(
|
||||
"text-[11px] font-bold transition-colors",
|
||||
active ? "text-blue-600" : done ? "text-slate-700" : "text-slate-400"
|
||||
)}>{phase.label}</p>
|
||||
{active && (
|
||||
<p className="text-[10px] text-slate-400 mt-0.5">{phase.desc}</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Progress bar */}
|
||||
<div className="space-y-2">
|
||||
<div className="flex justify-between items-center">
|
||||
<span className="text-sm font-bold text-slate-700">Progreso Global</span>
|
||||
<span className="text-sm font-black text-slate-900">{progress} / {total}</span>
|
||||
</div>
|
||||
<div className="h-3 bg-slate-100 rounded-full overflow-hidden">
|
||||
<div
|
||||
className="h-full bg-gradient-to-r from-blue-500 to-blue-400 rounded-full transition-all duration-700"
|
||||
style={{ width: `${pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex justify-between text-[10px] font-bold text-slate-400">
|
||||
<span>{Math.round(pct)}% completado</span>
|
||||
<span>{total - progress} restantes</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Live stats */}
|
||||
<div className="grid grid-cols-3 gap-3">
|
||||
{[
|
||||
{ label: "Exitosos", value: batch?.success_count || 0, color: "text-emerald-600 bg-emerald-50" },
|
||||
{ label: "Errores", value: batch?.error_count || 0, color: "text-red-500 bg-red-50" },
|
||||
{ label: "Tokens", value: (batch?.total_tokens || 0).toLocaleString(), color: "text-blue-600 bg-blue-50" },
|
||||
].map(s => (
|
||||
<div key={s.label} className={cn("rounded-xl p-3 text-center", s.color.split(" ")[1])}>
|
||||
<p className="text-lg font-black">{s.value}</p>
|
||||
<p className={cn("text-[10px] font-bold uppercase tracking-wide", s.color.split(" ")[0])}>{s.label}</p>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function EmptyProcess() {
|
||||
return (
|
||||
<div className="flex flex-col items-center justify-center py-16 text-center">
|
||||
<div className="w-20 h-20 bg-slate-100 rounded-3xl flex items-center justify-center mb-4">
|
||||
<FileSearch className="w-10 h-10 text-slate-300" />
|
||||
</div>
|
||||
<h3 className="text-lg font-bold text-slate-700 mb-1">Sin procesos activos</h3>
|
||||
<p className="text-sm text-slate-400 max-w-xs">Inicia un proceso desde la aplicación desktop para verlo aquí en tiempo real.</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function RecentBatchRow({ batch }) {
|
||||
const pct = Math.min(((batch.current_radicado_index || 0) / (batch.total_radicados || 1)) * 100, 100);
|
||||
return (
|
||||
<Link
|
||||
href="/processes"
|
||||
className="flex items-center gap-4 p-4 hover:bg-slate-50 rounded-xl transition-colors group"
|
||||
>
|
||||
<div className={cn(
|
||||
"w-9 h-9 rounded-xl flex items-center justify-center flex-shrink-0",
|
||||
batch.status === "completed" ? "bg-emerald-50" : "bg-red-50"
|
||||
)}>
|
||||
{batch.status === "completed"
|
||||
? <CheckCircle2 className="w-4 h-4 text-emerald-600" />
|
||||
: <AlertCircle className="w-4 h-4 text-red-500" />
|
||||
}
|
||||
</div>
|
||||
<div className="flex-1 min-w-0">
|
||||
<p className="text-sm font-semibold text-slate-800 truncate">{batch.root_path.split(/[\\/]/).pop()}</p>
|
||||
<div className="flex items-center gap-2 mt-0.5">
|
||||
<div className="flex-1 h-1 bg-slate-100 rounded-full overflow-hidden">
|
||||
<div className="h-full bg-blue-400 rounded-full" style={{ width: `${pct}%` }} />
|
||||
</div>
|
||||
<span className="text-[10px] font-bold text-slate-400 flex-shrink-0">{batch.success_count || 0} ok</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className="text-right flex-shrink-0">
|
||||
<p className="text-xs font-bold text-slate-900">${(batch.total_cost_usd || 0).toFixed(3)}</p>
|
||||
<p className="text-[10px] text-slate-400">{(batch.total_tokens || 0).toLocaleString()} tok.</p>
|
||||
</div>
|
||||
<ChevronRight className="w-4 h-4 text-slate-300 group-hover:text-blue-500 transition-colors" />
|
||||
</Link>
|
||||
);
|
||||
}
|
||||
|
||||
export default function HomePage() {
|
||||
const [batches, setBatches] = useState([]);
|
||||
const [wsStatus, setWsStatus] = useState("connecting");
|
||||
|
||||
const fetchBatches = useCallback(() => {
|
||||
fetch(`${API_URL}/batches`).then(r => r.json()).then(setBatches).catch(console.error);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
fetchBatches();
|
||||
const ws = new WebSocket("ws://localhost:8000/ws/tracking");
|
||||
ws.onopen = () => setWsStatus("connected");
|
||||
ws.onclose = () => { setWsStatus("disconnected"); setTimeout(fetchBatches, 5000); };
|
||||
ws.onmessage = (event) => {
|
||||
const { event: evt, data } = JSON.parse(event.data);
|
||||
if (evt === "batch_created") setBatches(prev => [data, ...prev]);
|
||||
else if (evt === "batch_updated") setBatches(prev => prev.map(b => b.id === data.id ? data : b));
|
||||
};
|
||||
return () => ws.close();
|
||||
}, [fetchBatches]);
|
||||
|
||||
const activeProcess = useMemo(() => batches.find(b => b.status === "processing"), [batches]);
|
||||
const recentDone = useMemo(() => batches.filter(b => b.status !== "processing").slice(0, 5), [batches]);
|
||||
const completed = useMemo(() => batches.filter(b => b.status === "completed"), [batches]);
|
||||
const stats = useMemo(() => ({
|
||||
total: batches.length,
|
||||
processed: completed.reduce((a, b) => a + (b.success_count || 0), 0),
|
||||
cost: completed.reduce((a, b) => a + (b.total_cost_usd || 0), 0),
|
||||
tokens: completed.reduce((a, b) => a + (b.total_tokens || 0), 0),
|
||||
}), [batches, completed]);
|
||||
|
||||
return (
|
||||
<div className="p-6 max-w-7xl mx-auto space-y-6">
|
||||
{/* Status pill */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h1 className="text-2xl font-black text-slate-900">Dashboard</h1>
|
||||
<p className="text-sm text-slate-400 mt-0.5">Monitoreo en tiempo real del procesamiento</p>
|
||||
</div>
|
||||
<div className={cn(
|
||||
"flex items-center gap-2 px-3 py-1.5 rounded-full text-[11px] font-bold",
|
||||
wsStatus === "connected" ? "bg-emerald-50 text-emerald-600" : "bg-slate-100 text-slate-500"
|
||||
)}>
|
||||
{wsStatus === "connected"
|
||||
? <Wifi className="w-3.5 h-3.5" />
|
||||
: <WifiOff className="w-3.5 h-3.5" />
|
||||
}
|
||||
{wsStatus === "connected" ? "En Vivo" : "Sin conexión"}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* KPIs */}
|
||||
<div className="grid grid-cols-2 lg:grid-cols-4 gap-4">
|
||||
<KpiCard label="Lotes totales" value={stats.total} icon={History} color="blue" />
|
||||
<KpiCard label="Radicados procesados" value={stats.processed.toLocaleString()} icon={Activity} color="emerald" sub={`en ${completed.length} lotes`} />
|
||||
<KpiCard label="Tokens consumidos" value={(stats.tokens / 1000000).toFixed(2) + "M"} icon={TrendingUp} color="amber" />
|
||||
<KpiCard label="Inversión total" value={`$${stats.cost.toFixed(2)}`} icon={DollarSign} color="violet" sub="USD acumulado" />
|
||||
</div>
|
||||
|
||||
{/* Main content: Process live + Recents */}
|
||||
<div className="grid grid-cols-1 lg:grid-cols-5 gap-6">
|
||||
{/* Live process */}
|
||||
<div className="lg:col-span-3">
|
||||
<Card className="h-full">
|
||||
<div className="p-5 border-b border-slate-100 flex items-center justify-between">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className={cn(
|
||||
"w-2.5 h-2.5 rounded-full",
|
||||
activeProcess ? "bg-blue-500 animate-pulse" : "bg-slate-300"
|
||||
)} />
|
||||
<h2 className="font-bold text-slate-800">
|
||||
{activeProcess ? "Proceso en Ejecución" : "Sin proceso activo"}
|
||||
</h2>
|
||||
</div>
|
||||
{activeProcess && (
|
||||
<div className="flex items-center gap-1.5 bg-blue-50 text-blue-600 text-[11px] font-bold px-2.5 py-1 rounded-full">
|
||||
<span className="w-1.5 h-1.5 bg-blue-500 rounded-full animate-ping" />
|
||||
EN VIVO
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="p-5">
|
||||
{activeProcess
|
||||
? <PipelineVisual batch={activeProcess} />
|
||||
: <EmptyProcess />
|
||||
}
|
||||
</div>
|
||||
{activeProcess && (
|
||||
<div className="px-5 pb-5">
|
||||
<div className="bg-slate-50 rounded-xl p-4 flex items-start gap-3">
|
||||
<FolderOpen className="w-4 h-4 text-slate-400 mt-0.5 flex-shrink-0" />
|
||||
<div className="min-w-0">
|
||||
<p className="text-[10px] font-bold text-slate-400 uppercase tracking-wider mb-1">Ruta de origen</p>
|
||||
<p className="text-xs font-mono text-slate-600 break-all">{activeProcess.root_path}</p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-3 mt-3">
|
||||
<div className="flex items-center gap-2 text-xs text-slate-500">
|
||||
<span className="font-semibold">Equipo:</span>
|
||||
<span className="font-mono text-slate-700">{activeProcess.hostname || "—"}</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2 text-xs text-slate-500">
|
||||
<span className="font-semibold">Usuario:</span>
|
||||
<span className="font-mono text-slate-700">{activeProcess.username || "—"}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
{/* Right column */}
|
||||
<div className="lg:col-span-2 space-y-4">
|
||||
{/* Cost live if active */}
|
||||
{activeProcess && (
|
||||
<Card className="p-5">
|
||||
<h3 className="text-xs font-bold text-slate-500 uppercase tracking-wider mb-4">Costo en Vivo</h3>
|
||||
<div className="text-center">
|
||||
<p className="text-4xl font-black text-slate-900 tracking-tight">
|
||||
${(activeProcess.total_cost_usd || 0).toFixed(4)}
|
||||
</p>
|
||||
<p className="text-sm text-slate-400 mt-1">USD hasta ahora</p>
|
||||
<div className="mt-4 pt-4 border-t border-slate-100 flex justify-around">
|
||||
<div className="text-center">
|
||||
<p className="text-lg font-black text-blue-600">{(activeProcess.total_tokens || 0).toLocaleString()}</p>
|
||||
<p className="text-[10px] font-bold text-slate-400 uppercase">Tokens</p>
|
||||
</div>
|
||||
<div className="text-center">
|
||||
<p className="text-lg font-black text-emerald-600">{activeProcess.success_count || 0}</p>
|
||||
<p className="text-[10px] font-bold text-slate-400 uppercase">Exitosos</p>
|
||||
</div>
|
||||
<div className="text-center">
|
||||
<p className="text-lg font-black text-red-500">{activeProcess.error_count || 0}</p>
|
||||
<p className="text-[10px] font-bold text-slate-400 uppercase">Errores</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{/* Recent */}
|
||||
<Card>
|
||||
<div className="p-5 border-b border-slate-100 flex items-center justify-between">
|
||||
<h3 className="font-bold text-slate-800">Recientes</h3>
|
||||
<Link href="/processes" className="text-[11px] font-bold text-blue-600 hover:text-blue-700 flex items-center gap-1">
|
||||
Ver todos <ArrowRight className="w-3.5 h-3.5" />
|
||||
</Link>
|
||||
</div>
|
||||
<div className="p-2">
|
||||
{recentDone.length > 0
|
||||
? recentDone.map(b => <RecentBatchRow key={b.id} batch={b} />)
|
||||
: (
|
||||
<p className="text-center text-sm text-slate-400 py-8">
|
||||
Aún no hay procesos completados.
|
||||
</p>
|
||||
)
|
||||
}
|
||||
</div>
|
||||
</Card>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
"use client";
|
||||
import { useEffect, useState, useCallback, useMemo } from "react";
|
||||
import {
|
||||
CheckCircle2, AlertCircle, RefreshCw, Download,
|
||||
Search, Clock, Monitor, User, FolderOpen,
|
||||
Copy, Check, ChevronDown, ChevronUp, List, TriangleAlert
|
||||
} from "lucide-react";
|
||||
import { clsx } from "clsx";
|
||||
import { twMerge } from "tailwind-merge";
|
||||
|
||||
function cn(...inputs) { return twMerge(clsx(inputs)); }
|
||||
|
||||
const API_URL = "http://localhost:8000";
|
||||
|
||||
function formatDate(iso) {
|
||||
if (!iso) return "-";
|
||||
return new Date(iso).toLocaleString("es-CO", {
|
||||
year: "numeric", month: "short", day: "numeric",
|
||||
hour: "2-digit", minute: "2-digit"
|
||||
});
|
||||
}
|
||||
|
||||
function StatusBadge({ status }) {
|
||||
const cfg = {
|
||||
processing: { label: "En Vivo", cls: "bg-blue-50 text-blue-600 border-blue-100", dot: "bg-blue-500 animate-pulse" },
|
||||
completed: { label: "Completado", cls: "bg-emerald-50 text-emerald-700 border-emerald-100", dot: "bg-emerald-500" },
|
||||
failed: { label: "Fallido", cls: "bg-red-50 text-red-700 border-red-100", dot: "bg-red-500" },
|
||||
};
|
||||
const c = cfg[status] || cfg.failed;
|
||||
return (
|
||||
<span className={cn("inline-flex items-center gap-1.5 px-2.5 py-0.5 rounded-full text-[11px] font-semibold border", c.cls)}>
|
||||
<span className={cn("w-1.5 h-1.5 rounded-full", c.dot)} />
|
||||
{c.label}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
function CopyButton({ text }) {
|
||||
const [copied, setCopied] = useState(false);
|
||||
const copy = () => {
|
||||
navigator.clipboard.writeText(text);
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 2000);
|
||||
};
|
||||
return (
|
||||
<button onClick={copy} className="p-1 text-slate-300 hover:text-blue-500 rounded transition-all" title="Copiar ruta">
|
||||
{copied ? <Check className="w-3.5 h-3.5 text-emerald-500" /> : <Copy className="w-3.5 h-3.5" />}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
const LOG_LEVEL_STYLE = {
|
||||
info: { cls: "text-slate-400", dot: "bg-slate-300", label: "INFO" },
|
||||
success: { cls: "text-emerald-600", dot: "bg-emerald-400", label: "OK" },
|
||||
error: { cls: "text-red-500", dot: "bg-red-500", label: "ERR" },
|
||||
warning: { cls: "text-amber-500", dot: "bg-amber-400", label: "WARN" },
|
||||
};
|
||||
|
||||
function LogPanel({ batchId, open }) {
|
||||
const [logs, setLogs] = useState(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!open || logs !== null) return;
|
||||
setLoading(true);
|
||||
fetch(`${API_URL}/batches/${batchId}/logs?limit=200`)
|
||||
.then(r => r.json())
|
||||
.then(data => { setLogs(data); setLoading(false); })
|
||||
.catch(() => { setLogs([]); setLoading(false); });
|
||||
}, [open, batchId, logs]);
|
||||
|
||||
if (!open) return null;
|
||||
|
||||
return (
|
||||
<div className="border-t border-slate-100 bg-[#0f1117] rounded-b-2xl overflow-hidden">
|
||||
{/* Terminal header */}
|
||||
<div className="flex items-center gap-2 px-4 py-2 border-b border-white/5">
|
||||
<span className="w-2.5 h-2.5 rounded-full bg-red-500/80" />
|
||||
<span className="w-2.5 h-2.5 rounded-full bg-amber-400/80" />
|
||||
<span className="w-2.5 h-2.5 rounded-full bg-emerald-500/80" />
|
||||
<span className="ml-2 text-[10px] font-mono text-slate-500 uppercase tracking-widest">Historial de logs</span>
|
||||
{logs && <span className="ml-auto text-[10px] font-mono text-slate-600">{logs.length} entradas</span>}
|
||||
</div>
|
||||
|
||||
<div className="max-h-72 overflow-y-auto p-3 space-y-0.5 font-mono text-[11px] scrollbar-thin scrollbar-thumb-white/10 scrollbar-track-transparent">
|
||||
{loading && (
|
||||
<p className="text-slate-500 py-6 text-center animate-pulse">Cargando logs…</p>
|
||||
)}
|
||||
{!loading && logs && logs.length === 0 && (
|
||||
<p className="text-slate-600 py-6 text-center">Sin registros para este proceso.</p>
|
||||
)}
|
||||
{!loading && logs && [...logs].reverse().map((log) => {
|
||||
const style = LOG_LEVEL_STYLE[log.level] || LOG_LEVEL_STYLE.info;
|
||||
const time = log.timestamp ? log.timestamp.split("T")[1]?.slice(0, 8) : "";
|
||||
return (
|
||||
<div key={log.id} className="flex items-start gap-2.5 py-0.5 hover:bg-white/[0.03] rounded px-1">
|
||||
<span className="text-slate-700 flex-shrink-0 tabular-nums">{time}</span>
|
||||
<span className={cn("flex-shrink-0 w-[30px] text-right font-bold text-[9px] uppercase", style.cls)}>{style.label}</span>
|
||||
{log.radicado && (
|
||||
<span className="flex-shrink-0 bg-white/5 text-slate-400 px-1.5 py-0 rounded text-[9px] leading-4 max-w-[120px] truncate">
|
||||
{log.radicado}
|
||||
</span>
|
||||
)}
|
||||
<span className={cn("flex-1 leading-relaxed", style.cls, log.level === "error" ? "font-semibold" : "")}>{log.message}</span>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function BatchCard({ batch }) {
|
||||
const progress = batch.current_radicado_index || 0;
|
||||
const total = batch.total_radicados || 1;
|
||||
const pct = Math.min((progress / total) * 100, 100);
|
||||
const isActive = batch.status === "processing";
|
||||
const [showLogs, setShowLogs] = useState(false);
|
||||
|
||||
return (
|
||||
<div className={cn(
|
||||
"bg-white rounded-2xl border shadow-sm overflow-hidden transition-all",
|
||||
isActive ? "border-blue-200 ring-1 ring-blue-100" : "border-slate-200/60"
|
||||
)}>
|
||||
{/* Header */}
|
||||
<div className="p-5 flex items-start gap-4">
|
||||
<div className={cn(
|
||||
"w-11 h-11 rounded-2xl flex items-center justify-center flex-shrink-0",
|
||||
batch.status === "completed" ? "bg-emerald-50" : batch.status === "failed" ? "bg-red-50" : "bg-blue-50"
|
||||
)}>
|
||||
{batch.status === "completed" ? <CheckCircle2 className="w-5 h-5 text-emerald-600" />
|
||||
: batch.status === "failed" ? <AlertCircle className="w-5 h-5 text-red-500" />
|
||||
: <RefreshCw className="w-5 h-5 text-blue-600 animate-spin" />}
|
||||
</div>
|
||||
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2 flex-wrap mb-1.5">
|
||||
<h3 className="text-base font-bold text-slate-900 truncate">
|
||||
{batch.root_path.split(/[\\/]/).pop()}
|
||||
</h3>
|
||||
<StatusBadge status={batch.status} />
|
||||
</div>
|
||||
|
||||
{/* Path */}
|
||||
<div className="flex items-center gap-1.5 mb-2 group">
|
||||
<FolderOpen className="w-3.5 h-3.5 text-slate-300 flex-shrink-0" />
|
||||
<p className="text-[11px] font-mono text-slate-400 truncate">{batch.root_path}</p>
|
||||
<CopyButton text={batch.root_path} />
|
||||
</div>
|
||||
|
||||
{/* Meta */}
|
||||
<div className="flex flex-wrap items-center gap-x-4 gap-y-1 text-[11px] text-slate-400 font-medium">
|
||||
<span className="flex items-center gap-1.5"><Clock className="w-3.5 h-3.5" /> {formatDate(batch.created_at)}</span>
|
||||
<span className="flex items-center gap-1.5"><Monitor className="w-3.5 h-3.5" /> {batch.hostname || "—"}</span>
|
||||
<span className="flex items-center gap-1.5"><User className="w-3.5 h-3.5" /> {batch.username || "—"}</span>
|
||||
<span className="font-mono text-slate-300 text-[10px]">#{batch.id.slice(0, 8)}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Error banner */}
|
||||
{batch.last_error && (
|
||||
<div className="mx-5 mb-4 flex items-start gap-2.5 bg-red-50 border border-red-100 rounded-xl px-3.5 py-3">
|
||||
<TriangleAlert className="w-4 h-4 text-red-500 flex-shrink-0 mt-0.5" />
|
||||
<div className="min-w-0">
|
||||
<p className="text-[11px] font-bold text-red-700 mb-0.5 uppercase tracking-wide">Último error</p>
|
||||
<p className="text-xs text-red-600 font-mono break-all leading-relaxed">{batch.last_error}</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Progress bar */}
|
||||
<div className="px-5 pb-4">
|
||||
<div className="flex justify-between text-[11px] font-bold text-slate-500 mb-1.5">
|
||||
<span>{isActive ? `Fase: ${batch.current_phase || "preparando"}` : "Completado"}</span>
|
||||
<span>{progress} / {total} radicados</span>
|
||||
</div>
|
||||
<div className="h-2 bg-slate-100 rounded-full overflow-hidden">
|
||||
<div
|
||||
className={cn("h-full rounded-full transition-all duration-700", batch.status === "failed" ? "bg-red-400" : "bg-gradient-to-r from-blue-500 to-blue-400")}
|
||||
style={{ width: `${pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Stats footer */}
|
||||
<div className="border-t border-slate-100 grid grid-cols-4 divide-x divide-slate-100">
|
||||
{[
|
||||
{ label: "Exitosos", value: batch.success_count || 0, color: "text-emerald-600" },
|
||||
{ label: "Errores", value: batch.error_count || 0, color: "text-red-500" },
|
||||
{ label: "Tokens", value: (batch.total_tokens || 0).toLocaleString(), color: "text-blue-600" },
|
||||
{ label: "Costo USD", value: `$${(batch.total_cost_usd || 0).toFixed(4)}`, color: "text-slate-700" },
|
||||
].map(s => (
|
||||
<div key={s.label} className="p-3 text-center">
|
||||
<p className={cn("text-sm font-black", s.color)}>{s.value}</p>
|
||||
<p className="text-[9px] font-bold text-slate-400 uppercase tracking-wider mt-0.5">{s.label}</p>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Actions bar */}
|
||||
<div className="border-t border-slate-100 px-5 py-3 flex items-center gap-3">
|
||||
{batch.status === "completed" && batch.csv_output_path && (
|
||||
<a
|
||||
href={`${API_URL}/download/${batch.id}`}
|
||||
className="inline-flex items-center gap-2 bg-emerald-600 hover:bg-emerald-700 text-white text-xs font-bold px-4 py-2 rounded-xl transition-colors shadow-sm shadow-emerald-200"
|
||||
>
|
||||
<Download className="w-3.5 h-3.5" /> Descargar CSV
|
||||
</a>
|
||||
)}
|
||||
<button
|
||||
onClick={() => setShowLogs(v => !v)}
|
||||
className={cn(
|
||||
"inline-flex items-center gap-2 text-xs font-bold px-4 py-2 rounded-xl border transition-all",
|
||||
showLogs
|
||||
? "bg-slate-900 text-white border-slate-700"
|
||||
: "bg-white text-slate-600 border-slate-200 hover:border-slate-300 hover:bg-slate-50"
|
||||
)}
|
||||
>
|
||||
<List className="w-3.5 h-3.5" />
|
||||
{showLogs ? "Ocultar logs" : "Ver logs"}
|
||||
{showLogs ? <ChevronUp className="w-3.5 h-3.5" /> : <ChevronDown className="w-3.5 h-3.5" />}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Log panel */}
|
||||
<LogPanel batchId={batch.id} open={showLogs} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function ProcessesPage() {
|
||||
const [batches, setBatches] = useState([]);
|
||||
const [search, setSearch] = useState("");
|
||||
const [filter, setFilter] = useState("all");
|
||||
|
||||
const fetchBatches = useCallback(() => {
|
||||
fetch(`${API_URL}/batches`).then(r => r.json()).then(setBatches).catch(console.error);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
fetchBatches();
|
||||
const ws = new WebSocket("ws://localhost:8000/ws/tracking");
|
||||
ws.onopen = fetchBatches;
|
||||
ws.onmessage = (event) => {
|
||||
const { event: evt, data } = JSON.parse(event.data);
|
||||
if (evt === "batch_created") setBatches(prev => [data, ...prev]);
|
||||
else if (evt === "batch_updated") setBatches(prev => prev.map(b => b.id === data.id ? data : b));
|
||||
};
|
||||
return () => ws.close();
|
||||
}, [fetchBatches]);
|
||||
|
||||
const filtered = useMemo(() => {
|
||||
return batches
|
||||
.filter(b => filter === "all" || b.status === filter)
|
||||
.filter(b => b.root_path.toLowerCase().includes(search.toLowerCase()));
|
||||
}, [batches, search, filter]);
|
||||
|
||||
const counts = useMemo(() => ({
|
||||
all: batches.length,
|
||||
processing: batches.filter(b => b.status === "processing").length,
|
||||
completed: batches.filter(b => b.status === "completed").length,
|
||||
failed: batches.filter(b => b.status === "failed").length,
|
||||
}), [batches]);
|
||||
|
||||
return (
|
||||
<div className="p-6 max-w-5xl mx-auto space-y-6">
|
||||
{/* Header */}
|
||||
<div>
|
||||
<h1 className="text-2xl font-black text-slate-900">Procesos</h1>
|
||||
<p className="text-sm text-slate-400 mt-0.5">Historial completo de lotes de extracción</p>
|
||||
</div>
|
||||
|
||||
{/* Filters */}
|
||||
<div className="flex flex-col sm:flex-row gap-3">
|
||||
<div className="relative flex-1">
|
||||
<Search className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-slate-400" />
|
||||
<input
|
||||
type="text"
|
||||
placeholder="Buscar por ruta o carpeta..."
|
||||
className="w-full pl-10 pr-4 py-2.5 bg-white border border-slate-200 rounded-xl text-sm focus:outline-none focus:ring-2 focus:ring-blue-500/20 focus:border-blue-500 transition-all"
|
||||
value={search}
|
||||
onChange={e => setSearch(e.target.value)}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
{[
|
||||
{ key: "all", label: `Todos (${counts.all})` },
|
||||
{ key: "processing", label: `En vivo (${counts.processing})` },
|
||||
{ key: "completed", label: `Completados (${counts.completed})` },
|
||||
{ key: "failed", label: `Fallidos (${counts.failed})` },
|
||||
].map(f => (
|
||||
<button
|
||||
key={f.key}
|
||||
onClick={() => setFilter(f.key)}
|
||||
className={cn(
|
||||
"px-3 py-2 rounded-xl text-xs font-bold transition-all whitespace-nowrap",
|
||||
filter === f.key ? "bg-blue-600 text-white shadow-lg shadow-blue-200" : "bg-white border border-slate-200 text-slate-600 hover:bg-slate-50"
|
||||
)}
|
||||
>
|
||||
{f.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* List */}
|
||||
<div className="space-y-4">
|
||||
{filtered.length > 0 ? (
|
||||
filtered.map(b => <BatchCard key={b.id} batch={b} />)
|
||||
) : (
|
||||
<div className="text-center py-20 bg-white rounded-2xl border border-slate-200/60">
|
||||
<Search className="w-10 h-10 text-slate-200 mx-auto mb-3" />
|
||||
<p className="text-slate-500 font-semibold">No se encontraron procesos</p>
|
||||
<p className="text-sm text-slate-400 mt-1">Prueba con otro filtro o búsqueda</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user