feat: Enhance CSV cleaning service with new endpoints for CSV analysis and preview functionality. Introduce support for job management linked to extraction batches, and update the cleaning rules structure. Add new utility functions for handling CSV data and improve documentation for clarity on usage and available endpoints.

This commit is contained in:
erik
2026-07-11 00:27:05 -05:00
parent cf0ebc72ba
commit b770761832
73 changed files with 7614 additions and 2671 deletions
+1
View File
@@ -140,6 +140,7 @@ celerybeat.pid
# Environments
.env
.env.minio
.venv
env/
venv/
+7 -3
View File
@@ -167,7 +167,9 @@ Table extraction_rows {
Table cleaning_jobs {
id varchar(36) [pk, note: 'UUID v4']
extraction_batch_id varchar(36) [not null, ref: > extraction_batches.id]
extraction_batch_id varchar(36) [null, ref: > extraction_batches.id, note: 'Null si source=csv']
job_id varchar(36) [null, ref: > jobs.id, note: 'Trabajo Desktop al que pertenece']
source varchar(32) [not null, default: 'extraction', note: 'extraction | csv']
status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed']
request jsonb [not null, default: '{}', note: 'CleaningRequest (rules_per_column, etc.)']
log jsonb [not null, default: '[]', note: 'Lista de mensajes de auditoría del job']
@@ -177,16 +179,18 @@ Table cleaning_jobs {
indexes {
extraction_batch_id
job_id
source
status
}
Note: 'Ejecución de reglas de limpieza sobre las filas de un lote de extracción.'
Note: 'Ejecución de reglas de limpieza sobre filas de extracción o CSV subido.'
}
Table cleaned_rows {
id integer [pk, increment]
cleaning_job_id varchar(36) [not null, ref: > cleaning_jobs.id]
extraction_batch_id varchar(36) [not null, ref: > extraction_batches.id]
extraction_batch_id varchar(36) [null, ref: > extraction_batches.id]
row_index integer [not null, default: 0, note: 'Conserva el orden del lote original']
row_json jsonb [not null, default: '{}', note: 'Fila tras aplicar reglas de limpieza']
created_at timestamptz [not null]
+7 -1
View File
@@ -66,7 +66,13 @@ def _client() -> httpx.AsyncClient:
def _is_upload_path(path: str) -> bool:
return path.startswith("/count/") or "/ingest" in path or "/upload" in path
return (
path.startswith("/count/")
or "/ingest" in path
or "/upload" in path
or path.startswith("/cleaning/analyze-csv")
or path.startswith("/cleaning/jobs/from-csv")
)
def _timeout_for_path(path: str) -> httpx.Timeout:
+12 -5
View File
@@ -18,7 +18,11 @@ Recibe lotes de filas extraídas por el servicio `extraction` y aplica reglas co
| `GET` | `/health` | Liveness |
| `GET` | `/rules` | Lista de reglas de limpieza disponibles |
| `GET` | `/batches/{batch_id}/analysis` | Análisis de columnas de un lote de extracción |
| `POST` | `/jobs` | Crear job de limpieza |
| `POST` | `/analyze-csv` | Análisis de columnas desde CSV (multipart) |
| `POST` | `/preview` | Vista previa Antes/Después sobre muestra (sin persistir) |
| `POST` | `/jobs` | Limpieza desde `extraction_batch_id` (+ `job_id` opcional) |
| `POST` | `/jobs/from-csv` | Limpieza desde CSV (multipart + reglas JSON) |
| `GET` | `/jobs?job_id=` | Listar jobs (filtro opcional por trabajo Desktop) |
| `GET` | `/jobs/{job_id}` | Estado y detalle del job |
| `GET` | `/jobs/{job_id}/rows` | Filas limpias del job |
@@ -44,7 +48,7 @@ Las reglas disponibles se listan en `GET /rules`.
## Reglas de limpieza
Definidas en `domain/csv_cleaning.py`. Cada regla es una función que transforma los valores de una columna. Se pueden encadenar múltiples reglas por columna.
Definidas en el paquete `domain/csv_cleaning/` (registro en `rules/builtins.py`). Cada regla transforma los valores de una columna; se pueden encadenar varias por columna. Para agregar una regla nueva, ver `rules/registry.py`.
---
@@ -72,11 +76,14 @@ cleaning_jobs — job de limpieza (FK → extraction_batches)
```
api/
main.py — FastAPI app con todos los endpoints
main.py — FastAPI app con todos los endpoints
domain/
csv_cleaning.py — reglas de limpieza y función de análisis
csv_cleaning/ — paquete de limpieza (análisis + reglas)
analyze.py — resumen por columna
apply.py — aplicación de reglas
rules/ — registro e implementaciones de reglas
infrastructure/
database.py — modelos SQLAlchemy y sesión
database.py — modelos SQLAlchemy y sesión
Dockerfile
requirements.txt
```
+363 -25
View File
@@ -6,16 +6,24 @@ import json
from typing import Any
import pandas as pd
from fastapi import FastAPI, HTTPException
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
from pydantic import BaseModel, Field
from sqlalchemy import delete, select
from domain.csv_cleaning import CLEANING_RULES, analyze_dataframe, apply_rules
from domain.csv_cleaning import (
CLEANING_RULES,
analyze_dataframe,
apply_rules,
load_csv_bytes,
sample_dataframe_rows,
)
from infrastructure.database import (
CleanedRow,
CleaningJob,
ExtractionBatch,
ExtractionRow,
Job,
Task,
init_db,
session_scope,
utcnow,
@@ -36,12 +44,22 @@ app = FastAPI(
class CleaningRequest(BaseModel):
extraction_batch_id: str
extraction_batch_id: str | None = None
job_id: str | None = None
rules_per_column: dict[str, list[str]] = Field(default_factory=dict)
cop_mod_fecha_column: dict[str, str] = Field(default_factory=dict)
cop_mod_config: dict[str, Any] | None = None
class PreviewRequest(BaseModel):
rows: list[dict[str, Any]] = Field(default_factory=list)
extraction_batch_id: str | None = None
rules_per_column: dict[str, list[str]] = Field(default_factory=dict)
cop_mod_fecha_column: dict[str, str] = Field(default_factory=dict)
cop_mod_config: dict[str, Any] | None = None
sample_size: int = 8
def dataframe_to_records(df: pd.DataFrame) -> list[dict[str, Any]]:
return json.loads(df.to_json(orient="records", date_format="iso"))
@@ -50,6 +68,8 @@ def serialize_job(job: CleaningJob) -> dict[str, Any]:
return {
"id": job.id,
"extraction_batch_id": job.extraction_batch_id,
"job_id": job.job_id,
"source": job.source,
"status": job.status,
"request": job.request,
"log": job.log,
@@ -59,6 +79,144 @@ def serialize_job(job: CleaningJob) -> dict[str, Any]:
}
def _validate_job_id(session: Any, job_id: str | None) -> None:
if not job_id:
return
if session.get(Job, job_id) is None:
raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado")
def _create_cleaning_task(
session: Any,
*,
job_id: str | None,
cleaning_job_id: str,
source: str,
status: str,
request: dict[str, Any],
error: str | None = None,
title: str | None = None,
) -> str | None:
"""Crea Task(type=cleaning) vinculada al job Desktop. Devuelve task.id o None."""
if not job_id:
return None
now = utcnow()
task = Task(
job_id=job_id,
type="cleaning",
status=status,
title=title or f"Limpieza ({source})",
request=request,
result={"cleaning_job_id": cleaning_job_id},
error=error,
started_at=now,
finished_at=now,
updated_at=now,
)
session.add(task)
session.flush()
return task.id
def _persist_cleaned_rows(
session: Any,
*,
job_id: str,
extraction_batch_id: str | None,
output_rows: list[dict[str, Any]],
) -> None:
session.execute(delete(CleanedRow).where(CleanedRow.cleaning_job_id == job_id))
for idx, row in enumerate(output_rows, start=1):
session.add(
CleanedRow(
cleaning_job_id=job_id,
extraction_batch_id=extraction_batch_id,
row_index=idx,
row_json=row,
)
)
def _run_rules_on_dataframe(
df: pd.DataFrame,
rules_per_column: dict[str, list[str]],
cop_mod_fecha_column: dict[str, str],
cop_mod_config: dict[str, Any] | None,
) -> tuple[list[dict[str, Any]], list[str]]:
result_df, log = apply_rules(
df,
rules_per_column,
cop_mod_fecha_column=cop_mod_fecha_column,
cop_mod_config=cop_mod_config,
)
return dataframe_to_records(result_df), log
def _parse_json_form(raw: str | None, default: Any) -> Any:
if not raw or not raw.strip():
return default
try:
return json.loads(raw)
except json.JSONDecodeError as exc:
raise HTTPException(status_code=400, detail=f"JSON inválido: {exc}") from exc
def _analysis_payload(df: pd.DataFrame, **extra: Any) -> dict[str, Any]:
return {
**extra,
"row_count": len(df),
"columns": [summary.__dict__ for summary in analyze_dataframe(df)],
"sample_rows": sample_dataframe_rows(df, 8),
}
def _cell_str(value: Any) -> str:
if value is None:
return ""
text = str(value)
if text in ("None", "nan", "NaT"):
return ""
if len(text) > 80:
return text[:77] + "..."
return text
def _preview_from_dataframe(
df: pd.DataFrame,
rules_per_column: dict[str, list[str]],
cop_mod_fecha_column: dict[str, str],
cop_mod_config: dict[str, Any] | None,
) -> dict[str, Any]:
rows_before = dataframe_to_records(df)
result_df, _log = apply_rules(
df,
rules_per_column,
cop_mod_fecha_column=cop_mod_fecha_column,
cop_mod_config=cop_mod_config,
)
rows_after = dataframe_to_records(result_df)
# Emparejar por índice de fila de muestra (misma posición)
n = min(len(rows_before), len(rows_after))
all_cols = list(dict.fromkeys([*df.columns.astype(str), *result_df.columns.astype(str)]))
columns_preview: list[dict[str, Any]] = []
for col in all_cols:
before_vals = [_cell_str(rows_before[i].get(col)) for i in range(n)]
after_vals = [_cell_str(rows_after[i].get(col)) for i in range(n)]
columns_preview.append({
"name": col,
"before": before_vals,
"after": after_vals,
"changed": before_vals != after_vals,
})
return {
"rows_before": rows_before,
"rows_after": rows_after,
"columns": columns_preview,
}
@app.get("/health")
def health() -> dict[str, str]:
return {"status": "ok"}
@@ -90,18 +248,75 @@ def analyze_batch(batch_id: str) -> dict[str, Any]:
records = [row[0] for row in rows]
df = pd.DataFrame(records)
return {
"batch_id": batch_id,
"columns": [summary.__dict__ for summary in analyze_dataframe(df)],
}
return _analysis_payload(df, batch_id=batch_id)
@app.post("/analyze-csv")
async def analyze_csv(file: UploadFile = File(...)) -> dict[str, Any]:
content = await file.read()
if not content:
raise HTTPException(status_code=400, detail="El archivo CSV está vacío")
try:
df, delimiter = load_csv_bytes(content, file.filename or "upload.csv")
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=400, detail=f"No se pudo leer el CSV: {exc}") from exc
if df.empty:
raise HTTPException(status_code=400, detail="El CSV no tiene filas")
return _analysis_payload(df, filename=file.filename, delimiter=delimiter)
@app.post("/preview")
def preview_cleaning(request: PreviewRequest) -> dict[str, Any]:
"""Aplica reglas sobre una muestra pequeña y devuelve Antes/Después."""
records: list[dict[str, Any]] = list(request.rows or [])
if not records and request.extraction_batch_id:
with session_scope() as session:
batch = session.get(ExtractionBatch, request.extraction_batch_id)
if batch is None:
raise HTTPException(status_code=404, detail="Lote de extraccion no encontrado")
rows = session.execute(
select(ExtractionRow.row_json)
.where(ExtractionRow.batch_id == request.extraction_batch_id)
.order_by(ExtractionRow.row_index, ExtractionRow.id)
).all()
all_records = [row[0] for row in rows]
if not all_records:
raise HTTPException(status_code=400, detail="El lote no tiene filas")
df_full = pd.DataFrame(all_records)
records = sample_dataframe_rows(df_full, max(1, min(request.sample_size, 20)))
if not records:
raise HTTPException(
status_code=400,
detail="Se requieren rows o extraction_batch_id con filas",
)
df = pd.DataFrame(records)
try:
return _preview_from_dataframe(
df,
request.rules_per_column,
request.cop_mod_fecha_column,
request.cop_mod_config,
)
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=500, detail=str(exc)) from exc
@app.post("/jobs")
def run_cleaning(request: CleaningRequest) -> dict[str, Any]:
if not request.extraction_batch_id:
raise HTTPException(
status_code=400,
detail="extraction_batch_id es requerido (use /jobs/from-csv para CSV)",
)
with session_scope() as session:
batch = session.get(ExtractionBatch, request.extraction_batch_id)
if batch is None:
raise HTTPException(status_code=404, detail="Lote de extraccion no encontrado")
_validate_job_id(session, request.job_id)
extracted_rows = session.execute(
select(ExtractionRow.row_json)
@@ -114,6 +329,8 @@ def run_cleaning(request: CleaningRequest) -> dict[str, Any]:
job = CleaningJob(
extraction_batch_id=request.extraction_batch_id,
job_id=request.job_id,
source="extraction",
status="running",
request=request.model_dump(),
updated_at=utcnow(),
@@ -121,36 +338,45 @@ def run_cleaning(request: CleaningRequest) -> dict[str, Any]:
session.add(job)
session.flush()
job_id = job.id
batch_id = request.extraction_batch_id
desktop_job_id = request.job_id
request_dump = request.model_dump()
try:
df = pd.DataFrame(records)
result_df, log = apply_rules(
output_rows, log = _run_rules_on_dataframe(
df,
request.rules_per_column,
cop_mod_fecha_column=request.cop_mod_fecha_column,
cop_mod_config=request.cop_mod_config,
request.cop_mod_fecha_column,
request.cop_mod_config,
)
output_rows = dataframe_to_records(result_df)
with session_scope() as session:
session.execute(delete(CleanedRow).where(CleanedRow.cleaning_job_id == job_id))
for idx, row in enumerate(output_rows, start=1):
session.add(
CleanedRow(
cleaning_job_id=job_id,
extraction_batch_id=request.extraction_batch_id,
row_index=idx,
row_json=row,
)
)
_persist_cleaned_rows(
session,
job_id=job_id,
extraction_batch_id=batch_id,
output_rows=output_rows,
)
job = session.get(CleaningJob, job_id)
if job is None:
raise RuntimeError("No se encontro el job de limpieza recien creado.")
job.status = "completed"
job.log = log
job.updated_at = utcnow()
_create_cleaning_task(
session,
job_id=desktop_job_id,
cleaning_job_id=job_id,
source="extraction",
status="completed",
request=request_dump,
title="Limpieza desde extracción",
)
session.flush()
return serialize_job(job)
except HTTPException:
raise
except Exception as exc:
with session_scope() as session:
job = session.get(CleaningJob, job_id)
@@ -158,15 +384,127 @@ def run_cleaning(request: CleaningRequest) -> dict[str, Any]:
job.status = "failed"
job.error = str(exc)
job.updated_at = utcnow()
_create_cleaning_task(
session,
job_id=desktop_job_id,
cleaning_job_id=job_id,
source="extraction",
status="failed",
request=request_dump,
error=str(exc),
title="Limpieza desde extracción",
)
raise HTTPException(status_code=500, detail=str(exc)) from exc
@app.post("/jobs/from-csv")
async def run_cleaning_from_csv(
file: UploadFile = File(...),
job_id: str | None = Form(None),
rules_per_column: str | None = Form(None),
cop_mod_fecha_column: str | None = Form(None),
cop_mod_config: str | None = Form(None),
) -> dict[str, Any]:
content = await file.read()
if not content:
raise HTTPException(status_code=400, detail="El archivo CSV está vacío")
rules = _parse_json_form(rules_per_column, {})
fecha_map = _parse_json_form(cop_mod_fecha_column, {})
cop_cfg = _parse_json_form(cop_mod_config, None)
if not isinstance(rules, dict):
raise HTTPException(status_code=400, detail="rules_per_column debe ser un objeto JSON")
if not isinstance(fecha_map, dict):
raise HTTPException(status_code=400, detail="cop_mod_fecha_column debe ser un objeto JSON")
try:
df, delimiter = load_csv_bytes(content, file.filename or "upload.csv")
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=400, detail=f"No se pudo leer el CSV: {exc}") from exc
if df.empty:
raise HTTPException(status_code=400, detail="El CSV no tiene filas")
request_dump: dict[str, Any] = {
"source": "csv",
"filename": file.filename,
"delimiter": delimiter,
"job_id": job_id,
"rules_per_column": rules,
"cop_mod_fecha_column": fecha_map,
"cop_mod_config": cop_cfg,
}
with session_scope() as session:
_validate_job_id(session, job_id)
job = CleaningJob(
extraction_batch_id=None,
job_id=job_id,
source="csv",
status="running",
request=request_dump,
updated_at=utcnow(),
)
session.add(job)
session.flush()
cleaning_job_id = job.id
desktop_job_id = job_id
try:
output_rows, log = _run_rules_on_dataframe(df, rules, fecha_map, cop_cfg)
with session_scope() as session:
_persist_cleaned_rows(
session,
job_id=cleaning_job_id,
extraction_batch_id=None,
output_rows=output_rows,
)
job = session.get(CleaningJob, cleaning_job_id)
if job is None:
raise RuntimeError("No se encontro el job de limpieza recien creado.")
job.status = "completed"
job.log = log
job.updated_at = utcnow()
_create_cleaning_task(
session,
job_id=desktop_job_id,
cleaning_job_id=cleaning_job_id,
source="csv",
status="completed",
request=request_dump,
title=f"Limpieza CSV — {file.filename or 'upload.csv'}",
)
session.flush()
return serialize_job(job)
except HTTPException:
raise
except Exception as exc:
with session_scope() as session:
job = session.get(CleaningJob, cleaning_job_id)
if job is not None:
job.status = "failed"
job.error = str(exc)
job.updated_at = utcnow()
_create_cleaning_task(
session,
job_id=desktop_job_id,
cleaning_job_id=cleaning_job_id,
source="csv",
status="failed",
request=request_dump,
error=str(exc),
title=f"Limpieza CSV — {file.filename or 'upload.csv'}",
)
raise HTTPException(status_code=500, detail=str(exc)) from exc
@app.get("/jobs")
def list_jobs() -> list[dict[str, Any]]:
def list_jobs(job_id: str | None = None) -> list[dict[str, Any]]:
with session_scope() as session:
jobs = session.scalars(
select(CleaningJob).order_by(CleaningJob.created_at.desc())
).all()
stmt = select(CleaningJob).order_by(CleaningJob.created_at.desc())
if job_id:
stmt = stmt.where(CleaningJob.job_id == job_id)
jobs = session.scalars(stmt).all()
return [serialize_job(job) for job in jobs]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,64 @@
"""Limpieza de CSV con análisis inteligente local y reglas configurables.
API pública (estable):
- ``analyze_dataframe`` — resumen por columna
- ``CLEANING_RULES`` / ``register_rule`` / ``get_rule`` — catálogo de reglas
- ``apply_rules`` — aplica reglas por columna
- ``load_csv`` / ``load_csv_bytes`` / ``detect_delimiter`` — I/O
Para agregar una regla nueva, ver ``rules/registry.py`` y ``rules/builtins.py``.
"""
from __future__ import annotations
from .analyze import analyze_dataframe, sample_dataframe_rows
from .apply import apply_rules
from .cop_mod import (
DEFAULT_MODERADORA_BY_YEAR,
merge_moderadora_config,
normalize_moderadora_by_year_dict,
)
from .documents import homologate_doc_type, split_doc_tipo_y_numero_texto
from .io import detect_delimiter, load_csv, load_csv_bytes
from .models import (
TYPE_BOOLEAN,
TYPE_CATEGORICAL,
TYPE_DATE,
TYPE_DECIMAL,
TYPE_EMPTY,
TYPE_INTEGER,
TYPE_MIXED,
TYPE_TEXT,
ColumnSummary,
)
from .notas import extract_autorizacion_nota, extract_paciente_documento_nota
from .rules import CLEANING_RULES, get_rule, register_rule
from .type_inference import applicable_rule_ids_for_type
__all__ = [
"CLEANING_RULES",
"ColumnSummary",
"DEFAULT_MODERADORA_BY_YEAR",
"TYPE_BOOLEAN",
"TYPE_CATEGORICAL",
"TYPE_DATE",
"TYPE_DECIMAL",
"TYPE_EMPTY",
"TYPE_INTEGER",
"TYPE_MIXED",
"TYPE_TEXT",
"analyze_dataframe",
"applicable_rule_ids_for_type",
"apply_rules",
"detect_delimiter",
"extract_autorizacion_nota",
"extract_paciente_documento_nota",
"get_rule",
"homologate_doc_type",
"load_csv",
"load_csv_bytes",
"merge_moderadora_config",
"normalize_moderadora_by_year_dict",
"register_rule",
"sample_dataframe_rows",
"split_doc_tipo_y_numero_texto",
]
@@ -0,0 +1,94 @@
"""Análisis de DataFrames: resumen por columna y muestras."""
from __future__ import annotations
import json
import random
from typing import Any
import pandas as pd
from .models import (
TYPE_CATEGORICAL,
TYPE_DATE,
TYPE_DECIMAL,
TYPE_INTEGER,
TYPE_TEXT,
ColumnSummary,
)
from .numbers import _to_numeric_lenient
from .type_inference import _applicable_rules_for_type, _infer_column_type
def analyze_dataframe(df: pd.DataFrame) -> list[ColumnSummary]:
"""Genera un resumen por columna del DataFrame."""
summaries: list[ColumnSummary] = []
total_rows = len(df)
rng = random.Random(42)
for col in df.columns:
series = df[col]
null_count = int(series.isna().sum() + (series.astype(str).str.strip() == "").sum())
# Evitar doble conteo: considerar vacío solo si no-NaN
null_count = int((series.isna() | (series.astype(str).str.strip() == "")).sum())
null_pct = (null_count / total_rows * 100) if total_rows else 0.0
non_null = series.dropna().astype(str)
non_null = non_null[non_null.str.strip() != ""]
example = ""
samples: list[str] = []
if not non_null.empty:
values = non_null.tolist()
k = min(5, len(values))
picked = rng.sample(values, k) if len(values) >= k else list(values)
for raw in picked:
text = str(raw)
if len(text) > 80:
text = text[:77] + "..."
samples.append(text)
example = samples[0] if samples else ""
unique_count = int(non_null.nunique())
unique_pct = (unique_count / total_rows * 100) if total_rows else 0.0
inferred_type, date_fmt = _infer_column_type(series, str(col))
summary = ColumnSummary(
name=str(col),
inferred_type=inferred_type,
example=example,
samples=samples,
null_count=null_count,
null_pct=null_pct,
unique_count=unique_count,
unique_pct=unique_pct,
detected_date_format=date_fmt,
)
if inferred_type in (TYPE_INTEGER, TYPE_DECIMAL):
numeric = _to_numeric_lenient(non_null)
if numeric.notna().any():
summary.min = float(numeric.min())
summary.max = float(numeric.max())
summary.mean = float(numeric.mean())
elif inferred_type in (TYPE_TEXT, TYPE_CATEGORICAL):
if not non_null.empty:
summary.avg_length = float(non_null.str.len().mean())
counts = non_null.value_counts().head(3)
summary.top_values = [(str(v), int(c)) for v, c in counts.items()]
summary.applicable_rules = _applicable_rules_for_type(inferred_type)
summaries.append(summary)
return summaries
def sample_dataframe_rows(df: pd.DataFrame, n: int = 8) -> list[dict[str, Any]]:
"""Devuelve hasta `n` filas aleatorias como dicts JSON-serializables."""
if df.empty or n <= 0:
return []
take = min(n, len(df))
rng = random.Random(42)
indices = rng.sample(list(range(len(df))), take)
subset = df.iloc[indices]
return json.loads(subset.to_json(orient="records", date_format="iso"))
@@ -0,0 +1,105 @@
"""Aplicación de reglas seleccionadas sobre un DataFrame."""
from __future__ import annotations
import logging
from typing import Any, Optional
import pandas as pd
from .cop_mod import _apply_cop_mod_split, merge_moderadora_config
from .documents import _apply_doc_id_split_columns
from .notas import _apply_nota_extract_columns
from .rules import get_rule_fn, get_rule_name
logger = logging.getLogger(__name__)
def apply_rules(
df: pd.DataFrame,
rules_per_column: dict[str, list[str]],
*,
cop_mod_fecha_column: Optional[dict[str, str]] = None,
cop_mod_config: Optional[dict[str, Any]] = None,
) -> tuple[pd.DataFrame, list[str]]:
"""Aplica las reglas seleccionadas a cada columna.
`rules_per_column` es un dict de `columna -> lista de IDs de reglas en el
orden a aplicar`. La regla `cop_mod_split` se difiere a una segunda pasada
tras el resto, para que la columna de fecha pueda normalizarse antes.
`cop_mod_fecha_column`: mapa columna de importe -> nombre de columna Fecha Servicio.
`cop_mod_config`: opcional; puede incluir ``moderadora_by_year`` (dict año → lista).
"""
result = df.copy()
log: list[str] = []
moderadora_years = merge_moderadora_config(cop_mod_config)
def _run_rules_for_column(col: str, rule_ids: list[str]) -> None:
if not rule_ids:
return
if col not in result.columns:
log.append(f"[WARN] Columna no encontrada: {col}")
return
split_has_homologation = (
"doc_id_split_tipo_numero" in rule_ids and "doc_type_homologate" in rule_ids
)
for rule_id in rule_ids:
if rule_id == "doc_id_split_tipo_numero":
try:
_apply_doc_id_split_columns(
result, col, log, homologate_type=split_has_homologation
)
except Exception as exc: # noqa: BLE001
logger.exception("Error aplicando regla %s a %s", rule_id, col)
log.append(f"[ERROR] '{col}' regla '{rule_id}': {exc}")
continue
if rule_id == "nota_paciente_autorizacion":
try:
_apply_nota_extract_columns(result, col, log)
except Exception as exc: # noqa: BLE001
logger.exception("Error aplicando regla %s a %s", rule_id, col)
log.append(f"[ERROR] '{col}' regla '{rule_id}': {exc}")
continue
if rule_id == "doc_type_homologate" and split_has_homologation:
continue
fn = get_rule_fn(rule_id)
if fn is None:
log.append(f"[WARN] Regla desconocida '{rule_id}' en columna '{col}'")
continue
try:
result[col] = fn(result[col])
log.append(f"[OK] '{col}' <- {get_rule_name(rule_id)}")
except Exception as exc: # noqa: BLE001
logger.exception("Error aplicando regla %s a %s", rule_id, col)
log.append(f"[ERROR] '{col}' regla '{rule_id}': {exc}")
# Fase 1: todas las reglas excepto cop_mod_split
for col, rule_ids in rules_per_column.items():
filtered = [r for r in rule_ids if r != "cop_mod_split"]
_run_rules_for_column(col, filtered)
# Fase 2: Separa COP/MOD sobre el DataFrame ya transformado
fecha_map = cop_mod_fecha_column or {}
for col, rule_ids in rules_per_column.items():
if "cop_mod_split" not in rule_ids:
continue
fecha_col = fecha_map.get(col)
if not fecha_col:
log.append(
f"[WARN] COP/MOD en '{col}': indique la columna «Fecha Servicio» en la tarjeta; se omite."
)
continue
try:
_apply_cop_mod_split(
result,
col,
fecha_col,
log,
moderadora_by_year=moderadora_years,
)
except Exception as exc: # noqa: BLE001
logger.exception("Error aplicando COP/MOD a %s", col)
log.append(f"[ERROR] '{col}' regla 'cop_mod_split': {exc}")
return result, log
@@ -0,0 +1,210 @@
"""Heurísticas por nombre de columna y presentación del contenido."""
from __future__ import annotations
import re
from typing import Optional
import pandas as pd
from .nullish import _is_nullish_scalar
def _column_name_suggests_date_like(name: str) -> bool:
"""Columnas cuyo nombre pide tratar el contenido como fecha a priori."""
n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip()
if not n:
return False
if re.search(
r"\b(fecha|date|año|mes|dia|día|f[_-]?(alta|baja|proc|corte|pago|ini|fin|"
r"desde|hasta|vigenc|vence|venc|caduc|f\s?ec)\b|periodo|anio|ano|expir)",
n,
) or n.startswith("fec_") or n.startswith("fecha"):
return True
return False
def _column_name_suggests_monetary_value(name: str) -> bool:
"""importe, valor, neto, IVA, etc. — no interpretar cifras como fechas.
Encabezados de tipo `fecha_…` no se tratan como moneda salvo mezcla con monto/valor.
"""
n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip()
if not n:
return False
if _column_name_suggests_date_like(name) and not re.search(
r"\b(monto|valor|importe|pago|total|bruto|neto|import|unit|precio|copago)\b", n
):
return False
for key in (
"monto", "importe", "import", "bruto", "neto", "subtotal", "sub total",
"total", "unitario", "unidad", "valor", "precio", "pago", "copago", "sald", "dcto",
"franq", "cuota", "iva", "tarif", "costo", "carg", "reten", "retencion",
):
if key in n:
if key == "iva" and re.search(r"(priva|giva|diva|viva|cliva)\b", n):
continue
if key == "carg" and re.search(r"descarg|recarg|abord", n):
continue
return True
return False
def _column_name_suggests_code_id(name: str) -> bool:
"""Código, ref., radicado, tipoynúm de documento — no forzar 6-8 cifras como fechas (DDMMYY, etc.)."""
n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip()
if not n or _column_name_suggests_date_like(name):
return False
for key in (
"codigo", "código", "codig", "códig", "cód_",
"sku", "ean", "upc", "isbn", "upn",
"ref", "referen", "radic", "nro_fac", "nro fac", "nit", "cédul", "cedul",
"autoriz", "correlat", "secuenc", "línea de", "linea de", "cupo",
"tipy", "tipi", "tipynum", "typnum", "type_n", "n_doc", "n_docu",
"núm doc", "n_docu", "doc_id", "docum", "id_doc",
):
if key in n:
return True
if re.search(
r"tipo[_\s\\-]*(&|/|u|e|y)[_\s\\-]*num|tipo[_\s\\-]+doc|tipo_y_n",
n,
re.IGNORECASE,
) or n.startswith("tip_"):
return True
if n.startswith("cod_") or n.startswith("cód_") or n.startswith("ref_"):
return True
return bool(re.search(r"\b(nro[._]fac|#?\s*autor|#?\s*ref)\b", n, re.IGNORECASE))
def _column_name_suggests_quantity_or_number_not_bool(name: str) -> bool:
"""Nombres típicos de cantidad, importe, fechas: no inferir bool por 0/1."""
n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip()
if not n:
return False
if _column_name_suggests_monetary_value(name) or _column_name_suggests_code_id(name):
return True
for key in (
"fecha",
"monto",
"importe",
"bruto",
"neto",
"subtotal",
"unitario",
"unidad",
"valor",
"precio",
"pago",
"copago",
"cantid",
"quantity",
"qty",
"nro",
"núm",
"n°",
):
if key in n:
return True
return False
def _raw_strings_suggest_monetary_presentation(non_null: pd.Series) -> bool:
"""$ €, miles 1,234,12 o .00: presentación monetaria aunque el float sea "entero"."""
if non_null is None or len(non_null) == 0:
return False
n_hit = 0
n_seen = 0
for raw in non_null.head(500):
if _is_nullish_scalar(raw):
continue
t = str(raw)
s = t.strip()
n_seen += 1
if re.search(r"[$\u00a3\u20ac\uffe5\uffe6\ufe69\uff04]", t):
n_hit += 1
continue
if re.search(r"\b\d{1,3}(?:[.,\u00a0]\d{3})+\b", s) or re.search(
r"(?<!\d)\d+[.,][0-9]{1,2}\b", s
):
n_hit += 1
if n_seen == 0:
return False
return n_hit / n_seen >= 0.10
def _cell_looks_cifra_con_moneda_iso_al_final(s: str) -> bool:
"""Solo cifra/símbolo (no prefijo de documento) y código de moneda al final.
Excluye filas al estilo `CC12…` (típico alfanumérico).
"""
t = str(s).strip()
u = t.upper()
if not re.search(
r"(?i)(USD|COP|EUR|CLP|ARS|GBP|MXN|PEN|DOP|UYU|VES|CRC|PYG|BRL|BOB|GTQ)\b\s*\Z", u
):
return False
if re.match(
r"^(?i)(CC|CE|NIT|NUIP|TI|PA|MS|RC|DNI|NUIP|CN|SC|AS|PE|PT)\D*[\d0]", u
):
return False
return re.match(
r"^[\$€0-9\s,.\-+%/]", t,
) is not None
def _raw_strings_suggest_alnum_mixed_presentation(non_null: pd.Series) -> bool:
"""Letras y dígitos juntos (p. ej. `CC32…`, `A1-2B`); no es puro 'entero/decimal' lógicamente.
Excluido: columnas con apariencia monetaria, y celdas solo cifra + ISO al final (100 USD).
"""
if _raw_strings_suggest_monetary_presentation(non_null):
return False
n_m = 0
n_s = 0
for raw in non_null.head(500):
if _is_nullish_scalar(raw):
continue
t = str(raw).strip()
n_s += 1
if not re.search(r"\d", t) or not re.search(
r"[A-Za-zÑñ\u00c0-\u024f]", t, re.IGNORECASE
):
continue
if _cell_looks_cifra_con_moneda_iso_al_final(t):
continue
n_m += 1
if n_s == 0:
return False
return n_m / n_s >= 0.20
def _cell_looks_like_documento_id_tipo_nit(s: str) -> bool:
"""Cadenas estilo `CC32324029`, `CE 1.234.567` — no son fechas (evita 8 cifras como DDMM).
Al menos una letra al inicio y 5-16 dígitos en la celda (no `Jan`+`2020` = 4 dígitos).
"""
t = str(s).strip()
if not t or len(t) < 3:
return False
if not re.match(r"^[A-Za-z]{1,3}", t):
return False
dig = re.sub(r"[^\d]", "", t)
if len(dig) < 5 or len(dig) > 16:
return False
return True
def _content_looks_like_doc_id_letter_prefix_column(non_null: pd.Series) -> bool:
"""Muestras con prefijo documental (CC, NIT, …) y ≥5 dígitos → no columna de fecha."""
n_ok = 0
n_seen = 0
for raw in non_null.head(500):
if _is_nullish_scalar(raw):
continue
s = str(raw).strip()
n_seen += 1
if _cell_looks_like_documento_id_tipo_nit(s):
n_ok += 1
if n_seen == 0:
return False
return n_ok / n_seen >= 0.20
@@ -0,0 +1,162 @@
"""Separación de copago / cuota moderadora por año."""
from __future__ import annotations
import re
from typing import Any, Literal, Optional
import numpy as np
import pandas as pd
from .dates import (
_detect_compact_date_formats,
_parse_any_date_to_tuple,
detect_date_format,
)
from .numbers import _infer_decimal_hint_from_column, _parse_lenient_one
DEFAULT_MODERADORA_BY_YEAR: dict[int, set[int]] = {
2023: {4100, 16400, 43000},
2024: {4500, 18200, 47700},
2025: {4700, 19200, 50300},
2026: {5000, 20100, 52800},
}
def normalize_moderadora_by_year_dict(raw: Any) -> dict[int, set[int]]:
"""Normaliza un dict JSON año → lista de enteros (cuotas moderadora)."""
if not isinstance(raw, dict):
return {}
out: dict[int, set[int]] = {}
for k, v in raw.items():
try:
yi = int(k)
except (TypeError, ValueError):
continue
if not isinstance(v, (list, tuple)):
continue
nums: set[int] = set()
for x in v:
try:
nums.add(int(round(float(x))))
except (TypeError, ValueError):
continue
if nums:
out[yi] = nums
return out
def merge_moderadora_config(
cop_mod_config: Optional[dict[str, Any]],
) -> dict[int, set[int]]:
"""Fusiona defaults con `moderadora_by_year` opcional del usuario."""
merged = {y: set(vals) for y, vals in DEFAULT_MODERADORA_BY_YEAR.items()}
if cop_mod_config and isinstance(cop_mod_config.get("moderadora_by_year"), dict):
custom = normalize_moderadora_by_year_dict(
cop_mod_config["moderadora_by_year"]
)
for y, vals in custom.items():
merged[y] = set(vals)
return merged
def _alloc_cop_mod_column_names(
col_name: str, existing: set[str]
) -> tuple[str, str]:
"""Nombres únicos para copago y moderadora."""
base = re.sub(r"[^\w]+", "_", str(col_name).strip(), flags=re.ASCII) or "col"
for i in range(0, 200):
suf = f"_{i}" if i else ""
ccol, mcol = f"{base}{suf}_copago", f"{base}{suf}_moderadora"
if ccol not in existing and mcol not in existing:
return ccol, mcol
raise ValueError("No se pudieron asignar nombres de columnas únicos.")
def _amount_cell_to_optional_int(
val: Any,
decimal_hint: Optional[Literal[",", "."]],
) -> Optional[int]:
try:
f = _parse_lenient_one(val, decimal_hint)
except Exception:
return None
if f is None or (isinstance(f, float) and not np.isfinite(f)) or (
isinstance(f, (float, np.floating)) and np.isnan(f)
):
return None
try:
return int(round(float(f)))
except (TypeError, ValueError, OverflowError):
return None
def _apply_cop_mod_split(
result: pd.DataFrame,
amount_col: str,
fecha_column: str,
log: list[str],
*,
moderadora_by_year: dict[int, set[int]],
) -> None:
"""Añade columnas copago y moderadora según cuotas por año de la fecha."""
if fecha_column not in result.columns:
log.append(
f"[WARN] COP/MOD: columna de fecha '{fecha_column}' no existe; se omite."
)
return
if amount_col not in result.columns:
log.append(f"[WARN] COP/MOD: columna de importe '{amount_col}' no existe; se omite.")
return
fec_ser = result[fecha_column]
amt_ser = result[amount_col]
stripped = fec_ser.astype(str).str.replace(r"[\s\/\-\.]", "", regex=True)
stripped = stripped.where(fec_ser.notna(), other=pd.NA)
sep_fmt = detect_date_format(fec_ser) or detect_date_format(stripped)
compact8_fmt, compact6_fmt = _detect_compact_date_formats(fec_ser)
amt_hint = _infer_decimal_hint_from_column(amt_ser.astype(str))
ccol, mcol = _alloc_cop_mod_column_names(str(amount_col), set(result.columns))
cop_vals: list[Any] = []
mod_vals: list[Any] = []
for i in range(len(result)):
amt_raw = amt_ser.iloc[i]
fec_raw = fec_ser.iloc[i]
amt_i = _amount_cell_to_optional_int(amt_raw, amt_hint)
if amt_i is None:
cop_vals.append(0)
mod_vals.append(0)
continue
parsed_d = _parse_any_date_to_tuple(
fec_raw,
sep_fmt=sep_fmt,
compact8_fmt=compact8_fmt,
compact6_fmt=compact6_fmt,
)
if parsed_d is None:
cop_vals.append(amt_i)
mod_vals.append(0)
continue
_d, _m, year = parsed_d
tiers = moderadora_by_year.get(year)
if tiers is None:
cop_vals.append(amt_i)
mod_vals.append(0)
continue
if amt_i in tiers:
cop_vals.append(0)
mod_vals.append(amt_i)
else:
cop_vals.append(amt_i)
mod_vals.append(0)
result[ccol] = cop_vals
result[mcol] = mod_vals
log.append(
f"[OK] '{amount_col}' -> '{ccol}', '{mcol}' <- Separa COP/MOD "
f"(fecha: '{fecha_column}')"
)
@@ -0,0 +1,459 @@
"""Detección y normalización de fechas."""
from __future__ import annotations
import re
import unicodedata
from datetime import date
from typing import Any, Optional
import pandas as pd
from .nullish import _is_nullish_scalar
_DATE_NUM_RE = re.compile(r"^\s*(\d{1,4})[\s\/\-\.](\d{1,2})[\s\/\-\.](\d{1,4})\s*$")
_DATE_COMPACT_RE = re.compile(r"^\s*(\d{6}|\d{8})\s*$")
# Hora opcional: HH[:MM[:SS]][ AM|PM] al final de la cadena
_TIME_TAIL_RE = re.compile(
r"[\sT]+(\d{1,2})(?::(\d{1,2})(?::(\d{1,2}))?)?\s*([APap]\.?\s*[Mm]\.?)?\s*$"
)
_MONTH_NAME_TO_NUM = {
"enero": 1,
"ene": 1,
"febrero": 2,
"feb": 2,
"marzo": 3,
"mar": 3,
"abril": 4,
"abr": 4,
"mayo": 5,
"may": 5,
"junio": 6,
"jun": 6,
"julio": 7,
"jul": 7,
"agosto": 8,
"ago": 8,
"septiembre": 9,
"setiembre": 9,
"sep": 9,
"sept": 9,
"octubre": 10,
"oct": 10,
"noviembre": 11,
"nov": 11,
"diciembre": 12,
"dic": 12,
}
def _split_date_time(value: str) -> tuple[str, Optional[str]]:
"""Separa la parte de fecha de la parte de hora si está presente.
Devuelve `(date_part, time_part_normalized)`. `time_part_normalized` es
None si no hay hora. La hora se normaliza a `HH:MM[:SS][ AM|PM]`.
"""
if value is None:
return "", None
s = str(value).strip()
if not s:
return "", None
m = _TIME_TAIL_RE.search(s)
if not m:
return s, None
hh = int(m.group(1))
mm = int(m.group(2)) if m.group(2) else 0
ss = m.group(3)
ampm = m.group(4)
if not (0 <= hh <= 23 and 0 <= mm <= 59):
return s, None
if ss is not None:
try:
ssi = int(ss)
if not (0 <= ssi <= 59):
return s, None
except ValueError:
return s, None
# Validar contra AM/PM (con AM/PM, hora 1-12)
if ampm and not (1 <= hh <= 12):
return s, None
# Quitar separadores habituales entre fecha y hora (p. ej. "12/27/24, 12:00 AM").
date_part = s[: m.start()].rstrip(" \t,;T")
if not date_part:
return s, None
parts: list[str] = [f"{hh:02d}:{mm:02d}"]
if ss is not None:
parts[0] += f":{int(ss):02d}"
if ampm:
norm_ampm = ampm.replace(".", "").replace(" ", "").upper()
if norm_ampm in ("AM", "PM"):
parts.append(norm_ampm)
time_norm = " ".join(parts)
return date_part, time_norm
def _extract_date_parts(value: str) -> Optional[tuple[int, int, int, str]]:
"""Intenta extraer (a, b, c, layout) de una fecha en cualquier formato común.
`layout` es uno de: 'sep' (con separadores), 'compact8' (YYYYMMDD o
DDMMYYYY), 'compact6' (DDMMYY o YYMMDD). No interpreta todavía qué
componente es día/mes/año; eso lo decide `detect_date_format` viendo la
columna completa. Acepta valores con hora al final (ignorada para los
componentes de fecha).
"""
if value is None:
return None
s = str(value).strip()
if not s or s.lower() in ("nan", "none", "nat"):
return None
date_part, _ = _split_date_time(s)
if date_part:
s = date_part
m = _DATE_NUM_RE.match(s)
if m:
try:
a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3))
return (a, b, c, "sep")
except ValueError:
return None
m2 = _DATE_COMPACT_RE.match(s)
if m2:
digits = m2.group(1)
if len(digits) == 8:
return (int(digits[0:2]), int(digits[2:4]), int(digits[4:8]), "compact8")
if len(digits) == 6:
return (int(digits[0:2]), int(digits[2:4]), int(digits[4:6]), "compact6")
return None
def detect_date_format(series: pd.Series, sample_size: int = 500) -> Optional[str]:
"""Analiza la serie y determina el formato de fecha dominante.
Devuelve un string con el layout detectado (p. ej. "DD/MM/YYYY",
"MM/DD/YYYY", "YYYYMMDD", "DDMMYYYY") o None si no parece fechas.
"""
if series is None or len(series) == 0:
return None
non_null = series.dropna()
if non_null.empty:
return None
if len(non_null) > sample_size:
non_null = non_null.sample(sample_size, random_state=42)
layouts = {"sep": 0, "compact8": 0, "compact6": 0}
sep_first_gt12 = 0 # primer componente > 12 → probable día
sep_second_gt12 = 0 # segundo componente > 12 → probable mes solo posible si MM/DD
sep_third_4digits = 0 # tercer componente de 4 dígitos → año al final
sep_first_4digits = 0 # primer componente de 4 dígitos → año al inicio
compact8_year_first = 0 # primeros 4 dígitos > 1900 → YYYYMMDD
compact8_year_last = 0 # últimos 4 dígitos > 1900 → DDMMYYYY
total_parsed = 0
for val in non_null:
parsed = _extract_date_parts(val)
if parsed is None:
continue
total_parsed += 1
a, b, c, layout = parsed
layouts[layout] = layouts.get(layout, 0) + 1
if layout == "sep":
# año puede estar al inicio o al final según cantidad de dígitos
if _digit_count(a) == 4:
sep_first_4digits += 1
if _digit_count(c) == 4:
sep_third_4digits += 1
if a > 12:
sep_first_gt12 += 1
if b > 12:
sep_second_gt12 += 1
elif layout == "compact8":
# a son los primeros 2 dígitos, c son los últimos 4
if c >= 1900:
compact8_year_last += 1
# a concatenado con b son los primeros 4 dígitos
first4 = a * 100 + b
if first4 >= 1900:
compact8_year_first += 1
if total_parsed == 0:
return None
# Requerimos que >=70% de los no nulos parseen como fecha para considerarla
# columna de fechas.
if total_parsed / len(non_null) < 0.7:
return None
dominant = max(layouts, key=lambda k: layouts[k])
if dominant == "compact8":
if compact8_year_first > compact8_year_last:
return "YYYYMMDD"
return "DDMMYYYY"
if dominant == "compact6":
return "DDMMYY"
# Caso con separadores
if sep_first_4digits > sep_third_4digits:
return "YYYY/MM/DD"
# año al final
if sep_first_gt12 > 0 and sep_second_gt12 == 0:
return "DD/MM/YYYY"
if sep_second_gt12 > 0 and sep_first_gt12 == 0:
return "MM/DD/YYYY"
# ambiguo → preferencia hispana
return "DD/MM/YYYY"
def _digit_count(n: int) -> int:
return len(str(abs(n)))
def _parse_date_with_format(value: str, fmt: str) -> Optional[tuple[int, int, int]]:
"""Convierte un valor de fecha a (día, mes, año) según el layout detectado."""
parsed = _extract_date_parts(value)
if parsed is None:
return None
a, b, c, layout = parsed
if fmt == "DD/MM/YYYY":
d, m, y = a, b, c
elif fmt == "MM/DD/YYYY":
m, d, y = a, b, c
elif fmt == "YYYY/MM/DD":
y, m, d = a, b, c
elif fmt == "YYYYMMDD" and layout == "compact8":
# reconstruir: primer4 = a*100 + b, luego mes y día
y = a * 100 + b
m = c // 100
d = c % 100
elif fmt == "DDMMYYYY" and layout == "compact8":
d = a
m = b
y = c
elif fmt == "DDMMYY" and layout == "compact6":
d = a
m = b
y = 2000 + c if c < 50 else 1900 + c
else:
# intentar como sep genérico
d, m, y = a, b, c
if not (1 <= m <= 12 and 1 <= d <= 31 and 1000 <= y <= 9999):
# año en 2 dígitos
if y < 100:
y = 2000 + y if y < 50 else 1900 + y
if not (1 <= m <= 12 and 1 <= d <= 31 and 1000 <= y <= 9999):
return None
return (d, m, y)
def _normalize_date_text(value: Any) -> str:
text = unicodedata.normalize("NFKD", str(value or ""))
text = "".join(c for c in text if not unicodedata.combining(c))
return text.lower()
def _valid_date_tuple(d: int, m: int, y: int) -> Optional[tuple[int, int, int]]:
if y < 100:
y = 2000 + y if y < 50 else 1900 + y
if not (1000 <= y <= 9999):
return None
try:
date(y, m, d)
except ValueError:
return None
return (d, m, y)
def _split_glued_year_suffix_hhmm(digits: str) -> Optional[tuple[int, int, int]]:
"""Detecta `YYYY` + `HHMM` en una sola tira de 8 cifras (p. ej. `20241344`).
Tras quitar separadores puede quedar pegado al mes (p. ej. `15may20241344`).
Devuelve (año, hora, minuto) si la parte horaria es válida; si no, None.
"""
if len(digits) != 8 or not digits.isdigit():
return None
y = int(digits[:4])
hh = int(digits[4:6])
mm = int(digits[6:8])
if not (1000 <= y <= 9999 and 0 <= hh <= 23 and 0 <= mm <= 59):
return None
return (y, hh, mm)
def _parse_text_month_candidate(
first: tuple[int, str],
month: int,
second: tuple[int, str],
*,
layout: Literal["around", "after", "before"],
) -> Optional[tuple[int, int, int]]:
a, astr = first
b, bstr = second
if layout == "around":
if len(astr) == 4:
return _valid_date_tuple(b, month, a)
if len(bstr) == 4:
return _valid_date_tuple(a, month, b)
return _valid_date_tuple(a, month, b)
if layout == "after":
if len(astr) == 4:
return _valid_date_tuple(b, month, a)
return _valid_date_tuple(a, month, b)
if len(astr) == 4:
return _valid_date_tuple(b, month, a)
return _valid_date_tuple(a, month, b)
def _parse_date_with_text_month(raw: Any) -> Optional[tuple[int, int, int]]:
if _is_nullish_scalar(raw):
return None
date_part, _ = _split_date_time(str(raw))
normalized = _normalize_date_text(date_part or raw)
# `\d{8}` antes que `\d{1,4}` para no partir año+hora pegados tras strip agresivo.
tokens = re.findall(r"[a-z]+|\d{8}|\d{1,4}", normalized)
if not tokens:
return None
for i, token in enumerate(tokens):
month = _MONTH_NAME_TO_NUM.get(token)
if month is None:
continue
nums_before = [
(int(t), t) for t in tokens[:i] if t.isdigit()
]
nums_after = [
(int(t), t) for t in tokens[i + 1 :] if t.isdigit()
]
candidates: list[Optional[tuple[int, int, int]]] = []
if nums_before and len(nums_after) == 1:
glued = _split_glued_year_suffix_hhmm(nums_after[0][1])
if glued is not None:
y, _hh, _mm = glued
candidates.append(
_parse_text_month_candidate(
nums_before[-1], month, (y, str(y)), layout="around"
)
)
if nums_before and nums_after:
candidates.append(
_parse_text_month_candidate(
nums_before[-1], month, nums_after[0], layout="around"
)
)
if len(nums_after) >= 2:
candidates.append(
_parse_text_month_candidate(
nums_after[0], month, nums_after[1], layout="after"
)
)
if len(nums_before) >= 2:
candidates.append(
_parse_text_month_candidate(
nums_before[-2], month, nums_before[-1], layout="before"
)
)
for parsed in candidates:
if parsed is not None:
return parsed
return None
def _detect_compact_date_formats(
series: pd.Series,
) -> tuple[Optional[str], Optional[str]]:
cser = series.map(_normalize_messy_date_to_compact)
c8s = pd.Series(
[x for x in cser.tolist() if isinstance(x, str) and len(x) == 8]
)
c6s = pd.Series(
[x for x in cser.tolist() if isinstance(x, str) and len(x) == 6]
)
fmt8 = detect_date_format(c8s) if not c8s.empty else None
fmt6 = detect_date_format(c6s) if not c6s.empty else None
return fmt8, fmt6
def _parse_any_date_to_tuple(
raw: Any,
*,
sep_fmt: Optional[str],
compact8_fmt: Optional[str],
compact6_fmt: Optional[str],
) -> Optional[tuple[int, int, int]]:
parsed_text = _parse_date_with_text_month(raw)
if parsed_text is not None:
return parsed_text
if _is_nullish_scalar(raw):
return None
raw_text = str(raw)
date_part, _ = _split_date_time(raw_text)
target = date_part if date_part else raw_text
if sep_fmt:
parsed = _parse_date_with_format(target, sep_fmt)
if parsed is not None:
return parsed
for fallback_fmt in ("DD/MM/YYYY", "MM/DD/YYYY", "YYYY/MM/DD"):
if fallback_fmt == sep_fmt:
continue
parsed = _parse_date_with_format(target, fallback_fmt)
if parsed is not None:
return parsed
compact = _normalize_messy_date_to_compact(raw)
if not isinstance(compact, str):
return None
if len(compact) == 8:
for fmt in (compact8_fmt, "DDMMYYYY", "YYYYMMDD"):
if not fmt:
continue
parsed = _parse_date_with_format(compact, fmt)
if parsed is not None:
return parsed
if len(compact) == 6:
parsed = _parse_date_with_format(compact, compact6_fmt or "DDMMYY")
if parsed is not None:
return parsed
return None
def _normalize_messy_date_to_compact(raw: Any) -> Optional[str]:
"""Quita ruido (* + _ / espacios, etc.); deja 6 o 8 dígitos o None.
Soporta separadores entremezclados; hora al final vía `_split_date_time`.
Con 9+ dígitos, heurística: tomar 8 iniciales (p. ej. 9.º = 0 colado) y
validar luego con `_parse_date_with_format`; con más de 8 sobrantes,
8 iniciales.
"""
if _is_nullish_scalar(raw):
return None
s = str(raw).strip()
if s.lower() in ("nan", "none", "nat", "<na>"):
return None
date_part, _ = _split_date_time(s)
body = date_part if date_part else s
d = re.sub(r"[^\d]", "", body)
if not d:
return None
n = len(d)
if n in (6, 8):
return d
if n < 6 or n == 7:
return None
# 9+ dígitos: 8 iniciales (p. ej. 9.º 0 sobrante, o ruido al final)
return d[:8]
@@ -0,0 +1,175 @@
"""Homologación y división de tipo/número de documento."""
from __future__ import annotations
import re
import unicodedata
from typing import Any, Optional
import pandas as pd
from .nullish import _is_nullish_scalar
_DOC_TYPE_ALIASES: dict[str, tuple[str, ...]] = {
"CC": ("CC", "Cedula", "CD", "C", "C.C", "C C", "CCC"),
"TI": ("TI", "Tarjeta", "Tarjeta Identidad", "T1", "T", "TL"),
"CN": ("CN", "Certificado", "Nacido Vivo", "Certificado Nacido Vivo"),
"CE": ("CE", "Cedula de Extranjeria", "Extranjera"),
"ME": ("ME", "Menor sin Identificacion"),
"PS": ("PS", "Pasaporte"),
"PT": ("PT", "Permiso por proteccion Temporal"),
"RC": ("RC", "Registro Civil", "Registro", "TC"),
"SC": ("SC", "Salvo Conducto", "Salvoconducto"),
"PE": ("PE", "Permiso especial de permanencia", "Permiso"),
}
def _ascii_upper(value: Any) -> str:
text = unicodedata.normalize("NFKD", str(value or ""))
text = "".join(c for c in text if not unicodedata.combining(c))
return text.upper()
def _normalize_doc_type_key(value: Any) -> str:
"""Clave comparable: sin acentos, espacios, puntos ni símbolos."""
return re.sub(r"[^A-Z0-9]", "", _ascii_upper(value))
def _clean_doc_type_text(value: Any) -> Optional[str]:
cleaned = re.sub(r"[^A-Z0-9]+", " ", _ascii_upper(value)).strip()
cleaned = re.sub(r"\s+", " ", cleaned)
return cleaned or None
_DOC_TYPE_ALIAS_BY_KEY: dict[str, str] = {
_normalize_doc_type_key(alias): canonical
for canonical, aliases in _DOC_TYPE_ALIASES.items()
for alias in aliases
}
_DOC_TYPE_ALIAS_DISPLAY_BY_KEY: dict[str, str] = {}
for _canonical, _aliases in _DOC_TYPE_ALIASES.items():
for _alias in _aliases:
_alias_key = _normalize_doc_type_key(_alias)
_alias_display = _clean_doc_type_text(_alias)
if _alias_key and _alias_display:
_DOC_TYPE_ALIAS_DISPLAY_BY_KEY.setdefault(_alias_key, _alias_display)
_DOC_TYPE_ALIAS_KEYS_SORTED: list[str] = sorted(
_DOC_TYPE_ALIAS_BY_KEY,
key=lambda k: (len(k), k),
reverse=True,
)
def homologate_doc_type(value: Any) -> Optional[str]:
"""Homologa un tipo documental a sigla; si no existe, conserva texto limpio."""
if _is_nullish_scalar(value):
return None
key = _normalize_doc_type_key(value)
if not key or key.lower() in ("nan", "none", "nat"):
return None
return _DOC_TYPE_ALIAS_BY_KEY.get(key) or _clean_doc_type_text(value)
def _compact_doc_text(value: Any) -> str:
return _normalize_doc_type_key(value)
def _looks_like_short_alias_prefix(compact: str, alias_key: str) -> bool:
"""Evita que alias de 1-2 chars capturen palabras como CODIGO o TOTAL."""
if not compact.startswith(alias_key):
return False
rest = compact[len(alias_key):]
if not rest:
return True
return bool(rest[0].isdigit())
def _match_known_doc_type_prefix(compact: str) -> Optional[tuple[str, str]]:
for alias_key in _DOC_TYPE_ALIAS_KEYS_SORTED:
if len(alias_key) <= 2:
if not _looks_like_short_alias_prefix(compact, alias_key):
continue
elif not compact.startswith(alias_key):
continue
return _DOC_TYPE_ALIAS_BY_KEY[alias_key], alias_key
return None
def split_doc_tipo_y_numero_texto(
raw: Any,
*,
homologate: bool = True,
) -> tuple[Optional[str], Optional[str]]:
"""Separa tipo documental y número, tolerando nombres largos y símbolos.
Si `homologate` es True, tipos conocidos se devuelven como sigla (CC, TI,
etc.). Los tipos desconocidos se conservan limpios en mayúsculas.
"""
if _is_nullish_scalar(raw) and not (isinstance(raw, str) and not str(raw).strip()):
return (None, None)
t = str(raw).strip()
if t.lower() in ("nan", "none", "nat", "<na>"):
return (None, None)
if t and t[0].isdigit():
return (None, None)
compact = _compact_doc_text(t)
if not compact:
return (None, None)
match = _match_known_doc_type_prefix(compact)
if match is not None:
canonical, alias_key = match
rest = compact[len(alias_key):]
nums = re.sub(r"[^\d]", "", rest)
if homologate:
return (canonical, nums if nums else None)
raw_type = _DOC_TYPE_ALIAS_DISPLAY_BY_KEY.get(alias_key) or canonical
return (raw_type, nums if nums else None)
m = re.match(
r"^([^\d]+?)\s*([0-9].*)\s*$",
t,
)
if m:
to_t = _clean_doc_type_text(m.group(1))
nums = re.sub(r"[^\d]", "", m.group(2))
return (homologate_doc_type(to_t) if homologate else to_t, nums if nums else None)
if not re.search(r"\d", t):
to_t = homologate_doc_type(t) if homologate else _clean_doc_type_text(t)
return (to_t, None)
return (None, None)
def _alloc_doc_split_column_names(
col_name: str, existing: set[str]
) -> tuple[str, str]:
"""Nombres únicos para las dos columnas derivadas."""
base = re.sub(r"[^\w]+", "_", str(col_name).strip(), flags=re.ASCII) or "col"
for i in range(0, 200):
suf = f"_{i}" if i else ""
t, n = f"{base}{suf}_doc_tipo", f"{base}{suf}_doc_numeros"
if t not in existing and n not in existing:
return t, n
raise ValueError("No se pudieron asignar nombres de columnas únicos.")
def _apply_doc_id_split_columns(
result: pd.DataFrame, col: str, log: list[str], *, homologate_type: bool = True
) -> None:
tcol, ncol = _alloc_doc_split_column_names(str(col), set(result.columns))
st_tipo: list[Optional[str]] = []
st_num: list[Any] = []
for v in result[col].tolist():
tt, nn = split_doc_tipo_y_numero_texto(v, homologate=homologate_type)
st_tipo.append(tt)
st_num.append(nn if nn is not None else pd.NA)
result[tcol] = st_tipo
result[ncol] = st_num
log.append(
f"[OK] '{str(col)}' -> columnas '{tcol}' y '{ncol}' <- Dividir tipo y número (documento)"
)
+132
View File
@@ -0,0 +1,132 @@
"""Carga robusta de CSV (delimitador, encoding)."""
from __future__ import annotations
import csv as _csv
from collections import Counter
from typing import Optional
import pandas as pd
_CANDIDATE_DELIMITERS = ["|", ";", "\t", ",", ":"]
def detect_delimiter(filepath: str) -> str:
"""Detecta el delimitador del CSV de forma robusta.
Estrategia:
1. Intentar `csv.Sniffer` con un sample grande.
2. Si falla o el resultado parece incorrecto, contar ocurrencias por línea
de cada delimitador candidato y elegir el que produce el mayor número
de columnas estable entre líneas.
"""
encodings = ("utf-8", "utf-8-sig", "latin-1", "cp1252")
sample = ""
for enc in encodings:
try:
with open(filepath, "r", encoding=enc, errors="replace") as f:
sample = f.read(65536)
break
except Exception:
continue
if not sample:
return ","
try:
dialect = _csv.Sniffer().sniff(sample, delimiters="".join(_CANDIDATE_DELIMITERS))
sniffed = dialect.delimiter
except _csv.Error:
sniffed = None
lines = [ln for ln in sample.splitlines() if ln.strip()]
if not lines:
return sniffed or ","
# Para cada candidato, calcular el número promedio de columnas y la
# consistencia (cuántas líneas tienen el mismo conteo dominante).
scores: dict[str, tuple[int, int]] = {}
for delim in _CANDIDATE_DELIMITERS:
counts = [ln.count(delim) for ln in lines[:50]]
if max(counts) == 0:
continue
# Modo (valor más frecuente)
common = Counter(counts).most_common(1)[0]
mode_count, mode_freq = common
if mode_count == 0:
continue
scores[delim] = (mode_count, mode_freq)
if scores:
# Elegir el delimitador con mayor número de columnas (mode_count) y,
# como desempate, mayor consistencia (mode_freq).
best = max(scores.items(), key=lambda kv: (kv[1][0], kv[1][1]))
best_delim = best[0]
# Si el sniffer dio algo razonable y coincide, prevalece; si difiere y
# nuestro candidato es claramente mejor, usamos el nuestro.
if sniffed and sniffed in scores:
sniffed_score = scores[sniffed]
if sniffed_score[0] >= best[1][0] - 1:
return sniffed
return best_delim
return sniffed or ","
def load_csv(filepath: str) -> tuple[pd.DataFrame, str]:
"""Carga un CSV con detección de delimitador y tolerancia a filas malas.
Devuelve `(df, delimiter)`. Lanza excepción si todos los intentos fallan.
"""
delimiter = detect_delimiter(filepath)
encodings_to_try = ["utf-8", "utf-8-sig", "latin-1", "cp1252"]
last_error: Optional[Exception] = None
for enc in encodings_to_try:
try:
df = pd.read_csv(
filepath,
delimiter=delimiter,
dtype=str,
encoding=enc,
engine="python",
on_bad_lines="skip",
)
return df, delimiter
except Exception as exc: # noqa: BLE001
last_error = exc
continue
# Último intento con quoting permisivo y sin separador detectado
try:
df = pd.read_csv(
filepath,
sep=None,
dtype=str,
engine="python",
on_bad_lines="skip",
)
return df, "(auto)"
except Exception as exc: # noqa: BLE001
last_error = exc
raise RuntimeError(f"No se pudo leer el CSV: {last_error}")
def load_csv_bytes(content: bytes, filename: str = "upload.csv") -> tuple[pd.DataFrame, str]:
"""Carga un CSV desde bytes (upload HTTP) vía archivo temporal."""
import os
import tempfile
suffix = os.path.splitext(filename)[1] or ".csv"
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
tmp.write(content)
tmp_path = tmp.name
try:
return load_csv(tmp_path)
finally:
try:
os.unlink(tmp_path)
except OSError:
pass
@@ -0,0 +1,67 @@
"""Tipos de columna y resumen de análisis."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Optional
TYPE_INTEGER = "integer"
TYPE_DECIMAL = "decimal"
TYPE_DATE = "date"
TYPE_BOOLEAN = "boolean"
TYPE_CATEGORICAL = "categorical"
TYPE_TEXT = "text"
TYPE_MIXED = "mixed"
TYPE_EMPTY = "empty"
@dataclass
class ColumnSummary:
"""Resumen inteligente para una columna del DataFrame."""
name: str
inferred_type: str
example: str = ""
samples: list[str] = field(default_factory=list)
null_count: int = 0
null_pct: float = 0.0
unique_count: int = 0
unique_pct: float = 0.0
min: Optional[float] = None
max: Optional[float] = None
mean: Optional[float] = None
avg_length: Optional[float] = None
top_values: list[tuple[str, int]] = field(default_factory=list)
detected_date_format: Optional[str] = None
applicable_rules: list[str] = field(default_factory=list)
def info_text(self) -> str:
"""Texto humano con información adicional (para subtítulo de la UI)."""
parts: list[str] = []
parts.append(f"nulos: {self.null_count} ({self.null_pct:.0f}%)")
parts.append(f"únicos: {self.unique_count}")
if self.inferred_type in (TYPE_INTEGER, TYPE_DECIMAL):
if self.min is not None:
parts.append(f"min: {_fmt_num(self.min)}")
if self.max is not None:
parts.append(f"max: {_fmt_num(self.max)}")
if self.mean is not None:
parts.append(f"media: {_fmt_num(self.mean)}")
elif self.inferred_type == TYPE_DATE:
if self.detected_date_format:
parts.append(f"formato: {self.detected_date_format}")
elif self.inferred_type in (TYPE_TEXT, TYPE_CATEGORICAL):
if self.avg_length is not None:
parts.append(f"long. prom: {self.avg_length:.1f}")
if self.top_values:
tops = ", ".join(f'"{v}" ({c})' for v, c in self.top_values[:3])
parts.append(f"top: {tops}")
return " · ".join(parts)
def _fmt_num(x: float) -> str:
if x is None:
return ""
if isinstance(x, float) and x.is_integer():
return str(int(x))
if isinstance(x, float):
return f"{x:.2f}"
return str(x)
@@ -0,0 +1,73 @@
"""Extracción de documento y autorización desde notas de texto."""
from __future__ import annotations
import re
from typing import Any, Optional
import pandas as pd
from .nullish import _is_nullish_scalar
def _normalize_nota_text(raw: Any) -> str:
if _is_nullish_scalar(raw):
return ""
try:
if pd.isna(raw):
return ""
except TypeError:
pass
s = str(raw).strip()
if not s or s.lower() in ("nan", "none", "nat", "<na>"):
return ""
return re.sub(r"\s+", " ", s)
def extract_paciente_documento_nota(raw: Any) -> Optional[str]:
"""Número de documento tras ``Paciente:`` y antes del guión del nombre."""
t = _normalize_nota_text(raw)
if not t:
return None
m = _PACIENTE_DOC_NOTA_RE.search(t)
return m.group(1) if m else None
def extract_autorizacion_nota(raw: Any) -> Optional[str]:
"""Número tras ``Autorización No.`` (opcional por fila)."""
t = _normalize_nota_text(raw)
if not t:
return None
m = _AUTORIZACION_NOTA_RE.search(t)
return m.group(1) if m else None
def _alloc_nota_extract_column_names(
col_name: str, existing: set[str]
) -> tuple[str, str]:
"""Nombres únicos para documento paciente y autorización."""
base = re.sub(r"[^\w]+", "_", str(col_name).strip(), flags=re.ASCII) or "col"
for i in range(0, 200):
suf = f"_{i}" if i else ""
dcol, acol = f"{base}{suf}_nota_paciente_doc", f"{base}{suf}_nota_autorizacion"
if dcol not in existing and acol not in existing:
return dcol, acol
raise ValueError("No se pudieron asignar nombres de columnas únicos.")
def _apply_nota_extract_columns(result: pd.DataFrame, col: str, log: list[str]) -> None:
"""Añade columnas con documento (Paciente:) y número de autorización si existe."""
dcol, acol = _alloc_nota_extract_column_names(str(col), set(result.columns))
docs: list[Any] = []
auths: list[Any] = []
for v in result[col].tolist():
dd = extract_paciente_documento_nota(v)
aa = extract_autorizacion_nota(v)
docs.append(dd if dd is not None else pd.NA)
auths.append(aa if aa is not None else pd.NA)
result[dcol] = docs
result[acol] = auths
log.append(
f"[OK] '{str(col)}' -> columnas '{dcol}' y '{acol}' <- Extraer datos de notas"
)
@@ -0,0 +1,24 @@
"""Detección de valores nulos / vacíos."""
from __future__ import annotations
from typing import Any
import numpy as np
import pandas as pd
def _is_nullish_scalar(x: Any) -> bool:
if x is None or x is pd.NA:
return True
if isinstance(x, (bool, np.bool_)):
return False
if isinstance(x, (int, float, np.integer, np.floating)) and not isinstance(
x, (bool, np.bool_)
):
if isinstance(x, float) and (np.isnan(x) or pd.isna(x)):
return True
return False
t = str(x).strip().lower()
if not t or t in ("nan", "none", "nat", "null", "<na>"):
return True
return False
@@ -0,0 +1,258 @@
"""Parsing numérico tolerante (miles, decimales, moneda)."""
from __future__ import annotations
import re
from typing import Any, Literal, Optional
import numpy as np
import pandas as pd
from .nullish import _is_nullish_scalar
_HINT_MAJORITY = 0.6
_MIN_HINT_VOTES = 1
def _strip_currency(s: str) -> str:
t = s.replace("\u00a0", " ").replace("\u202f", " ")
t = re.sub(r"[\s]+", "", t)
t = t.strip()
if not t or t.lower() in ("nan", "none", "nat"):
return ""
neg = False
if t[0:1] == "-":
neg, t = not neg, t[1:].lstrip()
if t[0:1] == "(" and t.endswith(")"):
neg, t = not neg, t[1:-1]
t = re.sub(
r"^[\$€\u00a3\u0e3f\uffe5\uffe6\uff04\ufe69]+",
"",
t,
count=1,
)
t = re.sub(r"[\$€%]+$", "", t)
if not t and neg:
return "-0"
return ("-" if neg else "") + t
def _extract_numeric_chunk(s: str) -> str:
t = s.strip()
t = t.replace("$", "").replace("€", "").replace("\u00a3", "")
t = t.replace(" ", "")
cands1 = re.findall(
r"[+\-]?(?:\d{1,3}(?:[.,]\d{3})*|\d+)(?:[.,]\d+)?", t
)
cands2 = re.findall(
r"[+\-]?\d[.,\d ]*\d", t
)
cands = cands1 + cands2
if not cands:
m = re.search(r"[\d.,\-]+", t)
if m:
c = m.group(0).replace("(", "-").replace(")", "")
return c.lstrip("+")
return ""
best = max(cands, key=len) if cands else ""
best = best.replace(" ", "").replace("(", "-").replace(")", "")
if not re.search(r"\d", best):
return ""
return best.lstrip("+")
def _as_float_s(s: str) -> Optional[float]:
if not s:
return None
try:
f = float(s)
if not np.isfinite(f) or abs(f) > 1e100:
return None
return f
except (ValueError, TypeError, OverflowError):
return None
def _parse_both_separators(body: str) -> Optional[float]:
"""El separador más a la DERECHA entre `,` y `.` es el decimal; el otro, miles."""
if "," not in body or "." not in body:
return None
t = body.strip()
if re.search(r"[^0-9,.\-]", t):
t2 = re.sub(r"[^0-9,.\-]", "", t)
t = t2
if not t:
return None
neg = t[0:1] == "-"
b = t[1:].lstrip() if neg else t
if not b:
return None
i_comma, i_dot = b.rfind(","), b.rfind(".")
if min(i_comma, i_dot) < 0:
return None
dpos = max(i_comma, i_dot)
th = "," if b[dpos] == "." else "."
intp, fr = b[:dpos], b[dpos + 1 :]
if not re.fullmatch(r"\d{1,4}$", fr):
return None
intc = intp.replace(th, "")
if not intc.isdigit():
return None
v = _as_float_s(f"{intc}.{fr}")
if v is None:
return None
if neg and v and v is not None:
return -abs(v)
return v
def _only_commas(w: str, dec_hint: Optional[Literal[",", "."]]) -> Optional[float]:
neg = w[0:1] == "-"
wn = w[1:].lstrip() if neg else w
if wn and re.search(r"[^0-9,]", wn):
wn = re.sub(r"[^0-9,]", "", wn)
n = wn.count(",")
if n == 0 and wn.isdigit():
v = float(wn)
return -v if neg else v
if n == 0 or "." in wn:
return None
if n == 1:
i = wn.rfind(",")
h, t = wn[:i], wn[i + 1 :]
if not t.isdigit() or not h.isdigit():
return None
if len(t) in (1, 2):
v = _as_float_s(f"{h}.{t}")
if v is None:
return None
return -v if neg else v
if len(t) == 3:
if dec_hint is None or dec_hint == ",":
v = _as_float_s(f"{h}{t}")
if v is None:
return None
return -v if neg else v
if dec_hint == ".":
v = _as_float_s(wn.replace(",", ""))
if v is None:
return None
return -v if neg else v
v = _as_float_s(wn.replace(",", ""))
if v is None:
return None
return -v if neg else v
def _only_dots(w: str, dec_hint: Optional[Literal[",", "."]]) -> Optional[float]:
neg = w[0:1] == "-"
wn = w[1:].lstrip() if neg else w
if wn and re.search(r"[^0-9.]", wn):
wn = re.sub(r"[^0-9.]", "", wn)
n = wn.count(".")
if n == 0 and wn.isdigit():
v = float(wn)
return -v if neg else v
if n == 0 or "," in wn:
return None
if n == 1:
i = wn.rfind(".")
h, t = wn[:i], wn[i + 1 :]
if not t.isdigit() or not h.isdigit():
return None
if len(t) in (1, 2):
v = _as_float_s(f"{h}.{t}")
if v is None:
return None
return -v if neg else v
if len(t) == 3:
if dec_hint is None or dec_hint == ".":
v = _as_float_s(f"{h}{t}")
if v is None:
return None
return -v if neg else v
if dec_hint == ",":
v = _as_float_s(wn.replace(".", ""))
if v is None:
return None
return -v if neg else v
v = _as_float_s(wn.replace(".", ""))
if v is None:
return None
return -v if neg else v
def _parse_lenient_one(
raw: Any,
decimal_hint: Optional[Literal[",", "."]] = None,
) -> float:
if _is_nullish_scalar(raw):
return float("nan")
t = _strip_currency(str(raw))
if not t or t == "-":
return float("nan")
c = _extract_numeric_chunk(t) or t
c = c.strip()
if not c or c == "-":
return float("nan")
n_comma, n_dot = c.count(","), c.count(".")
if n_comma and n_dot:
f = _parse_both_separators(c)
elif n_comma and not n_dot:
f = _only_commas(c, decimal_hint)
elif n_dot and not n_comma:
f = _only_dots(c, decimal_hint)
else:
c2 = re.sub(r"[^0-9.\-]", "", c)
f = _as_float_s(c2) if c2 and re.fullmatch(r"-?\d+$", c2) else None
if f is not None and np.isfinite(f):
return f
return float("nan")
def _infer_decimal_hint_from_column(
series: pd.Series,
) -> Optional[Literal[",", "."]]:
"""Voto con celdas que traen `,` y `.`: gana el de más a la derecha = decimal."""
votes: list[Literal[",", "."]] = []
for raw in series.dropna():
if _is_nullish_scalar(raw):
continue
t = _strip_currency(str(raw))
c = _extract_numeric_chunk(t) or t
c = c.strip()
if "," not in c or "." not in c or _parse_both_separators(c) is None:
continue
if c.rfind(",") > c.rfind("."):
votes.append(",")
else:
votes.append(".")
n = len(votes)
if n < _MIN_HINT_VOTES:
return None
a = sum(1 for v in votes if v == ",")
b = sum(1 for v in votes if v == ".")
if a + b == 0:
return None
if a / n >= _HINT_MAJORITY and a > b:
return ","
if b / n >= _HINT_MAJORITY and b > a:
return "."
return None
def _to_numeric_lenient(
series: pd.Series,
decimal_hint: Optional[Literal[",", "."]] = None,
) -> pd.Series:
"""Convierte a flotante; pista de columna por voto (si `decimal_hint` es None)."""
hint = decimal_hint if decimal_hint is not None else _infer_decimal_hint_from_column(
series
)
def _cell(x: Any) -> float:
if _is_nullish_scalar(x):
return float("nan")
return _parse_lenient_one(x, hint)
return series.map(_cell)
@@ -0,0 +1,13 @@
"""Reglas de limpieza registradas."""
from __future__ import annotations
from . import builtins as _builtins # noqa: F401 — registra reglas al importar
from .registry import CLEANING_RULES, get_rule, get_rule_fn, get_rule_name, register_rule
__all__ = [
"CLEANING_RULES",
"get_rule",
"get_rule_fn",
"get_rule_name",
"register_rule",
]
@@ -0,0 +1,135 @@
"""Registro de todas las reglas built-in.
Añadir una regla nueva: implementa la función y llama a register_rule abajo.
"""
from __future__ import annotations
from . import dates as date_rules
from . import documents as doc_rules
from . import numeric as num_rules
from . import special
from . import text as text_rules
from .registry import rebuild_cleaning_rules, register_rule
register_rule(
"strip_special_spaces",
name="Eliminar caracteres especiales y espacios",
description=(
"Deja solo caracteres alfanuméricos (A-Z, a-z, 0-9). No usar antes de "
"«Normalizar fecha» en columnas con fecha y hora: pega cifras y puede impedir "
"interpretar el valor (aplique primero la normalización de fecha o use la regla "
"que conserva separadores)."
),
fn=text_rules._rule_strip_special_spaces,
)
register_rule(
"strip_special_keep_date_seps",
name="Quitar símbolos; conserva /, -, : y espacios (fechas o texto con fechas)",
description=(
"Quita comas, $, comillas, etc. y deja letras, números, /, -, : (hora) y espacios. "
"Adecuado p. ej. en columna 'Fecha Servicio'."
),
fn=text_rules._rule_strip_special_keep_date_seps,
)
register_rule(
"strip_spaces",
name="Eliminar únicamente espacios",
description="Elimina todos los caracteres de espacio en blanco.",
fn=text_rules._rule_strip_spaces,
)
register_rule(
"to_integer",
name="Convertir a Entero",
description=(
"Convierte a enteros con detección de miles/decimal (`,`/`.`) y moneda; "
"redondea decimales."
),
fn=num_rules._rule_to_integer,
)
register_rule(
"date_to_ddmmyyyy",
name="Normalizar fecha → DD/MM/AAAA",
description=(
"Normaliza fechas numéricas, con separadores, con ruido o con meses en "
"texto (ej. 26-abril-2026) al formato DD/MM/AAAA."
),
fn=date_rules._rule_date_to_ddmmyyyy,
)
register_rule(
"date_numeric_to_ddmmyyyy",
name="Conversión de formato numérico (Fechas) → DD/MM/AAAA",
description=(
"Detecta el formato numérico predominante (YYYYMMDD, DDMMYYYY, etc.) "
"y normaliza a DD/MM/AAAA."
),
fn=date_rules._rule_date_numeric_to_ddmmyyyy,
)
register_rule(
"date_alpha_to_ddmmyyyy",
name="Conversión de formato alfanumérico (Fechas) → DD/MM/AAAA",
description=(
"Detecta el formato con separadores predominante "
"(DD/MM/YYYY, MM/DD/YYYY, YYYY/MM/DD, etc.) y normaliza a DD/MM/AAAA."
),
fn=date_rules._rule_date_alpha_to_ddmmyyyy,
)
register_rule(
"date_messy_text_to_ddmmyyyy",
name="Texto a fecha (ruido en fechas) → DD/MM/AAAA",
description=(
"Quita ruido entre cifras (*, +, _, /, etc.), detecta el orden en la "
"columna (DDMMYYYY, YYYYMMDD, DDMMYY) y unifica a DD/MM/AAAA."
),
fn=date_rules._rule_date_messy_text_to_ddmmyyyy,
)
register_rule(
"round_decimals_custom",
name="Aproximación Decimales (>0,5 sube, ≤0,5 baja)",
description=(
"Usa el mismo análisis de miles, decimales y moneda que «Entero». Tras leer "
"el número, si la parte fraccionaria de |valor| es mayor a 0,5 se aproxima al "
"entero hacia lejos de cero; en caso contrario, hacia cero. El 0,5 exacto se baja."
),
fn=num_rules._rule_round_decimals_custom,
)
register_rule(
"doc_id_split_tipo_numero",
name="Dividir tipo y número (documento)",
description=(
"Añade dos columnas (nombres únicos p. ej. TipoID_doc_tipo, TipoID_doc_numeros): "
"prefijo con letras (NIT, CC, CE, …) y cifras sin separadores. No modifica la "
"columna de origen."
),
fn=special._rule_passthrough_series,
)
register_rule(
"doc_type_homologate",
name="Homologar tipo de documento",
description=(
"Convierte tipos de documento escritos como nombre largo, sigla o texto con "
"símbolos a su abreviatura estándar (CC, TI, CE, RC, etc.)."
),
fn=doc_rules._rule_doc_type_homologate,
)
register_rule(
"cop_mod_split",
name="Separa COP/MOD (copago / moderadora)",
description=(
"Según el año en la columna Fecha Servicio que elija en la tarjeta, clasifica "
"el importe en cuota moderadora (valores por año en Configuración) o copago. "
"Se ejecuta al final del proceso de limpieza para usar fechas ya normalizadas."
),
fn=special._rule_passthrough_series,
)
register_rule(
"nota_paciente_autorizacion",
name="Extraer datos de notas (documento y autorización)",
description=(
"Desde texto tipo nota hospitalaria, extrae el número tras «Paciente:» (antes del "
"guión del nombre) y, si existe, el de «Autorización No.». Añade dos columnas; "
"no modifica el texto original."
),
fn=special._rule_passthrough_series,
)
rebuild_cleaning_rules()
@@ -0,0 +1,141 @@
"""Reglas de normalización de fechas a DD/MM/AAAA."""
from __future__ import annotations
from typing import Any, Optional
import pandas as pd
from ..dates import (
_detect_compact_date_formats,
_normalize_messy_date_to_compact,
_parse_any_date_to_tuple,
_parse_date_with_format,
_parse_date_with_text_month,
detect_date_format,
)
from ..nullish import _is_nullish_scalar
def _rule_date_to_ddmmyyyy(series: pd.Series) -> pd.Series:
"""Normaliza fechas numéricas, alfanuméricas, con meses en texto y con ruido."""
stripped = series.astype(str).str.replace(r"[\s\/\-\.]", "", regex=True)
stripped = stripped.where(series.notna(), other=pd.NA)
sep_fmt = detect_date_format(series) or detect_date_format(stripped)
compact8_fmt, compact6_fmt = _detect_compact_date_formats(series)
def _fn(x: Any):
if pd.isna(x) or str(x).strip() == "":
return x
parsed = _parse_any_date_to_tuple(
x,
sep_fmt=sep_fmt,
compact8_fmt=compact8_fmt,
compact6_fmt=compact6_fmt,
)
if parsed is None:
return x
d, m, y = parsed
return f"{d:02d}/{m:02d}/{y:04d}"
return series.map(_fn)
def _rule_date_numeric_to_ddmmyyyy(series: pd.Series) -> pd.Series:
"""Entrada: fechas en formato numérico (ej. 20240315 o 15032024).
Salida: "DD/MM/AAAA" tras detectar el formato predominante.
"""
# Para esta regla consideramos los valores "numéricos" (quitamos separadores
# comunes si los hay para detectar un YYYYMMDD/DDMMYYYY confiable).
stripped = series.astype(str).str.replace(r"[\s\/\-\.]", "", regex=True)
stripped = stripped.where(series.notna(), other=pd.NA)
fmt = detect_date_format(stripped)
if fmt is None:
# fallback: intentar con detector normal
fmt = detect_date_format(series)
return _format_dates_to_ddmmyyyy(series, fmt)
def _rule_date_alpha_to_ddmmyyyy(series: pd.Series) -> pd.Series:
"""Entrada: fechas en formato alfanumérico con separadores.
Salida: "DD/MM/AAAA".
"""
fmt = detect_date_format(series)
return _format_dates_to_ddmmyyyy(series, fmt)
def _format_dates_to_ddmmyyyy(series: pd.Series, fmt: Optional[str]) -> pd.Series:
"""Convierte cada valor al formato DD/MM/AAAA usando `fmt` como layout de
entrada detectado. La parte de hora (si existe) se descarta. Mantiene el
valor original si no puede parsear la fecha.
"""
if fmt is None:
return series.copy()
def _fn(x):
if pd.isna(x) or str(x).strip() == "":
return x
raw = str(x)
date_part, _time_part = _split_date_time(raw)
target = date_part if date_part else raw
parsed = _parse_date_with_format(target, fmt)
if parsed is None:
return x
d, m, y = parsed
return f"{d:02d}/{m:02d}/{y:04d}"
return series.map(_fn)
def _rule_date_messy_text_to_ddmmyyyy(series: pd.Series) -> pd.Series:
"""Fechas con símbolos y basura entre cifras → DD/MM/AAAA.
Normaliza a solo 6/8 dígitos, vota el formato con `detect_date_format` por
separado en compact8 y compact6; si el detector no devuelve nada, usa
`DDMMYYYY` / `DDMMYY` (alineado con la preferencia hispana en fechas
numéricas).
"""
cser = series.map(_normalize_messy_date_to_compact)
c8s = pd.Series(
[x for x in cser.tolist() if isinstance(x, str) and len(x) == 8]
)
c6s = pd.Series(
[x for x in cser.tolist() if isinstance(x, str) and len(x) == 6]
)
if c8s.empty and c6s.empty:
return series.copy()
fmt8: Optional[str] = None
fmt6: Optional[str] = None
if not c8s.empty:
fmt8 = detect_date_format(c8s) or "DDMMYYYY"
if not c6s.empty:
fmt6 = detect_date_format(c6s) or "DDMMYY"
def _one(x: Any, c: Any) -> Any:
if _is_nullish_scalar(x):
return x
if not isinstance(c, str) or len(c) not in (6, 8):
return x
f = (fmt8 or "DDMMYYYY") if len(c) == 8 else (fmt6 or "DDMMYY")
t = _parse_date_with_format(c, f)
if t is None and len(c) == 8:
t = _parse_date_with_format(c, "DDMMYYYY") or _parse_date_with_format(
c, "YYYYMMDD"
)
if t is None and len(c) == 6:
t = _parse_date_with_format(c, "DDMMYY")
if t is None:
return x
d, m, y = t
return f"{d:02d}/{m:02d}/{y:04d}"
return pd.Series(
[
_one(v, cser.iloc[i] if i < len(cser) else None) for i, v in enumerate(series)
],
index=series.index,
dtype=object,
)
@@ -0,0 +1,23 @@
"""Reglas de documento (homologación; split se aplica en apply.py)."""
from __future__ import annotations
from typing import Any
import pandas as pd
from ..documents import homologate_doc_type, split_doc_tipo_y_numero_texto
from ..nullish import _is_nullish_scalar
def _rule_doc_type_homologate(series: pd.Series) -> pd.Series:
"""Homologa una columna que contiene tipos de documento."""
def _fn(x: Any):
if _is_nullish_scalar(x):
return x
tipo, _num = split_doc_tipo_y_numero_texto(x, homologate=True)
if tipo is not None:
return tipo
return homologate_doc_type(x)
return series.map(_fn)
@@ -0,0 +1,57 @@
"""Reglas numéricas (entero, redondeo)."""
from __future__ import annotations
import math
import numpy as np
import pandas as pd
from ..nullish import _is_nullish_scalar
from ..numbers import _to_numeric_lenient
def _rule_to_integer(series: pd.Series) -> pd.Series:
"""Convierte a entero (Int64 nullable) con parser tolerante."""
numeric = _to_numeric_lenient(series.astype(str))
return numeric.round().astype("Int64")
def _round_to_int_half_tie_le(x: float) -> int:
"""Fracc. > 0,5 hacia arriba; 0,5 o menos hacia abajo. Banda 1e-9 en 0,5 (float).
Misma noción que en la descripción de la regla. Trabaja sobre |x| y
recompone el signo.
"""
sign = -1 if x < 0 else 1
if x == 0.0 or x == -0.0:
return 0
ax = abs(float(x))
ip = math.trunc(ax)
frac = ax - ip
if frac < 0.5 - 1e-9:
out = ip
elif frac > 0.5 + 1e-9:
out = ip + 1
else:
out = ip
return -out if sign < 0 else out
def _rule_round_decimals_custom(series: pd.Series) -> pd.Series:
"""Mismo parser tolerante que convertir a entero; aprox. con fracc. >0,5 arriba."""
numeric = _to_numeric_lenient(series.astype(str))
def _fn(x):
if _is_nullish_scalar(x) or (isinstance(x, (float, np.floating)) and (not np.isfinite(x) or np.isnan(x))):
return pd.NA
try:
f = float(x)
except (TypeError, ValueError, OverflowError):
return pd.NA
if not np.isfinite(f):
return pd.NA
if f == 0.0 or f == -0.0:
return 0
return _round_to_int_half_tie_le(f)
return numeric.map(_fn).astype("Int64")
@@ -0,0 +1,80 @@
"""Registro de reglas de limpieza.
Para agregar una regla nueva:
1. Implementa ``fn(series: pd.Series) -> pd.Series`` (o passthrough si es especial).
2. Llama a ``register_rule(...)`` en ``rules/builtins.py``.
3. Si la regla es especial (añade columnas, usa config), maneja el id en ``apply.py``.
4. Incluye el id en ``_applicable_rules_for_type`` (type_inference.py) si debe sugerirse.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Callable, Optional
import pandas as pd
RuleFn = Callable[[pd.Series], pd.Series]
@dataclass(frozen=True)
class RuleSpec:
id: str
name: str
description: str
fn: RuleFn
def as_dict(self) -> dict[str, Any]:
return {
"id": self.id,
"name": self.name,
"description": self.description,
"fn": self.fn,
}
_REGISTRY: dict[str, RuleSpec] = {}
_ORDER: list[str] = []
def register_rule(
rule_id: str,
*,
name: str,
description: str,
fn: RuleFn,
) -> RuleSpec:
"""Registra una regla. Si el id ya existe, lo reemplaza manteniendo el orden."""
spec = RuleSpec(id=rule_id, name=name, description=description, fn=fn)
if rule_id not in _REGISTRY:
_ORDER.append(rule_id)
_REGISTRY[rule_id] = spec
return spec
def get_rule(rule_id: str) -> Optional[dict[str, Any]]:
spec = _REGISTRY.get(rule_id)
return spec.as_dict() if spec else None
def get_rule_fn(rule_id: str) -> Optional[RuleFn]:
spec = _REGISTRY.get(rule_id)
return spec.fn if spec else None
def get_rule_name(rule_id: str) -> str:
spec = _REGISTRY.get(rule_id)
return spec.name if spec else rule_id
def list_rules() -> list[dict[str, Any]]:
return [_REGISTRY[rid].as_dict() for rid in _ORDER if rid in _REGISTRY]
# Compat: lista mutable usada por la API (se rellena al importar builtins).
CLEANING_RULES: list[dict[str, Any]] = []
def rebuild_cleaning_rules() -> list[dict[str, Any]]:
CLEANING_RULES.clear()
CLEANING_RULES.extend(list_rules())
return CLEANING_RULES
@@ -0,0 +1,8 @@
"""Marcadores passthrough para reglas que no transforman la serie in-place."""
from __future__ import annotations
import pandas as pd
def _rule_passthrough_series(s: pd.Series) -> pd.Series:
return s.copy()
@@ -0,0 +1,40 @@
"""Reglas de limpieza de texto / espacios."""
from __future__ import annotations
import re
import pandas as pd
def _rule_strip_special_spaces(series: pd.Series) -> pd.Series:
"""Elimina todo lo que no sea alfanumérico (incluye espacios)."""
pattern = re.compile(r"[^A-Za-z0-9]", flags=re.UNICODE)
def _fn(x):
if pd.isna(x):
return x
return pattern.sub("", str(x))
return series.map(_fn)
def _rule_strip_special_keep_date_seps(series: pd.Series) -> pd.Series:
"""Conserva alfanuméricos, separadores de fecha '/', '-' y hora ':', y espacio."""
pattern = re.compile(r"[^A-Za-z0-9\/\-: ]", flags=re.UNICODE)
def _fn(x):
if pd.isna(x):
return x
return pattern.sub("", str(x))
return series.map(_fn)
def _rule_strip_spaces(series: pd.Series) -> pd.Series:
"""Elimina únicamente espacios (incluidos tabs y saltos de línea)."""
def _fn(x):
if pd.isna(x):
return x
return re.sub(r"\s+", "", str(x))
return series.map(_fn)
@@ -0,0 +1,143 @@
"""Inferencia de tipo de columna y reglas aplicables."""
from __future__ import annotations
from typing import Optional
import pandas as pd
from .column_hints import (
_column_name_suggests_code_id,
_column_name_suggests_monetary_value,
_column_name_suggests_quantity_or_number_not_bool,
_content_looks_like_doc_id_letter_prefix_column,
_raw_strings_suggest_alnum_mixed_presentation,
_raw_strings_suggest_monetary_presentation,
)
from .dates import _normalize_messy_date_to_compact, detect_date_format
from .models import (
TYPE_BOOLEAN,
TYPE_CATEGORICAL,
TYPE_DATE,
TYPE_DECIMAL,
TYPE_EMPTY,
TYPE_INTEGER,
TYPE_MIXED,
TYPE_TEXT,
)
from .numbers import _to_numeric_lenient
_BOOLEAN_VALUES = {"true", "false", "si", "sí", "no", "1", "0", "yes"}
def _detect_messy_text_date_format_guess(non_null: pd.Series) -> Optional[str]:
"""Fechas con ruido: compacta a dígitos; si hay consenso, reutiliza el detector.
(Depende de `_normalize_messy_date_to_compact`, definida más abajo en el módulo.)
"""
# resolución en tiempo de ejecución
norm = non_null.map(_normalize_messy_date_to_compact) # type: ignore[name-defined]
compacts: list[str] = []
for c in norm:
if isinstance(c, str) and len(c) in (6, 8):
compacts.append(c)
n = max(len(non_null), 1)
if not compacts or len(compacts) < max(2, int(0.25 * n)):
return None
s_compact = pd.Series(compacts, dtype=object)
return detect_date_format(s_compact)
def _infer_column_type(
series: pd.Series, column_name: str = ""
) -> tuple[str, Optional[str]]:
"""Devuelve (tipo, formato_fecha detectado; solo si aplica a tipo fecha)."""
non_null = series.dropna().astype(str).str.strip()
non_null = non_null[non_null != ""]
if non_null.empty:
return TYPE_EMPTY, None
cname = str(column_name or "")
# Boolean: no aplicar 0/1/ sí-no a columnas cuyo nombre pide importe, cantid., fecha…
if not _column_name_suggests_quantity_or_number_not_bool(cname):
lowered = non_null.str.lower()
if lowered.isin(_BOOLEAN_VALUES).all() and lowered.nunique() <= 3:
return TYPE_BOOLEAN, None
is_money = _column_name_suggests_monetary_value(cname) or _raw_strings_suggest_monetary_presentation(
non_null
)
is_code = _column_name_suggests_code_id(cname)
is_doc_pattern = _content_looks_like_doc_id_letter_prefix_column(non_null)
# Fecha: no forzar en importe/código, ni con datos tipo `CC`+cifra (6–8 dígitos sueltos)
# ni cuando el contenido parece moneda (p. ej. 113,400.00 → no es DDMMYYYY).
if not is_money and not is_code and not is_doc_pattern:
date_fmt = detect_date_format(non_null)
if date_fmt:
return (TYPE_DATE, date_fmt)
messy_date_fmt = _detect_messy_text_date_format_guess(non_null)
if messy_date_fmt:
return (TYPE_DATE, messy_date_fmt)
numeric_candidate = _to_numeric_lenient(non_null, decimal_hint=None)
numeric_ratio = numeric_candidate.notna().mean()
if numeric_ratio >= 0.9:
if _raw_strings_suggest_alnum_mixed_presentation(non_null):
return (TYPE_MIXED, None)
is_all_integer = (numeric_candidate.dropna() % 1 == 0).all()
if is_all_integer and not _raw_strings_suggest_monetary_presentation(non_null):
return (TYPE_INTEGER, None)
return (TYPE_DECIMAL, None)
unique_ratio = non_null.nunique() / len(non_null)
if non_null.nunique() <= 15 and unique_ratio < 0.3:
return TYPE_CATEGORICAL, None
if 0.1 < numeric_ratio < 0.9:
return TYPE_MIXED, None
return TYPE_TEXT, None
def _applicable_rules_for_type(inferred_type: str) -> list[str]:
"""Identificadores de reglas a mostrar según el tipo inferido (sin redundancias)."""
base = ["strip_special_spaces", "strip_special_keep_date_seps", "strip_spaces"]
if inferred_type == TYPE_INTEGER:
return base + ["cop_mod_split"]
if inferred_type == TYPE_DECIMAL:
return base + [
"to_integer",
"round_decimals_custom",
"doc_id_split_tipo_numero",
"doc_type_homologate",
"cop_mod_split",
]
if inferred_type == TYPE_DATE:
return base + [
"doc_id_split_tipo_numero",
"doc_type_homologate",
"date_to_ddmmyyyy",
"nota_paciente_autorizacion",
]
if inferred_type == TYPE_MIXED:
return base + [
"to_integer",
"round_decimals_custom",
"doc_id_split_tipo_numero",
"doc_type_homologate",
"date_to_ddmmyyyy",
"cop_mod_split",
"nota_paciente_autorizacion",
]
# Texto / categórico / boolean / empty → todas las de limpieza de texto + las
# conversoras (el usuario sabrá si aplica).
return base + [
"to_integer",
"round_decimals_custom",
"doc_id_split_tipo_numero",
"doc_type_homologate",
"date_to_ddmmyyyy",
"cop_mod_split",
"nota_paciente_autorizacion",
]
applicable_rule_ids_for_type = _applicable_rules_for_type
+26 -2
View File
@@ -54,10 +54,26 @@ class ExtractionBatch(Base):
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
class Job(Base):
"""Contenedor de trabajo del usuario (lectura/validación de FK)."""
__tablename__ = "jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
job_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
class Task(Base):
"""Módulo/resultado dentro de un Job; un job puede tener N tasks (una por servicio)."""
__tablename__ = "tasks"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
job_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("jobs.id"), nullable=True, index=True
)
type: Mapped[str] = mapped_column(String(64), index=True)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
title: Mapped[str | None] = mapped_column(Text, nullable=True)
@@ -88,7 +104,13 @@ class CleaningJob(Base):
__tablename__ = "cleaning_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
extraction_batch_id: Mapped[str] = mapped_column(ForeignKey("extraction_batches.id"), index=True)
extraction_batch_id: Mapped[str | None] = mapped_column(
ForeignKey("extraction_batches.id"), nullable=True, index=True
)
job_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("jobs.id"), nullable=True, index=True
)
source: Mapped[str] = mapped_column(String(32), default="extraction", index=True)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
log: Mapped[list[str]] = mapped_column(JSON, default=list)
@@ -102,7 +124,9 @@ class CleanedRow(Base):
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
cleaning_job_id: Mapped[str] = mapped_column(ForeignKey("cleaning_jobs.id"), index=True)
extraction_batch_id: Mapped[str] = mapped_column(ForeignKey("extraction_batches.id"), index=True)
extraction_batch_id: Mapped[str | None] = mapped_column(
ForeignKey("extraction_batches.id"), nullable=True, index=True
)
row_index: Mapped[int] = mapped_column(Integer, default=0)
row_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
+1
View File
@@ -3,5 +3,6 @@ numpy==2.2.6
pandas==2.2.3
psycopg[binary]==3.2.3
pydantic==2.11.9
python-multipart==0.0.20
SQLAlchemy==2.0.36
uvicorn[standard]==0.32.1
@@ -0,0 +1,90 @@
"""cleaning_jobs: batch_id nullable, job_id, source; cleaned_rows.batch_id nullable.
Revision ID: 0003
Revises: 0002
Create Date: 2026-07-10
"""
from __future__ import annotations
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0003"
down_revision: Union[str, None] = "0002"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _col_exists(conn: sa.engine.Connection, table: str, column: str) -> bool:
inspector = sa.inspect(conn)
return any(c["name"] == column for c in inspector.get_columns(table))
def upgrade() -> None:
conn = op.get_bind()
if _col_exists(conn, "cleaning_jobs", "extraction_batch_id"):
op.alter_column(
"cleaning_jobs",
"extraction_batch_id",
existing_type=sa.String(36),
nullable=True,
)
if not _col_exists(conn, "cleaning_jobs", "job_id"):
op.add_column(
"cleaning_jobs",
sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True),
)
op.create_index("ix_cleaning_jobs_job_id", "cleaning_jobs", ["job_id"])
if not _col_exists(conn, "cleaning_jobs", "source"):
op.add_column(
"cleaning_jobs",
sa.Column(
"source",
sa.String(32),
nullable=False,
server_default="extraction",
),
)
op.create_index("ix_cleaning_jobs_source", "cleaning_jobs", ["source"])
if _col_exists(conn, "cleaned_rows", "extraction_batch_id"):
op.alter_column(
"cleaned_rows",
"extraction_batch_id",
existing_type=sa.String(36),
nullable=True,
)
def downgrade() -> None:
conn = op.get_bind()
if _col_exists(conn, "cleaned_rows", "extraction_batch_id"):
op.alter_column(
"cleaned_rows",
"extraction_batch_id",
existing_type=sa.String(36),
nullable=False,
)
if _col_exists(conn, "cleaning_jobs", "source"):
op.drop_index("ix_cleaning_jobs_source", table_name="cleaning_jobs")
op.drop_column("cleaning_jobs", "source")
if _col_exists(conn, "cleaning_jobs", "job_id"):
op.drop_index("ix_cleaning_jobs_job_id", table_name="cleaning_jobs")
op.drop_column("cleaning_jobs", "job_id")
if _col_exists(conn, "cleaning_jobs", "extraction_batch_id"):
op.alter_column(
"cleaning_jobs",
"extraction_batch_id",
existing_type=sa.String(36),
nullable=False,
)
+11
View File
@@ -0,0 +1,11 @@
# Credenciales locales para publicar en MinIO (NO subir al git).
# Copia este archivo a .env.minio y rellena los valores.
MINIO_ENDPOINT=https://s3.builker.xyz
MINIO_BUCKET=apps-versions
MINIO_PREFIX=IPSExtractor
MINIO_ACCESS_KEY=
MINIO_SECRET_KEY=
# Opcional (DD/MM/YYYY). Si se omite, usa version.json o la fecha de hoy.
# RELEASE_DATE=06/07/2026
+47 -5
View File
@@ -106,17 +106,59 @@ npm run dev
## Build para producción
```bash
npm run build # compila el renderer (TypeScript + Vite)
npm run dist # genera el instalador Windows NSIS
# 1) Sube la versión en package.json (ej. 1.0.1) y la fecha en src/shared/version.json
# 2) Configura credenciales una vez: copia .env.minio.example → .env.minio
# 3) Genera el instalador y súbelo a MinIO automáticamente:
npm run dist
```
El instalador se crea en `release/IPSExtractor-Desktop-1.0.0-Setup.exe`.
El instalador queda en `release/` y se publica en:
`s3.builker.xyz/apps-versions/IPSExtractor/{version}/`
Si solo quieres el `.exe` local **sin** subir a MinIO:
```bash
npm run dist:local
```
### Versión de la app
| Campo | Archivo |
|-------|---------|
| Número (`1.0.0`) | `package.json` → `version` |
| Fecha de release | `src/shared/version.json` → `releaseDate` |
| Fecha de release | `src/shared/version.json` → `releaseDate` (ISO) |
La versión se muestra en el menú lateral y en la esquina superior derecha. Al publicar una nueva versión, actualiza ambos archivos y ejecuta `npm run dist`.
La versión se muestra en el menú lateral y en la esquina superior derecha.
### Auto-update (MinIO)
Al iniciar la app **instalada**, consulta:
`https://s3.builker.xyz/apps-versions/IPSExtractor/latest.json`
Si hay una versión mayor, pregunta al usuario, descarga el Setup.exe, verifica `sha256` y lanza el instalador NSIS.
Layout en el bucket `apps-versions`:
```text
IPSExtractor/
latest.json
1.0.1/IPSExtractor-Desktop-1.0.1-Setup.exe
1.0.1/DATA.txt ← fecha DD/MM/YYYY, ej. 06/07/2026
```
**Requisito:** el prefijo `IPSExtractor/` debe permitir `GetObject` anónimo en la API (`s3.builker.xyz`). La política se configura en la consola (`minio.builker.xyz`).
Credenciales de publicación (solo en tu PC):
```bash
cp .env.minio.example .env.minio
# Edita MINIO_ACCESS_KEY y MINIO_SECRET_KEY
```
`npm run dist` las usa automáticamente al final. También puedes republicar sin rebuild:
```bash
npm run publish:minio
```
@@ -0,0 +1,224 @@
/**
* Auto-update desde MinIO (apps-versions/IPSExtractor).
* Sin credenciales: lee latest.json y el Setup.exe con GetObject público.
*/
const { app } = require("electron");
const crypto = require("node:crypto");
const fs = require("node:fs");
const fsp = require("node:fs/promises");
const http = require("node:http");
const https = require("node:https");
const os = require("node:os");
const path = require("node:path");
const { spawn } = require("node:child_process");
const { URL } = require("node:url");
const { version: CURRENT_VERSION } = require("../package.json");
const LATEST_JSON_URL =
process.env.IPS_UPDATE_LATEST_URL ||
"https://s3.builker.xyz/apps-versions/IPSExtractor/latest.json";
const FETCH_TIMEOUT_MS = 12_000;
/**
* @typedef {{
* version: string;
* releaseDate?: string;
* releaseDateDisplay?: string;
* fileName: string;
* url: string;
* sha256: string;
* size?: number;
* }} UpdateManifest
*/
/**
* @param {string} a
* @param {string} b
* @returns {number} -1 if a<b, 0 if equal, 1 if a>b
*/
function compareSemver(a, b) {
const pa = String(a).replace(/^v/i, "").split(".").map((x) => Number.parseInt(x, 10) || 0);
const pb = String(b).replace(/^v/i, "").split(".").map((x) => Number.parseInt(x, 10) || 0);
const len = Math.max(pa.length, pb.length);
for (let i = 0; i < len; i++) {
const x = pa[i] ?? 0;
const y = pb[i] ?? 0;
if (x < y) return -1;
if (x > y) return 1;
}
return 0;
}
/**
* @param {string} urlStr
* @param {{ onProgress?: (received: number, total: number) => void, destPath?: string }} [opts]
* @returns {Promise<{ buffer?: Buffer, destPath?: string, headers: import('http').IncomingHttpHeaders }>}
*/
function download(urlStr, opts = {}) {
return new Promise((resolve, reject) => {
const url = new URL(urlStr);
const lib = url.protocol === "http:" ? http : https;
const req = lib.get(
url,
{
timeout: FETCH_TIMEOUT_MS,
headers: { "User-Agent": `IPSExtractor-Desktop/${CURRENT_VERSION}` },
},
(res) => {
if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
res.resume();
download(res.headers.location, opts).then(resolve, reject);
return;
}
if (res.statusCode !== 200) {
res.resume();
reject(new Error(`HTTP ${res.statusCode} al descargar ${urlStr}`));
return;
}
const total = Number(res.headers["content-length"] || 0);
let received = 0;
if (opts.destPath) {
const out = fs.createWriteStream(opts.destPath);
res.on("data", (chunk) => {
received += chunk.length;
opts.onProgress?.(received, total);
});
res.pipe(out);
out.on("finish", () => resolve({ destPath: opts.destPath, headers: res.headers }));
out.on("error", reject);
res.on("error", reject);
return;
}
const chunks = [];
res.on("data", (chunk) => {
chunks.push(chunk);
received += chunk.length;
opts.onProgress?.(received, total);
});
res.on("end", () => resolve({ buffer: Buffer.concat(chunks), headers: res.headers }));
res.on("error", reject);
},
);
req.on("timeout", () => {
req.destroy();
reject(new Error("Timeout al contactar el servidor de versiones"));
});
req.on("error", reject);
});
}
/**
* @returns {Promise<{ updateAvailable: boolean, currentVersion: string, manifest?: UpdateManifest, error?: string }>}
*/
async function checkForUpdates() {
try {
const { buffer } = await download(LATEST_JSON_URL);
if (!buffer) {
return { updateAvailable: false, currentVersion: CURRENT_VERSION, error: "Respuesta vacía" };
}
/** @type {UpdateManifest} */
const manifest = JSON.parse(buffer.toString("utf8"));
if (!manifest?.version || !manifest?.url || !manifest?.fileName) {
return {
updateAvailable: false,
currentVersion: CURRENT_VERSION,
error: "Manifiesto inválido",
};
}
// Si falta la fecha de DATA.txt en el manifiesto, intentar leerla junto al exe
if (!manifest.releaseDateDisplay) {
try {
const dataUrl = manifest.url.replace(/[^/]+$/, "DATA.txt");
const { buffer: dataBuf } = await download(dataUrl);
if (dataBuf) {
const display = dataBuf.toString("utf8").trim().split(/\r?\n/)[0]?.trim();
if (display) {
manifest.releaseDateDisplay = display;
const iso = display.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/);
if (iso && !manifest.releaseDate) {
manifest.releaseDate = `${iso[3]}-${iso[2].padStart(2, "0")}-${iso[1].padStart(2, "0")}`;
}
}
}
} catch {
/* opcional */
}
}
const newer = compareSemver(CURRENT_VERSION, manifest.version) < 0;
return {
updateAvailable: newer,
currentVersion: CURRENT_VERSION,
manifest,
};
} catch (err) {
return {
updateAvailable: false,
currentVersion: CURRENT_VERSION,
error: err instanceof Error ? err.message : String(err),
};
}
}
/**
* @param {UpdateManifest} manifest
* @param {(payload: { phase: string, received?: number, total?: number, percent?: number }) => void} [onProgress]
*/
async function downloadAndInstall(manifest, onProgress) {
if (!manifest?.url || !manifest?.fileName) {
throw new Error("Manifiesto incompleto");
}
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ipsextractor-update-"));
const destPath = path.join(tmpDir, path.basename(manifest.fileName));
onProgress?.({ phase: "downloading", received: 0, total: manifest.size || 0, percent: 0 });
await download(manifest.url, {
destPath,
onProgress: (received, total) => {
const t = total || manifest.size || 0;
const percent = t > 0 ? Math.min(100, Math.round((received / t) * 100)) : 0;
onProgress?.({ phase: "downloading", received, total: t, percent });
},
});
onProgress?.({ phase: "verifying" });
if (manifest.sha256) {
const hash = crypto.createHash("sha256");
const fileBuf = await fsp.readFile(destPath);
hash.update(fileBuf);
const digest = hash.digest("hex").toLowerCase();
if (digest !== String(manifest.sha256).toLowerCase()) {
await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => undefined);
throw new Error("El instalador descargado no coincide con el hash esperado (sha256).");
}
}
onProgress?.({ phase: "installing" });
// NSIS silent install; la app debe cerrarse para que el instalador reemplace archivos.
const child = spawn(destPath, ["/S"], {
detached: true,
stdio: "ignore",
windowsHide: false,
});
child.unref();
setTimeout(() => {
app.quit();
}, 400);
return { ok: true, destPath };
}
module.exports = {
CURRENT_VERSION,
LATEST_JSON_URL,
compareSemver,
checkForUpdates,
downloadAndInstall,
};
Binary file not shown.

After

Width:  |  Height:  |  Size: 56 KiB

+37
View File
@@ -2,10 +2,15 @@ const { app, BrowserWindow, dialog, ipcMain } = require("electron");
const fs = require("node:fs/promises");
const path = require("node:path");
const { indexFolder } = require("./folderIndex.cjs");
const autoUpdate = require("./autoUpdate.cjs");
const { version: appVersion } = require("../package.json");
const DOCUMENT_EXTENSIONS = new Set([".pdf", ".tif", ".tiff", ".png", ".jpg", ".jpeg"]);
const APP_ICON = path.join(__dirname, "icon.png");
/** @type {BrowserWindow | null} */
let mainWindow = null;
function createWindow() {
const win = new BrowserWindow({
@@ -14,6 +19,7 @@ function createWindow() {
minWidth: 1080,
minHeight: 720,
title: `IPSExtractor Desktop v${appVersion}`,
icon: APP_ICON,
backgroundColor: "#f6f7f9",
webPreferences: {
preload: path.join(__dirname, "preload.cjs"),
@@ -21,6 +27,10 @@ function createWindow() {
nodeIntegration: false,
},
});
mainWindow = win;
win.on("closed", () => {
if (mainWindow === win) mainWindow = null;
});
if (!app.isPackaged) {
win.loadURL("http://127.0.0.1:5173");
@@ -30,7 +40,34 @@ function createWindow() {
}
}
function sendUpdateProgress(payload) {
if (mainWindow && !mainWindow.isDestroyed()) {
mainWindow.webContents.send("updater:progress", payload);
}
}
ipcMain.handle("updater:check", async () => {
if (!app.isPackaged) {
return {
updateAvailable: false,
currentVersion: appVersion,
error: "Auto-update desactivado en desarrollo",
};
}
return autoUpdate.checkForUpdates();
});
ipcMain.handle("updater:install", async (_event, manifest) => {
if (!app.isPackaged) {
throw new Error("Auto-update solo disponible en la app instalada");
}
return autoUpdate.downloadAndInstall(manifest, sendUpdateProgress);
});
app.whenReady().then(() => {
if (process.platform === "win32") {
app.setAppUserModelId("com.ipsextractor.desktop");
}
createWindow();
app.on("activate", () => {
if (BrowserWindow.getAllWindows().length === 0) createWindow();
@@ -9,4 +9,11 @@ contextBridge.exposeInMainWorld("desktopApi", {
indexFolder: (rootPath, structureMode, docTypes, keywords) =>
ipcRenderer.invoke("fs:index-folder", rootPath, structureMode, docTypes, keywords),
readFile: (filePath) => ipcRenderer.invoke("fs:read-file", filePath),
checkForUpdates: () => ipcRenderer.invoke("updater:check"),
installUpdate: (manifest) => ipcRenderer.invoke("updater:install", manifest),
onUpdateProgress: (callback) => {
const handler = (_event, payload) => callback(payload);
ipcRenderer.on("updater:progress", handler);
return () => ipcRenderer.removeListener("updater:progress", handler);
},
});
+1
View File
@@ -3,6 +3,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" type="image/png" href="/app-icon.png" />
<title>IPSExtractor Desktop</title>
</head>
<body>
+398 -3
View File
@@ -1,13 +1,14 @@
{
"name": "ipsextractor-desktop",
"version": "0.1.0",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "ipsextractor-desktop",
"version": "0.1.0",
"version": "1.0.0",
"dependencies": {
"@aws-sdk/client-s3": "^3.1085.0",
"lucide-react": "^0.468.0",
"react": "^18.3.1",
"react-dom": "^18.3.1"
@@ -58,6 +59,314 @@
"dev": true,
"license": "ISC"
},
"node_modules/@aws-sdk/checksums": {
"version": "3.1000.16",
"resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.16.tgz",
"integrity": "sha512-EKnvkXSmz3IpA99tCNuI+dLFXyZyClSm8zns9sB/elvkU+MTuomAs6toJMPMBf98/fICG/urXDkzGz0/c3yyAQ==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.975.1",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/client-s3": {
"version": "3.1085.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1085.0.tgz",
"integrity": "sha512-O0xe8sR50AYkwxlvRRsV0qytEO2dtXQTQ1CF3YBBdE5xtVkbu27H0vGa1mjQi1/+fbYM80AWEIPai5jZmXyubw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/checksums": "^3.1000.16",
"@aws-sdk/core": "^3.975.1",
"@aws-sdk/credential-provider-node": "^3.972.66",
"@aws-sdk/middleware-sdk-s3": "^3.972.62",
"@aws-sdk/signature-v4-multi-region": "^3.996.39",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/fetch-http-handler": "^5.6.4",
"@smithy/node-http-handler": "^4.9.4",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/core": {
"version": "3.975.1",
"resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.975.1.tgz",
"integrity": "sha512-8qh/6EYb7hl/ZwVfQufhbMEZs1gQIc7GbdrIf4eprQJ7cv042+74nE6l3YDfyWNzb9iPXb8fRyYSHkNIk5eE6Q==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/types": "^3.974.0",
"@aws-sdk/xml-builder": "^3.972.34",
"@aws/lambda-invoke-store": "^0.3.0",
"@smithy/core": "^3.29.2",
"@smithy/signature-v4": "^5.6.3",
"@smithy/types": "^4.16.0",
"bowser": "^2.11.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-env": {
"version": "3.972.57",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.57.tgz",
"integrity": "sha512-1RfJaF7SW1TOnvNGU7kaYjwUf5H3sfm+synGH1bHhRlqcnxCt3szebH3dmKEyY4tuGcbQ6ffzUT89cRitBV8OQ==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.975.1",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-http": {
"version": "3.972.59",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.59.tgz",
"integrity": "sha512-sRCkpTiFnCdQvuaRVjQ6SVoHu6i7RUpurVo1c4F81HWhPvUJ7Wdp5MNtSdX1O29CNXc8em3O5m52hCjVtAD9SA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.975.1",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/fetch-http-handler": "^5.6.4",
"@smithy/node-http-handler": "^4.9.4",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-ini": {
"version": "3.973.1",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.1.tgz",
"integrity": "sha512-6d8H6ZAh3ZPKZ6fe1nG2OWeZEZPtt9ravoD1dezPdPtsSkJRoxGAnFSHwKT3E/Te6fHE30zRzjV6TD12rvF6yQ==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.975.1",
"@aws-sdk/credential-provider-env": "^3.972.57",
"@aws-sdk/credential-provider-http": "^3.972.59",
"@aws-sdk/credential-provider-login": "^3.972.63",
"@aws-sdk/credential-provider-process": "^3.972.57",
"@aws-sdk/credential-provider-sso": "^3.973.1",
"@aws-sdk/credential-provider-web-identity": "^3.972.63",
"@aws-sdk/nested-clients": "^3.997.31",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/credential-provider-imds": "^4.4.7",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-login": {
"version": "3.972.63",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.63.tgz",
"integrity": "sha512-GREWRrMj0XnNKMaVa/Mauoaui26qBEHu71WWqXbwZOu/jFQOnPZjTf7u0KtGKC8VGa6VUs9kDWGgocrKNLS9vw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.975.1",
"@aws-sdk/nested-clients": "^3.997.31",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-node": {
"version": "3.972.66",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.66.tgz",
"integrity": "sha512-f+qjRXZpz7sgzbc4QB+6nLKfyKFgRRXzWdXbsKPv/VhVRyHsDyq4yBWC/B75BAJpFIcUeI2XR/3gdWJ677zB4A==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/credential-provider-env": "^3.972.57",
"@aws-sdk/credential-provider-http": "^3.972.59",
"@aws-sdk/credential-provider-ini": "^3.973.1",
"@aws-sdk/credential-provider-process": "^3.972.57",
"@aws-sdk/credential-provider-sso": "^3.973.1",
"@aws-sdk/credential-provider-web-identity": "^3.972.63",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/credential-provider-imds": "^4.4.7",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-process": {
"version": "3.972.57",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.57.tgz",
"integrity": "sha512-TiVQhuU0pbhIZAUZacbPHMyzrIdiH+lnx+PMY/Pu/b93dJrq3wdZwzUJ0TPpvNxaqbHsxJvQZW3/h/beLiKq7Q==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.975.1",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-sso": {
"version": "3.973.1",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.1.tgz",
"integrity": "sha512-3foTZUJ4821Ij60X7K3NJroygiZLnbBmarN+T//O2cjkISan90zElN3NBmgSlDrTQ7Gs6z/yO8V7h60QNcDZHQ==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.975.1",
"@aws-sdk/nested-clients": "^3.997.31",
"@aws-sdk/token-providers": "3.1083.0",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-web-identity": {
"version": "3.972.63",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.63.tgz",
"integrity": "sha512-8qZLFhM69eKcS37m459ctPR05Qimycm/74OPVioe6wNZabMT54GYhwBju0+J656RkMasNSawWQu+c8CmBe3TUQ==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.975.1",
"@aws-sdk/nested-clients": "^3.997.31",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/middleware-sdk-s3": {
"version": "3.972.62",
"resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.62.tgz",
"integrity": "sha512-k8JJwYXVYlOOjWnPZDThQS1xDFJgi5Dokt73qFlDtrZAbdcint5aIdjB9XgJAAQVP5OoqcefQmh1FYXiPpvsvw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.975.1",
"@aws-sdk/signature-v4-multi-region": "^3.996.39",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/nested-clients": {
"version": "3.997.31",
"resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.31.tgz",
"integrity": "sha512-BDHTpwcsZHEBNEJzOg/B1BkFYJxAXY50dau/NyVWs3d51F0WgIUGSWZot/Os+N3KpDhXeaXnz37mWffAvduREw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.975.1",
"@aws-sdk/signature-v4-multi-region": "^3.996.39",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/fetch-http-handler": "^5.6.4",
"@smithy/node-http-handler": "^4.9.4",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/signature-v4-multi-region": {
"version": "3.996.39",
"resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.39.tgz",
"integrity": "sha512-8+srXqYIF8KYMLC4FxMLEM5Ek7kUNibJu1R4m8/fUhhNYIZZz26oGtKkCr8I/HiG2fFQxBvaGgQZT4/mqRCSnA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/types": "^3.974.0",
"@smithy/signature-v4": "^5.6.3",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/token-providers": {
"version": "3.1083.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1083.0.tgz",
"integrity": "sha512-s0woKnxuHrExLc5L2ArIH5BMkbonHPtt+5hSBM8oknp9M6QTuUmmAmJ2E0EdzCGONrO+8+ADPqvv6UX0nNcc7A==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.975.1",
"@aws-sdk/nested-clients": "^3.997.31",
"@aws-sdk/types": "^3.974.0",
"@smithy/core": "^3.29.2",
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/types": {
"version": "3.974.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.0.tgz",
"integrity": "sha512-QIBrw90CDm4O0UaIIzkU6DrFdeJzEb2Va5EPEVpyldj6sHJxB6cshhStJuhZxk3wR3PmjJlYsjPmY1kNb+KGBg==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/xml-builder": {
"version": "3.972.34",
"resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.34.tgz",
"integrity": "sha512-wHhWL1y7sN3enBA8POrPpQM5jCcmu2ozyhbRei4c8OjVcEaEs6yLucLa/pla457ggS/ysuy7bosagz3HaJkZXA==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.16.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws/lambda-invoke-store": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz",
"integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==",
"license": "Apache-2.0",
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@babel/code-frame": {
"version": "7.29.7",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz",
@@ -2008,6 +2317,87 @@
"url": "https://github.com/sindresorhus/is?sponsor=1"
}
},
"node_modules/@smithy/core": {
"version": "3.29.3",
"resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.3.tgz",
"integrity": "sha512-L+Ys6ecjk5vwPMAKHBpPKlJ3DkqwNcnfEISXBZIsVvWG/XKXfsAP8mwIYlTeLcd2ElHdesPI8OuOmJSFAPhm6A==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/credential-provider-imds": {
"version": "4.4.8",
"resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.8.tgz",
"integrity": "sha512-q9J7JTiXrAhB8sDp4px97uEPT7CwKH61Co78grdNQvU8QZAdiuaSRhP0tUVf2ogy36RZTrlMU1rBmDEH+cnkiA==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.29.3",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/fetch-http-handler": {
"version": "5.6.5",
"resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.5.tgz",
"integrity": "sha512-SuqeisTyPoiIPtIYru/sGxGyXzmZ+8nnFOhC+qRPglt06Ebd1yH//CDltZB2J/3WBNVhwfUaZ0EtHB3cm2X32g==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.29.3",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/node-http-handler": {
"version": "4.9.5",
"resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.5.tgz",
"integrity": "sha512-bNqdxTQTxmLbomSmlkZFz8L6B/feQ2HHzw4L2zY7Ecp2XffYAZq2uzdWDdxJHJFbEvqd+SRuluJso0P8+xPdbw==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.29.3",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/signature-v4": {
"version": "5.6.4",
"resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.4.tgz",
"integrity": "sha512-B89bpf2t/y/wia6LZ+4JfHXYQT9PnVftsH05rgJKKIStS7r/4XSs9HOjtPoLtgcA6HCW9jVqX5DBbq7E0PAkiQ==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.29.3",
"@smithy/types": "^4.16.1",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/types": {
"version": "4.16.1",
"resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz",
"integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==",
"license": "Apache-2.0",
"dependencies": {
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@standard-schema/spec": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz",
@@ -2957,6 +3347,12 @@
"license": "MIT",
"optional": true
},
"node_modules/bowser": {
"version": "2.14.1",
"resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz",
"integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==",
"license": "MIT"
},
"node_modules/brace-expansion": {
"version": "5.0.7",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz",
@@ -7751,7 +8147,6 @@
"version": "2.8.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
"dev": true,
"license": "0BSD"
},
"node_modules/type-fest": {
+12 -6
View File
@@ -1,6 +1,6 @@
{
"name": "ipsextractor-desktop",
"version": "1.0.0",
"version": "1.0.1",
"private": true,
"description": "Desktop extraction client for IPSExtractor microservices.",
"main": "electron/main.cjs",
@@ -10,22 +10,25 @@
"preview": "vite preview",
"test": "vitest run",
"test:watch": "vitest",
"dist": "npm run build && electron-builder --win",
"dist:win": "npm run dist"
"dist": "npm run build && electron-builder --win && npm run publish:minio",
"dist:win": "npm run dist",
"dist:local": "npm run build && electron-builder --win",
"publish:minio": "node scripts/publish-version.mjs"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.1085.0",
"lucide-react": "^0.468.0",
"react": "^18.3.1",
"react-dom": "^18.3.1"
},
"devDependencies": {
"@vitejs/plugin-react": "^4.3.4",
"@testing-library/jest-dom": "^6.6.3",
"@testing-library/react": "^16.1.0",
"@testing-library/user-event": "^14.5.2",
"@types/node": "^22.10.2",
"@types/react": "^18.3.17",
"@types/react-dom": "^18.3.5",
"@vitejs/plugin-react": "^4.3.4",
"concurrently": "^9.1.0",
"electron": "^33.2.1",
"electron-builder": "^25.1.8",
@@ -40,7 +43,8 @@
"productName": "IPSExtractor Desktop",
"artifactName": "IPSExtractor-Desktop-${version}-Setup.${ext}",
"directories": {
"output": "release"
"output": "release",
"buildResources": "resources"
},
"files": [
"dist/**/*",
@@ -48,9 +52,11 @@
"package.json"
],
"mac": {
"category": "public.app-category.productivity"
"category": "public.app-category.productivity",
"icon": "resources/icon.png"
},
"win": {
"icon": "resources/icon.ico",
"target": [
{
"target": "nsis",
Binary file not shown.

After

Width:  |  Height:  |  Size: 162 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 52 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 89 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 162 KiB

@@ -0,0 +1,188 @@
/**
* Publica el instalador NSIS en MinIO y actualiza latest.json + DATA.txt.
*
* Uso:
* set MINIO_ACCESS_KEY=...
* set MINIO_SECRET_KEY=...
* npm run publish:minio
*
* Opcional:
* MINIO_ENDPOINT=https://s3.builker.xyz
* MINIO_BUCKET=apps-versions
* MINIO_PREFIX=IPSExtractor
* RELEASE_DATE=06/07/2026 (formato DATA.txt DD/MM/YYYY; default: hoy o version.json)
*/
import { createHash } from "node:crypto";
import { createReadStream, existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
const __dirname = dirname(fileURLToPath(import.meta.url));
const root = join(__dirname, "..");
/** Carga KEY=VALUE desde un archivo local (sin dependencia dotenv). */
function loadEnvFile(filePath) {
if (!existsSync(filePath)) return;
const text = readFileSync(filePath, "utf8");
for (const line of text.split(/\r?\n/)) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith("#")) continue;
const eq = trimmed.indexOf("=");
if (eq <= 0) continue;
const key = trimmed.slice(0, eq).trim();
let value = trimmed.slice(eq + 1).trim();
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
if (!(key in process.env) || process.env[key] === "") {
process.env[key] = value;
}
}
}
loadEnvFile(join(root, ".env.minio"));
loadEnvFile(join(root, ".env"));
const pkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8"));
const version = pkg.version;
const endpoint = (process.env.MINIO_ENDPOINT || "https://s3.builker.xyz").replace(/\/$/, "");
const bucket = process.env.MINIO_BUCKET || "apps-versions";
const prefix = (process.env.MINIO_PREFIX || "IPSExtractor").replace(/^\/|\/$/g, "");
const accessKey = process.env.MINIO_ACCESS_KEY || process.env.AWS_ACCESS_KEY_ID;
const secretKey = process.env.MINIO_SECRET_KEY || process.env.AWS_SECRET_ACCESS_KEY;
if (!accessKey || !secretKey) {
console.error(
"Faltan credenciales. Define MINIO_ACCESS_KEY y MINIO_SECRET_KEY (o AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY).",
);
process.exit(1);
}
const setupName = `IPSExtractor-Desktop-${version}-Setup.exe`;
const setupPath = join(root, "release", setupName);
if (!existsSync(setupPath)) {
console.error(`No se encontró ${setupPath}. Ejecuta antes: npm run dist`);
process.exit(1);
}
function todayDisplay() {
const d = new Date();
const dd = String(d.getDate()).padStart(2, "0");
const mm = String(d.getMonth() + 1).padStart(2, "0");
const yyyy = d.getFullYear();
return `${dd}/${mm}/${yyyy}`;
}
function isoFromDisplay(display) {
// DD/MM/YYYY → YYYY-MM-DD
const m = String(display).trim().match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/);
if (!m) return null;
const [, dd, mm, yyyy] = m;
return `${yyyy}-${mm.padStart(2, "0")}-${dd.padStart(2, "0")}`;
}
function displayFromIso(iso) {
const m = String(iso).trim().match(/^(\d{4})-(\d{2})-(\d{2})$/);
if (!m) return null;
const [, yyyy, mm, dd] = m;
return `${dd}/${mm}/${yyyy}`;
}
function resolveReleaseDateDisplay() {
if (process.env.RELEASE_DATE) return String(process.env.RELEASE_DATE).trim();
try {
const meta = JSON.parse(readFileSync(join(root, "src/shared/version.json"), "utf8"));
if (meta.releaseDate) {
return displayFromIso(meta.releaseDate) || todayDisplay();
}
} catch {
/* ignore */
}
return todayDisplay();
}
async function sha256File(filePath) {
return new Promise((resolve, reject) => {
const hash = createHash("sha256");
const stream = createReadStream(filePath);
stream.on("data", (chunk) => hash.update(chunk));
stream.on("error", reject);
stream.on("end", () => resolve(hash.digest("hex")));
});
}
async function upload(client, key, body, contentType) {
await client.send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: body,
ContentType: contentType,
}),
);
console.log(` ↑ s3://${bucket}/${key}`);
}
async function main() {
const releaseDateDisplay = resolveReleaseDateDisplay();
const releaseDateIso = isoFromDisplay(releaseDateDisplay) || new Date().toISOString().slice(0, 10);
const sha256 = await sha256File(setupPath);
const { size } = statSync(setupPath);
const publicBase = `${endpoint}/${bucket}/${prefix}`;
const objectKeyExe = `${prefix}/${version}/${setupName}`;
const objectKeyData = `${prefix}/${version}/DATA.txt`;
const objectKeyLatest = `${prefix}/latest.json`;
const publicUrl = `${publicBase}/${version}/${setupName}`;
const latest = {
version,
releaseDate: releaseDateIso,
releaseDateDisplay,
fileName: setupName,
url: publicUrl,
sha256,
size,
};
// Actualiza version.json local para alinear la UI con la fecha publicada
writeFileSync(
join(root, "src/shared/version.json"),
`${JSON.stringify({ releaseDate: releaseDateIso }, null, 2)}\n`,
"utf8",
);
const client = new S3Client({
region: process.env.MINIO_REGION || "us-east-1",
endpoint,
forcePathStyle: true,
credentials: {
accessKeyId: accessKey,
secretAccessKey: secretKey,
},
});
console.log(`Publicando v${version} → ${endpoint}/${bucket}/${prefix}/`);
await upload(client, objectKeyExe, createReadStream(setupPath), "application/octet-stream");
await upload(client, objectKeyData, `${releaseDateDisplay}\n`, "text/plain; charset=utf-8");
await upload(
client,
objectKeyLatest,
Buffer.from(`${JSON.stringify(latest, null, 2)}\n`, "utf8"),
"application/json",
);
console.log("\nListo.");
console.log(` latest.json: ${publicBase}/latest.json`);
console.log(` DATA.txt: ${publicBase}/${version}/DATA.txt → ${releaseDateDisplay}`);
console.log(` Setup: ${publicUrl}`);
}
main().catch((err) => {
console.error(err);
process.exit(1);
});
+26 -141
View File
@@ -3,164 +3,49 @@ import userEvent from "@testing-library/user-event";
import { describe, expect, it, vi } from "vitest";
import App from "./App";
async function advanceToDiscover(user: ReturnType<typeof userEvent.setup>) {
await user.click(screen.getByRole("button", { name: /explorar/i }));
await user.click(screen.getByRole("button", { name: /siguiente/i }));
}
async function discoverFields(user: ReturnType<typeof userEvent.setup>) {
await advanceToDiscover(user);
await user.click(screen.getByRole("button", { name: /descubrir campos/i }));
}
describe("App", () => {
it("renders the extraction wizard", () => {
it("renders home shell with jobs and configuration entry", () => {
vi.spyOn(globalThis, "fetch").mockResolvedValue({
ok: true,
text: async () => "[]",
} as Response);
render(<App />);
expect(screen.getByText("IPSExtractor")).toBeInTheDocument();
expect(screen.getByRole("button", { name: /^extraccion$/i })).toBeInTheDocument();
expect(screen.getByRole("button", { name: /tareas/i })).toBeInTheDocument();
expect(screen.getByRole("button", { name: /configuracion/i })).toBeInTheDocument();
expect(screen.getByText("Origen de los documentos")).toBeInTheDocument();
expect(screen.queryByText("IPS_DOCUMENT_ROOT_LOCAL")).not.toBeInTheDocument();
expect(screen.queryByText("IPS_DOCUMENT_ROOT_CONTAINER")).not.toBeInTheDocument();
expect(screen.queryByText("Ruta enviada a Docker/API")).not.toBeInTheDocument();
expect(screen.getByRole("button", { name: /siguiente/i })).toBeDisabled();
expect(screen.getByRole("button", { name: /^trabajos$/i })).toBeInTheDocument();
expect(screen.getByRole("button", { name: /configuraci[oó]n/i })).toBeInTheDocument();
});
it("shows API URL settings in the configuration page", async () => {
it("opens configuration as a submenu with connection settings", async () => {
const user = userEvent.setup();
vi.spyOn(globalThis, "fetch").mockResolvedValue({
ok: true,
text: async () => "[]",
} as Response);
render(<App />);
await user.click(screen.getByRole("button", { name: /configuracion/i }));
expect(screen.getByText("Conexion API")).toBeInTheDocument();
await user.click(screen.getByRole("button", { name: /configuraci[oó]n/i }));
expect(
screen.getByRole("navigation", { name: /secciones de configuraci[oó]n/i }),
).toBeInTheDocument();
expect(screen.getByRole("button", { name: /^conexi[oó]n$/i })).toBeInTheDocument();
expect(screen.getByRole("button", { name: /^limpieza$/i })).toBeInTheDocument();
expect(screen.getByRole("button", { name: /^backend$/i })).toBeInTheDocument();
expect(screen.getByRole("heading", { name: /^conexi[oó]n$/i })).toBeInTheDocument();
const input = screen.getByLabelText("URL del API Gateway");
await user.clear(input);
await user.type(input, "http://localhost:9000");
expect(input).toHaveValue("http://localhost:9000");
});
it("adds and removes document types", async () => {
it("shows cleaning COP/MOD settings section", async () => {
const user = userEvent.setup();
render(<App />);
await user.click(screen.getByRole("button", { name: /agregar tipo/i }));
expect(screen.getByLabelText("Nombre tipo 2")).toBeInTheDocument();
const removeButtons = screen.getAllByLabelText("Eliminar tipo");
await user.click(removeButtons[1]);
expect(screen.queryByLabelText("Nombre tipo 2")).not.toBeInTheDocument();
});
it("does not show keywords in general folder mode", async () => {
const user = userEvent.setup();
render(<App />);
await user.click(screen.getByRole("button", { name: /carpeta general/i }));
expect(screen.queryByLabelText(/keywords/i)).not.toBeInTheDocument();
expect(screen.queryByText(/^Keywords$/)).not.toBeInTheDocument();
expect(screen.getByText(/se procesaran todos los pdf y tiff/i)).toBeInTheDocument();
});
it("shows schema fields from a mocked API response", async () => {
const user = userEvent.setup();
vi.spyOn(window.desktopApi!, "selectDirectory").mockResolvedValue("C:\\docs\\lote");
vi.spyOn(globalThis, "fetch").mockResolvedValue({
ok: true,
text: async () => JSON.stringify({ schema_by_type: { Factura: { numero: "str" } }, sample_by_type: { Factura: { numero: "A1" } } }),
text: async () => "[]",
} as Response);
render(<App />);
await discoverFields(user);
await user.click(screen.getByRole("button", { name: /siguiente/i }));
expect(await screen.findByRole("checkbox", { name: "Seleccionar numero" })).toBeInTheDocument();
});
it("blocks advancing while no fields are selected", async () => {
const user = userEvent.setup();
vi.spyOn(window.desktopApi!, "selectDirectory").mockResolvedValue("C:\\docs\\lote");
vi.spyOn(globalThis, "fetch").mockResolvedValue({
ok: true,
text: async () => JSON.stringify({ schema_by_type: { Factura: { numero: "str" } }, sample_by_type: { Factura: { numero: "A1" } } }),
} as Response);
render(<App />);
await discoverFields(user);
await user.click(screen.getByRole("button", { name: /siguiente/i }));
expect(screen.getByRole("button", { name: /siguiente/i })).toBeDisabled();
});
it("creates an extraction task with mocked API responses", async () => {
const user = userEvent.setup();
vi.spyOn(window.desktopApi!, "selectDirectory").mockResolvedValue("C:\\docs\\lote");
vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
const url = String(input);
if (url.includes("/extraction/schemas/upload")) {
return {
ok: true,
text: async () => JSON.stringify({ schema_by_type: { Factura: { numero: "str" } }, sample_by_type: { Factura: { numero: "A1" } } }),
} as Response;
}
if (url.includes("/tasks/extraction")) {
return {
ok: true,
text: async () => JSON.stringify({
id: "task-1",
type: "extraction",
status: "running",
title: "Extraccion desktop",
request: { source: "desktop" },
result: { extraction_batch_id: "batch-1", processed: 0, total: 1 },
error: null,
owner_id: null,
created_at: "now",
updated_at: "now",
started_at: "now",
finished_at: null,
}),
} as Response;
}
if (url.includes("/tasks/task-1/ingest")) {
return {
ok: true,
text: async () => JSON.stringify({ processed: 1, total: 1, task: { id: "task-1", status: "completed", result: { extraction_batch_id: "batch-1" } } }),
} as Response;
}
if (url.includes("/tasks/task-1") && !url.includes("/rows") && !url.includes("/ingest")) {
return {
ok: true,
text: async () => JSON.stringify({
id: "task-1",
type: "extraction",
status: "completed",
title: "Extraccion desktop",
request: { source: "desktop" },
result: { extraction_batch_id: "batch-1", processed: 1, total: 1 },
error: null,
owner_id: null,
created_at: "now",
updated_at: "now",
started_at: "now",
finished_at: "now",
}),
} as Response;
}
if (url.includes("/tasks?")) {
return {
ok: true,
text: async () => JSON.stringify([{ id: "task-1", type: "extraction", status: "completed", title: "Extraccion desktop", request: {}, result: { extraction_batch_id: "batch-1" }, error: null, owner_id: null, created_at: "now", updated_at: "now", started_at: "now", finished_at: "now" }]),
} as Response;
}
if (url.includes("/tasks/task-1/rows")) {
return {
ok: true,
text: async () => JSON.stringify({ task_id: "task-1", batch_id: "batch-1", rows: [{ radicado: "R1", numero: "A1" }], limit: 500, offset: 0 }),
} as Response;
}
return { ok: true, text: async () => "{}" } as Response;
});
render(<App />);
await discoverFields(user);
await user.click(screen.getByRole("button", { name: /siguiente/i }));
await user.click(await screen.findByRole("checkbox", { name: "Seleccionar numero" }));
await user.click(screen.getByRole("button", { name: /siguiente/i }));
await user.click(screen.getByRole("button", { name: /ejecutar extraccion/i }));
expect(await screen.findByText("Tareas y resultados")).toBeInTheDocument();
expect((await screen.findAllByText("completed")).length).toBeGreaterThan(0);
expect(await screen.findByText("A1")).toBeInTheDocument();
await user.click(screen.getByRole("button", { name: /configuraci[oó]n/i }));
await user.click(screen.getByRole("button", { name: /^limpieza$/i }));
expect(screen.getByRole("heading", { name: /limpieza.*cop/i })).toBeInTheDocument();
expect(screen.getByLabelText("2024")).toBeInTheDocument();
});
});
+104 -141
View File
@@ -1,17 +1,14 @@
import {
BarChart2,
Briefcase,
CheckCircle2,
Cpu,
Database,
Hash,
Loader2,
Server,
Link2,
Settings,
XCircle,
Sparkles,
} from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import type { ExtractionFormState, Job } from "../shared/types";
import type { Job } from "../shared/types";
import {
APP_RELEASE_DATE,
APP_VERSION,
@@ -19,14 +16,18 @@ import {
formatVersionLabel,
} from "../shared/appVersion";
import { ApiClient } from "./lib/apiClient";
import { DEFAULT_API_BASE_URL, loadApiBaseUrl, migrateApiBaseUrlInStorage } from "./lib/apiDefaults";
import { loadApiBaseUrl, migrateApiBaseUrlInStorage } from "./lib/apiDefaults";
import { HomeView } from "./views/HomeView";
import { JobView } from "./views/JobView";
import { SettingsView, type SettingsSection } from "./views/SettingsView";
import type { JobSection } from "./types/jobSection";
import appIcon from "./assets/app-icon.png";
import { UpdatePrompt, useAutoUpdateCheck } from "./components/UpdatePrompt";
// Registrar módulos al cargar la app
import "./modules/counting/CountingModule";
import "./modules/extraction/ExtractionModule";
import "./modules/cleaning/CleaningModule";
type Status = "idle" | "ok" | "error" | "busy";
type View = "home" | "job" | "settings";
@@ -34,15 +35,19 @@ export type { JobSection } from "./types/jobSection";
function App() {
const [view, setView] = useState<View>("home");
const [settingsReturnView, setSettingsReturnView] = useState<"home" | "job">("home");
const [settingsSection, setSettingsSection] = useState<SettingsSection>("connection");
const [activeJob, setActiveJob] = useState<Job | null>(null);
const [jobSection, setJobSection] = useState<JobSection>("counting");
const [jobSectionContext, setJobSectionContext] = useState<Record<string, unknown>>({});
const [apiBaseUrl, setApiBaseUrl] = useState(loadApiBaseUrl);
const [connectionStatus, setConnectionStatus] = useState<Status>("idle");
const [connectionMessage, setConnectionMessage] = useState("Sin verificar");
const [connectionMessage, setConnectionMessage] = useState("Pulsa «Verificar conexión» para comprobar el gateway.");
const [healthJson, setHealthJson] = useState<unknown>(null);
const [configJson, setConfigJson] = useState<unknown>(null);
const [modelsJson, setModelsJson] = useState<unknown>(null);
const [busy, setBusy] = useState(false);
const update = useAutoUpdateCheck();
const client = useMemo(() => new ApiClient(apiBaseUrl), [apiBaseUrl]);
@@ -54,21 +59,27 @@ function App() {
}
}, []); // eslint-disable-line react-hooks/exhaustive-deps
useEffect(() => { localStorage.setItem("apiBaseUrl", apiBaseUrl); }, [apiBaseUrl]);
useEffect(() => {
localStorage.setItem("apiBaseUrl", apiBaseUrl);
}, [apiBaseUrl]);
async function verifyConnection() {
setConnectionStatus("busy");
setConnectionMessage("Verificando gateway...");
setConnectionMessage("Comprobando gateway y servicios…");
setBusy(true);
try {
const [health, config, models] = await Promise.all([
client.health(), client.config(), client.models(),
client.health(),
client.config(),
client.models(),
]);
setHealthJson(health);
setConfigJson(config);
setModelsJson(models);
setConnectionStatus("ok");
setConnectionMessage(`Conectado: ${JSON.stringify(health)}`);
setConnectionMessage("Gateway respondió correctamente.");
} catch (err) {
setHealthJson(null);
setConnectionStatus("error");
setConnectionMessage(err instanceof Error ? err.message : "No se pudo conectar");
} finally {
@@ -88,27 +99,73 @@ function App() {
setJobSectionContext(context);
}
function openSettings(section: SettingsSection = "connection") {
setSettingsReturnView(view === "job" ? "job" : "home");
setSettingsSection(section);
setView("settings");
}
function leaveSettings() {
if (settingsReturnView === "job" && activeJob) {
setView("job");
} else {
setView("home");
}
}
const isJobOpen = view === "job" && activeJob;
const isSettingsOpen = view === "settings";
return (
<main className="app-frame">
<aside className="sidebar">
<div className="brand">
<div className="brand-mark">IP</div>
<img className="brand-mark" src={appIcon} alt="IPSExtractor" />
<div className="brand-text">
<strong>IPSExtractor</strong>
<span>Desktop</span>
</div>
</div>
{isJobOpen ? (
{isSettingsOpen ? (
/* ── Submenú de configuración ── */
<>
<button type="button" className="side-job-back" onClick={leaveSettings}>
← {settingsReturnView === "job" && activeJob ? "Trabajo" : "Trabajos"}
</button>
<div className="side-job-name">Configuración</div>
<div className="side-section-divider" />
<nav className="side-nav" aria-label="Secciones de configuración">
<button
type="button"
className={settingsSection === "connection" ? "active" : ""}
onClick={() => setSettingsSection("connection")}
>
<Link2 size={17} />
<span>Conexión</span>
</button>
<button
type="button"
className={settingsSection === "cleaning" ? "active" : ""}
onClick={() => setSettingsSection("cleaning")}
>
<Sparkles size={17} />
<span>Limpieza</span>
</button>
<button
type="button"
className={settingsSection === "backend" ? "active" : ""}
onClick={() => setSettingsSection("backend")}
>
<Cpu size={17} />
<span>Backend</span>
</button>
</nav>
</>
) : isJobOpen ? (
/* ── Submenú del job activo ── */
<>
<button
type="button"
className="side-job-back"
onClick={() => setView("home")}
>
<button type="button" className="side-job-back" onClick={() => setView("home")}>
← Trabajos
</button>
<div className="side-job-name" title={activeJob.title ?? ""}>
@@ -132,6 +189,14 @@ function App() {
<Cpu size={17} />
<span>Extracción</span>
</button>
<button
type="button"
className={jobSection === "cleaning" ? "active" : ""}
onClick={() => handleActivateSection("cleaning")}
>
<Sparkles size={17} />
<span>Limpieza</span>
</button>
<button
type="button"
className={jobSection === "results" ? "active" : ""}
@@ -141,12 +206,9 @@ function App() {
<span>Resultados</span>
</button>
</nav>
<div className="side-section-divider" />
<nav className="side-nav side-nav--bottom">
<button
type="button"
onClick={() => setView("settings")}
>
<div className="side-section-divider" />
<button type="button" onClick={() => openSettings()}>
<Settings size={17} />
<span>Configuración</span>
</button>
@@ -163,11 +225,7 @@ function App() {
<Briefcase size={18} />
<span>Trabajos</span>
</button>
<button
type="button"
className={view === "settings" ? "active" : ""}
onClick={() => setView("settings")}
>
<button type="button" onClick={() => openSettings()}>
<Settings size={18} />
<span>Configuración</span>
</button>
@@ -194,14 +252,12 @@ function App() {
</div>
</aside>
<section className="workspace workspace--full">
<section className={`workspace workspace--full${isSettingsOpen ? " workspace--settings" : ""}`}>
<header className="app-topbar">
<span className="app-version-badge">{formatVersionLabel()}</span>
</header>
{view === "home" && (
<HomeView client={client} onOpenJob={openJob} />
)}
{view === "home" && <HomeView client={client} onOpenJob={openJob} />}
{view === "job" && activeJob && (
<JobView
@@ -216,10 +272,12 @@ function App() {
{view === "settings" && (
<SettingsView
section={settingsSection}
apiBaseUrl={apiBaseUrl}
onApiBaseUrlChange={setApiBaseUrl}
connectionStatus={connectionStatus}
connectionMessage={connectionMessage}
healthJson={healthJson}
verifyConnection={verifyConnection}
busy={busy}
configJson={configJson}
@@ -227,116 +285,21 @@ function App() {
/>
)}
</section>
{update.manifest && (
<UpdatePrompt
open={update.open}
currentVersion={update.currentVersion}
manifest={update.manifest}
onClose={update.close}
onConfirm={() => void update.confirmInstall()}
installing={update.installing}
progress={update.progress}
error={update.error}
/>
)}
</main>
);
}
type SettingsTab = "config" | "models";
function SettingsView({
apiBaseUrl,
onApiBaseUrlChange,
connectionStatus,
connectionMessage,
verifyConnection,
busy,
configJson,
modelsJson,
}: {
apiBaseUrl: string;
onApiBaseUrlChange: (url: string) => void;
connectionStatus: Status;
connectionMessage: string;
verifyConnection: () => Promise<void>;
busy: boolean;
configJson: unknown;
modelsJson: unknown;
}) {
const [backendTab, setBackendTab] = useState<SettingsTab>("config");
const connectionLabel =
connectionStatus === "ok"
? "Conectado"
: connectionStatus === "busy"
? "Verificando..."
: connectionStatus === "error"
? "Sin conexión"
: "Sin verificar";
const backendData = backendTab === "config" ? configJson : modelsJson;
return (
<div className="settings-layout">
<aside className="settings-sidebar">
<div className="settings-sidebar-header"><h2>Conexión API</h2></div>
<div className="settings-sidebar-body">
<div className={`settings-status-card settings-status-card--${connectionStatus}`}>
<div className={`settings-status-icon settings-status-icon--${connectionStatus}`}>
{connectionStatus === "ok" ? <CheckCircle2 size={20} />
: connectionStatus === "error" ? <XCircle size={20} />
: connectionStatus === "busy" ? <Loader2 size={20} className="spin" />
: <Server size={20} />}
</div>
<div className="settings-status-text">
<strong>{connectionLabel}</strong>
<span>{connectionMessage}</span>
</div>
</div>
<div className="settings-field">
<label className="wiz-field-label" htmlFor="api-base-url">URL del API Gateway</label>
<input
id="api-base-url"
value={apiBaseUrl}
onChange={(e) => onApiBaseUrlChange(e.target.value)}
placeholder={DEFAULT_API_BASE_URL}
/>
<p className="wiz-hint">Los cambios se guardan automáticamente.</p>
</div>
<button type="button" className="btn btn-primary" onClick={verifyConnection} disabled={busy}>
<Server size={15} />
{busy ? "Verificando..." : "Verificar conexión"}
</button>
</div>
</aside>
<div className="settings-main">
<div className="settings-main-header">
<div className="settings-main-title">
<Database size={16} />
<h3>Configuración del backend</h3>
</div>
<div className="settings-tabs" role="tablist">
<button
type="button" role="tab"
aria-selected={backendTab === "config"}
className={backendTab === "config" ? "active" : ""}
onClick={() => setBackendTab("config")}
>Configuración</button>
<button
type="button" role="tab"
aria-selected={backendTab === "models"}
className={backendTab === "models" ? "active" : ""}
onClick={() => setBackendTab("models")}
>Modelos</button>
</div>
</div>
<div className="settings-json-area">
{backendData != null ? (
<pre className="settings-json-box">{JSON.stringify(backendData, null, 2)}</pre>
) : (
<div className="settings-empty">
<Server size={36} strokeWidth={1.2} />
<p>Sin datos del backend</p>
<span>Verifica la conexión para cargar la {backendTab === "config" ? "configuración" : "lista de modelos"}.</span>
</div>
)}
</div>
</div>
</div>
);
}
export default App;
Binary file not shown.

After

Width:  |  Height:  |  Size: 56 KiB

@@ -13,7 +13,10 @@ function pct(processed: number, total: number): number {
export function BackgroundProgressBanner({ progress, compact = false }: Props) {
const percent = pct(progress.processed, progress.total);
const moduleLabel = progress.module === "counting" ? "Conteo" : "Extracción";
const moduleLabel =
progress.module === "counting" ? "Conteo"
: progress.module === "cleaning" ? "Limpieza"
: "Extracción";
return (
<div className={`bg-progress-banner${compact ? " bg-progress-banner--compact" : ""}`}>
@@ -0,0 +1,44 @@
import { Check } from "lucide-react";
export type CleaningStep = 1 | 2 | 3;
const STEPS: Array<{ id: CleaningStep; label: string }> = [
{ id: 1, label: "Fuente" },
{ id: 2, label: "Reglas" },
{ id: 3, label: "Resultados" },
];
interface CleaningStepperProps {
current: CleaningStep;
maxReached: CleaningStep;
onStepClick?: (step: CleaningStep) => void;
}
export function CleaningStepper({ current, maxReached, onStepClick }: CleaningStepperProps) {
return (
<nav className="stepper" aria-label="Pasos de limpieza">
{STEPS.map((step, index) => {
const isDone = step.id < current;
const isActive = step.id === current;
const isClickable = step.id <= maxReached && !isActive;
const state = isDone ? "done" : isActive ? "active" : step.id <= maxReached ? "available" : "todo";
return (
<div className="stepper-item" key={step.id}>
<button
type="button"
className={`stepper-dot ${state}`}
disabled={!isClickable}
onClick={() => isClickable && onStepClick?.(step.id)}
aria-current={isActive ? "step" : undefined}
>
{isDone ? <Check size={14} strokeWidth={3} /> : step.id}
</button>
<span className={`stepper-label ${state}`}>{step.label}</span>
{index < STEPS.length - 1 && <div className={`stepper-connector ${isDone ? "done" : ""}`} />}
</div>
);
})}
</nav>
);
}
@@ -24,6 +24,7 @@ import type {
Task,
} from "../../shared/types";
import { ApiClient } from "../lib/apiClient";
import { downloadCsv, toCsv, buildExportCsvFilename } from "../lib/csv";
import { CountResultsTable } from "./CountResultsTable";
import { CountingStepper, type CountingStep } from "./CountingStepper";
@@ -377,25 +378,16 @@ export function CountingWizard({ client, localRoot, containerRoot, job, onStartE
function handleExportCsv() {
if (!savedFiles.length) return;
const headers = ["filename", "radicado", "service_count", "auto_ajustado", "status", "is_valid", "error"];
const rows = savedFiles.map((f) =>
[
`"${f.filename}"`,
`"${f.radicado ?? ""}"`,
f.service_count ?? "",
autoAdjustedFilenames.has(f.filename) ? "si" : "",
f.status,
f.is_valid === null ? "" : String(f.is_valid),
`"${(f.error ?? "").replace(/"/g, '""')}"`,
].join(","),
);
const csv = [headers.join(","), ...rows].join("\n");
const blob = new Blob([csv], { type: "text/csv" });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = `conteo-${new Date().toISOString().slice(0, 10)}.csv`;
a.click();
URL.revokeObjectURL(url);
const rows = savedFiles.map((f) => [
f.filename,
f.radicado ?? "",
f.service_count ?? "",
autoAdjustedFilenames.has(f.filename) ? "si" : "",
f.status,
f.is_valid === null ? "" : String(f.is_valid),
f.error ?? "",
]);
downloadCsv(buildExportCsvFilename("conteo", job?.title), toCsv(headers, rows));
}
function resetWizard() {
@@ -0,0 +1,147 @@
import { Search } from "lucide-react";
import { useMemo, useState } from "react";
type Row = Record<string, unknown>;
type FilterKey = "all" | "complete" | "incomplete";
interface Props {
rows: Row[];
columns: string[];
}
function cellValue(row: Row, col: string): string {
const v = row[col];
if (v === null || v === undefined) return "";
return String(v);
}
function isEmpty(row: Row, col: string): boolean {
return cellValue(row, col).trim() === "";
}
function isComplete(row: Row, columns: string[]): boolean {
return columns.every((c) => !isEmpty(row, c));
}
export function ExtractionResultsTable({ rows, columns }: Props) {
const [filter, setFilter] = useState<FilterKey>("all");
const [search, setSearch] = useState("");
const summary = useMemo(() => {
const complete = rows.filter((r) => isComplete(r, columns)).length;
return {
total: rows.length,
complete,
incomplete: rows.length - complete,
};
}, [rows, columns]);
const filtered = useMemo(() => {
let list = rows;
if (filter === "complete") list = list.filter((r) => isComplete(r, columns));
else if (filter === "incomplete") list = list.filter((r) => !isComplete(r, columns));
const q = search.trim().toLowerCase();
if (q) {
list = list.filter((r) =>
columns.some((c) => cellValue(r, c).toLowerCase().includes(q)),
);
}
return list;
}, [rows, columns, filter, search]);
const FILTERS: { key: FilterKey; label: string; count: number }[] = [
{ key: "all", label: "Todos", count: summary.total },
{ key: "complete", label: "Completos", count: summary.complete },
{ key: "incomplete", label: "Incompletos", count: summary.incomplete },
];
return (
<div className="extraction-results">
<div className="wiz-stats-row count-results-summary">
<div className="wiz-stat">
<span className="wiz-stat-value">{summary.total}</span>
<span className="wiz-stat-label">Total</span>
</div>
<div className="wiz-stat count-stat--valid">
<span className="wiz-stat-value">{summary.complete}</span>
<span className="wiz-stat-label">Completos</span>
</div>
<div className="wiz-stat count-stat--invalid">
<span className="wiz-stat-value">{summary.incomplete}</span>
<span className="wiz-stat-label">Incompletos</span>
</div>
</div>
<div className="count-toolbar">
<div className="count-filter-tabs">
{FILTERS.map(({ key, label, count }) => (
<button
key={key}
type="button"
className={`count-filter-tab ${filter === key ? "active" : ""}`}
onClick={() => setFilter(key)}
>
{label}
<span className="count-filter-count">{count}</span>
</button>
))}
</div>
<div className="count-toolbar-right">
<div className="wiz-search-box">
<Search size={14} />
<input
placeholder="Buscar en filas…"
value={search}
onChange={(e) => setSearch(e.target.value)}
/>
</div>
</div>
</div>
<div className="results-table-wrapper extraction-table-wrap">
<table className="results-table">
<thead>
<tr>
{columns.map((c) => (
<th key={c}>{c}</th>
))}
</tr>
</thead>
<tbody>
{filtered.length === 0 ? (
<tr>
<td colSpan={Math.max(columns.length, 1)} className="extraction-empty">
{search
? "No hay filas que coincidan con la búsqueda."
: "No hay filas en esta categoría."}
</td>
</tr>
) : (
filtered.map((row, i) => (
<tr key={i} className={!isComplete(row, columns) ? "extraction-row--incomplete" : undefined}>
{columns.map((c) => (
<td key={c}>
{isEmpty(row, c) ? (
<span className="extraction-empty-cell">—</span>
) : (
cellValue(row, c)
)}
</td>
))}
</tr>
))
)}
</tbody>
</table>
</div>
{filtered.length !== rows.length && (
<p className="extraction-filter-hint">
Mostrando {filtered.length} de {rows.length} filas
</p>
)}
</div>
);
}
@@ -15,6 +15,7 @@ import {
RefreshCw,
Search,
Settings2,
Shuffle,
Trash2,
Zap,
} from "lucide-react";
@@ -36,6 +37,9 @@ interface ExtractionWizardProps {
statusText: string;
selectRoot: () => Promise<void>;
discoverSchema: () => Promise<void>;
rediscoverWithOtherSample?: () => Promise<void>;
discoverySampleLabels?: string[];
canRediscoverWithOtherSample?: boolean;
estimate: () => Promise<void>;
runExtraction: () => Promise<void>;
selection: FieldSelection;
@@ -104,6 +108,9 @@ export function ExtractionWizard({
statusText,
selectRoot,
discoverSchema,
rediscoverWithOtherSample,
discoverySampleLabels = [],
canRediscoverWithOtherSample = false,
estimate,
runExtraction,
selection,
@@ -373,7 +380,39 @@ export function ExtractionWizard({
{fieldCount > 0 && (
<div className="wiz-success-msg">
<Check size={16} />
{fieldCount} campos detectados. Continua al siguiente paso para elegir cuales exportar.
{fieldCount} campos detectados
{discoverySampleLabels.length > 0
? ` con «${discoverySampleLabels.join(", ")}».`
: "."}{" "}
Continúa al siguiente paso para elegir cuáles exportar.
</div>
)}
{schemaResponse && rediscoverWithOtherSample && (
<div className="wiz-rediscover-card">
<div className="wiz-rediscover-text">
<strong>¿Algo no salió como esperabas?</strong>
<span>
Vuelve a descubrir los campos usando otro archivo aleatorio de la carpeta.
{discoverySampleLabels.length > 0 && (
<> Muestra actual: <em>{discoverySampleLabels.join(", ")}</em>.</>
)}
</span>
</div>
<button
type="button"
className="btn btn-sm"
onClick={() => void rediscoverWithOtherSample()}
disabled={busy || !canRediscoverWithOtherSample}
title={
canRediscoverWithOtherSample
? "Descubrir con otro archivo"
: "Solo hay un documento en la carpeta"
}
>
{busy ? <Loader2 className="spin" size={14} /> : <Shuffle size={14} />}
Probar con otro archivo
</button>
</div>
)}
</div>
@@ -424,6 +463,34 @@ export function ExtractionWizard({
/>
</div>
<div className="wiz-split-aside">
{schemaResponse && rediscoverWithOtherSample && (
<div className="wiz-rediscover-card wiz-rediscover-card--compact">
<div className="wiz-rediscover-text">
<strong>¿Algo no salió como esperabas?</strong>
<span>
Reintenta el descubrimiento con otro archivo aleatorio
{discoverySampleLabels.length > 0 && (
<> (ahora: <em>{discoverySampleLabels.join(", ")}</em>)</>
)}
.
</span>
</div>
<button
type="button"
className="btn btn-sm"
onClick={() => void rediscoverWithOtherSample()}
disabled={busy || !canRediscoverWithOtherSample}
title={
canRediscoverWithOtherSample
? "Descubrir con otro archivo"
: "Solo hay un documento en la carpeta"
}
>
{busy ? <Loader2 className="spin" size={14} /> : <Shuffle size={14} />}
Probar con otro archivo
</button>
</div>
)}
<h3 className="wiz-section-title">
Vista previa
{selection.fields.length > 0 && (
@@ -0,0 +1,172 @@
import { Download, Loader2, X } from "lucide-react";
import { useEffect, useState } from "react";
import type { UpdateManifest, UpdateProgress } from "../../shared/types";
type Props = {
open: boolean;
currentVersion: string;
manifest: UpdateManifest;
onClose: () => void;
onConfirm: () => void;
installing: boolean;
progress: UpdateProgress | null;
error: string | null;
};
function formatBytes(n: number | undefined): string {
if (!n || n <= 0) return "";
if (n < 1024) return `${n} B`;
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
return `${(n / (1024 * 1024)).toFixed(1)} MB`;
}
export function UpdatePrompt({
open,
currentVersion,
manifest,
onClose,
onConfirm,
installing,
progress,
error,
}: Props) {
if (!open) return null;
const dateLabel = manifest.releaseDateDisplay || manifest.releaseDate || "";
const percent = progress?.percent ?? 0;
const phaseLabel =
progress?.phase === "verifying"
? "Verificando archivo…"
: progress?.phase === "installing"
? "Iniciando instalador…"
: progress?.phase === "downloading"
? `Descargando… ${percent}%`
: null;
return (
<div className="update-modal-backdrop" role="presentation">
<div className="update-modal" role="dialog" aria-labelledby="update-modal-title">
<div className="update-modal-header">
<h2 id="update-modal-title">Nueva versión disponible</h2>
{!installing && (
<button type="button" className="btn btn-sm btn-ghost" onClick={onClose} aria-label="Cerrar">
<X size={16} />
</button>
)}
</div>
<p className="update-modal-body">
Hay una versión nueva: <strong>v{manifest.version}</strong>
{dateLabel ? <> (lanzamiento {dateLabel})</> : null}.
<br />
Tienes instalada la <strong>v{currentVersion}</strong>.
</p>
{manifest.size != null && manifest.size > 0 && (
<p className="form-hint">Tamaño aproximado: {formatBytes(manifest.size)}</p>
)}
{installing && (
<div className="update-progress">
<div className="update-progress-bar">
<div
className="update-progress-fill"
style={{ width: `${progress?.phase === "downloading" ? percent : 100}%` }}
/>
</div>
<span className="form-hint">
{phaseLabel || "Preparando…"}
{progress?.phase === "downloading" && progress.total
? ` · ${formatBytes(progress.received)} / ${formatBytes(progress.total)}`
: ""}
</span>
</div>
)}
{error && (
<div className="wiz-toast wiz-toast--error" style={{ marginTop: 12 }}>
{error}
</div>
)}
<div className="update-modal-actions">
<button type="button" className="btn" onClick={onClose} disabled={installing}>
Ahora no
</button>
<button type="button" className="btn btn-primary" onClick={onConfirm} disabled={installing}>
{installing ? <Loader2 size={15} className="spin" /> : <Download size={15} />}
{installing ? "Instalando…" : "Descargar e instalar"}
</button>
</div>
</div>
</div>
);
}
/** Comprueba actualizaciones al montar (solo si existe desktopApi). */
export function useAutoUpdateCheck() {
const [open, setOpen] = useState(false);
const [currentVersion, setCurrentVersion] = useState("");
const [manifest, setManifest] = useState<UpdateManifest | null>(null);
const [installing, setInstalling] = useState(false);
const [progress, setProgress] = useState<UpdateProgress | null>(null);
const [error, setError] = useState<string | null>(null);
useEffect(() => {
const api = window.desktopApi;
if (!api?.checkForUpdates) return;
let cancelled = false;
const timer = window.setTimeout(() => {
void (async () => {
try {
const result = await api.checkForUpdates();
if (cancelled || !result.updateAvailable || !result.manifest) return;
setCurrentVersion(result.currentVersion);
setManifest(result.manifest);
setOpen(true);
} catch {
/* silencioso */
}
})();
}, 2000);
return () => {
cancelled = true;
window.clearTimeout(timer);
};
}, []);
useEffect(() => {
const api = window.desktopApi;
if (!api?.onUpdateProgress || !installing) return;
return api.onUpdateProgress((p) => setProgress(p));
}, [installing]);
async function confirmInstall() {
if (!manifest || !window.desktopApi?.installUpdate) return;
setInstalling(true);
setError(null);
setProgress({ phase: "downloading", percent: 0 });
try {
await window.desktopApi.installUpdate(manifest);
// La app debería cerrarse desde main; si no, mostramos mensaje.
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
setInstalling(false);
}
}
return {
open,
currentVersion,
manifest,
installing,
progress,
error,
close: () => {
if (!installing) setOpen(false);
},
confirmInstall,
};
}
@@ -1,4 +1,10 @@
import type {
CleaningAnalysisResponse,
CleaningJob,
CleaningPreviewPayload,
CleaningPreviewResponse,
CleaningRowsResponse,
CleaningRuleInfo,
CountJobFile,
CountSyncResponse,
CreateCountJobPayload,
@@ -7,6 +13,7 @@ import type {
ExtractionFromCountPayload,
Job,
RowsResponse,
RunCleaningPayload,
SchemaResponse,
Task,
UpdateJobPayload,
@@ -51,7 +58,9 @@ export class ApiClient {
return (
path.startsWith("/count/") ||
path.includes("/ingest") ||
path.includes("/upload")
path.includes("/upload") ||
path.startsWith("/cleaning/analyze-csv") ||
path.startsWith("/cleaning/jobs/from-csv")
);
}
@@ -218,6 +227,51 @@ export class ApiClient {
return this.postJson<Task>("/tasks/extraction/from-count", payload);
}
// ── Cleaning ───────────────────────────────────────────────────────────────
async cleaningRules(): Promise<CleaningRuleInfo[]> {
return this.getJson<CleaningRuleInfo[]>("/cleaning/rules");
}
async analyzeCleaningBatch(batchId: string): Promise<CleaningAnalysisResponse> {
return this.getJson<CleaningAnalysisResponse>(
`/cleaning/batches/${encodeURIComponent(batchId)}/analysis`,
);
}
async analyzeCleaningCsv(file: File | Blob, filename = "upload.csv"): Promise<CleaningAnalysisResponse> {
const form = new FormData();
form.append("file", file, filename);
return this.postForm<CleaningAnalysisResponse>("/cleaning/analyze-csv", form);
}
async runCleaning(payload: RunCleaningPayload): Promise<CleaningJob> {
return this.postJson<CleaningJob>("/cleaning/jobs", payload);
}
async previewCleaning(payload: CleaningPreviewPayload): Promise<CleaningPreviewResponse> {
return this.postJson<CleaningPreviewResponse>("/cleaning/preview", payload);
}
async runCleaningFromCsv(form: FormData): Promise<CleaningJob> {
return this.postForm<CleaningJob>("/cleaning/jobs/from-csv", form);
}
async cleaningJobs(jobId?: string): Promise<CleaningJob[]> {
const qs = jobId ? `?job_id=${encodeURIComponent(jobId)}` : "";
return this.getJson<CleaningJob[]>(`/cleaning/jobs${qs}`);
}
async cleaningJob(cleaningJobId: string): Promise<CleaningJob> {
return this.getJson<CleaningJob>(`/cleaning/jobs/${encodeURIComponent(cleaningJobId)}`);
}
async cleaningJobRows(cleaningJobId: string, limit = 500, offset = 0): Promise<CleaningRowsResponse> {
return this.getJson<CleaningRowsResponse>(
`/cleaning/jobs/${encodeURIComponent(cleaningJobId)}/rows?limit=${limit}&offset=${offset}`,
);
}
private async handle<T>(response: Response): Promise<T> {
const text = await response.text();
let data: unknown = null;
@@ -0,0 +1,32 @@
import { describe, expect, it, beforeEach } from "vitest";
import {
defaultCopModConfig,
loadCopModConfig,
saveCopModConfig,
CLEANING_COPMOD_STORAGE_KEY,
} from "./cleaningSettings";
describe("cleaningSettings", () => {
beforeEach(() => {
localStorage.removeItem(CLEANING_COPMOD_STORAGE_KEY);
});
it("returns defaults when empty", () => {
const cfg = loadCopModConfig();
expect(cfg.moderadora_by_year["2024"]).toEqual([4500, 18200, 47700]);
});
it("persists and reloads custom years", () => {
saveCopModConfig({
moderadora_by_year: { "2030": [1000, 2000] },
});
const cfg = loadCopModConfig();
expect(cfg.moderadora_by_year).toEqual({ "2030": [1000, 2000] });
});
it("defaultCopModConfig is a deep copy", () => {
const a = defaultCopModConfig();
a.moderadora_by_year["2024"].push(999);
expect(defaultCopModConfig().moderadora_by_year["2024"]).not.toContain(999);
});
});
@@ -0,0 +1,66 @@
/** Preferencias locales de limpieza (COP/MOD). */
export const CLEANING_COPMOD_STORAGE_KEY = "cleaningCopModConfig";
/** Valores por defecto alineados con el backend (`DEFAULT_MODERADORA_BY_YEAR`). */
export const DEFAULT_MODERADORA_BY_YEAR: Record<string, number[]> = {
"2023": [4100, 16400, 43000],
"2024": [4500, 18200, 47700],
"2025": [4700, 19200, 50300],
"2026": [5000, 20100, 52800],
};
export type CopModConfig = {
moderadora_by_year: Record<string, number[]>;
};
export function defaultCopModConfig(): CopModConfig {
return {
moderadora_by_year: Object.fromEntries(
Object.entries(DEFAULT_MODERADORA_BY_YEAR).map(([y, vals]) => [y, [...vals]]),
),
};
}
function normalizeYearMap(raw: unknown): Record<string, number[]> {
if (!raw || typeof raw !== "object") return {};
const out: Record<string, number[]> = {};
for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {
const year = String(Number.parseInt(k, 10));
if (!Number.isFinite(Number(year)) || year === "NaN") continue;
if (!Array.isArray(v)) continue;
const nums = v
.map((x) => Math.round(Number(x)))
.filter((n) => Number.isFinite(n) && n >= 0);
if (nums.length > 0) {
out[year] = [...new Set(nums)].sort((a, b) => a - b);
}
}
return out;
}
export function loadCopModConfig(): CopModConfig {
try {
const raw = localStorage.getItem(CLEANING_COPMOD_STORAGE_KEY);
if (!raw) return defaultCopModConfig();
const parsed = JSON.parse(raw) as { moderadora_by_year?: unknown };
const years = normalizeYearMap(parsed?.moderadora_by_year);
if (Object.keys(years).length === 0) return defaultCopModConfig();
return { moderadora_by_year: years };
} catch {
return defaultCopModConfig();
}
}
export function saveCopModConfig(config: CopModConfig): void {
const years = normalizeYearMap(config.moderadora_by_year);
localStorage.setItem(
CLEANING_COPMOD_STORAGE_KEY,
JSON.stringify({ moderadora_by_year: years }),
);
}
/** Payload listo para enviar al API de cleaning. */
export function copModConfigForApi(config?: CopModConfig | null): CopModConfig {
return config ?? loadCopModConfig();
}
@@ -1,18 +1,40 @@
import { describe, expect, it } from "vitest";
import { rowsToCsv } from "./csv";
import {
buildExportCsvFilename,
csvEscape,
rowsToCsv,
sanitizeFilenamePart,
toCsv,
} from "./csv";
describe("csv export", () => {
it("escapes commas, quotes and newlines", () => {
it("usa pipe como separador", () => {
const csv = toCsv(["a", "b"], [["1", "2"], ["3", "4"]]);
expect(csv).toBe("a|b\n1|2\n3|4");
});
it("escapa pipes, comillas y saltos de línea", () => {
const csv = rowsToCsv(
[
{
radicado: "R1",
nombre: 'Ana, "IPS"',
nombre: 'Ana| "IPS"',
nota: "linea 1\nlinea 2",
},
],
{ nombre_original: "nombre" },
);
expect(csv).toContain('R1,"Ana, ""IPS""","linea 1\nlinea 2"');
expect(csv).toContain('R1|"Ana| ""IPS"""|"linea 1\nlinea 2"');
});
it("no escapa comas (no son el delimitador)", () => {
expect(csvEscape("Ana, IPS")).toBe("Ana, IPS");
});
it("incluye el título del trabajo en el nombre de descarga", () => {
expect(sanitizeFilenamePart("Lote Abril / 2026")).toBe("Lote-Abril-2026");
expect(buildExportCsvFilename("limpieza", "Lote Abril", "validos")).toMatch(
/^limpieza-Lote-Abril-validos-\d{4}-\d{2}-\d{2}\.csv$/,
);
});
});
+56 -6
View File
@@ -1,4 +1,6 @@
const NEEDS_QUOTES = /[",\r\n]/;
export const CSV_DELIMITER = "|";
const NEEDS_QUOTES = /["|\r\n]/;
export function csvEscape(value: unknown): string {
if (value === null || value === undefined) return "";
@@ -7,13 +9,21 @@ export function csvEscape(value: unknown): string {
return `"${text.replace(/"/g, '""')}"`;
}
/** Serializa encabezado + filas con separador pipe (`|`). */
export function toCsv(headers: string[], rows: Array<Array<unknown>>): string {
const lines = [
headers.map(csvEscape).join(CSV_DELIMITER),
...rows.map((row) => row.map(csvEscape).join(CSV_DELIMITER)),
];
return lines.join("\n");
}
export function rowsToCsv(rows: Array<Record<string, unknown>>, aliasMap: Record<string, string> = {}): string {
const header = inferHeader(rows, aliasMap);
const lines = [header.map(csvEscape).join(",")];
for (const row of rows) {
lines.push(header.map((column) => csvEscape(row[column])).join(","));
}
return `${lines.join("\r\n")}\r\n`;
return toCsv(
header,
rows.map((row) => header.map((column) => row[column])),
);
}
export function inferHeader(rows: Array<Record<string, unknown>>, aliasMap: Record<string, string> = {}): string[] {
@@ -38,3 +48,43 @@ export function inferHeader(rows: Array<Record<string, unknown>>, aliasMap: Reco
}
return header;
}
export function downloadCsv(filename: string, csv: string): void {
const blob = new Blob([csv], { type: "text/csv;charset=utf-8" });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = filename;
a.click();
URL.revokeObjectURL(url);
}
/** Normaliza el título del trabajo para usarlo en nombres de archivo. */
export function sanitizeFilenamePart(value: string | null | undefined, fallback = "sin-titulo"): string {
const cleaned = String(value ?? "")
.normalize("NFKD")
.replace(/[\u0300-\u036f]/g, "")
.replace(/[^a-zA-Z0-9._-]+/g, "-")
.replace(/-+/g, "-")
.replace(/^-|-$/g, "")
.slice(0, 80);
return cleaned || fallback;
}
/**
* Nombre de descarga: `{servicio}-{titulo-trabajo}[-{extra}]-{YYYY-MM-DD}.csv`
* Ej.: `limpieza-lote-abril-2026-07-10.csv`
*/
export function buildExportCsvFilename(
service: string,
jobTitle?: string | null,
extra?: string | null,
): string {
const parts = [
sanitizeFilenamePart(service, "export"),
sanitizeFilenamePart(jobTitle, "sin-titulo"),
];
if (extra) parts.push(sanitizeFilenamePart(extra));
parts.push(new Date().toISOString().slice(0, 10));
return `${parts.join("-")}.csv`;
}
@@ -0,0 +1,18 @@
import { CleaningWizardModule } from "./CleaningWizardModule";
import { CleaningResults } from "./CleaningResults";
import type { ModuleDescriptor } from "../registry";
import { registerModule } from "../registry";
import type { Task } from "../../../shared/types";
const CleaningModule: ModuleDescriptor = {
type: "cleaning",
label: "Limpieza",
iconName: "Sparkles",
hasTask: (tasks: Task[]) => tasks.some((t) => t.type === "cleaning"),
Wizard: CleaningWizardModule,
Results: CleaningResults,
};
registerModule(CleaningModule);
export { CleaningModule };
@@ -0,0 +1,125 @@
import { Download, Loader2, RefreshCw, Sparkles } from "lucide-react";
import { useEffect, useState } from "react";
import type { CleaningJob } from "../../../shared/types";
import { ExtractionResultsTable } from "../../components/ExtractionResultsTable";
import { buildExportCsvFilename, downloadCsv, toCsv } from "../../lib/csv";
import type { ModuleResultsProps } from "../registry";
type Row = Record<string, unknown>;
export function CleaningResults({ client, task, job }: ModuleResultsProps) {
const [cleaningJob, setCleaningJob] = useState<CleaningJob | null>(null);
const [rows, setRows] = useState<Row[]>([]);
const [columns, setColumns] = useState<string[]>([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
async function load() {
try {
const cleaningJobId = String(
(task.result as Record<string, unknown>)?.cleaning_job_id ?? "",
);
if (!cleaningJobId) {
setError("La tarea no tiene cleaning_job_id en result");
return;
}
const jobData = await client.cleaningJob(cleaningJobId);
setCleaningJob(jobData);
if (jobData.status === "completed") {
const response = await client.cleaningJobRows(cleaningJobId, 500, 0);
const data = response.rows ?? [];
setRows(data);
setColumns(Array.from(new Set(data.flatMap((r) => Object.keys(r)))));
}
} catch (e) {
setError(String(e instanceof Error ? e.message : e));
} finally {
setLoading(false);
}
}
useEffect(() => {
void load();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [task.id]);
function exportCsv() {
const csv = toCsv(
columns,
rows.map((r) => columns.map((h) => r[h])),
);
downloadCsv(buildExportCsvFilename("limpieza", job.title), csv);
}
if (loading) {
return (
<div className="module-loading">
<Loader2 size={22} className="spin" />
<span>Cargando resultados de limpieza…</span>
</div>
);
}
if (error) {
return (
<div className="wiz-toast wiz-toast--error" style={{ margin: 16 }}>
{error}
<button
className="btn btn-sm"
onClick={() => {
setError(null);
setLoading(true);
void load();
}}
>
<RefreshCw size={13} /> Reintentar
</button>
</div>
);
}
return (
<div className="module-results">
<div className="module-results-header">
<div className="module-results-title">
<h3>
<Sparkles size={14} style={{ marginRight: 6, verticalAlign: "middle" }} />
{task.title ?? "Limpieza"}
</h3>
<span className={`task-badge task-badge--${cleaningJob?.status ?? task.status}`}>
{cleaningJob?.status ?? task.status}
</span>
{cleaningJob?.source && (
<span style={{ fontSize: 12, color: "var(--text-muted)" }}>
· {cleaningJob.source === "csv" ? "CSV" : "extracción"}
</span>
)}
</div>
{rows.length > 0 && (
<button type="button" className="btn btn-sm btn-ghost" onClick={exportCsv}>
<Download size={13} /> Descargar CSV ({rows.length} filas)
</button>
)}
</div>
{cleaningJob?.log && cleaningJob.log.length > 0 && (
<details style={{ margin: "0 0 12px", fontSize: 12 }}>
<summary style={{ cursor: "pointer" }}>Log ({cleaningJob.log.length})</summary>
<ul style={{ margin: "8px 0 0", paddingLeft: 18, color: "var(--text-muted)" }}>
{cleaningJob.log.map((line, i) => (
<li key={i}>{line}</li>
))}
</ul>
</details>
)}
{rows.length === 0 ? (
<div className="wiz-step wiz-step-centered" style={{ padding: 32 }}>
<p className="wiz-desc">No se encontraron filas limpiadas.</p>
</div>
) : (
<ExtractionResultsTable rows={rows} columns={columns} />
)}
</div>
);
}
@@ -0,0 +1,967 @@
import {
AlertCircle,
ArrowRight,
CheckCircle2,
Cpu,
Download,
FileSpreadsheet,
Loader2,
Play,
Sparkles,
Upload,
} from "lucide-react";
import { useEffect, useRef, useState } from "react";
import type {
BackgroundProgress,
CleaningAnalysisResponse,
CleaningColumnAnalysis,
CleaningJob,
CleaningPreviewColumn,
CleaningPreviewResponse,
CleaningRuleInfo,
Job,
Task,
} from "../../../shared/types";
import { CleaningStepper, type CleaningStep } from "../../components/CleaningStepper";
import { ExtractionResultsTable } from "../../components/ExtractionResultsTable";
import type { ApiClient } from "../../lib/apiClient";
import { loadCopModConfig } from "../../lib/cleaningSettings";
import { downloadCsv, toCsv, buildExportCsvFilename } from "../../lib/csv";
type SourceMode = "extraction" | "csv";
interface Props {
client: ApiClient;
job: Job;
extractionBatchId?: string;
extractionTaskId?: string;
onTaskCreated?: (task: Task) => void;
onProgress?: (progress: BackgroundProgress) => void;
}
function batchIdFromTask(task: Task): string {
return String((task.result as Record<string, unknown>)?.extraction_batch_id ?? "");
}
function formatTaskDate(iso: string): string {
try {
return new Date(iso).toLocaleString("es-CO", {
day: "2-digit",
month: "short",
year: "numeric",
hour: "2-digit",
minute: "2-digit",
});
} catch {
return iso;
}
}
export function CleaningWizard({
client,
job,
extractionBatchId: initialBatchId,
extractionTaskId: initialTaskId,
onTaskCreated,
onProgress,
}: Props) {
const hasHandoff = Boolean(initialBatchId || initialTaskId);
const [step, setStep] = useState<CleaningStep>(1);
const [maxReached, setMaxReached] = useState<CleaningStep>(1);
const [source, setSource] = useState<SourceMode>(hasHandoff ? "extraction" : "extraction");
const [extractionTasks, setExtractionTasks] = useState<Task[]>([]);
const [loadingTasks, setLoadingTasks] = useState(true);
const [selectedTaskId, setSelectedTaskId] = useState<string | null>(initialTaskId ?? null);
const [batchId, setBatchId] = useState(initialBatchId ?? "");
const [csvFile, setCsvFile] = useState<File | null>(null);
const [rulesCatalog, setRulesCatalog] = useState<CleaningRuleInfo[]>([]);
const [analysis, setAnalysis] = useState<CleaningAnalysisResponse | null>(null);
const [rulesPerColumn, setRulesPerColumn] = useState<Record<string, string[]>>({});
const [copModFechaColumn, setCopModFechaColumn] = useState<Record<string, string>>({});
const [loading, setLoading] = useState(false);
const [running, setRunning] = useState(false);
const [error, setError] = useState<string | null>(null);
const [resultJob, setResultJob] = useState<CleaningJob | null>(null);
const [resultRows, setResultRows] = useState<Record<string, unknown>[]>([]);
const [handoffAnalyzed, setHandoffAnalyzed] = useState(false);
const [preview, setPreview] = useState<CleaningPreviewResponse | null>(null);
const [previewLoading, setPreviewLoading] = useState(false);
const [previewError, setPreviewError] = useState<string | null>(null);
const [activeColumn, setActiveColumn] = useState<string | null>(null);
const [sampleMode, setSampleMode] = useState<"before" | "after">("before");
const fileInputRef = useRef<HTMLInputElement>(null);
const previewTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
function goToStep(next: CleaningStep) {
setStep(next);
setMaxReached((m) => (next > m ? next : m));
}
useEffect(() => {
client.cleaningRules().then(setRulesCatalog).catch(() => setRulesCatalog([]));
}, [client]);
useEffect(() => {
let cancelled = false;
async function loadExtractions() {
setLoadingTasks(true);
try {
const tasks = await client.jobTasks(job.id);
if (cancelled) return;
const extractions = tasks
.filter((t) => t.type === "extraction" && t.status === "completed")
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
setExtractionTasks(extractions);
// Preselección por handoff o la más reciente
if (initialTaskId && extractions.some((t) => t.id === initialTaskId)) {
const t = extractions.find((x) => x.id === initialTaskId)!;
setSelectedTaskId(t.id);
setBatchId(batchIdFromTask(t) || initialBatchId || "");
setSource("extraction");
} else if (initialBatchId) {
const match = extractions.find((t) => batchIdFromTask(t) === initialBatchId);
if (match) {
setSelectedTaskId(match.id);
setBatchId(initialBatchId);
} else {
setBatchId(initialBatchId);
}
setSource("extraction");
} else if (extractions.length > 0 && !selectedTaskId) {
const newest = extractions[0];
setSelectedTaskId(newest.id);
setBatchId(batchIdFromTask(newest));
}
} catch {
if (!cancelled) setExtractionTasks([]);
} finally {
if (!cancelled) setLoadingTasks(false);
}
}
void loadExtractions();
return () => { cancelled = true; };
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [client, job.id, initialBatchId, initialTaskId]);
// Handoff: analizar automáticamente al llegar desde Resultados
useEffect(() => {
if (!hasHandoff || handoffAnalyzed || loadingTasks) return;
const id = batchId || initialBatchId;
if (!id) return;
setHandoffAnalyzed(true);
void loadAnalysis("extraction", id, null);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [hasHandoff, handoffAnalyzed, loadingTasks, batchId, initialBatchId]);
function selectExtraction(task: Task) {
setSelectedTaskId(task.id);
setBatchId(batchIdFromTask(task));
setSource("extraction");
setError(null);
}
function ruleName(id: string): string {
return rulesCatalog.find((r) => r.id === id)?.name ?? id;
}
function guessFechaColumn(columns: CleaningColumnAnalysis[], amountCol: string): string {
const candidates = columns.filter(
(c) =>
c.name !== amountCol &&
(c.inferred_type === "date" ||
/fecha|date|servicio|fec[_ ]/i.test(c.name)),
);
const preferred =
candidates.find((c) => /fecha\s*servicio|fec[_ ]?serv/i.test(c.name)) ??
candidates.find((c) => c.inferred_type === "date") ??
candidates[0];
return preferred?.name ?? "";
}
function toggleRule(column: string, ruleId: string) {
setRulesPerColumn((prev) => {
const current = prev[column] ?? [];
const turningOn = !current.includes(ruleId);
const next = turningOn
? [...current, ruleId]
: current.filter((r) => r !== ruleId);
return { ...prev, [column]: next };
});
if (ruleId === "cop_mod_split") {
setCopModFechaColumn((prev) => {
const current = rulesPerColumn[column] ?? [];
const turningOn = !current.includes(ruleId);
if (!turningOn) {
const next = { ...prev };
delete next[column];
return next;
}
if (prev[column]) return prev;
const guess = analysis ? guessFechaColumn(analysis.columns, column) : "";
return guess ? { ...prev, [column]: guess } : prev;
});
}
setSampleMode("after");
}
function suggestRules(columns: CleaningColumnAnalysis[]) {
const suggested: Record<string, string[]> = {};
for (const col of columns) {
const applicable = col.applicable_rules ?? [];
suggested[col.name] = applicable.includes("strip_spaces") ? ["strip_spaces"] : [];
}
setRulesPerColumn(suggested);
setCopModFechaColumn({});
}
function cleaningExtraPayload(): {
cop_mod_fecha_column?: Record<string, string>;
cop_mod_config: ReturnType<typeof loadCopModConfig>;
} {
const fechaEntries = Object.entries(copModFechaColumn).filter(
([col, fecha]) =>
Boolean(fecha) && (rulesPerColumn[col] ?? []).includes("cop_mod_split"),
);
return {
...(fechaEntries.length > 0
? { cop_mod_fecha_column: Object.fromEntries(fechaEntries) }
: {}),
cop_mod_config: loadCopModConfig(),
};
}
function previewColumn(name: string): CleaningPreviewColumn | undefined {
return preview?.columns.find((c) => c.name === name);
}
function displaySamples(col: CleaningColumnAnalysis): string[] {
if (col.samples && col.samples.length > 0) return col.samples;
if (col.example) return [String(col.example)];
return [];
}
/** Columnas nuevas respecto al análisis (p. ej. *_doc_tipo / *_doc_numeros). */
function derivedColumnNames(sourceCol: string): string[] {
if (!preview) return [];
const original = new Set(analysis?.columns.map((c) => c.name) ?? []);
return preview.columns
.map((c) => c.name)
.filter(
(name) =>
!original.has(name) &&
name.toLowerCase().startsWith(`${sourceCol.toLowerCase()}_`),
);
}
/** Orden: columna activa → columnas derivadas → resto (sin cortar). */
function orderedSampleColumnNames(mode: "before" | "after"): string[] {
const analysisNames = analysis?.columns.map((c) => c.name) ?? [];
const names =
mode === "after" && preview
? preview.columns.map((c) => c.name)
: analysisNames;
const focus = activeColumn;
if (!focus || !names.includes(focus)) return names;
const derived = names.filter(
(n) => n !== focus && n.toLowerCase().startsWith(`${focus.toLowerCase()}_`),
);
const rest = names.filter((n) => n !== focus && !derived.includes(n));
return [focus, ...derived, ...rest];
}
// Preview en vivo al cambiar reglas (debounce 300ms)
useEffect(() => {
if (step !== 2 || !analysis) return;
const sampleRows = analysis.sample_rows ?? [];
if (previewTimerRef.current) clearTimeout(previewTimerRef.current);
previewTimerRef.current = setTimeout(() => {
void (async () => {
setPreviewLoading(true);
setPreviewError(null);
try {
const extra = cleaningExtraPayload();
const payload =
sampleRows.length > 0
? { rows: sampleRows, rules_per_column: rulesPerColumn, ...extra }
: batchId
? {
extraction_batch_id: batchId,
rules_per_column: rulesPerColumn,
...extra,
}
: null;
if (!payload) {
setPreview(null);
return;
}
const data = await client.previewCleaning(payload);
setPreview(data);
} catch (e) {
setPreviewError(String(e instanceof Error ? e.message : e));
} finally {
setPreviewLoading(false);
}
})();
}, 300);
return () => {
if (previewTimerRef.current) clearTimeout(previewTimerRef.current);
};
}, [step, analysis, rulesPerColumn, copModFechaColumn, batchId, client]);
async function loadAnalysis(mode: SourceMode, batch?: string, file?: File | null) {
setLoading(true);
setError(null);
setAnalysis(null);
onProgress?.({
module: "cleaning",
total: 1,
processed: 0,
failed: 0,
currentFile: mode === "csv" ? (file?.name ?? "CSV") : "extracción",
running: true,
label: "Analizando columnas…",
});
try {
let data: CleaningAnalysisResponse;
if (mode === "extraction") {
if (!batch) throw new Error("Selecciona una extracción de la lista");
data = await client.analyzeCleaningBatch(batch);
} else {
if (!file) throw new Error("Selecciona un archivo CSV");
data = await client.analyzeCleaningCsv(file, file.name);
}
setAnalysis(data);
suggestRules(data.columns);
setActiveColumn(data.columns[0]?.name ?? null);
setSampleMode("before");
setPreview(null);
goToStep(2);
} catch (e) {
setError(String(e instanceof Error ? e.message : e));
} finally {
setLoading(false);
onProgress?.({
module: "cleaning",
total: 1,
processed: 1,
failed: 0,
currentFile: "",
running: false,
label: "",
});
}
}
async function handleAnalyze() {
await loadAnalysis(source, batchId, csvFile);
}
async function handleRun() {
setRunning(true);
setError(null);
setResultJob(null);
setResultRows([]);
onProgress?.({
module: "cleaning",
total: 1,
processed: 0,
failed: 0,
currentFile: source === "csv" ? (csvFile?.name ?? "CSV") : "extracción",
running: true,
label: "Aplicando reglas de limpieza…",
});
try {
let cleaningJob: CleaningJob;
if (source === "extraction") {
if (!batchId) throw new Error("Selecciona una extracción");
cleaningJob = await client.runCleaning({
extraction_batch_id: batchId,
job_id: job.id,
rules_per_column: rulesPerColumn,
...cleaningExtraPayload(),
});
} else {
if (!csvFile) throw new Error("Selecciona un archivo CSV");
const form = new FormData();
const extra = cleaningExtraPayload();
form.append("file", csvFile, csvFile.name);
form.append("job_id", job.id);
form.append("rules_per_column", JSON.stringify(rulesPerColumn));
if (extra.cop_mod_fecha_column) {
form.append("cop_mod_fecha_column", JSON.stringify(extra.cop_mod_fecha_column));
}
form.append("cop_mod_config", JSON.stringify(extra.cop_mod_config));
cleaningJob = await client.runCleaningFromCsv(form);
}
setResultJob(cleaningJob);
if (cleaningJob.status === "completed") {
const rowsResp = await client.cleaningJobRows(cleaningJob.id, 500, 0);
setResultRows(rowsResp.rows ?? []);
}
try {
const tasks = await client.jobTasks(job.id);
const latest = tasks
.filter((t) => t.type === "cleaning")
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())[0];
if (latest) onTaskCreated?.(latest);
} catch { /* silent */ }
goToStep(3);
} catch (e) {
setError(String(e instanceof Error ? e.message : e));
} finally {
setRunning(false);
onProgress?.({
module: "cleaning",
total: 1,
processed: 1,
failed: 0,
currentFile: "",
running: false,
label: "",
});
}
}
function exportCsv() {
if (resultRows.length === 0) return;
const columns = Array.from(new Set(resultRows.flatMap((r) => Object.keys(r))));
const csv = toCsv(
columns,
resultRows.map((r) => columns.map((h) => r[h])),
);
downloadCsv(buildExportCsvFilename("limpieza", job.title), csv);
}
function resetWizard() {
setStep(1);
setMaxReached(1);
setResultJob(null);
setResultRows([]);
setAnalysis(null);
setHandoffAnalyzed(true); // no re-auto-analizar
setError(null);
}
const resultColumns =
resultRows.length > 0
? Array.from(new Set(resultRows.flatMap((r) => Object.keys(r))))
: [];
const selectedExtraction = extractionTasks.find((t) => t.id === selectedTaskId) ?? null;
const canAnalyze =
source === "extraction" ? Boolean(batchId) : Boolean(csvFile);
return (
<div className="wiz">
<CleaningStepper current={step} maxReached={maxReached} onStepClick={goToStep} />
<div className="wiz-body">
{error && (
<div className="wiz-toast wiz-toast--error" style={{ marginBottom: 12 }}>
<AlertCircle size={14} /> {error}
</div>
)}
{/* ── PASO 1: Fuente ── */}
{step === 1 && (
<div className="wiz-step">
<div className="wiz-section">
<h3 className="wiz-section-title">Origen de los datos</h3>
<p className="wiz-desc">
Elige una extracción completada de este trabajo o sube un CSV para limpiar.
</p>
</div>
<div className="wiz-section">
<div className="wiz-checks-row" style={{ gap: 10, flexWrap: "wrap" }}>
<button
type="button"
className={`btn${source === "extraction" ? " btn-primary" : ""}`}
onClick={() => setSource("extraction")}
>
<Cpu size={14} /> Desde extracción
</button>
<button
type="button"
className={`btn${source === "csv" ? " btn-primary" : ""}`}
onClick={() => setSource("csv")}
>
<Upload size={14} /> Subir CSV
</button>
</div>
</div>
{source === "extraction" ? (
<div className="wiz-section">
<label className="wiz-field-label">Extracciones de este trabajo</label>
{loadingTasks ? (
<div className="module-loading" style={{ padding: 24 }}>
<Loader2 size={18} className="spin" /> Cargando extracciones…
</div>
) : extractionTasks.length === 0 ? (
<div className="wiz-found-badge" style={{ flexDirection: "column", alignItems: "flex-start", gap: 8 }}>
<span>No hay extracciones completadas en este trabajo.</span>
<button type="button" className="btn btn-sm" onClick={() => setSource("csv")}>
<Upload size={13} /> Subir un CSV en su lugar
</button>
</div>
) : (
<div className="cleaning-source-list" role="listbox" aria-label="Extracciones">
{extractionTasks.map((task, index) => {
const selected = task.id === selectedTaskId;
return (
<button
key={task.id}
type="button"
role="option"
aria-selected={selected}
className={`cleaning-source-item${selected ? " cleaning-source-item--selected" : ""}`}
onClick={() => selectExtraction(task)}
>
<div className="cleaning-source-item-main">
<FileSpreadsheet size={16} />
<div>
<div className="cleaning-source-item-title">
{task.title ?? `Extracción ${extractionTasks.length - index}`}
</div>
<div className="cleaning-source-item-meta">
{formatTaskDate(task.created_at)}
{index === 0 ? " · más reciente" : ""}
</div>
</div>
</div>
<span className={`task-badge task-badge--${task.status}`}>{task.status}</span>
</button>
);
})}
</div>
)}
{selectedExtraction && (
<p className="wiz-hint" style={{ marginTop: 8 }}>
Seleccionada: <strong>{selectedExtraction.title ?? "Extracción"}</strong>
</p>
)}
</div>
) : (
<div className="wiz-section">
<label className="wiz-field-label">Archivo CSV</label>
<input
ref={fileInputRef}
type="file"
accept=".csv,text/csv"
style={{ display: "none" }}
onChange={(e) => setCsvFile(e.target.files?.[0] ?? null)}
/>
<div className="folder-picker">
<div className="folder-picker-display">
{csvFile ? (
<>
<Upload size={16} className="folder-picker-icon" />
<span className="folder-picker-path">{csvFile.name}</span>
</>
) : (
<span className="folder-picker-placeholder">Ningún archivo seleccionado</span>
)}
</div>
<button type="button" className="btn btn-primary" onClick={() => fileInputRef.current?.click()}>
<Upload size={15} /> Explorar
</button>
</div>
</div>
)}
<div className="wiz-nav" style={{ marginTop: 8 }}>
<button
type="button"
className="btn btn-primary"
disabled={loading || !canAnalyze}
onClick={() => void handleAnalyze()}
>
{loading ? <Loader2 size={14} className="spin" /> : <Play size={14} />}
Analizar columnas
</button>
</div>
</div>
)}
{/* ── PASO 2: Reglas ── */}
{step === 2 && (
<div className="wiz-step wiz-step--cleaning-rules">
<div className="cleaning-rules-toolbar">
<div>
<h3 className="wiz-section-title" style={{ margin: 0 }}>Reglas por columna</h3>
<p className="wiz-desc" style={{ margin: "4px 0 0" }}>
Elige una columna, aplica reglas y mira el cambio al instante.
{analysis?.row_count != null ? ` · ${analysis.row_count} filas` : ""}
{previewLoading ? " · actualizando…" : ""}
</p>
</div>
{previewLoading && <Loader2 size={16} className="spin" />}
</div>
{loading && !analysis ? (
<div className="module-loading">
<Loader2 size={20} className="spin" /> Analizando…
</div>
) : analysis ? (
<div className="cleaning-rules-workspace">
{/* Lista compacta de columnas */}
<aside className="cleaning-col-nav" aria-label="Columnas">
{analysis.columns.map((col) => {
const selected = (rulesPerColumn[col.name] ?? []).length;
const changed = Boolean(previewColumn(col.name)?.changed);
const isActive = activeColumn === col.name;
return (
<button
key={col.name}
type="button"
className={`cleaning-col-nav-item${isActive ? " cleaning-col-nav-item--active" : ""}${changed ? " cleaning-col-nav-item--changed" : ""}`}
onClick={() => setActiveColumn(col.name)}
>
<span className="cleaning-col-nav-name">{col.name}</span>
<span className="cleaning-col-nav-meta">
<span className="cleaning-type-badge">{col.inferred_type}</span>
{selected > 0 && (
<span className="cleaning-col-nav-count">{selected}</span>
)}
</span>
</button>
);
})}
</aside>
{/* Detalle de la columna activa */}
<div className="cleaning-col-detail">
{previewError && (
<div className="wiz-toast wiz-toast--error" style={{ marginBottom: 8 }}>
<AlertCircle size={14} /> {previewError}
</div>
)}
{(() => {
const col = analysis.columns.find((c) => c.name === activeColumn) ?? analysis.columns[0];
if (!col) return <p className="wiz-desc">Sin columnas.</p>;
const applicable = col.applicable_rules ?? [];
const selected = rulesPerColumn[col.name] ?? [];
const beforeFromSamples = displaySamples(col);
const pv = previewColumn(col.name);
const beforeVals = pv?.before?.length ? pv.before.slice(0, 4) : beforeFromSamples.slice(0, 4);
const derived = derivedColumnNames(col.name);
const afterVals = pv?.after?.length
? pv.after.slice(0, 4)
: selected.length === 0
? beforeVals
: [];
const sourceUnchanged =
beforeVals.length > 0 &&
afterVals.length > 0 &&
beforeVals.every((v, i) => String(v) === String(afterVals[i] ?? ""));
const changed = Boolean(pv?.changed) || derived.length > 0;
return (
<>
<div className="cleaning-col-detail-header">
<div>
<h4 className="cleaning-col-detail-title">{col.name}</h4>
<div className="cleaning-rule-card-meta">
<span className="cleaning-type-badge">{col.inferred_type}</span>
{col.null_count != null && (
<span>nulos {col.null_count}{col.null_pct != null ? ` (${Math.round(col.null_pct)}%)` : ""}</span>
)}
{col.unique_count != null && <span>únicos {col.unique_count}</span>}
</div>
</div>
</div>
<div className={`cleaning-preview-compare cleaning-preview-compare--compact${changed ? " cleaning-preview-compare--changed" : ""}`}>
<div className="cleaning-preview-side">
<div className="cleaning-preview-label">Antes</div>
<div className="cleaning-preview-values">
{beforeVals.length === 0 ? (
<span className="cleaning-preview-empty">Sin valores</span>
) : (
beforeVals.map((v, i) => (
<code key={i} className="cleaning-preview-chip">{v || "—"}</code>
))
)}
</div>
</div>
<ArrowRight size={14} className="cleaning-preview-arrow" />
<div className="cleaning-preview-side">
<div className={`cleaning-preview-label${changed ? " cleaning-preview-label--after" : ""}`}>
Después
</div>
<div className="cleaning-preview-values">
{derived.length > 0 ? (
derived.map((dName) => {
const dPv = previewColumn(dName);
const sample = (dPv?.after ?? []).find((v) => v) ?? (dPv?.after?.[0] ?? "");
return (
<code key={dName} className="cleaning-preview-chip cleaning-preview-chip--changed" title={dName}>
<span className="cleaning-preview-chip-col">{dName.replace(/^.*_doc_/, "_doc_")}</span>
{sample || "—"}
</code>
);
})
) : afterVals.length === 0 ? (
<span className="cleaning-preview-empty">
{previewLoading ? "…" : "Activa una regla"}
</span>
) : sourceUnchanged ? (
<span className="cleaning-preview-empty">Sin cambio en origen</span>
) : (
afterVals.map((v, i) => {
const before = beforeVals[i] ?? "";
const isDiff = String(v) !== String(before);
return (
<code
key={i}
className={`cleaning-preview-chip${isDiff ? " cleaning-preview-chip--changed" : ""}`}
>
{v || "—"}
</code>
);
})
)}
</div>
</div>
</div>
<div className="cleaning-rules-block">
<div className="cleaning-preview-label">Reglas</div>
{applicable.length === 0 ? (
<span className="wiz-hint">Sin reglas para este tipo</span>
) : (
<div className="cleaning-rule-chips">
{applicable.map((ruleId) => {
const on = selected.includes(ruleId);
const info = rulesCatalog.find((r) => r.id === ruleId);
return (
<button
key={ruleId}
type="button"
className={`btn btn-sm${on ? " btn-primary" : ""}`}
title={info?.description || ruleId}
onClick={() => toggleRule(col.name, ruleId)}
>
{ruleName(ruleId)}
</button>
);
})}
</div>
)}
{selected.length > 0 && (
<p className="wiz-hint cleaning-rule-desc">
{selected
.map((id) => rulesCatalog.find((r) => r.id === id)?.description || id)
.join(" · ")}
</p>
)}
{selected.includes("cop_mod_split") && (
<div className="cleaning-copmod-fecha">
<label className="wiz-field-label" htmlFor={`copmod-fecha-${col.name}`}>
Fecha Servicio (para COP/MOD)
</label>
<select
id={`copmod-fecha-${col.name}`}
className="form-input"
value={copModFechaColumn[col.name] ?? ""}
onChange={(e) =>
setCopModFechaColumn((prev) => ({
...prev,
[col.name]: e.target.value,
}))
}
>
<option value="">Selecciona columna de fecha…</option>
{analysis.columns
.filter((c) => c.name !== col.name)
.map((c) => (
<option key={c.name} value={c.name}>
{c.name}
{c.inferred_type === "date" ? " (fecha)" : ""}
</option>
))}
</select>
<p className="form-hint">
Las cuotas por año se editan en Configuración → Limpieza.
</p>
</div>
)}
</div>
</>
);
})()}
{/* Una sola tabla de muestra: Antes | Después */}
{((analysis.sample_rows?.length ?? 0) > 0 || (preview?.rows_after.length ?? 0) > 0) && (
<div className="cleaning-sample-dock">
<div className="cleaning-sample-dock-header">
<strong>Muestra</strong>
<div className="cleaning-sample-tabs" role="tablist">
<button
type="button"
role="tab"
aria-selected={sampleMode === "before"}
className={`cleaning-sample-tab${sampleMode === "before" ? " cleaning-sample-tab--active" : ""}`}
onClick={() => setSampleMode("before")}
>
Antes
</button>
<button
type="button"
role="tab"
aria-selected={sampleMode === "after"}
className={`cleaning-sample-tab${sampleMode === "after" ? " cleaning-sample-tab--active" : ""}`}
onClick={() => setSampleMode("after")}
disabled={!preview}
>
Después
</button>
</div>
</div>
<div className="cleaning-sample-table-scroll">
<table className="results-table cleaning-preview-table">
<thead>
<tr>
{orderedSampleColumnNames(sampleMode).map((name) => {
const isNew = !(analysis.columns.some((c) => c.name === name));
const changed = preview?.columns.find((x) => x.name === name)?.changed;
return (
<th
key={name}
className={
sampleMode === "after" && (changed || isNew)
? "cleaning-th-changed"
: undefined
}
>
{name}
</th>
);
})}
</tr>
</thead>
<tbody>
{(sampleMode === "after" && preview
? preview.rows_after
: analysis.sample_rows ?? preview?.rows_before ?? []
).slice(0, 4).map((row, i) => (
<tr key={i}>
{orderedSampleColumnNames(sampleMode).map((name) => {
const val = row[name];
const before = preview?.rows_before[i]?.[name];
const isNew = !(analysis.columns.some((c) => c.name === name));
const diff =
sampleMode === "after" &&
(isNew || String(before ?? "") !== String(val ?? ""));
return (
<td key={name} className={diff ? "cleaning-td-changed" : undefined}>
{val == null || val === ""
? <span className="cleaning-preview-empty">—</span>
: String(val).slice(0, 36)}
</td>
);
})}
</tr>
))}
</tbody>
</table>
</div>
</div>
)}
</div>
</div>
) : (
<p className="wiz-desc">No hay análisis. Vuelve al paso Fuente.</p>
)}
<div className="wiz-nav cleaning-rules-nav">
<button type="button" className="btn" onClick={() => goToStep(1)} disabled={running}>
Atrás
</button>
<button
type="button"
className="btn btn-primary"
disabled={running || !analysis}
onClick={() => void handleRun()}
>
{running ? <Loader2 size={14} className="spin" /> : <Sparkles size={14} />}
Ejecutar limpieza
</button>
</div>
</div>
)}
{/* ── PASO 3: Resultados ── */}
{step === 3 && (
<div className="wiz-step">
<div className="wiz-section">
<h3 className="wiz-section-title">Resultado de la limpieza</h3>
</div>
{resultJob?.status === "completed" ? (
<>
<div className="wiz-found-badge" style={{ marginBottom: 12 }}>
<CheckCircle2 size={15} />
<span>
Limpieza completada
{resultJob.log?.length ? ` · ${resultJob.log.length} operaciones` : null}
{resultRows.length > 0 ? ` · ${resultRows.length} filas` : null}
</span>
</div>
{resultJob.log && resultJob.log.length > 0 && (
<details className="wiz-advanced" style={{ marginBottom: 12 }}>
<summary className="wiz-advanced-trigger">Ver log de operaciones</summary>
<div className="wiz-advanced-body">
<ul style={{ margin: 0, paddingLeft: 18, color: "var(--text-muted)", fontSize: 12 }}>
{resultJob.log.map((line, i) => (
<li key={i}>{line}</li>
))}
</ul>
</div>
</details>
)}
<div className="module-results-header" style={{ marginBottom: 8 }}>
<span style={{ fontSize: 13, color: "var(--text-muted)" }}>
Vista previa{resultRows.length > 100 ? " (primeras 100)" : ""}
</span>
{resultRows.length > 0 && (
<button type="button" className="btn btn-sm btn-ghost" onClick={exportCsv}>
<Download size={13} /> Descargar CSV
</button>
)}
</div>
{resultRows.length > 0 ? (
<ExtractionResultsTable rows={resultRows} columns={resultColumns} />
) : (
<p className="wiz-desc">Sin filas en el resultado.</p>
)}
</>
) : (
<div className="wiz-toast wiz-toast--error">
<AlertCircle size={14} />
{resultJob?.error ?? "La limpieza no se completó"}
</div>
)}
<div className="wiz-nav" style={{ marginTop: 16, gap: 8 }}>
<button type="button" className="btn" onClick={() => goToStep(2)}>
Atrás
</button>
<button type="button" className="btn btn-primary" onClick={resetWizard}>
Nueva limpieza
</button>
</div>
</div>
)}
</div>
</div>
);
}
@@ -0,0 +1,26 @@
import type { BackgroundProgress } from "../../../shared/types";
import type { ModuleWizardProps } from "../registry";
import { CleaningWizard } from "./CleaningWizard";
interface Props extends ModuleWizardProps {
onProgress?: (p: BackgroundProgress) => void;
}
export function CleaningWizardModule({
client,
job,
onTaskCreated,
onProgress,
context,
}: Props) {
return (
<CleaningWizard
client={client}
job={job}
onTaskCreated={onTaskCreated}
onProgress={onProgress}
extractionBatchId={context?.extractionBatchId as string | undefined}
extractionTaskId={context?.extractionTaskId as string | undefined}
/>
);
}
@@ -2,6 +2,7 @@ import { ArrowRight, Download, Loader2, RefreshCw } from "lucide-react";
import { useEffect, useState } from "react";
import type { CountJobFile } from "../../../shared/types";
import { CountResultsTable } from "../../components/CountResultsTable";
import { downloadCsv, toCsv, buildExportCsvFilename } from "../../lib/csv";
import type { ModuleResultsProps } from "../registry";
export function CountingResults({
@@ -47,25 +48,16 @@ export function CountingResults({
: files;
const headers = ["filename", "radicado", "service_count", "status", "is_valid", "error"];
const rows = subset_files.map((f) =>
[
`"${f.filename}"`,
`"${f.radicado ?? ""}"`,
f.service_count ?? "",
f.status,
f.is_valid === null ? "" : String(f.is_valid),
`"${(f.error ?? "").replace(/"/g, '""')}"`,
].join(","),
);
const csv = [headers.join(","), ...rows].join("\n");
const blob = new Blob([csv], { type: "text/csv" });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
const rows = subset_files.map((f) => [
f.filename,
f.radicado ?? "",
f.service_count ?? "",
f.status,
f.is_valid === null ? "" : String(f.is_valid),
f.error ?? "",
]);
const label = subset === "valid" ? "validos" : subset === "invalid" ? "invalidos" : "todos";
a.download = `conteo-${label}-${new Date().toISOString().slice(0, 10)}.csv`;
a.click();
URL.revokeObjectURL(url);
downloadCsv(buildExportCsvFilename("conteo", job.title, label), toCsv(headers, rows));
}
const validCount = files.filter(isFileValid).length;
@@ -1,11 +1,13 @@
import { Download, Loader2, RefreshCw } from "lucide-react";
import { ArrowRight, Download, Loader2, RefreshCw, Sparkles } from "lucide-react";
import { useEffect, useState } from "react";
import type { ExtractionBatch } from "../../../shared/types";
import { ExtractionResultsTable } from "../../components/ExtractionResultsTable";
import { downloadCsv, toCsv, buildExportCsvFilename } from "../../lib/csv";
import type { ModuleResultsProps } from "../registry";
type Row = Record<string, unknown>;
export function ExtractionResults({ client, task }: ModuleResultsProps) {
export function ExtractionResults({ client, task, job, onActivateModule }: ModuleResultsProps) {
const [batch, setBatch] = useState<ExtractionBatch | null>(null);
const [rows, setRows] = useState<Row[]>([]);
const [columns, setColumns] = useState<string[]>([]);
@@ -54,25 +56,24 @@ export function ExtractionResults({ client, task }: ModuleResultsProps) {
}, [task.id]); // eslint-disable-line react-hooks/exhaustive-deps
function exportCsv() {
const headers = columns;
const csvRows = rows.map((r) =>
headers
.map((h) => {
const v = r[h];
if (v === null || v === undefined) return "";
const s = String(v);
return `"${s.replace(/"/g, '""')}"`;
})
.join(","),
const csv = toCsv(
columns,
rows.map((r) => columns.map((h) => r[h])),
);
const csv = [headers.join(","), ...csvRows].join("\n");
const blob = new Blob([csv], { type: "text/csv" });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = `extraccion-${new Date().toISOString().slice(0, 10)}.csv`;
a.click();
URL.revokeObjectURL(url);
downloadCsv(buildExportCsvFilename("extraccion", job.title), csv);
}
function startCleaning() {
const batchId = String(
(task.result as Record<string, unknown>)?.extraction_batch_id
?? batch?.id
?? "",
);
onActivateModule?.("cleaning", {
extractionBatchId: batchId || undefined,
extractionTaskId: task.id,
jobId: job.id,
});
}
if (loading && !batch) {
@@ -109,15 +110,22 @@ export function ExtractionResults({ client, task }: ModuleResultsProps) {
{batch?.status ?? "pendiente"}
</span>
</div>
{rows.length > 0 && (
<button type="button" className="btn btn-sm btn-ghost" onClick={exportCsv}>
<Download size={13} /> Descargar CSV ({rows.length} filas)
</button>
)}
<div style={{ display: "flex", gap: 8 }}>
{rows.length > 0 && (
<button type="button" className="btn btn-sm btn-ghost" onClick={exportCsv}>
<Download size={13} /> Descargar CSV ({rows.length} filas)
</button>
)}
{rows.length > 0 && !isRunning && onActivateModule && (
<button type="button" className="btn btn-sm btn-primary" onClick={startCleaning}>
<Sparkles size={13} /> Limpiar datos <ArrowRight size={13} />
</button>
)}
</div>
</div>
{isRunning && (
<div className="wiz-progress" style={{ margin: "0 0 12px" }}>
<div className="wiz-progress" style={{ margin: "0 16px 12px" }}>
<div className="wiz-progress-header">
<span>Extracción en progreso…</span>
<Loader2 size={13} className="spin" />
@@ -132,32 +140,9 @@ export function ExtractionResults({ client, task }: ModuleResultsProps) {
<div className="wiz-step wiz-step-centered" style={{ padding: 32 }}>
<p className="wiz-desc">No se encontraron filas extraídas.</p>
</div>
) : (
<div className="results-table-wrapper" style={{ overflowX: "auto" }}>
<table className="results-table">
<thead>
<tr>
{columns.map((c) => (
<th key={c}>{c}</th>
))}
</tr>
</thead>
<tbody>
{rows.map((row, i) => (
<tr key={i}>
{columns.map((c) => (
<td key={c}>
{row[c] === null || row[c] === undefined
? <span style={{ color: "var(--text-muted)" }}>—</span>
: String(row[c])}
</td>
))}
</tr>
))}
</tbody>
</table>
</div>
)}
) : rows.length > 0 ? (
<ExtractionResultsTable rows={rows} columns={columns} />
) : null}
</div>
);
}
@@ -30,6 +30,7 @@ import { defaultAlias } from "../../lib/schemaDisplay";
import { loadApiBaseUrl } from "../../lib/apiDefaults";
import { splitRadicados } from "../../lib/pathMapping";
import type { ModuleWizardProps } from "../registry";
import { ExtractionResults } from "./ExtractionResults";
declare const desktopApi: import("../../../shared/types").DesktopBridge;
@@ -71,6 +72,7 @@ export function JobExtractionWizard({
client,
job,
onTaskCreated,
onActivateModule,
context,
countingTaskId: countingTaskIdProp,
validCounts: validCountsProp,
@@ -95,6 +97,9 @@ export function JobExtractionWizard({
const [extractionStep, setExtractionStep] = useState<ExtractionStep>(() => countingTaskId ? 2 : 1);
const [maxReachedStep, setMaxReachedStep] = useState<ExtractionStep>(() => countingTaskId ? 2 : 1);
const [taskCreated, setTaskCreated] = useState<Task | null>(null);
/** Rutas de muestras ya usadas en descubrimiento (para no repetir al reintentar). */
const [usedDiscoveryPaths, setUsedDiscoveryPaths] = useState<string[]>([]);
const [discoverySampleLabels, setDiscoverySampleLabels] = useState<string[]>([]);
const flattened = useMemo(
() => (schemaResponse ? flattenSchema(schemaResponse) : { schema: {}, sample: null }),
@@ -151,6 +156,8 @@ export function JobExtractionWizard({
setSchemaResponse(null);
setSelection({ fields: [], aliasMap: {} });
setEstimateJson(null);
setUsedDiscoveryPaths([]);
setDiscoverySampleLabels([]);
setMaxReachedStep(1);
if (extractionStep > 1) setExtractionStep(1);
}
@@ -167,7 +174,6 @@ export function JobExtractionWizard({
resetSchemaState();
setMaxReachedStep(1);
setStatusText(`${indexed.length} documento(s) listos`);
// Actualizar carpeta del job
try { await client.updateJob(job.id, { folder_path: selected }); } catch { /* ignore */ }
} catch (err) {
setManifest([]);
@@ -190,41 +196,91 @@ export function JobExtractionWizard({
}
}
async function discoverSchema() {
await runTask("Descubriendo campos...", async () => {
if (!window.desktopApi) throw new Error("Requiere Electron.");
if (manifest.length === 0) throw new Error("Selecciona una carpeta con documentos.");
function pickSampleEntry(
candidates: ManifestEntry[],
exclude: Set<string>,
preferRandom: boolean,
): ManifestEntry {
if (candidates.length === 0) {
throw new Error("No hay documentos candidatos para la muestra.");
}
const unused = candidates.filter((e) => !exclude.has(e.localPath));
const pool = unused.length > 0 ? unused : candidates;
if (!preferRandom && unused.length === candidates.length) {
return candidates[0];
}
return pool[Math.floor(Math.random() * pool.length)]!;
}
const responses: SchemaResponse[] = [];
if (form.mode === "general") {
const entry = manifest[0];
const fileData = await window.desktopApi.readFile(entry.localPath);
const blob = new Blob([fileData.data]);
responses.push(await client.discoverSchemaUpload(buildSchemaUploadForm(form, blob, fileData.filename)));
} else {
for (const docType of validDocTypes(form.docTypes)) {
const entry = manifest.find((item) => item.docType === docType.name);
if (!entry) throw new Error(`No se encontró muestra para ${docType.name}.`);
async function discoverSchema(options?: { randomOther?: boolean }) {
const randomOther = Boolean(options?.randomOther);
await runTask(
randomOther ? "Probando con otro archivo…" : "Descubriendo campos...",
async () => {
if (!window.desktopApi) throw new Error("Requiere Electron.");
if (manifest.length === 0) throw new Error("Selecciona una carpeta con documentos.");
const exclude = new Set(randomOther ? usedDiscoveryPaths : []);
const responses: SchemaResponse[] = [];
const picked: ManifestEntry[] = [];
if (form.mode === "general") {
const entry = pickSampleEntry(manifest, exclude, randomOther);
picked.push(entry);
const fileData = await window.desktopApi.readFile(entry.localPath);
const blob = new Blob([fileData.data]);
responses.push(
await client.discoverSchemaUpload(buildSchemaUploadForm(form, blob, fileData.filename, docType.name)),
await client.discoverSchemaUpload(
buildSchemaUploadForm(form, blob, fileData.filename),
),
);
} else {
for (const docType of validDocTypes(form.docTypes)) {
const typed = manifest.filter((item) => item.docType === docType.name);
if (typed.length === 0) {
throw new Error(`No se encontró muestra para ${docType.name}.`);
}
const entry = pickSampleEntry(typed, exclude, randomOther);
picked.push(entry);
const fileData = await window.desktopApi.readFile(entry.localPath);
const blob = new Blob([fileData.data]);
responses.push(
await client.discoverSchemaUpload(
buildSchemaUploadForm(form, blob, fileData.filename, docType.name),
),
);
}
}
}
const schema = mergeSchemaResponses(responses);
setSchemaResponse(schema);
const flat = flattenSchema(schema);
setSelection({
fields: [],
aliasMap: Object.fromEntries(Object.keys(flat.schema).map((p) => [p, defaultAlias(p)])),
});
setEstimateJson(null);
setStatusText(`${Object.keys(flat.schema).length} campo(s) descubiertos`);
setMaxReachedStep((cur) => (cur < 3 ? 3 : cur));
setExtractionStep(3);
});
const schema = mergeSchemaResponses(responses);
setSchemaResponse(schema);
const flat = flattenSchema(schema);
setSelection({
fields: [],
aliasMap: Object.fromEntries(Object.keys(flat.schema).map((p) => [p, defaultAlias(p)])),
});
setEstimateJson(null);
const labels = picked.map((e) => e.filename);
setDiscoverySampleLabels(labels);
setUsedDiscoveryPaths((prev) => {
const next = new Set(randomOther ? prev : []);
for (const e of picked) next.add(e.localPath);
return [...next];
});
setStatusText(
`${Object.keys(flat.schema).length} campo(s) descubiertos` +
(labels.length ? ` · muestra: ${labels.join(", ")}` : ""),
);
setMaxReachedStep((cur) => (cur < 3 ? 3 : cur));
if (!randomOther) {
setExtractionStep(3);
}
},
);
}
async function rediscoverWithOtherSample() {
await discoverSchema({ randomOther: true });
}
async function estimate() {
@@ -419,14 +475,26 @@ export function JobExtractionWizard({
});
}
// Si ya se creó tarea, muestra mensaje y el módulo Results tomará el relevo
// Tras crear la tarea: mostrar resultados (con polling) en esta misma pestaña
if (taskCreated) {
return (
<div className="module-loading" style={{ padding: 48 }}>
<p style={{ fontSize: 15, fontWeight: 600, margin: 0 }}>✓ Extracción iniciada</p>
<p style={{ color: "var(--text-muted)", margin: 0 }}>
Los resultados aparecerán automáticamente en esta pestaña.
</p>
<div className="job-extraction-results-pane">
{(ingestProgress || busy) && (
<div className="wiz-toast" style={{ margin: "0 0 12px" }}>
<CheckCircle2 size={14} />
{statusText || "Extracción en curso…"}
</div>
)}
<ExtractionResults
client={client}
job={job}
task={taskCreated}
onTaskUpdated={(t) => {
setTaskCreated(t);
onTaskCreated(t);
}}
onActivateModule={onActivateModule}
/>
</div>
);
}
@@ -461,7 +529,16 @@ export function JobExtractionWizard({
error={error}
statusText={statusText}
selectRoot={selectRoot}
discoverSchema={discoverSchema}
discoverSchema={() => void discoverSchema()}
rediscoverWithOtherSample={() => void rediscoverWithOtherSample()}
discoverySampleLabels={discoverySampleLabels}
canRediscoverWithOtherSample={
form.mode === "general"
? manifest.length > 1
: validDocTypes(form.docTypes).some(
(dt) => manifest.filter((m) => m.docType === dt.name).length > 1,
)
}
estimate={estimate}
runExtraction={runExtraction}
selection={selection}
+771 -18
View File
@@ -73,15 +73,12 @@ button:disabled {
}
.brand-mark {
align-items: center;
background: linear-gradient(135deg, #2e7db2, #3b9ad9);
background: #0b0b0b;
border-radius: 10px;
color: #ffffff;
display: flex;
font-size: 13px;
font-weight: 800;
display: block;
flex-shrink: 0;
height: 34px;
justify-content: center;
object-fit: contain;
width: 34px;
}
@@ -289,6 +286,212 @@ button:disabled {
flex: 1;
}
.settings-page {
display: flex;
flex: 1;
flex-direction: column;
min-height: 0;
overflow: hidden;
}
.settings-page-header {
border-bottom: 1px solid #e8edf4;
flex-shrink: 0;
padding: 20px 28px 16px;
}
.settings-page-header h2 {
font-size: 18px;
font-weight: 700;
margin: 0 0 6px;
}
.settings-page-header p {
color: #64748b;
font-size: 13px;
line-height: 1.45;
margin: 0;
max-width: 640px;
}
.settings-page-header-row {
align-items: flex-start;
display: flex;
gap: 16px;
justify-content: space-between;
}
.settings-page-actions {
align-items: center;
display: flex;
flex-shrink: 0;
gap: 8px;
}
.settings-saved-badge {
background: #dcfce7;
border-radius: 999px;
color: #15803d;
font-size: 11px;
font-weight: 600;
padding: 4px 10px;
}
.settings-page-body {
display: flex;
flex: 1;
flex-direction: column;
gap: 16px;
min-height: 0;
overflow-y: auto;
padding: 20px 28px 28px;
}
.settings-page-body--fill {
overflow: hidden;
}
.settings-panel {
background: #ffffff;
border: 1px solid #e3e8ef;
border-radius: 12px;
display: flex;
flex-direction: column;
gap: 14px;
max-width: 720px;
padding: 18px 20px;
}
.settings-panel-title {
font-size: 14px;
font-weight: 700;
margin: 0;
}
.copmod-year-list {
display: flex;
flex-direction: column;
gap: 10px;
}
.copmod-year-row {
align-items: center;
display: grid;
gap: 10px;
grid-template-columns: 64px minmax(0, 1fr) auto;
}
.copmod-year-label {
color: #334155;
font-size: 13px;
font-weight: 700;
}
.copmod-add-row {
align-items: center;
display: flex;
gap: 10px;
margin-top: 4px;
}
.copmod-add-year {
max-width: 120px;
}
.cleaning-copmod-fecha {
display: flex;
flex-direction: column;
gap: 6px;
margin-top: 12px;
max-width: 420px;
}
.settings-status-summary {
color: #15803d;
font-size: 12px;
font-weight: 600;
}
.settings-status-card--error .settings-status-summary {
color: #b91c1c;
}
.settings-service-grid {
display: grid;
gap: 10px;
grid-template-columns: repeat(auto-fill, minmax(160px, 1fr));
max-width: 720px;
}
.settings-service-chip {
align-items: center;
background: #ffffff;
border: 1px solid #e3e8ef;
border-radius: 10px;
display: flex;
gap: 10px;
padding: 12px 14px;
}
.settings-service-chip--ok {
background: #f0fdf6;
border-color: #bbf7d0;
}
.settings-service-chip--err {
background: #fff5f5;
border-color: #fecaca;
}
.settings-service-chip-dot {
background: #94a3b8;
border-radius: 50%;
flex-shrink: 0;
height: 8px;
width: 8px;
}
.settings-service-chip--ok .settings-service-chip-dot {
background: #16a34a;
}
.settings-service-chip--err .settings-service-chip-dot {
background: #dc2626;
}
.settings-service-chip-text {
display: flex;
flex-direction: column;
gap: 2px;
min-width: 0;
}
.settings-service-chip-text strong {
color: #18212f;
font-size: 13px;
font-weight: 700;
}
.settings-service-chip-text span {
color: #64748b;
font-size: 11.5px;
}
.settings-url-row {
align-items: stretch;
display: flex;
gap: 10px;
}
.settings-url-row .form-input {
flex: 1;
min-width: 0;
}
.settings-url-row .btn {
flex-shrink: 0;
}
.topbar-inline {
align-items: center;
display: flex;
@@ -1520,7 +1723,7 @@ th {
align-items: center;
display: flex;
flex-shrink: 0;
padding: 0 24px 24px;
padding: 20px 24px 24px;
}
.stepper-item {
@@ -1620,6 +1823,15 @@ th {
padding: 0 2px 8px;
}
.job-extraction-results-pane {
display: flex;
flex: 1;
flex-direction: column;
min-height: 0;
overflow: auto;
padding: 8px 4px 16px;
}
.wiz-step {
display: flex;
flex-direction: column;
@@ -1995,6 +2207,52 @@ th {
padding: 10px 16px;
}
.wiz-rediscover-card {
align-items: flex-start;
background: #f8fafc;
border: 1px solid #e3e8ef;
border-radius: 10px;
display: flex;
flex-wrap: wrap;
gap: 12px;
justify-content: space-between;
margin-top: 14px;
max-width: 560px;
padding: 14px 16px;
}
.wiz-rediscover-card--compact {
margin-bottom: 12px;
margin-top: 0;
max-width: none;
}
.wiz-rediscover-text {
display: flex;
flex: 1;
flex-direction: column;
gap: 4px;
min-width: 180px;
}
.wiz-rediscover-text strong {
color: #18212f;
font-size: 13px;
font-weight: 700;
}
.wiz-rediscover-text span {
color: #64748b;
font-size: 12px;
line-height: 1.45;
}
.wiz-rediscover-text em {
color: #334155;
font-style: normal;
font-weight: 600;
}
/* ═══════════════════════════════════════════
Review grid (step 4)
═══════════════════════════════════════════ */
@@ -2496,9 +2754,13 @@ th {
.count-results {
display: flex;
flex: 1 1 auto;
flex-direction: column;
gap: 16px;
margin-top: 12px;
min-height: 0;
overflow: hidden;
padding: 0 16px 16px;
}
/* Summary stat overrides */
@@ -2626,7 +2888,9 @@ th {
.count-table-wrap {
border: 1px solid #e8edf4;
border-radius: 10px;
overflow: hidden;
flex: 1 1 auto;
min-height: 0;
overflow: auto;
}
.count-table {
@@ -3332,7 +3596,7 @@ th {
background: none;
border: none;
border-radius: 6px;
color: var(--text-muted);
color: #8a9ab4;
cursor: pointer;
display: flex;
font-size: 12.5px;
@@ -3344,27 +3608,33 @@ th {
}
.side-job-back:hover {
background: var(--border);
color: var(--text);
background: rgba(255, 255, 255, 0.05);
color: #c8d5e4;
}
.side-job-name {
color: var(--text);
font-size: 13px;
font-weight: 700;
margin: 4px 12px 6px;
color: #c8d5e4;
font-size: 12.5px;
font-weight: 600;
line-height: 1.35;
margin: 2px 12px 10px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.side-section-divider {
background: var(--border);
background: rgba(255, 255, 255, 0.06);
flex-shrink: 0;
height: 1px;
margin: 4px 10px;
margin: 8px 10px;
}
.side-nav--bottom {
display: flex;
flex-direction: column;
flex-shrink: 0;
gap: 2px;
margin-top: auto;
}
@@ -3700,8 +3970,49 @@ th {
.task-badge--failed { background: #fee2e2; color: #b91c1c; }
/* ── Results table (extracción) ── */
.extraction-results {
display: flex;
flex: 1 1 auto;
flex-direction: column;
gap: 16px;
min-height: 0;
overflow: hidden;
padding: 12px 16px 16px;
}
.extraction-table-wrap {
border: 1px solid #e8edf4;
border-radius: 10px;
flex: 1 1 auto;
min-height: 0;
overflow: auto;
padding: 0;
}
.extraction-empty {
color: var(--text-muted);
padding: 28px 12px !important;
text-align: center;
}
.extraction-empty-cell {
color: var(--text-muted);
}
.extraction-row--incomplete td {
background: #fefce8;
}
.extraction-filter-hint {
color: var(--text-muted);
flex-shrink: 0;
font-size: 12px;
margin: 0;
}
.results-table-wrapper {
flex: 1;
min-height: 0;
overflow: auto;
padding: 0 16px 16px;
}
@@ -3953,3 +4264,445 @@ th {
margin-left: auto;
padding: 3px 10px;
}
/* ── Cleaning wizard: lista de extracciones / reglas ── */
.cleaning-source-list {
display: flex;
flex-direction: column;
gap: 8px;
max-height: min(280px, 36vh);
overflow-y: auto;
}
.cleaning-source-item {
align-items: center;
background: #fff;
border: 1px solid #e2e8f0;
border-radius: 10px;
cursor: pointer;
display: flex;
gap: 12px;
justify-content: space-between;
padding: 12px 14px;
text-align: left;
transition: border-color 0.15s, background 0.15s;
width: 100%;
}
.cleaning-source-item:hover {
border-color: #94a3b8;
background: #f8fafc;
}
.cleaning-source-item--selected {
border-color: #2563eb;
background: #eff6ff;
box-shadow: 0 0 0 1px #2563eb;
}
.cleaning-source-item-main {
align-items: flex-start;
display: flex;
gap: 10px;
min-width: 0;
}
.cleaning-source-item-title {
color: #0f172a;
font-size: 13px;
font-weight: 600;
}
.cleaning-source-item-meta {
color: #64748b;
font-size: 12px;
margin-top: 2px;
}
/* ── Paso Reglas: lista fija + detalle con scroll ── */
.wiz-step--cleaning-rules {
display: flex;
flex: 1;
flex-direction: column;
gap: 12px;
min-height: 0;
}
.cleaning-rules-toolbar {
align-items: flex-start;
display: flex;
flex-shrink: 0;
gap: 12px;
justify-content: space-between;
}
.cleaning-rules-workspace {
display: grid;
flex: 1;
gap: 12px;
grid-template-columns: minmax(160px, 220px) minmax(0, 1fr);
min-height: 0;
overflow: hidden;
}
.cleaning-col-nav {
background: #f8fafc;
border: 1px solid #e2e8f0;
border-radius: 10px;
display: flex;
flex-direction: column;
gap: 2px;
min-height: 0;
overflow-y: auto;
padding: 6px;
}
.cleaning-col-nav-item {
align-items: center;
background: transparent;
border: none;
border-radius: 8px;
color: #334155;
cursor: pointer;
display: flex;
gap: 8px;
justify-content: space-between;
padding: 8px 10px;
text-align: left;
width: 100%;
}
.cleaning-col-nav-item:hover {
background: #eef2f7;
}
.cleaning-col-nav-item--active {
background: #fff;
box-shadow: 0 0 0 1px #2563eb;
color: #0f172a;
}
.cleaning-col-nav-item--changed:not(.cleaning-col-nav-item--active) {
background: #eff6ff;
}
.cleaning-col-nav-name {
font-size: 12px;
font-weight: 600;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.cleaning-col-nav-meta {
align-items: center;
display: flex;
flex-shrink: 0;
gap: 4px;
}
.cleaning-col-nav-count {
background: #2563eb;
border-radius: 999px;
color: #fff;
font-size: 10px;
font-weight: 700;
min-width: 18px;
padding: 1px 6px;
text-align: center;
}
.cleaning-col-detail {
background: #fff;
border: 1px solid #e2e8f0;
border-radius: 10px;
display: flex;
flex-direction: column;
gap: 12px;
min-height: 0;
min-width: 0;
overflow-x: hidden;
overflow-y: auto;
padding: 12px 14px;
}
.cleaning-col-detail-header,
.cleaning-preview-compare--compact,
.cleaning-rules-block,
.cleaning-copmod-fecha {
flex-shrink: 0;
}
.cleaning-col-detail-title {
font-size: 15px;
font-weight: 700;
margin: 0;
}
.cleaning-rules-block {
display: flex;
flex-direction: column;
gap: 6px;
}
.cleaning-rule-desc {
margin: 0;
}
.cleaning-preview-compare--compact {
background: #f8fafc;
border: 1px solid #e8edf4;
border-radius: 8px;
margin: 0;
padding: 10px;
}
.cleaning-preview-compare--changed {
background: #f0f7ff;
border-color: #bfdbfe;
}
.cleaning-sample-dock {
border-top: 1px solid #e8edf4;
display: flex;
flex: 0 0 auto;
flex-direction: column;
gap: 8px;
min-height: 140px;
padding-top: 10px;
}
.cleaning-sample-dock-header {
align-items: center;
display: flex;
flex-shrink: 0;
gap: 12px;
justify-content: space-between;
}
.cleaning-sample-tabs {
background: #eef2f7;
border-radius: 8px;
display: inline-flex;
padding: 2px;
}
.cleaning-sample-tab {
background: transparent;
border: none;
border-radius: 6px;
color: #64748b;
cursor: pointer;
font-size: 12px;
font-weight: 600;
padding: 4px 10px;
}
.cleaning-sample-tab:disabled {
cursor: not-allowed;
opacity: 0.45;
}
.cleaning-sample-tab--active {
background: #fff;
box-shadow: 0 1px 2px rgba(15, 23, 42, 0.08);
color: #0f172a;
}
.cleaning-sample-table-scroll {
max-height: 220px;
min-height: 0;
overflow: auto;
}
.cleaning-rules-nav {
flex-shrink: 0;
gap: 8px;
margin-top: 0;
}
.cleaning-type-badge {
background: #e2e8f0;
border-radius: 999px;
color: #334155;
font-size: 10px;
font-weight: 600;
padding: 2px 7px;
}
.cleaning-rule-card-meta {
align-items: center;
color: #64748b;
display: flex;
flex-wrap: wrap;
font-size: 12px;
gap: 8px;
margin-top: 4px;
}
.cleaning-rule-chips {
display: flex;
flex-wrap: wrap;
gap: 6px;
}
.cleaning-preview-compare {
align-items: center;
display: grid;
gap: 8px;
grid-template-columns: 1fr auto 1fr;
}
.cleaning-preview-side {
min-width: 0;
}
.cleaning-preview-label {
color: #64748b;
font-size: 11px;
font-weight: 700;
letter-spacing: 0.02em;
margin-bottom: 6px;
text-transform: uppercase;
}
.cleaning-preview-label--after {
color: #1d4ed8;
}
.cleaning-preview-arrow {
color: #94a3b8;
flex-shrink: 0;
}
.cleaning-preview-values {
display: flex;
flex-wrap: wrap;
gap: 4px;
}
.cleaning-preview-chip {
background: #f1f5f9;
border: 1px solid #e2e8f0;
border-radius: 6px;
color: #0f172a;
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 11px;
max-width: 100%;
overflow: hidden;
padding: 3px 7px;
text-overflow: ellipsis;
white-space: nowrap;
}
.cleaning-preview-chip--changed {
background: #dbeafe;
border-color: #93c5fd;
color: #1e3a8a;
font-weight: 600;
}
.cleaning-preview-chip-col {
color: #64748b;
font-weight: 500;
margin-right: 6px;
}
.cleaning-preview-empty {
color: #94a3b8;
font-size: 12px;
}
.cleaning-preview-table {
font-size: 12px;
}
.cleaning-th-changed,
.cleaning-td-changed {
background: #eff6ff;
}
@media (max-width: 900px) {
.cleaning-rules-workspace {
grid-template-columns: 1fr;
overflow: auto;
}
.cleaning-col-nav {
display: grid;
grid-auto-flow: column;
grid-auto-columns: max-content;
max-height: none;
overflow-x: auto;
overflow-y: hidden;
}
}
/* ── Auto-update modal ── */
.update-modal-backdrop {
align-items: center;
background: rgba(15, 23, 42, 0.45);
display: flex;
inset: 0;
justify-content: center;
position: fixed;
z-index: 1000;
}
.update-modal {
background: #ffffff;
border-radius: 12px;
box-shadow: 0 20px 50px rgba(15, 23, 42, 0.25);
display: flex;
flex-direction: column;
gap: 12px;
max-width: 440px;
padding: 20px 22px;
width: calc(100% - 32px);
}
.update-modal-header {
align-items: center;
display: flex;
justify-content: space-between;
gap: 12px;
}
.update-modal-header h2 {
font-size: 16px;
font-weight: 700;
margin: 0;
}
.update-modal-body {
color: #334155;
font-size: 13.5px;
line-height: 1.5;
margin: 0;
}
.update-modal-actions {
display: flex;
gap: 8px;
justify-content: flex-end;
margin-top: 8px;
}
.update-progress {
display: flex;
flex-direction: column;
gap: 6px;
}
.update-progress-bar {
background: #e2e8f0;
border-radius: 999px;
height: 8px;
overflow: hidden;
}
.update-progress-fill {
background: #2563eb;
height: 100%;
transition: width 0.2s ease;
}
@@ -1 +1 @@
export type JobSection = "counting" | "extraction" | "results";
export type JobSection = "counting" | "extraction" | "cleaning" | "results";
@@ -8,14 +8,17 @@ import {
Loader2,
Plus,
RefreshCw,
Sparkles,
} from "lucide-react";
import { useEffect, useState } from "react";
import type { BackgroundProgress, CountJobFile, Job, Task } from "../../shared/types";
import type { BackgroundProgress, CleaningJob, CountJobFile, Job, Task } from "../../shared/types";
import { CountResultsTable } from "../components/CountResultsTable";
import { ExtractionResultsTable } from "../components/ExtractionResultsTable";
import { BackgroundProgressBanner } from "../components/BackgroundProgressBanner";
import type { ApiClient } from "../lib/apiClient";
import { downloadCsv, toCsv, buildExportCsvFilename } from "../lib/csv";
type ResultTab = "counting" | "extraction";
type ResultTab = "counting" | "extraction" | "cleaning";
interface Props {
client: ApiClient;
@@ -23,9 +26,11 @@ interface Props {
tasks: Task[];
countingProgress?: BackgroundProgress | null;
extractionProgress?: BackgroundProgress | null;
cleaningProgress?: BackgroundProgress | null;
onRefresh: () => Promise<void>;
onStartCounting: () => void;
onStartExtraction: (ctx?: Record<string, unknown>) => void;
onStartCleaning: (ctx?: Record<string, unknown>) => void;
}
export function JobResultsView({
@@ -34,9 +39,11 @@ export function JobResultsView({
tasks,
countingProgress,
extractionProgress,
cleaningProgress,
onRefresh,
onStartCounting,
onStartExtraction,
onStartCleaning,
}: Props) {
const [activeTab, setActiveTab] = useState<ResultTab>("counting");
const [refreshing, setRefreshing] = useState(false);
@@ -45,6 +52,8 @@ export function JobResultsView({
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
const extractionTasks = tasks.filter((t) => t.type === "extraction")
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
const cleaningTasks = tasks.filter((t) => t.type === "cleaning")
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
async function refresh() {
setRefreshing(true);
@@ -54,7 +63,7 @@ export function JobResultsView({
return (
<div className="job-results-view">
{/* ── Sub-tabs: Conteo | Extracción ── */}
{/* ── Sub-tabs: Conteo | Extracción | Limpieza ── */}
<div className="job-results-tabs">
<button
type="button"
@@ -86,6 +95,21 @@ export function JobResultsView({
</span>
)}
</button>
<button
type="button"
className={`job-results-tab${activeTab === "cleaning" ? " job-results-tab--active" : ""}`}
onClick={() => setActiveTab("cleaning")}
>
<Sparkles size={15} />
Limpieza
{(cleaningTasks.length > 0 || cleaningProgress?.running) && (
<span className={`job-results-tab-count${cleaningProgress?.running ? " job-results-tab-count--live" : ""}`}>
{cleaningProgress?.running
? `${cleaningProgress.processed}/${cleaningProgress.total}`
: cleaningTasks.length}
</span>
)}
</button>
<div style={{ flex: 1 }} />
<button
type="button"
@@ -103,6 +127,7 @@ export function JobResultsView({
{activeTab === "counting" && (
<CountingResultsSection
client={client}
job={job}
tasks={countingTasks}
liveProgress={countingProgress}
onStartCounting={onStartCounting}
@@ -114,9 +139,20 @@ export function JobResultsView({
{activeTab === "extraction" && (
<ExtractionResultsSection
client={client}
job={job}
tasks={extractionTasks}
liveProgress={extractionProgress}
onStartExtraction={() => onStartExtraction()}
onStartCleaning={onStartCleaning}
/>
)}
{activeTab === "cleaning" && (
<CleaningResultsSection
client={client}
job={job}
tasks={cleaningTasks}
liveProgress={cleaningProgress}
onStartCleaning={() => onStartCleaning()}
/>
)}
</div>
@@ -129,12 +165,14 @@ export function JobResultsView({
══════════════════════════════════════════ */
function CountingResultsSection({
client,
job,
tasks,
liveProgress,
onStartCounting,
onExtractFromCount,
}: {
client: ApiClient;
job: Job;
tasks: Task[];
liveProgress?: BackgroundProgress | null;
onStartCounting: () => void;
@@ -175,17 +213,15 @@ function CountingResultsSection({
function exportCsv(subset: "valid" | "invalid") {
const subset_files = subset === "valid" ? files.filter(isFileValid) : files.filter((f) => !isFileValid(f));
const headers = ["filename", "radicado", "service_count", "status", "is_valid", "error"];
const rows = subset_files.map((f) =>
[`"${f.filename}"`, `"${f.radicado ?? ""}"`, f.service_count ?? "", f.status, f.is_valid === null ? "" : String(f.is_valid), `"${(f.error ?? "").replace(/"/g, '""')}"`].join(","),
);
const csv = [headers.join(","), ...rows].join("\n");
const blob = new Blob([csv], { type: "text/csv" });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = `conteo-${subset}-${new Date().toISOString().slice(0, 10)}.csv`;
a.click();
URL.revokeObjectURL(url);
const rows = subset_files.map((f) => [
f.filename,
f.radicado ?? "",
f.service_count ?? "",
f.status,
f.is_valid === null ? "" : String(f.is_valid),
f.error ?? "",
]);
downloadCsv(buildExportCsvFilename("conteo", job.title, subset), toCsv(headers, rows));
}
if (tasks.length === 0 && !liveProgress?.running) {
@@ -291,14 +327,18 @@ type Row = Record<string, unknown>;
function ExtractionResultsSection({
client,
job,
tasks,
liveProgress,
onStartExtraction,
onStartCleaning,
}: {
client: ApiClient;
job: Job;
tasks: Task[];
liveProgress?: BackgroundProgress | null;
onStartExtraction: () => void;
onStartCleaning: (ctx?: Record<string, unknown>) => void;
}) {
const [selectedTask, setSelectedTask] = useState<Task | null>(tasks[0] ?? null);
const [rows, setRows] = useState<Row[]>([]);
@@ -344,20 +384,11 @@ function ExtractionResultsSection({
}, [selectedTask?.id]); // eslint-disable-line react-hooks/exhaustive-deps
function exportCsv() {
const csv = [columns.join(","), ...rows.map((r) =>
columns.map((c) => {
const v = r[c];
if (v === null || v === undefined) return "";
return `"${String(v).replace(/"/g, '""')}"`;
}).join(","),
)].join("\n");
const blob = new Blob([csv], { type: "text/csv" });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = `extraccion-${new Date().toISOString().slice(0, 10)}.csv`;
a.click();
URL.revokeObjectURL(url);
const csv = toCsv(
columns,
rows.map((r) => columns.map((c) => r[c])),
);
downloadCsv(buildExportCsvFilename("extraccion", job.title), csv);
}
if (tasks.length === 0 && !liveProgress?.running) {
@@ -422,6 +453,24 @@ function ExtractionResultsSection({
<Download size={13} /> CSV ({rows.length} filas)
</button>
)}
{rows.length > 0 && selectedTask?.status === "completed" && (
<button
type="button"
className="btn btn-sm btn-primary"
onClick={() => {
const batchId = String(
(selectedTask.result as Record<string, unknown>)?.extraction_batch_id ?? "",
);
onStartCleaning({
extractionBatchId: batchId || undefined,
extractionTaskId: selectedTask.id,
jobId: job.id,
});
}}
>
<Sparkles size={13} /> Limpiar datos <ArrowRight size={13} />
</button>
)}
<button type="button" className="btn btn-sm btn-ghost" onClick={onStartExtraction}>
<Plus size={13} /> Nueva extracción
</button>
@@ -468,26 +517,162 @@ function ExtractionResultsSection({
<p>No hay filas extraídas.</p>
</div>
) : (
<div style={{ overflowX: "auto", flex: 1 }}>
<table className="results-table">
<thead>
<tr>{columns.map((c) => <th key={c}>{c}</th>)}</tr>
</thead>
<tbody>
{rows.map((row, i) => (
<tr key={i}>
{columns.map((c) => (
<td key={c}>
{row[c] === null || row[c] === undefined
? <span style={{ color: "var(--text-muted)" }}>—</span>
: String(row[c])}
</td>
))}
</tr>
<ExtractionResultsTable rows={rows} columns={columns} />
)}
</div>
);
}
/* ══════════════════════════════════════════
Sección de resultados de limpieza
══════════════════════════════════════════ */
function CleaningResultsSection({
client,
job,
tasks,
liveProgress,
onStartCleaning,
}: {
client: ApiClient;
job: Job;
tasks: Task[];
liveProgress?: BackgroundProgress | null;
onStartCleaning: () => void;
}) {
const [selectedTask, setSelectedTask] = useState<Task | null>(tasks[0] ?? null);
const [cleaningJob, setCleaningJob] = useState<CleaningJob | null>(null);
const [rows, setRows] = useState<Row[]>([]);
const [columns, setColumns] = useState<string[]>([]);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
useEffect(() => {
setSelectedTask(tasks[0] ?? null);
}, [tasks.map((t) => t.id).join(",")]); // eslint-disable-line react-hooks/exhaustive-deps
useEffect(() => {
if (!selectedTask) {
setRows([]);
setColumns([]);
setCleaningJob(null);
return;
}
let cancelled = false;
async function load() {
setLoading(true);
setError(null);
try {
const cleaningJobId = String(
(selectedTask!.result as Record<string, unknown>)?.cleaning_job_id ?? "",
);
if (!cleaningJobId) {
setError("La tarea no tiene cleaning_job_id");
return;
}
const jobData = await client.cleaningJob(cleaningJobId);
if (cancelled) return;
setCleaningJob(jobData);
if (jobData.status === "completed") {
const resp = await client.cleaningJobRows(cleaningJobId, 500, 0);
if (cancelled) return;
const data = resp.rows ?? [];
setRows(data);
setColumns(Array.from(new Set(data.flatMap((r) => Object.keys(r)))));
}
} catch (e) {
if (!cancelled) setError(String(e instanceof Error ? e.message : e));
} finally {
if (!cancelled) setLoading(false);
}
}
void load();
return () => { cancelled = true; };
}, [selectedTask?.id, client]); // eslint-disable-line react-hooks/exhaustive-deps
function exportCsv() {
const csv = toCsv(
columns,
rows.map((r) => columns.map((h) => r[h])),
);
downloadCsv(buildExportCsvFilename("limpieza", job.title), csv);
}
if (tasks.length === 0 && !liveProgress?.running) {
return (
<div className="job-results-empty">
<Sparkles size={36} style={{ opacity: 0.3 }} />
<p>No hay resultados de limpieza todavía.</p>
<button type="button" className="btn btn-primary" onClick={onStartCleaning}>
<Plus size={14} /> Iniciar limpieza
</button>
</div>
);
}
if (tasks.length === 0 && liveProgress?.running) {
return (
<div className="results-section results-section--live-only">
<BackgroundProgressBanner progress={liveProgress} />
</div>
);
}
return (
<div className="results-section">
{liveProgress?.running && (
<div className="results-live-progress">
<BackgroundProgressBanner progress={liveProgress} />
</div>
)}
<div className="results-section-header">
<div className="results-task-selector">
{tasks.length > 1 && (
<select
className="form-input"
style={{ width: "auto", fontSize: 13 }}
value={selectedTask?.id ?? ""}
onChange={(e) => setSelectedTask(tasks.find((t) => t.id === e.target.value) ?? null)}
>
{tasks.map((t, i) => (
<option key={t.id} value={t.id}>
{t.title ?? `Limpieza ${tasks.length - i}`} — {new Date(t.created_at).toLocaleDateString("es-CO")}
</option>
))}
</tbody>
</table>
</select>
)}
{selectedTask && (
<span className={`task-badge task-badge--${cleaningJob?.status ?? selectedTask.status}`}>
{cleaningJob?.status ?? selectedTask.status}
</span>
)}
</div>
<div className="results-section-actions">
{rows.length > 0 && (
<button type="button" className="btn btn-sm btn-ghost" onClick={exportCsv}>
<Download size={13} /> CSV ({rows.length} filas)
</button>
)}
<button type="button" className="btn btn-sm btn-ghost" onClick={onStartCleaning}>
<Plus size={13} /> Nueva limpieza
</button>
</div>
</div>
{error && (
<div className="wiz-toast wiz-toast--error" style={{ margin: "12px 16px" }}>
<AlertCircle size={14} /> {error}
</div>
)}
{loading && rows.length === 0 ? (
<div className="module-loading"><Loader2 size={20} className="spin" /></div>
) : rows.length === 0 ? (
<div className="job-results-empty" style={{ padding: 40 }}>
<p>No hay filas limpiadas.</p>
</div>
) : (
<ExtractionResultsTable rows={rows} columns={columns} />
)}
</div>
);
@@ -5,6 +5,7 @@ import type { BackgroundProgress, Job, Task } from "../../shared/types";
import type { JobSection } from "../types/jobSection";
import { BackgroundProgressBanner } from "../components/BackgroundProgressBanner";
import { CountingWizardModule } from "../modules/counting/CountingWizardModule";
import { CleaningWizardModule } from "../modules/cleaning/CleaningWizardModule";
import { JobExtractionWizard } from "../modules/extraction/JobExtractionWizard";
import { JobResultsView } from "./JobResultsView";
@@ -24,10 +25,13 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
const [savingTitle, setSavingTitle] = useState(false);
const [countingProgress, setCountingProgress] = useState<BackgroundProgress | null>(null);
const [extractionProgress, setExtractionProgress] = useState<BackgroundProgress | null>(null);
const [cleaningProgress, setCleaningProgress] = useState<BackgroundProgress | null>(null);
const pollingRef = useRef<ReturnType<typeof setInterval> | null>(null);
const backgroundRunning =
countingProgress?.running === true || extractionProgress?.running === true;
countingProgress?.running === true ||
extractionProgress?.running === true ||
cleaningProgress?.running === true;
const loadTasks = useCallback(async () => {
try {
@@ -74,6 +78,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
const activeBackground =
countingProgress?.running ? countingProgress
: extractionProgress?.running ? extractionProgress
: cleaningProgress?.running ? cleaningProgress
: null;
return (
@@ -114,6 +119,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
<div className="job-content-header-section">
{section === "counting" && "Conteo de servicios"}
{section === "extraction" && "Extracción de datos"}
{section === "cleaning" && "Limpieza de datos"}
{section === "results" && "Resultados del trabajo"}
</div>
</header>
@@ -161,6 +167,18 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
/>
</div>
<div className={`job-section-pane${section === "cleaning" ? "" : " job-section-pane--hidden"}`}>
<CleaningWizardModule
key={`cleaning-${String(sectionContext.extractionTaskId ?? sectionContext.extractionBatchId ?? "default")}`}
client={client}
job={job}
onTaskCreated={handleTaskCreated}
onProgress={(p) => setCleaningProgress(p.running ? p : null)}
onActivateModule={(type, ctx) => onActivateSection(type as JobSection, ctx)}
context={sectionContext}
/>
</div>
<div className={`job-section-pane${section === "results" ? "" : " job-section-pane--hidden"}`}>
<JobResultsView
client={client}
@@ -168,9 +186,11 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
tasks={tasks}
countingProgress={countingProgress}
extractionProgress={extractionProgress}
cleaningProgress={cleaningProgress}
onRefresh={loadTasks}
onStartCounting={() => onActivateSection("counting")}
onStartExtraction={(ctx?: Record<string, unknown>) => onActivateSection("extraction", ctx)}
onStartCleaning={(ctx?: Record<string, unknown>) => onActivateSection("cleaning", ctx)}
/>
</div>
</div>
@@ -0,0 +1,471 @@
import {
CheckCircle2,
Database,
Loader2,
Plus,
RotateCcw,
Server,
Trash2,
XCircle,
} from "lucide-react";
import { useEffect, useState } from "react";
import { DEFAULT_API_BASE_URL } from "../lib/apiDefaults";
import {
type CopModConfig,
defaultCopModConfig,
loadCopModConfig,
saveCopModConfig,
} from "../lib/cleaningSettings";
export type SettingsSection = "connection" | "cleaning" | "backend";
type Status = "idle" | "ok" | "error" | "busy";
type BackendTab = "config" | "models";
type SettingsViewProps = {
section: SettingsSection;
apiBaseUrl: string;
onApiBaseUrlChange: (url: string) => void;
connectionStatus: Status;
connectionMessage: string;
healthJson?: unknown;
verifyConnection: () => Promise<void>;
busy: boolean;
configJson: unknown;
modelsJson: unknown;
};
const SERVICE_LABELS: Record<string, string> = {
extraction: "Extracción",
cleaning: "Limpieza",
counting: "Conteo",
};
type ServiceRow = { id: string; label: string; ok: boolean; detail: string };
function parseHealthServices(health: unknown): ServiceRow[] {
if (!health || typeof health !== "object") return [];
const services = (health as { services?: unknown }).services;
if (!services || typeof services !== "object") return [];
return Object.entries(services as Record<string, unknown>).map(([id, raw]) => {
const info = raw && typeof raw === "object" ? (raw as Record<string, unknown>) : {};
const ok = Boolean(info.ok);
const code = info.status_code != null ? String(info.status_code) : "";
const err = typeof info.error === "string" ? info.error : "";
return {
id,
label: SERVICE_LABELS[id] ?? id,
ok,
detail: ok ? (code ? `HTTP ${code}` : "OK") : err || (code ? `HTTP ${code}` : "Error"),
};
});
}
export function SettingsView({
section,
apiBaseUrl,
onApiBaseUrlChange,
connectionStatus,
connectionMessage,
healthJson,
verifyConnection,
busy,
configJson,
modelsJson,
}: SettingsViewProps) {
if (section === "connection") {
return (
<ConnectionSettings
apiBaseUrl={apiBaseUrl}
onApiBaseUrlChange={onApiBaseUrlChange}
connectionStatus={connectionStatus}
connectionMessage={connectionMessage}
healthJson={healthJson}
verifyConnection={verifyConnection}
busy={busy}
/>
);
}
if (section === "cleaning") {
return <CleaningSettings />;
}
return (
<BackendSettings
connectionStatus={connectionStatus}
configJson={configJson}
modelsJson={modelsJson}
verifyConnection={verifyConnection}
busy={busy}
/>
);
}
function ConnectionSettings({
apiBaseUrl,
onApiBaseUrlChange,
connectionStatus,
connectionMessage,
healthJson,
verifyConnection,
busy,
}: {
apiBaseUrl: string;
onApiBaseUrlChange: (url: string) => void;
connectionStatus: Status;
connectionMessage: string;
healthJson?: unknown;
verifyConnection: () => Promise<void>;
busy: boolean;
}) {
const connectionLabel =
connectionStatus === "ok"
? "Conectado"
: connectionStatus === "busy"
? "Verificando…"
: connectionStatus === "error"
? "Sin conexión"
: "Sin verificar";
const services = parseHealthServices(healthJson);
const allOk = services.length > 0 && services.every((s) => s.ok);
return (
<div className="settings-page">
<header className="settings-page-header">
<h2>Conexión</h2>
<p>URL del API Gateway y estado de los servicios del backend.</p>
</header>
<div className="settings-page-body">
<div className={`settings-status-card settings-status-card--${connectionStatus}`}>
<div className={`settings-status-icon settings-status-icon--${connectionStatus}`}>
{connectionStatus === "ok" ? (
<CheckCircle2 size={20} />
) : connectionStatus === "error" ? (
<XCircle size={20} />
) : connectionStatus === "busy" ? (
<Loader2 size={20} className="spin" />
) : (
<Server size={20} />
)}
</div>
<div className="settings-status-text">
<strong>{connectionLabel}</strong>
<span>{connectionMessage}</span>
{connectionStatus === "ok" && services.length > 0 && (
<span className="settings-status-summary">
{allOk
? `${services.length} servicios operativos`
: `${services.filter((s) => s.ok).length} de ${services.length} servicios OK`}
</span>
)}
</div>
</div>
{services.length > 0 && connectionStatus === "ok" && (
<div className="settings-service-grid" aria-label="Estado de servicios">
{services.map((svc) => (
<div
key={svc.id}
className={`settings-service-chip${svc.ok ? " settings-service-chip--ok" : " settings-service-chip--err"}`}
>
<span className="settings-service-chip-dot" aria-hidden />
<div className="settings-service-chip-text">
<strong>{svc.label}</strong>
<span>{svc.detail}</span>
</div>
</div>
))}
</div>
)}
<div className="settings-panel">
<div className="settings-field">
<label className="wiz-field-label" htmlFor="api-base-url">
URL del API Gateway
</label>
<div className="settings-url-row">
<input
id="api-base-url"
className="form-input"
value={apiBaseUrl}
onChange={(e) => onApiBaseUrlChange(e.target.value)}
placeholder={DEFAULT_API_BASE_URL}
spellCheck={false}
/>
<button
type="button"
className="btn btn-primary"
onClick={() => void verifyConnection()}
disabled={busy}
>
<Server size={15} />
{busy ? "Verificando…" : "Verificar"}
</button>
</div>
<p className="form-hint">Los cambios se guardan automáticamente en este equipo.</p>
</div>
</div>
</div>
</div>
);
}
function parseAmountList(text: string): number[] {
return [
...new Set(
text
.split(/[,;\s]+/)
.map((s) => s.trim())
.filter(Boolean)
.map((s) => Math.round(Number(s.replace(/\./g, "").replace(",", "."))))
.filter((n) => Number.isFinite(n) && n >= 0),
),
].sort((a, b) => a - b);
}
function CleaningSettings() {
const [config, setConfig] = useState<CopModConfig>(() => loadCopModConfig());
const [drafts, setDrafts] = useState<Record<string, string>>(() => {
const initial = loadCopModConfig();
return Object.fromEntries(
Object.entries(initial.moderadora_by_year).map(([y, vals]) => [y, vals.join(", ")]),
);
});
const [savedFlash, setSavedFlash] = useState(false);
const [newYear, setNewYear] = useState("");
const [hydrated, setHydrated] = useState(false);
useEffect(() => {
setHydrated(true);
}, []);
useEffect(() => {
if (!hydrated) return;
saveCopModConfig(config);
setSavedFlash(true);
const t = window.setTimeout(() => setSavedFlash(false), 1200);
return () => window.clearTimeout(t);
}, [config, hydrated]);
const years = Object.keys(config.moderadora_by_year).sort(
(a, b) => Number(a) - Number(b),
);
function commitYear(year: string, text: string) {
const nums = parseAmountList(text);
setDrafts((prev) => ({ ...prev, [year]: nums.join(", ") }));
setConfig((prev) => ({
moderadora_by_year: {
...prev.moderadora_by_year,
[year]: nums,
},
}));
}
function removeYear(year: string) {
setConfig((prev) => {
const next = { ...prev.moderadora_by_year };
delete next[year];
return { moderadora_by_year: next };
});
setDrafts((prev) => {
const next = { ...prev };
delete next[year];
return next;
});
}
function addYear() {
const y = String(Number.parseInt(newYear, 10));
if (!Number.isFinite(Number(y)) || y === "NaN") return;
if (config.moderadora_by_year[y]) {
setNewYear("");
return;
}
setConfig((prev) => ({
moderadora_by_year: { ...prev.moderadora_by_year, [y]: [] },
}));
setDrafts((prev) => ({ ...prev, [y]: "" }));
setNewYear("");
}
function resetDefaults() {
const defaults = defaultCopModConfig();
setConfig(defaults);
setDrafts(
Object.fromEntries(
Object.entries(defaults.moderadora_by_year).map(([y, vals]) => [y, vals.join(", ")]),
),
);
}
return (
<div className="settings-page">
<header className="settings-page-header">
<div className="settings-page-header-row">
<div>
<h2>Limpieza · COP / MOD</h2>
<p>
Cuotas moderadora por año. La regla «Separa COP/MOD» clasifica el importe
según el año de la fecha de servicio que elijas en el wizard.
</p>
</div>
<div className="settings-page-actions">
{savedFlash && <span className="settings-saved-badge">Guardado</span>}
<button type="button" className="btn btn-sm" onClick={resetDefaults}>
<RotateCcw size={14} />
Restaurar defaults
</button>
</div>
</div>
</header>
<div className="settings-page-body">
<div className="settings-panel">
<h3 className="settings-panel-title">Cuotas moderadora por año</h3>
<p className="form-hint" style={{ marginBottom: 12 }}>
Separa los valores con coma o espacio. Si el importe coincide con uno de
estos montos en el año de la fecha, se marca como moderadora; si no, como
copago.
</p>
<div className="copmod-year-list">
{years.length === 0 && (
<p className="wiz-desc">No hay años configurados. Añade uno o restaura los defaults.</p>
)}
{years.map((year) => (
<div key={year} className="copmod-year-row">
<label className="copmod-year-label" htmlFor={`copmod-year-${year}`}>
{year}
</label>
<input
id={`copmod-year-${year}`}
className="form-input"
value={drafts[year] ?? config.moderadora_by_year[year].join(", ")}
onChange={(e) =>
setDrafts((prev) => ({ ...prev, [year]: e.target.value }))
}
onBlur={(e) => commitYear(year, e.target.value)}
placeholder="ej. 5000, 20100, 52800"
/>
<button
type="button"
className="btn btn-sm btn-ghost"
aria-label={`Eliminar año ${year}`}
onClick={() => removeYear(year)}
>
<Trash2 size={14} />
</button>
</div>
))}
</div>
<div className="copmod-add-row">
<input
className="form-input copmod-add-year"
value={newYear}
onChange={(e) => setNewYear(e.target.value)}
placeholder="Año"
inputMode="numeric"
aria-label="Nuevo año"
onKeyDown={(e) => {
if (e.key === "Enter") {
e.preventDefault();
addYear();
}
}}
/>
<button type="button" className="btn btn-sm" onClick={addYear} disabled={!newYear.trim()}>
<Plus size={14} />
Añadir año
</button>
</div>
</div>
</div>
</div>
);
}
function BackendSettings({
connectionStatus,
configJson,
modelsJson,
verifyConnection,
busy,
}: {
connectionStatus: Status;
configJson: unknown;
modelsJson: unknown;
verifyConnection: () => Promise<void>;
busy: boolean;
}) {
const [backendTab, setBackendTab] = useState<BackendTab>("config");
const backendData = backendTab === "config" ? configJson : modelsJson;
return (
<div className="settings-page">
<header className="settings-page-header">
<div className="settings-page-header-row">
<div>
<h2>Backend</h2>
<p>Respuesta en vivo de <code>/config</code> y <code>/models</code> del gateway.</p>
</div>
<button
type="button"
className="btn btn-sm"
onClick={() => void verifyConnection()}
disabled={busy}
>
{busy ? <Loader2 size={14} className="spin" /> : <Server size={14} />}
Actualizar
</button>
</div>
</header>
<div className="settings-page-body settings-page-body--fill">
<div className="settings-main-header" style={{ padding: "0 0 12px", border: 0 }}>
<div className="settings-main-title">
<Database size={16} />
<h3>Datos del servidor</h3>
</div>
<div className="settings-tabs" role="tablist">
<button
type="button"
role="tab"
aria-selected={backendTab === "config"}
className={backendTab === "config" ? "active" : ""}
onClick={() => setBackendTab("config")}
>
Configuración
</button>
<button
type="button"
role="tab"
aria-selected={backendTab === "models"}
className={backendTab === "models" ? "active" : ""}
onClick={() => setBackendTab("models")}
>
Modelos
</button>
</div>
</div>
<div className="settings-json-area" style={{ padding: 0 }}>
{backendData != null ? (
<pre className="settings-json-box">{JSON.stringify(backendData, null, 2)}</pre>
) : (
<div className="settings-empty">
<Server size={36} strokeWidth={1.2} />
<p>Sin datos del backend</p>
<span>
{connectionStatus === "ok"
? "Pulsa Actualizar para cargar los datos."
: "Verifica la conexión en la sección Conexión."}
</span>
</div>
)}
</div>
</div>
</div>
);
}
+106 -1
View File
@@ -164,6 +164,83 @@ export interface CountSyncResponse {
failed: number;
}
export interface CleaningRuleInfo {
id: string;
name: string;
description: string;
}
export interface CleaningColumnAnalysis {
name: string;
inferred_type: string;
example?: string | null;
samples?: string[];
null_count?: number;
null_pct?: number;
unique_count?: number;
applicable_rules?: string[];
[key: string]: unknown;
}
export interface CleaningAnalysisResponse {
batch_id?: string;
filename?: string;
delimiter?: string;
row_count?: number;
columns: CleaningColumnAnalysis[];
sample_rows?: Array<Record<string, unknown>>;
}
export interface CleaningPreviewColumn {
name: string;
before: string[];
after: string[];
changed?: boolean;
}
export interface CleaningPreviewResponse {
rows_before: Array<Record<string, unknown>>;
rows_after: Array<Record<string, unknown>>;
columns: CleaningPreviewColumn[];
}
export interface CleaningPreviewPayload {
rows?: Array<Record<string, unknown>>;
extraction_batch_id?: string;
rules_per_column: Record<string, string[]>;
cop_mod_fecha_column?: Record<string, string>;
cop_mod_config?: Record<string, unknown> | null;
sample_size?: number;
}
export interface CleaningJob {
id: string;
extraction_batch_id: string | null;
job_id: string | null;
source: "extraction" | "csv" | string;
status: "pending" | "running" | "completed" | "failed" | string;
request: Record<string, unknown>;
log: string[];
error: string | null;
created_at: string;
updated_at: string;
}
export interface RunCleaningPayload {
extraction_batch_id: string;
job_id?: string;
rules_per_column: Record<string, string[]>;
cop_mod_fecha_column?: Record<string, string>;
cop_mod_config?: Record<string, unknown> | null;
}
export interface CleaningRowsResponse {
job_id: string;
rows: Array<Record<string, unknown>>;
limit: number;
offset: number;
}
export interface CountingFormState {
localRoot: string;
containerRoot: string;
@@ -174,7 +251,7 @@ export interface CountingFormState {
/** Estado de progreso de un módulo ejecutándose en segundo plano. */
export interface BackgroundProgress {
module: "counting" | "extraction";
module: "counting" | "extraction" | "cleaning";
total: number;
processed: number;
failed: number;
@@ -189,6 +266,31 @@ export interface RuntimeConfig {
documentRootContainer: string;
}
export interface UpdateManifest {
version: string;
releaseDate?: string;
/** Fecha de lanzamiento como en DATA.txt, ej. "06/07/2026" */
releaseDateDisplay?: string;
fileName: string;
url: string;
sha256: string;
size?: number;
}
export interface UpdateCheckResult {
updateAvailable: boolean;
currentVersion: string;
manifest?: UpdateManifest;
error?: string;
}
export interface UpdateProgress {
phase: "downloading" | "verifying" | "installing" | string;
received?: number;
total?: number;
percent?: number;
}
export interface DesktopBridge {
getRuntimeConfig(): Promise<RuntimeConfig>;
selectDirectory(): Promise<string | null>;
@@ -202,4 +304,7 @@ export interface DesktopBridge {
keywords: string[],
): Promise<ManifestEntry[]>;
readFile(filePath: string): Promise<FileUploadData>;
checkForUpdates(): Promise<UpdateCheckResult>;
installUpdate(manifest: UpdateManifest): Promise<{ ok: boolean; destPath?: string }>;
onUpdateProgress(callback: (progress: UpdateProgress) => void): () => void;
}
+1 -2
View File
@@ -1126,8 +1126,7 @@ def create_count_job(request: CreateCountJobRequest) -> dict[str, Any]:
job = session.get(Job, request.job_id)
if job is None:
raise HTTPException(status_code=404, detail="Job no encontrado")
if request.title:
job.title = request.title
# No sobrescribir el nombre que el usuario dio al trabajo.
job.status = "running"
job.updated_at = utcnow()
else:
+10 -2
View File
@@ -136,7 +136,13 @@ class CleaningJob(Base):
__tablename__ = "cleaning_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
extraction_batch_id: Mapped[str] = mapped_column(ForeignKey("extraction_batches.id"), index=True)
extraction_batch_id: Mapped[str | None] = mapped_column(
ForeignKey("extraction_batches.id"), nullable=True, index=True
)
job_id: Mapped[str | None] = mapped_column(
String(36), ForeignKey("jobs.id"), nullable=True, index=True
)
source: Mapped[str] = mapped_column(String(32), default="extraction", index=True)
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
log: Mapped[list[str]] = mapped_column(JSON, default=list)
@@ -150,7 +156,9 @@ class CleanedRow(Base):
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
cleaning_job_id: Mapped[str] = mapped_column(ForeignKey("cleaning_jobs.id"), index=True)
extraction_batch_id: Mapped[str] = mapped_column(ForeignKey("extraction_batches.id"), index=True)
extraction_batch_id: Mapped[str | None] = mapped_column(
ForeignKey("extraction_batches.id"), nullable=True, index=True
)
row_index: Mapped[int] = mapped_column(Integer, default=0)
row_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)