diff --git a/.gitignore b/.gitignore index aea078a..70546ac 100644 --- a/.gitignore +++ b/.gitignore @@ -140,6 +140,7 @@ celerybeat.pid # Environments .env +.env.minio .venv env/ venv/ diff --git a/docs/database.dbml b/docs/database.dbml index 2e29b1b..58929a0 100644 --- a/docs/database.dbml +++ b/docs/database.dbml @@ -167,7 +167,9 @@ Table extraction_rows { Table cleaning_jobs { id varchar(36) [pk, note: 'UUID v4'] - extraction_batch_id varchar(36) [not null, ref: > extraction_batches.id] + extraction_batch_id varchar(36) [null, ref: > extraction_batches.id, note: 'Null si source=csv'] + job_id varchar(36) [null, ref: > jobs.id, note: 'Trabajo Desktop al que pertenece'] + source varchar(32) [not null, default: 'extraction', note: 'extraction | csv'] status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed'] request jsonb [not null, default: '{}', note: 'CleaningRequest (rules_per_column, etc.)'] log jsonb [not null, default: '[]', note: 'Lista de mensajes de auditoría del job'] @@ -177,16 +179,18 @@ Table cleaning_jobs { indexes { extraction_batch_id + job_id + source status } - Note: 'Ejecución de reglas de limpieza sobre las filas de un lote de extracción.' + Note: 'Ejecución de reglas de limpieza sobre filas de extracción o CSV subido.' } Table cleaned_rows { id integer [pk, increment] cleaning_job_id varchar(36) [not null, ref: > cleaning_jobs.id] - extraction_batch_id varchar(36) [not null, ref: > extraction_batches.id] + extraction_batch_id varchar(36) [null, ref: > extraction_batches.id] row_index integer [not null, default: 0, note: 'Conserva el orden del lote original'] row_json jsonb [not null, default: '{}', note: 'Fila tras aplicar reglas de limpieza'] created_at timestamptz [not null] diff --git a/services/api_gateway/api/main.py b/services/api_gateway/api/main.py index 3048722..b5960d5 100644 --- a/services/api_gateway/api/main.py +++ b/services/api_gateway/api/main.py @@ -66,7 +66,13 @@ def _client() -> httpx.AsyncClient: def _is_upload_path(path: str) -> bool: - return path.startswith("/count/") or "/ingest" in path or "/upload" in path + return ( + path.startswith("/count/") + or "/ingest" in path + or "/upload" in path + or path.startswith("/cleaning/analyze-csv") + or path.startswith("/cleaning/jobs/from-csv") + ) def _timeout_for_path(path: str) -> httpx.Timeout: diff --git a/services/cleaning/README.md b/services/cleaning/README.md index e5b2b77..57138cb 100644 --- a/services/cleaning/README.md +++ b/services/cleaning/README.md @@ -18,7 +18,11 @@ Recibe lotes de filas extraídas por el servicio `extraction` y aplica reglas co | `GET` | `/health` | Liveness | | `GET` | `/rules` | Lista de reglas de limpieza disponibles | | `GET` | `/batches/{batch_id}/analysis` | Análisis de columnas de un lote de extracción | -| `POST` | `/jobs` | Crear job de limpieza | +| `POST` | `/analyze-csv` | Análisis de columnas desde CSV (multipart) | +| `POST` | `/preview` | Vista previa Antes/Después sobre muestra (sin persistir) | +| `POST` | `/jobs` | Limpieza desde `extraction_batch_id` (+ `job_id` opcional) | +| `POST` | `/jobs/from-csv` | Limpieza desde CSV (multipart + reglas JSON) | +| `GET` | `/jobs?job_id=` | Listar jobs (filtro opcional por trabajo Desktop) | | `GET` | `/jobs/{job_id}` | Estado y detalle del job | | `GET` | `/jobs/{job_id}/rows` | Filas limpias del job | @@ -44,7 +48,7 @@ Las reglas disponibles se listan en `GET /rules`. ## Reglas de limpieza -Definidas en `domain/csv_cleaning.py`. Cada regla es una función que transforma los valores de una columna. Se pueden encadenar múltiples reglas por columna. +Definidas en el paquete `domain/csv_cleaning/` (registro en `rules/builtins.py`). Cada regla transforma los valores de una columna; se pueden encadenar varias por columna. Para agregar una regla nueva, ver `rules/registry.py`. --- @@ -72,11 +76,14 @@ cleaning_jobs — job de limpieza (FK → extraction_batches) ``` api/ - main.py — FastAPI app con todos los endpoints + main.py — FastAPI app con todos los endpoints domain/ - csv_cleaning.py — reglas de limpieza y función de análisis + csv_cleaning/ — paquete de limpieza (análisis + reglas) + analyze.py — resumen por columna + apply.py — aplicación de reglas + rules/ — registro e implementaciones de reglas infrastructure/ - database.py — modelos SQLAlchemy y sesión + database.py — modelos SQLAlchemy y sesión Dockerfile requirements.txt ``` diff --git a/services/cleaning/api/main.py b/services/cleaning/api/main.py index 505da45..dc7e365 100644 --- a/services/cleaning/api/main.py +++ b/services/cleaning/api/main.py @@ -6,16 +6,24 @@ import json from typing import Any import pandas as pd -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, File, Form, HTTPException, UploadFile from pydantic import BaseModel, Field from sqlalchemy import delete, select -from domain.csv_cleaning import CLEANING_RULES, analyze_dataframe, apply_rules +from domain.csv_cleaning import ( + CLEANING_RULES, + analyze_dataframe, + apply_rules, + load_csv_bytes, + sample_dataframe_rows, +) from infrastructure.database import ( CleanedRow, CleaningJob, ExtractionBatch, ExtractionRow, + Job, + Task, init_db, session_scope, utcnow, @@ -36,12 +44,22 @@ app = FastAPI( class CleaningRequest(BaseModel): - extraction_batch_id: str + extraction_batch_id: str | None = None + job_id: str | None = None rules_per_column: dict[str, list[str]] = Field(default_factory=dict) cop_mod_fecha_column: dict[str, str] = Field(default_factory=dict) cop_mod_config: dict[str, Any] | None = None +class PreviewRequest(BaseModel): + rows: list[dict[str, Any]] = Field(default_factory=list) + extraction_batch_id: str | None = None + rules_per_column: dict[str, list[str]] = Field(default_factory=dict) + cop_mod_fecha_column: dict[str, str] = Field(default_factory=dict) + cop_mod_config: dict[str, Any] | None = None + sample_size: int = 8 + + def dataframe_to_records(df: pd.DataFrame) -> list[dict[str, Any]]: return json.loads(df.to_json(orient="records", date_format="iso")) @@ -50,6 +68,8 @@ def serialize_job(job: CleaningJob) -> dict[str, Any]: return { "id": job.id, "extraction_batch_id": job.extraction_batch_id, + "job_id": job.job_id, + "source": job.source, "status": job.status, "request": job.request, "log": job.log, @@ -59,6 +79,144 @@ def serialize_job(job: CleaningJob) -> dict[str, Any]: } +def _validate_job_id(session: Any, job_id: str | None) -> None: + if not job_id: + return + if session.get(Job, job_id) is None: + raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado") + + +def _create_cleaning_task( + session: Any, + *, + job_id: str | None, + cleaning_job_id: str, + source: str, + status: str, + request: dict[str, Any], + error: str | None = None, + title: str | None = None, +) -> str | None: + """Crea Task(type=cleaning) vinculada al job Desktop. Devuelve task.id o None.""" + if not job_id: + return None + now = utcnow() + task = Task( + job_id=job_id, + type="cleaning", + status=status, + title=title or f"Limpieza ({source})", + request=request, + result={"cleaning_job_id": cleaning_job_id}, + error=error, + started_at=now, + finished_at=now, + updated_at=now, + ) + session.add(task) + session.flush() + return task.id + + +def _persist_cleaned_rows( + session: Any, + *, + job_id: str, + extraction_batch_id: str | None, + output_rows: list[dict[str, Any]], +) -> None: + session.execute(delete(CleanedRow).where(CleanedRow.cleaning_job_id == job_id)) + for idx, row in enumerate(output_rows, start=1): + session.add( + CleanedRow( + cleaning_job_id=job_id, + extraction_batch_id=extraction_batch_id, + row_index=idx, + row_json=row, + ) + ) + + +def _run_rules_on_dataframe( + df: pd.DataFrame, + rules_per_column: dict[str, list[str]], + cop_mod_fecha_column: dict[str, str], + cop_mod_config: dict[str, Any] | None, +) -> tuple[list[dict[str, Any]], list[str]]: + result_df, log = apply_rules( + df, + rules_per_column, + cop_mod_fecha_column=cop_mod_fecha_column, + cop_mod_config=cop_mod_config, + ) + return dataframe_to_records(result_df), log + + +def _parse_json_form(raw: str | None, default: Any) -> Any: + if not raw or not raw.strip(): + return default + try: + return json.loads(raw) + except json.JSONDecodeError as exc: + raise HTTPException(status_code=400, detail=f"JSON inválido: {exc}") from exc + + +def _analysis_payload(df: pd.DataFrame, **extra: Any) -> dict[str, Any]: + return { + **extra, + "row_count": len(df), + "columns": [summary.__dict__ for summary in analyze_dataframe(df)], + "sample_rows": sample_dataframe_rows(df, 8), + } + + +def _cell_str(value: Any) -> str: + if value is None: + return "" + text = str(value) + if text in ("None", "nan", "NaT"): + return "" + if len(text) > 80: + return text[:77] + "..." + return text + + +def _preview_from_dataframe( + df: pd.DataFrame, + rules_per_column: dict[str, list[str]], + cop_mod_fecha_column: dict[str, str], + cop_mod_config: dict[str, Any] | None, +) -> dict[str, Any]: + rows_before = dataframe_to_records(df) + result_df, _log = apply_rules( + df, + rules_per_column, + cop_mod_fecha_column=cop_mod_fecha_column, + cop_mod_config=cop_mod_config, + ) + rows_after = dataframe_to_records(result_df) + + # Emparejar por índice de fila de muestra (misma posición) + n = min(len(rows_before), len(rows_after)) + all_cols = list(dict.fromkeys([*df.columns.astype(str), *result_df.columns.astype(str)])) + columns_preview: list[dict[str, Any]] = [] + for col in all_cols: + before_vals = [_cell_str(rows_before[i].get(col)) for i in range(n)] + after_vals = [_cell_str(rows_after[i].get(col)) for i in range(n)] + columns_preview.append({ + "name": col, + "before": before_vals, + "after": after_vals, + "changed": before_vals != after_vals, + }) + + return { + "rows_before": rows_before, + "rows_after": rows_after, + "columns": columns_preview, + } + + @app.get("/health") def health() -> dict[str, str]: return {"status": "ok"} @@ -90,18 +248,75 @@ def analyze_batch(batch_id: str) -> dict[str, Any]: records = [row[0] for row in rows] df = pd.DataFrame(records) - return { - "batch_id": batch_id, - "columns": [summary.__dict__ for summary in analyze_dataframe(df)], - } + return _analysis_payload(df, batch_id=batch_id) + + +@app.post("/analyze-csv") +async def analyze_csv(file: UploadFile = File(...)) -> dict[str, Any]: + content = await file.read() + if not content: + raise HTTPException(status_code=400, detail="El archivo CSV está vacío") + try: + df, delimiter = load_csv_bytes(content, file.filename or "upload.csv") + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=400, detail=f"No se pudo leer el CSV: {exc}") from exc + if df.empty: + raise HTTPException(status_code=400, detail="El CSV no tiene filas") + return _analysis_payload(df, filename=file.filename, delimiter=delimiter) + + +@app.post("/preview") +def preview_cleaning(request: PreviewRequest) -> dict[str, Any]: + """Aplica reglas sobre una muestra pequeña y devuelve Antes/Después.""" + records: list[dict[str, Any]] = list(request.rows or []) + + if not records and request.extraction_batch_id: + with session_scope() as session: + batch = session.get(ExtractionBatch, request.extraction_batch_id) + if batch is None: + raise HTTPException(status_code=404, detail="Lote de extraccion no encontrado") + rows = session.execute( + select(ExtractionRow.row_json) + .where(ExtractionRow.batch_id == request.extraction_batch_id) + .order_by(ExtractionRow.row_index, ExtractionRow.id) + ).all() + all_records = [row[0] for row in rows] + if not all_records: + raise HTTPException(status_code=400, detail="El lote no tiene filas") + df_full = pd.DataFrame(all_records) + records = sample_dataframe_rows(df_full, max(1, min(request.sample_size, 20))) + + if not records: + raise HTTPException( + status_code=400, + detail="Se requieren rows o extraction_batch_id con filas", + ) + + df = pd.DataFrame(records) + try: + return _preview_from_dataframe( + df, + request.rules_per_column, + request.cop_mod_fecha_column, + request.cop_mod_config, + ) + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=500, detail=str(exc)) from exc @app.post("/jobs") def run_cleaning(request: CleaningRequest) -> dict[str, Any]: + if not request.extraction_batch_id: + raise HTTPException( + status_code=400, + detail="extraction_batch_id es requerido (use /jobs/from-csv para CSV)", + ) + with session_scope() as session: batch = session.get(ExtractionBatch, request.extraction_batch_id) if batch is None: raise HTTPException(status_code=404, detail="Lote de extraccion no encontrado") + _validate_job_id(session, request.job_id) extracted_rows = session.execute( select(ExtractionRow.row_json) @@ -114,6 +329,8 @@ def run_cleaning(request: CleaningRequest) -> dict[str, Any]: job = CleaningJob( extraction_batch_id=request.extraction_batch_id, + job_id=request.job_id, + source="extraction", status="running", request=request.model_dump(), updated_at=utcnow(), @@ -121,36 +338,45 @@ def run_cleaning(request: CleaningRequest) -> dict[str, Any]: session.add(job) session.flush() job_id = job.id + batch_id = request.extraction_batch_id + desktop_job_id = request.job_id + request_dump = request.model_dump() try: df = pd.DataFrame(records) - result_df, log = apply_rules( + output_rows, log = _run_rules_on_dataframe( df, request.rules_per_column, - cop_mod_fecha_column=request.cop_mod_fecha_column, - cop_mod_config=request.cop_mod_config, + request.cop_mod_fecha_column, + request.cop_mod_config, ) - output_rows = dataframe_to_records(result_df) with session_scope() as session: - session.execute(delete(CleanedRow).where(CleanedRow.cleaning_job_id == job_id)) - for idx, row in enumerate(output_rows, start=1): - session.add( - CleanedRow( - cleaning_job_id=job_id, - extraction_batch_id=request.extraction_batch_id, - row_index=idx, - row_json=row, - ) - ) + _persist_cleaned_rows( + session, + job_id=job_id, + extraction_batch_id=batch_id, + output_rows=output_rows, + ) job = session.get(CleaningJob, job_id) if job is None: raise RuntimeError("No se encontro el job de limpieza recien creado.") job.status = "completed" job.log = log job.updated_at = utcnow() + _create_cleaning_task( + session, + job_id=desktop_job_id, + cleaning_job_id=job_id, + source="extraction", + status="completed", + request=request_dump, + title="Limpieza desde extracción", + ) session.flush() return serialize_job(job) + except HTTPException: + raise except Exception as exc: with session_scope() as session: job = session.get(CleaningJob, job_id) @@ -158,15 +384,127 @@ def run_cleaning(request: CleaningRequest) -> dict[str, Any]: job.status = "failed" job.error = str(exc) job.updated_at = utcnow() + _create_cleaning_task( + session, + job_id=desktop_job_id, + cleaning_job_id=job_id, + source="extraction", + status="failed", + request=request_dump, + error=str(exc), + title="Limpieza desde extracción", + ) + raise HTTPException(status_code=500, detail=str(exc)) from exc + + +@app.post("/jobs/from-csv") +async def run_cleaning_from_csv( + file: UploadFile = File(...), + job_id: str | None = Form(None), + rules_per_column: str | None = Form(None), + cop_mod_fecha_column: str | None = Form(None), + cop_mod_config: str | None = Form(None), +) -> dict[str, Any]: + content = await file.read() + if not content: + raise HTTPException(status_code=400, detail="El archivo CSV está vacío") + + rules = _parse_json_form(rules_per_column, {}) + fecha_map = _parse_json_form(cop_mod_fecha_column, {}) + cop_cfg = _parse_json_form(cop_mod_config, None) + if not isinstance(rules, dict): + raise HTTPException(status_code=400, detail="rules_per_column debe ser un objeto JSON") + if not isinstance(fecha_map, dict): + raise HTTPException(status_code=400, detail="cop_mod_fecha_column debe ser un objeto JSON") + + try: + df, delimiter = load_csv_bytes(content, file.filename or "upload.csv") + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=400, detail=f"No se pudo leer el CSV: {exc}") from exc + if df.empty: + raise HTTPException(status_code=400, detail="El CSV no tiene filas") + + request_dump: dict[str, Any] = { + "source": "csv", + "filename": file.filename, + "delimiter": delimiter, + "job_id": job_id, + "rules_per_column": rules, + "cop_mod_fecha_column": fecha_map, + "cop_mod_config": cop_cfg, + } + + with session_scope() as session: + _validate_job_id(session, job_id) + job = CleaningJob( + extraction_batch_id=None, + job_id=job_id, + source="csv", + status="running", + request=request_dump, + updated_at=utcnow(), + ) + session.add(job) + session.flush() + cleaning_job_id = job.id + desktop_job_id = job_id + + try: + output_rows, log = _run_rules_on_dataframe(df, rules, fecha_map, cop_cfg) + + with session_scope() as session: + _persist_cleaned_rows( + session, + job_id=cleaning_job_id, + extraction_batch_id=None, + output_rows=output_rows, + ) + job = session.get(CleaningJob, cleaning_job_id) + if job is None: + raise RuntimeError("No se encontro el job de limpieza recien creado.") + job.status = "completed" + job.log = log + job.updated_at = utcnow() + _create_cleaning_task( + session, + job_id=desktop_job_id, + cleaning_job_id=cleaning_job_id, + source="csv", + status="completed", + request=request_dump, + title=f"Limpieza CSV — {file.filename or 'upload.csv'}", + ) + session.flush() + return serialize_job(job) + except HTTPException: + raise + except Exception as exc: + with session_scope() as session: + job = session.get(CleaningJob, cleaning_job_id) + if job is not None: + job.status = "failed" + job.error = str(exc) + job.updated_at = utcnow() + _create_cleaning_task( + session, + job_id=desktop_job_id, + cleaning_job_id=cleaning_job_id, + source="csv", + status="failed", + request=request_dump, + error=str(exc), + title=f"Limpieza CSV — {file.filename or 'upload.csv'}", + ) raise HTTPException(status_code=500, detail=str(exc)) from exc @app.get("/jobs") -def list_jobs() -> list[dict[str, Any]]: +def list_jobs(job_id: str | None = None) -> list[dict[str, Any]]: with session_scope() as session: - jobs = session.scalars( - select(CleaningJob).order_by(CleaningJob.created_at.desc()) - ).all() + stmt = select(CleaningJob).order_by(CleaningJob.created_at.desc()) + if job_id: + stmt = stmt.where(CleaningJob.job_id == job_id) + jobs = session.scalars(stmt).all() return [serialize_job(job) for job in jobs] diff --git a/services/cleaning/domain/csv_cleaning.py b/services/cleaning/domain/csv_cleaning.py deleted file mode 100644 index 8371708..0000000 --- a/services/cleaning/domain/csv_cleaning.py +++ /dev/null @@ -1,2130 +0,0 @@ -"""Limpieza de CSV con análisis inteligente local y reglas configurables. - -Este módulo es pura lógica (sin GUI, sin LLM). Provee: -- `analyze_dataframe` para obtener un resumen por columna (tipo inferido, - ejemplo aleatorio, nulos, únicos, estadísticas específicas por tipo). -- Un registro `CLEANING_RULES` con reglas aplicables a columnas. -- `apply_rules` que aplica las reglas seleccionadas por columna y devuelve - el DataFrame resultante junto con un log de operaciones/advertencias. -""" -from __future__ import annotations - -import csv as _csv -import logging -import math -import random -import re -import unicodedata -from collections import Counter -from dataclasses import dataclass, field -from datetime import date -from typing import Any, Callable, Literal, Optional - -import numpy as np -import pandas as pd - -logger = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# CUOTA MODERADORA (COP/MOD) -# --------------------------------------------------------------------------- - -DEFAULT_MODERADORA_BY_YEAR: dict[int, set[int]] = { - 2023: {4100, 16400, 43000}, - 2024: {4500, 18200, 47700}, - 2025: {4700, 19200, 50300}, - 2026: {5000, 20100, 52800}, -} - - -def normalize_moderadora_by_year_dict(raw: Any) -> dict[int, set[int]]: - """Normaliza un dict JSON año → lista de enteros (cuotas moderadora).""" - if not isinstance(raw, dict): - return {} - out: dict[int, set[int]] = {} - for k, v in raw.items(): - try: - yi = int(k) - except (TypeError, ValueError): - continue - if not isinstance(v, (list, tuple)): - continue - nums: set[int] = set() - for x in v: - try: - nums.add(int(round(float(x)))) - except (TypeError, ValueError): - continue - if nums: - out[yi] = nums - return out - - -def merge_moderadora_config( - cop_mod_config: Optional[dict[str, Any]], -) -> dict[int, set[int]]: - """Fusiona defaults con `moderadora_by_year` opcional del usuario.""" - merged = {y: set(vals) for y, vals in DEFAULT_MODERADORA_BY_YEAR.items()} - if cop_mod_config and isinstance(cop_mod_config.get("moderadora_by_year"), dict): - custom = normalize_moderadora_by_year_dict( - cop_mod_config["moderadora_by_year"] - ) - for y, vals in custom.items(): - merged[y] = set(vals) - return merged - - -def _alloc_cop_mod_column_names( - col_name: str, existing: set[str] -) -> tuple[str, str]: - """Nombres únicos para copago y moderadora.""" - base = re.sub(r"[^\w]+", "_", str(col_name).strip(), flags=re.ASCII) or "col" - for i in range(0, 200): - suf = f"_{i}" if i else "" - ccol, mcol = f"{base}{suf}_copago", f"{base}{suf}_moderadora" - if ccol not in existing and mcol not in existing: - return ccol, mcol - raise ValueError("No se pudieron asignar nombres de columnas únicos.") - - -def _amount_cell_to_optional_int( - val: Any, - decimal_hint: Optional[Literal[",", "."]], -) -> Optional[int]: - try: - f = _parse_lenient_one(val, decimal_hint) - except Exception: - return None - if f is None or (isinstance(f, float) and not np.isfinite(f)) or ( - isinstance(f, (float, np.floating)) and np.isnan(f) - ): - return None - try: - return int(round(float(f))) - except (TypeError, ValueError, OverflowError): - return None - - -def _apply_cop_mod_split( - result: pd.DataFrame, - amount_col: str, - fecha_column: str, - log: list[str], - *, - moderadora_by_year: dict[int, set[int]], -) -> None: - """Añade columnas copago y moderadora según cuotas por año de la fecha.""" - if fecha_column not in result.columns: - log.append( - f"[WARN] COP/MOD: columna de fecha '{fecha_column}' no existe; se omite." - ) - return - if amount_col not in result.columns: - log.append(f"[WARN] COP/MOD: columna de importe '{amount_col}' no existe; se omite.") - return - - fec_ser = result[fecha_column] - amt_ser = result[amount_col] - stripped = fec_ser.astype(str).str.replace(r"[\s\/\-\.]", "", regex=True) - stripped = stripped.where(fec_ser.notna(), other=pd.NA) - sep_fmt = detect_date_format(fec_ser) or detect_date_format(stripped) - compact8_fmt, compact6_fmt = _detect_compact_date_formats(fec_ser) - amt_hint = _infer_decimal_hint_from_column(amt_ser.astype(str)) - - ccol, mcol = _alloc_cop_mod_column_names(str(amount_col), set(result.columns)) - cop_vals: list[Any] = [] - mod_vals: list[Any] = [] - - for i in range(len(result)): - amt_raw = amt_ser.iloc[i] - fec_raw = fec_ser.iloc[i] - amt_i = _amount_cell_to_optional_int(amt_raw, amt_hint) - if amt_i is None: - cop_vals.append(0) - mod_vals.append(0) - continue - - parsed_d = _parse_any_date_to_tuple( - fec_raw, - sep_fmt=sep_fmt, - compact8_fmt=compact8_fmt, - compact6_fmt=compact6_fmt, - ) - if parsed_d is None: - cop_vals.append(amt_i) - mod_vals.append(0) - continue - _d, _m, year = parsed_d - tiers = moderadora_by_year.get(year) - if tiers is None: - cop_vals.append(amt_i) - mod_vals.append(0) - continue - - if amt_i in tiers: - cop_vals.append(0) - mod_vals.append(amt_i) - else: - cop_vals.append(amt_i) - mod_vals.append(0) - - result[ccol] = cop_vals - result[mcol] = mod_vals - log.append( - f"[OK] '{amount_col}' -> '{ccol}', '{mcol}' <- Separa COP/MOD " - f"(fecha: '{fecha_column}')" - ) - - -# --------------------------------------------------------------------------- -# MODELO DE DATOS -# --------------------------------------------------------------------------- - -TYPE_INTEGER = "integer" -TYPE_DECIMAL = "decimal" -TYPE_DATE = "date" -TYPE_BOOLEAN = "boolean" -TYPE_CATEGORICAL = "categorical" -TYPE_TEXT = "text" -TYPE_MIXED = "mixed" -TYPE_EMPTY = "empty" - - -@dataclass -class ColumnSummary: - """Resumen inteligente para una columna del DataFrame.""" - name: str - inferred_type: str - example: str = "" - null_count: int = 0 - null_pct: float = 0.0 - unique_count: int = 0 - unique_pct: float = 0.0 - min: Optional[float] = None - max: Optional[float] = None - mean: Optional[float] = None - avg_length: Optional[float] = None - top_values: list[tuple[str, int]] = field(default_factory=list) - detected_date_format: Optional[str] = None - applicable_rules: list[str] = field(default_factory=list) - - def info_text(self) -> str: - """Texto humano con información adicional (para subtítulo de la UI).""" - parts: list[str] = [] - parts.append(f"nulos: {self.null_count} ({self.null_pct:.0f}%)") - parts.append(f"únicos: {self.unique_count}") - if self.inferred_type in (TYPE_INTEGER, TYPE_DECIMAL): - if self.min is not None: - parts.append(f"min: {_fmt_num(self.min)}") - if self.max is not None: - parts.append(f"max: {_fmt_num(self.max)}") - if self.mean is not None: - parts.append(f"media: {_fmt_num(self.mean)}") - elif self.inferred_type == TYPE_DATE: - if self.detected_date_format: - parts.append(f"formato: {self.detected_date_format}") - elif self.inferred_type in (TYPE_TEXT, TYPE_CATEGORICAL): - if self.avg_length is not None: - parts.append(f"long. prom: {self.avg_length:.1f}") - if self.top_values: - tops = ", ".join(f'"{v}" ({c})' for v, c in self.top_values[:3]) - parts.append(f"top: {tops}") - return " · ".join(parts) - - -def _fmt_num(x: float) -> str: - if x is None: - return "" - if isinstance(x, float) and x.is_integer(): - return str(int(x)) - if isinstance(x, float): - return f"{x:.2f}" - return str(x) - - -# --------------------------------------------------------------------------- -# DETECCIÓN DE FECHAS -# --------------------------------------------------------------------------- - -_DATE_NUM_RE = re.compile(r"^\s*(\d{1,4})[\s\/\-\.](\d{1,2})[\s\/\-\.](\d{1,4})\s*$") -_DATE_COMPACT_RE = re.compile(r"^\s*(\d{6}|\d{8})\s*$") -# Hora opcional: HH[:MM[:SS]][ AM|PM] al final de la cadena -_TIME_TAIL_RE = re.compile( - r"[\sT]+(\d{1,2})(?::(\d{1,2})(?::(\d{1,2}))?)?\s*([APap]\.?\s*[Mm]\.?)?\s*$" -) - -_MONTH_NAME_TO_NUM = { - "enero": 1, - "ene": 1, - "febrero": 2, - "feb": 2, - "marzo": 3, - "mar": 3, - "abril": 4, - "abr": 4, - "mayo": 5, - "may": 5, - "junio": 6, - "jun": 6, - "julio": 7, - "jul": 7, - "agosto": 8, - "ago": 8, - "septiembre": 9, - "setiembre": 9, - "sep": 9, - "sept": 9, - "octubre": 10, - "oct": 10, - "noviembre": 11, - "nov": 11, - "diciembre": 12, - "dic": 12, -} - - -def _split_date_time(value: str) -> tuple[str, Optional[str]]: - """Separa la parte de fecha de la parte de hora si está presente. - - Devuelve `(date_part, time_part_normalized)`. `time_part_normalized` es - None si no hay hora. La hora se normaliza a `HH:MM[:SS][ AM|PM]`. - """ - if value is None: - return "", None - s = str(value).strip() - if not s: - return "", None - - m = _TIME_TAIL_RE.search(s) - if not m: - return s, None - - hh = int(m.group(1)) - mm = int(m.group(2)) if m.group(2) else 0 - ss = m.group(3) - ampm = m.group(4) - if not (0 <= hh <= 23 and 0 <= mm <= 59): - return s, None - if ss is not None: - try: - ssi = int(ss) - if not (0 <= ssi <= 59): - return s, None - except ValueError: - return s, None - # Validar contra AM/PM (con AM/PM, hora 1-12) - if ampm and not (1 <= hh <= 12): - return s, None - - # Quitar separadores habituales entre fecha y hora (p. ej. "12/27/24, 12:00 AM"). - date_part = s[: m.start()].rstrip(" \t,;T") - if not date_part: - return s, None - - parts: list[str] = [f"{hh:02d}:{mm:02d}"] - if ss is not None: - parts[0] += f":{int(ss):02d}" - if ampm: - norm_ampm = ampm.replace(".", "").replace(" ", "").upper() - if norm_ampm in ("AM", "PM"): - parts.append(norm_ampm) - time_norm = " ".join(parts) - return date_part, time_norm - - -def _extract_date_parts(value: str) -> Optional[tuple[int, int, int, str]]: - """Intenta extraer (a, b, c, layout) de una fecha en cualquier formato común. - - `layout` es uno de: 'sep' (con separadores), 'compact8' (YYYYMMDD o - DDMMYYYY), 'compact6' (DDMMYY o YYMMDD). No interpreta todavía qué - componente es día/mes/año; eso lo decide `detect_date_format` viendo la - columna completa. Acepta valores con hora al final (ignorada para los - componentes de fecha). - """ - if value is None: - return None - s = str(value).strip() - if not s or s.lower() in ("nan", "none", "nat"): - return None - - date_part, _ = _split_date_time(s) - if date_part: - s = date_part - - m = _DATE_NUM_RE.match(s) - if m: - try: - a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)) - return (a, b, c, "sep") - except ValueError: - return None - - m2 = _DATE_COMPACT_RE.match(s) - if m2: - digits = m2.group(1) - if len(digits) == 8: - return (int(digits[0:2]), int(digits[2:4]), int(digits[4:8]), "compact8") - if len(digits) == 6: - return (int(digits[0:2]), int(digits[2:4]), int(digits[4:6]), "compact6") - - return None - - -def detect_date_format(series: pd.Series, sample_size: int = 500) -> Optional[str]: - """Analiza la serie y determina el formato de fecha dominante. - - Devuelve un string con el layout detectado (p. ej. "DD/MM/YYYY", - "MM/DD/YYYY", "YYYYMMDD", "DDMMYYYY") o None si no parece fechas. - """ - if series is None or len(series) == 0: - return None - - non_null = series.dropna() - if non_null.empty: - return None - - if len(non_null) > sample_size: - non_null = non_null.sample(sample_size, random_state=42) - - layouts = {"sep": 0, "compact8": 0, "compact6": 0} - sep_first_gt12 = 0 # primer componente > 12 → probable día - sep_second_gt12 = 0 # segundo componente > 12 → probable mes solo posible si MM/DD - sep_third_4digits = 0 # tercer componente de 4 dígitos → año al final - sep_first_4digits = 0 # primer componente de 4 dígitos → año al inicio - compact8_year_first = 0 # primeros 4 dígitos > 1900 → YYYYMMDD - compact8_year_last = 0 # últimos 4 dígitos > 1900 → DDMMYYYY - total_parsed = 0 - - for val in non_null: - parsed = _extract_date_parts(val) - if parsed is None: - continue - total_parsed += 1 - a, b, c, layout = parsed - layouts[layout] = layouts.get(layout, 0) + 1 - - if layout == "sep": - # año puede estar al inicio o al final según cantidad de dígitos - if _digit_count(a) == 4: - sep_first_4digits += 1 - if _digit_count(c) == 4: - sep_third_4digits += 1 - if a > 12: - sep_first_gt12 += 1 - if b > 12: - sep_second_gt12 += 1 - elif layout == "compact8": - # a son los primeros 2 dígitos, c son los últimos 4 - if c >= 1900: - compact8_year_last += 1 - # a concatenado con b son los primeros 4 dígitos - first4 = a * 100 + b - if first4 >= 1900: - compact8_year_first += 1 - - if total_parsed == 0: - return None - - # Requerimos que >=70% de los no nulos parseen como fecha para considerarla - # columna de fechas. - if total_parsed / len(non_null) < 0.7: - return None - - dominant = max(layouts, key=lambda k: layouts[k]) - - if dominant == "compact8": - if compact8_year_first > compact8_year_last: - return "YYYYMMDD" - return "DDMMYYYY" - if dominant == "compact6": - return "DDMMYY" - - # Caso con separadores - if sep_first_4digits > sep_third_4digits: - return "YYYY/MM/DD" - # año al final - if sep_first_gt12 > 0 and sep_second_gt12 == 0: - return "DD/MM/YYYY" - if sep_second_gt12 > 0 and sep_first_gt12 == 0: - return "MM/DD/YYYY" - # ambiguo → preferencia hispana - return "DD/MM/YYYY" - - -def _digit_count(n: int) -> int: - return len(str(abs(n))) - - -def _parse_date_with_format(value: str, fmt: str) -> Optional[tuple[int, int, int]]: - """Convierte un valor de fecha a (día, mes, año) según el layout detectado.""" - parsed = _extract_date_parts(value) - if parsed is None: - return None - a, b, c, layout = parsed - - if fmt == "DD/MM/YYYY": - d, m, y = a, b, c - elif fmt == "MM/DD/YYYY": - m, d, y = a, b, c - elif fmt == "YYYY/MM/DD": - y, m, d = a, b, c - elif fmt == "YYYYMMDD" and layout == "compact8": - # reconstruir: primer4 = a*100 + b, luego mes y día - y = a * 100 + b - m = c // 100 - d = c % 100 - elif fmt == "DDMMYYYY" and layout == "compact8": - d = a - m = b - y = c - elif fmt == "DDMMYY" and layout == "compact6": - d = a - m = b - y = 2000 + c if c < 50 else 1900 + c - else: - # intentar como sep genérico - d, m, y = a, b, c - - if not (1 <= m <= 12 and 1 <= d <= 31 and 1000 <= y <= 9999): - # año en 2 dígitos - if y < 100: - y = 2000 + y if y < 50 else 1900 + y - if not (1 <= m <= 12 and 1 <= d <= 31 and 1000 <= y <= 9999): - return None - return (d, m, y) - - -def _normalize_date_text(value: Any) -> str: - text = unicodedata.normalize("NFKD", str(value or "")) - text = "".join(c for c in text if not unicodedata.combining(c)) - return text.lower() - - -def _valid_date_tuple(d: int, m: int, y: int) -> Optional[tuple[int, int, int]]: - if y < 100: - y = 2000 + y if y < 50 else 1900 + y - if not (1000 <= y <= 9999): - return None - try: - date(y, m, d) - except ValueError: - return None - return (d, m, y) - - -def _split_glued_year_suffix_hhmm(digits: str) -> Optional[tuple[int, int, int]]: - """Detecta `YYYY` + `HHMM` en una sola tira de 8 cifras (p. ej. `20241344`). - - Tras quitar separadores puede quedar pegado al mes (p. ej. `15may20241344`). - Devuelve (año, hora, minuto) si la parte horaria es válida; si no, None. - """ - if len(digits) != 8 or not digits.isdigit(): - return None - y = int(digits[:4]) - hh = int(digits[4:6]) - mm = int(digits[6:8]) - if not (1000 <= y <= 9999 and 0 <= hh <= 23 and 0 <= mm <= 59): - return None - return (y, hh, mm) - - -def _parse_text_month_candidate( - first: tuple[int, str], - month: int, - second: tuple[int, str], - *, - layout: Literal["around", "after", "before"], -) -> Optional[tuple[int, int, int]]: - a, astr = first - b, bstr = second - if layout == "around": - if len(astr) == 4: - return _valid_date_tuple(b, month, a) - if len(bstr) == 4: - return _valid_date_tuple(a, month, b) - return _valid_date_tuple(a, month, b) - if layout == "after": - if len(astr) == 4: - return _valid_date_tuple(b, month, a) - return _valid_date_tuple(a, month, b) - if len(astr) == 4: - return _valid_date_tuple(b, month, a) - return _valid_date_tuple(a, month, b) - - -def _parse_date_with_text_month(raw: Any) -> Optional[tuple[int, int, int]]: - if _is_nullish_scalar(raw): - return None - date_part, _ = _split_date_time(str(raw)) - normalized = _normalize_date_text(date_part or raw) - # `\d{8}` antes que `\d{1,4}` para no partir año+hora pegados tras strip agresivo. - tokens = re.findall(r"[a-z]+|\d{8}|\d{1,4}", normalized) - if not tokens: - return None - - for i, token in enumerate(tokens): - month = _MONTH_NAME_TO_NUM.get(token) - if month is None: - continue - nums_before = [ - (int(t), t) for t in tokens[:i] if t.isdigit() - ] - nums_after = [ - (int(t), t) for t in tokens[i + 1 :] if t.isdigit() - ] - candidates: list[Optional[tuple[int, int, int]]] = [] - if nums_before and len(nums_after) == 1: - glued = _split_glued_year_suffix_hhmm(nums_after[0][1]) - if glued is not None: - y, _hh, _mm = glued - candidates.append( - _parse_text_month_candidate( - nums_before[-1], month, (y, str(y)), layout="around" - ) - ) - if nums_before and nums_after: - candidates.append( - _parse_text_month_candidate( - nums_before[-1], month, nums_after[0], layout="around" - ) - ) - if len(nums_after) >= 2: - candidates.append( - _parse_text_month_candidate( - nums_after[0], month, nums_after[1], layout="after" - ) - ) - if len(nums_before) >= 2: - candidates.append( - _parse_text_month_candidate( - nums_before[-2], month, nums_before[-1], layout="before" - ) - ) - for parsed in candidates: - if parsed is not None: - return parsed - return None - - -# --------------------------------------------------------------------------- -# ANÁLISIS DE COLUMNAS -# --------------------------------------------------------------------------- - -_BOOLEAN_VALUES = {"true", "false", "si", "sí", "no", "1", "0", "yes"} - - -# PARSING numérico tolerante (miles/decimal) sin fijar un solo locale -_HINT_MAJORITY = 0.6 -_MIN_HINT_VOTES = 1 - - -def _is_nullish_scalar(x: Any) -> bool: - if x is None or x is pd.NA: - return True - if isinstance(x, (bool, np.bool_)): - return False - if isinstance(x, (int, float, np.integer, np.floating)) and not isinstance( - x, (bool, np.bool_) - ): - if isinstance(x, float) and (np.isnan(x) or pd.isna(x)): - return True - return False - t = str(x).strip().lower() - if not t or t in ("nan", "none", "nat", "null", ""): - return True - return False - - -def _strip_currency(s: str) -> str: - t = s.replace("\u00a0", " ").replace("\u202f", " ") - t = re.sub(r"[\s]+", "", t) - t = t.strip() - if not t or t.lower() in ("nan", "none", "nat"): - return "" - neg = False - if t[0:1] == "-": - neg, t = not neg, t[1:].lstrip() - if t[0:1] == "(" and t.endswith(")"): - neg, t = not neg, t[1:-1] - t = re.sub( - r"^[\$€\u00a3\u0e3f\uffe5\uffe6\uff04\ufe69]+", - "", - t, - count=1, - ) - t = re.sub(r"[\$€%]+$", "", t) - if not t and neg: - return "-0" - return ("-" if neg else "") + t - - -def _extract_numeric_chunk(s: str) -> str: - t = s.strip() - t = t.replace("$", "").replace("€", "").replace("\u00a3", "") - t = t.replace(" ", "") - cands1 = re.findall( - r"[+\-]?(?:\d{1,3}(?:[.,]\d{3})*|\d+)(?:[.,]\d+)?", t - ) - cands2 = re.findall( - r"[+\-]?\d[.,\d ]*\d", t - ) - cands = cands1 + cands2 - if not cands: - m = re.search(r"[\d.,\-]+", t) - if m: - c = m.group(0).replace("(", "-").replace(")", "") - return c.lstrip("+") - return "" - best = max(cands, key=len) if cands else "" - best = best.replace(" ", "").replace("(", "-").replace(")", "") - if not re.search(r"\d", best): - return "" - return best.lstrip("+") - - -def _as_float_s(s: str) -> Optional[float]: - if not s: - return None - try: - f = float(s) - if not np.isfinite(f) or abs(f) > 1e100: - return None - return f - except (ValueError, TypeError, OverflowError): - return None - - -def _parse_both_separators(body: str) -> Optional[float]: - """El separador más a la DERECHA entre `,` y `.` es el decimal; el otro, miles.""" - if "," not in body or "." not in body: - return None - t = body.strip() - if re.search(r"[^0-9,.\-]", t): - t2 = re.sub(r"[^0-9,.\-]", "", t) - t = t2 - if not t: - return None - neg = t[0:1] == "-" - b = t[1:].lstrip() if neg else t - if not b: - return None - i_comma, i_dot = b.rfind(","), b.rfind(".") - if min(i_comma, i_dot) < 0: - return None - dpos = max(i_comma, i_dot) - th = "," if b[dpos] == "." else "." - intp, fr = b[:dpos], b[dpos + 1 :] - if not re.fullmatch(r"\d{1,4}$", fr): - return None - intc = intp.replace(th, "") - if not intc.isdigit(): - return None - v = _as_float_s(f"{intc}.{fr}") - if v is None: - return None - if neg and v and v is not None: - return -abs(v) - return v - - -def _only_commas(w: str, dec_hint: Optional[Literal[",", "."]]) -> Optional[float]: - neg = w[0:1] == "-" - wn = w[1:].lstrip() if neg else w - if wn and re.search(r"[^0-9,]", wn): - wn = re.sub(r"[^0-9,]", "", wn) - n = wn.count(",") - if n == 0 and wn.isdigit(): - v = float(wn) - return -v if neg else v - if n == 0 or "." in wn: - return None - if n == 1: - i = wn.rfind(",") - h, t = wn[:i], wn[i + 1 :] - if not t.isdigit() or not h.isdigit(): - return None - if len(t) in (1, 2): - v = _as_float_s(f"{h}.{t}") - if v is None: - return None - return -v if neg else v - if len(t) == 3: - if dec_hint is None or dec_hint == ",": - v = _as_float_s(f"{h}{t}") - if v is None: - return None - return -v if neg else v - if dec_hint == ".": - v = _as_float_s(wn.replace(",", "")) - if v is None: - return None - return -v if neg else v - v = _as_float_s(wn.replace(",", "")) - if v is None: - return None - return -v if neg else v - - -def _only_dots(w: str, dec_hint: Optional[Literal[",", "."]]) -> Optional[float]: - neg = w[0:1] == "-" - wn = w[1:].lstrip() if neg else w - if wn and re.search(r"[^0-9.]", wn): - wn = re.sub(r"[^0-9.]", "", wn) - n = wn.count(".") - if n == 0 and wn.isdigit(): - v = float(wn) - return -v if neg else v - if n == 0 or "," in wn: - return None - if n == 1: - i = wn.rfind(".") - h, t = wn[:i], wn[i + 1 :] - if not t.isdigit() or not h.isdigit(): - return None - if len(t) in (1, 2): - v = _as_float_s(f"{h}.{t}") - if v is None: - return None - return -v if neg else v - if len(t) == 3: - if dec_hint is None or dec_hint == ".": - v = _as_float_s(f"{h}{t}") - if v is None: - return None - return -v if neg else v - if dec_hint == ",": - v = _as_float_s(wn.replace(".", "")) - if v is None: - return None - return -v if neg else v - v = _as_float_s(wn.replace(".", "")) - if v is None: - return None - return -v if neg else v - - -def _parse_lenient_one( - raw: Any, - decimal_hint: Optional[Literal[",", "."]] = None, -) -> float: - if _is_nullish_scalar(raw): - return float("nan") - t = _strip_currency(str(raw)) - if not t or t == "-": - return float("nan") - c = _extract_numeric_chunk(t) or t - c = c.strip() - if not c or c == "-": - return float("nan") - n_comma, n_dot = c.count(","), c.count(".") - if n_comma and n_dot: - f = _parse_both_separators(c) - elif n_comma and not n_dot: - f = _only_commas(c, decimal_hint) - elif n_dot and not n_comma: - f = _only_dots(c, decimal_hint) - else: - c2 = re.sub(r"[^0-9.\-]", "", c) - f = _as_float_s(c2) if c2 and re.fullmatch(r"-?\d+$", c2) else None - if f is not None and np.isfinite(f): - return f - return float("nan") - - -def _infer_decimal_hint_from_column( - series: pd.Series, -) -> Optional[Literal[",", "."]]: - """Voto con celdas que traen `,` y `.`: gana el de más a la derecha = decimal.""" - votes: list[Literal[",", "."]] = [] - for raw in series.dropna(): - if _is_nullish_scalar(raw): - continue - t = _strip_currency(str(raw)) - c = _extract_numeric_chunk(t) or t - c = c.strip() - if "," not in c or "." not in c or _parse_both_separators(c) is None: - continue - if c.rfind(",") > c.rfind("."): - votes.append(",") - else: - votes.append(".") - n = len(votes) - if n < _MIN_HINT_VOTES: - return None - a = sum(1 for v in votes if v == ",") - b = sum(1 for v in votes if v == ".") - if a + b == 0: - return None - if a / n >= _HINT_MAJORITY and a > b: - return "," - if b / n >= _HINT_MAJORITY and b > a: - return "." - return None - - -def _to_numeric_lenient( - series: pd.Series, - decimal_hint: Optional[Literal[",", "."]] = None, -) -> pd.Series: - """Convierte a flotante; pista de columna por voto (si `decimal_hint` es None).""" - hint = decimal_hint if decimal_hint is not None else _infer_decimal_hint_from_column( - series - ) - - def _cell(x: Any) -> float: - if _is_nullish_scalar(x): - return float("nan") - return _parse_lenient_one(x, hint) - - return series.map(_cell) - - -def _column_name_suggests_date_like(name: str) -> bool: - """Columnas cuyo nombre pide tratar el contenido como fecha a priori.""" - n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip() - if not n: - return False - if re.search( - r"\b(fecha|date|año|mes|dia|día|f[_-]?(alta|baja|proc|corte|pago|ini|fin|" - r"desde|hasta|vigenc|vence|venc|caduc|f\s?ec)\b|periodo|anio|ano|expir)", - n, - ) or n.startswith("fec_") or n.startswith("fecha"): - return True - return False - - -def _column_name_suggests_monetary_value(name: str) -> bool: - """importe, valor, neto, IVA, etc. — no interpretar cifras como fechas. - - Encabezados de tipo `fecha_…` no se tratan como moneda salvo mezcla con monto/valor. - """ - n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip() - if not n: - return False - if _column_name_suggests_date_like(name) and not re.search( - r"\b(monto|valor|importe|pago|total|bruto|neto|import|unit|precio|copago)\b", n - ): - return False - for key in ( - "monto", "importe", "import", "bruto", "neto", "subtotal", "sub total", - "unitario", "unidad", "valor", "precio", "pago", "copago", "sald", "dcto", - "franq", "cuota", "iva", "tarif", "costo", "carg", "reten", "retencion", - ): - if key in n: - if key == "iva" and re.search(r"(priva|giva|diva|viva|cliva)\b", n): - continue - if key == "carg" and re.search(r"descarg|recarg|abord", n): - continue - return True - return False - - -def _column_name_suggests_code_id(name: str) -> bool: - """Código, ref., radicado, tipoynúm de documento — no forzar 6-8 cifras como fechas (DDMMYY, etc.).""" - n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip() - if not n or _column_name_suggests_date_like(name): - return False - for key in ( - "codigo", "código", "codig", "códig", "cód_", - "sku", "ean", "upc", "isbn", "upn", - "ref", "referen", "radic", "nro_fac", "nro fac", "nit", "cédul", "cedul", - "autoriz", "correlat", "secuenc", "línea de", "linea de", "cupo", - "tipy", "tipi", "tipynum", "typnum", "type_n", "n_doc", "n_docu", - "núm doc", "n_docu", "doc_id", "docum", "id_doc", - ): - if key in n: - return True - if re.search( - r"tipo[_\s\\-]*(&|/|u|e|y)[_\s\\-]*num|tipo[_\s\\-]+doc|tipo_y_n", - n, - re.IGNORECASE, - ) or n.startswith("tip_"): - return True - if n.startswith("cod_") or n.startswith("cód_") or n.startswith("ref_"): - return True - return bool(re.search(r"\b(nro[._]fac|#?\s*autor|#?\s*ref)\b", n, re.IGNORECASE)) - - -def _column_name_suggests_quantity_or_number_not_bool(name: str) -> bool: - """Nombres típicos de cantidad, importe, fechas: no inferir bool por 0/1.""" - n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip() - if not n: - return False - if _column_name_suggests_monetary_value(name) or _column_name_suggests_code_id(name): - return True - for key in ( - "fecha", - "monto", - "importe", - "bruto", - "neto", - "subtotal", - "unitario", - "unidad", - "valor", - "precio", - "pago", - "copago", - "cantid", - "quantity", - "qty", - "nro", - "núm", - "n°", - ): - if key in n: - return True - return False - - -def _raw_strings_suggest_monetary_presentation(non_null: pd.Series) -> bool: - """$ €, miles 1,234,12 o .00: presentación monetaria aunque el float sea "entero".""" - if non_null is None or len(non_null) == 0: - return False - n_hit = 0 - n_seen = 0 - for raw in non_null.head(500): - if _is_nullish_scalar(raw): - continue - t = str(raw) - s = t.strip() - n_seen += 1 - if re.search(r"[$\u00a3\u20ac\uffe5\uffe6\ufe69\uff04]", t): - n_hit += 1 - continue - if re.search(r"\b\d{1,3}(?:[.,\u00a0]\d{3})+\b", s) or re.search( - r"(?= 0.10 - - -def _cell_looks_cifra_con_moneda_iso_al_final(s: str) -> bool: - """Solo cifra/símbolo (no prefijo de documento) y código de moneda al final. - - Excluye filas al estilo `CC12…` (típico alfanumérico). - """ - t = str(s).strip() - u = t.upper() - if not re.search( - r"(?i)(USD|COP|EUR|CLP|ARS|GBP|MXN|PEN|DOP|UYU|VES|CRC|PYG|BRL|BOB|GTQ)\b\s*\Z", u - ): - return False - if re.match( - r"^(?i)(CC|CE|NIT|NUIP|TI|PA|MS|RC|DNI|NUIP|CN|SC|AS|PE|PT)\D*[\d0]", u - ): - return False - return re.match( - r"^[\$€0-9\s,.\-+%/]", t, - ) is not None - - -def _raw_strings_suggest_alnum_mixed_presentation(non_null: pd.Series) -> bool: - """Letras y dígitos juntos (p. ej. `CC32…`, `A1-2B`); no es puro 'entero/decimal' lógicamente. - - Excluido: columnas con apariencia monetaria, y celdas solo cifra + ISO al final (100 USD). - """ - if _raw_strings_suggest_monetary_presentation(non_null): - return False - n_m = 0 - n_s = 0 - for raw in non_null.head(500): - if _is_nullish_scalar(raw): - continue - t = str(raw).strip() - n_s += 1 - if not re.search(r"\d", t) or not re.search( - r"[A-Za-zÑñ\u00c0-\u024f]", t, re.IGNORECASE - ): - continue - if _cell_looks_cifra_con_moneda_iso_al_final(t): - continue - n_m += 1 - if n_s == 0: - return False - return n_m / n_s >= 0.20 - - -def _cell_looks_like_documento_id_tipo_nit(s: str) -> bool: - """Cadenas estilo `CC32324029`, `CE 1.234.567` — no son fechas (evita 8 cifras como DDMM). - - Al menos una letra al inicio y 5-16 dígitos en la celda (no `Jan`+`2020` = 4 dígitos). - """ - t = str(s).strip() - if not t or len(t) < 3: - return False - if not re.match(r"^[A-Za-z]{1,3}", t): - return False - dig = re.sub(r"[^\d]", "", t) - if len(dig) < 5 or len(dig) > 16: - return False - return True - - -def _content_looks_like_doc_id_letter_prefix_column(non_null: pd.Series) -> bool: - """Muestras con prefijo documental (CC, NIT, …) y ≥5 dígitos → no columna de fecha.""" - n_ok = 0 - n_seen = 0 - for raw in non_null.head(500): - if _is_nullish_scalar(raw): - continue - s = str(raw).strip() - n_seen += 1 - if _cell_looks_like_documento_id_tipo_nit(s): - n_ok += 1 - if n_seen == 0: - return False - return n_ok / n_seen >= 0.20 - - -def _detect_messy_text_date_format_guess(non_null: pd.Series) -> Optional[str]: - """Fechas con ruido: compacta a dígitos; si hay consenso, reutiliza el detector. - (Depende de `_normalize_messy_date_to_compact`, definida más abajo en el módulo.) - """ - # resolución en tiempo de ejecución - norm = non_null.map(_normalize_messy_date_to_compact) # type: ignore[name-defined] - compacts: list[str] = [] - for c in norm: - if isinstance(c, str) and len(c) in (6, 8): - compacts.append(c) - n = max(len(non_null), 1) - if not compacts or len(compacts) < max(2, int(0.25 * n)): - return None - s_compact = pd.Series(compacts, dtype=object) - return detect_date_format(s_compact) - - -def _infer_column_type( - series: pd.Series, column_name: str = "" -) -> tuple[str, Optional[str]]: - """Devuelve (tipo, formato_fecha detectado; solo si aplica a tipo fecha).""" - non_null = series.dropna().astype(str).str.strip() - non_null = non_null[non_null != ""] - if non_null.empty: - return TYPE_EMPTY, None - - cname = str(column_name or "") - - # Boolean: no aplicar 0/1/ sí-no a columnas cuyo nombre pide importe, cantid., fecha… - if not _column_name_suggests_quantity_or_number_not_bool(cname): - lowered = non_null.str.lower() - if lowered.isin(_BOOLEAN_VALUES).all() and lowered.nunique() <= 3: - return TYPE_BOOLEAN, None - - is_money = _column_name_suggests_monetary_value(cname) - is_code = _column_name_suggests_code_id(cname) - is_doc_pattern = _content_looks_like_doc_id_letter_prefix_column(non_null) - - # Fecha: no forzar en importe/código, ni con datos tipo `CC`+cifra (6–8 dígitos sueltos) - if not is_money and not is_code and not is_doc_pattern: - date_fmt = detect_date_format(non_null) - if date_fmt: - return (TYPE_DATE, date_fmt) - messy_date_fmt = _detect_messy_text_date_format_guess(non_null) - if messy_date_fmt: - return (TYPE_DATE, messy_date_fmt) - - numeric_candidate = _to_numeric_lenient(non_null, decimal_hint=None) - numeric_ratio = numeric_candidate.notna().mean() - if numeric_ratio >= 0.9: - if _raw_strings_suggest_alnum_mixed_presentation(non_null): - return (TYPE_MIXED, None) - is_all_integer = (numeric_candidate.dropna() % 1 == 0).all() - if is_all_integer and not _raw_strings_suggest_monetary_presentation(non_null): - return (TYPE_INTEGER, None) - return (TYPE_DECIMAL, None) - - unique_ratio = non_null.nunique() / len(non_null) - if non_null.nunique() <= 15 and unique_ratio < 0.3: - return TYPE_CATEGORICAL, None - if 0.1 < numeric_ratio < 0.9: - return TYPE_MIXED, None - return TYPE_TEXT, None - - -def _applicable_rules_for_type(inferred_type: str) -> list[str]: - """Identificadores de reglas a mostrar según el tipo inferido (sin redundancias).""" - base = ["strip_special_spaces", "strip_special_keep_date_seps", "strip_spaces"] - if inferred_type == TYPE_INTEGER: - return base + ["cop_mod_split"] - if inferred_type == TYPE_DECIMAL: - return base + [ - "to_integer", - "round_decimals_custom", - "doc_id_split_tipo_numero", - "doc_type_homologate", - "cop_mod_split", - ] - if inferred_type == TYPE_DATE: - return base + [ - "doc_id_split_tipo_numero", - "doc_type_homologate", - "date_to_ddmmyyyy", - "nota_paciente_autorizacion", - ] - if inferred_type == TYPE_MIXED: - return base + [ - "to_integer", - "round_decimals_custom", - "doc_id_split_tipo_numero", - "doc_type_homologate", - "date_to_ddmmyyyy", - "cop_mod_split", - "nota_paciente_autorizacion", - ] - # Texto / categórico / boolean / empty → todas las de limpieza de texto + las - # conversoras (el usuario sabrá si aplica). - return base + [ - "to_integer", - "round_decimals_custom", - "doc_id_split_tipo_numero", - "doc_type_homologate", - "date_to_ddmmyyyy", - "cop_mod_split", - "nota_paciente_autorizacion", - ] - - -applicable_rule_ids_for_type = _applicable_rules_for_type - - -def analyze_dataframe(df: pd.DataFrame) -> list[ColumnSummary]: - """Genera un resumen por columna del DataFrame.""" - summaries: list[ColumnSummary] = [] - total_rows = len(df) - rng = random.Random(42) - - for col in df.columns: - series = df[col] - null_count = int(series.isna().sum() + (series.astype(str).str.strip() == "").sum()) - # Evitar doble conteo: considerar vacío solo si no-NaN - null_count = int((series.isna() | (series.astype(str).str.strip() == "")).sum()) - null_pct = (null_count / total_rows * 100) if total_rows else 0.0 - - non_null = series.dropna().astype(str) - non_null = non_null[non_null.str.strip() != ""] - - example = "" - if not non_null.empty: - example = str(rng.choice(non_null.tolist())) - if len(example) > 80: - example = example[:77] + "..." - - unique_count = int(non_null.nunique()) - unique_pct = (unique_count / total_rows * 100) if total_rows else 0.0 - - inferred_type, date_fmt = _infer_column_type(series, str(col)) - - summary = ColumnSummary( - name=str(col), - inferred_type=inferred_type, - example=example, - null_count=null_count, - null_pct=null_pct, - unique_count=unique_count, - unique_pct=unique_pct, - detected_date_format=date_fmt, - ) - - if inferred_type in (TYPE_INTEGER, TYPE_DECIMAL): - numeric = _to_numeric_lenient(non_null) - if numeric.notna().any(): - summary.min = float(numeric.min()) - summary.max = float(numeric.max()) - summary.mean = float(numeric.mean()) - elif inferred_type in (TYPE_TEXT, TYPE_CATEGORICAL): - if not non_null.empty: - summary.avg_length = float(non_null.str.len().mean()) - counts = non_null.value_counts().head(3) - summary.top_values = [(str(v), int(c)) for v, c in counts.items()] - - summary.applicable_rules = _applicable_rules_for_type(inferred_type) - summaries.append(summary) - - return summaries - - -# --------------------------------------------------------------------------- -# REGLAS DE LIMPIEZA -# --------------------------------------------------------------------------- - - -def _rule_strip_special_spaces(series: pd.Series) -> pd.Series: - """Elimina todo lo que no sea alfanumérico (incluye espacios).""" - pattern = re.compile(r"[^A-Za-z0-9]", flags=re.UNICODE) - - def _fn(x): - if pd.isna(x): - return x - return pattern.sub("", str(x)) - - return series.map(_fn) - - -def _rule_strip_special_keep_date_seps(series: pd.Series) -> pd.Series: - """Conserva alfanuméricos, separadores de fecha '/', '-' y hora ':', y espacio.""" - pattern = re.compile(r"[^A-Za-z0-9\/\-: ]", flags=re.UNICODE) - - def _fn(x): - if pd.isna(x): - return x - return pattern.sub("", str(x)) - - return series.map(_fn) - - -def _rule_strip_spaces(series: pd.Series) -> pd.Series: - """Elimina únicamente espacios (incluidos tabs y saltos de línea).""" - def _fn(x): - if pd.isna(x): - return x - return re.sub(r"\s+", "", str(x)) - - return series.map(_fn) - - -def _rule_to_integer(series: pd.Series) -> pd.Series: - """Convierte a entero (Int64 nullable) con parser tolerante.""" - numeric = _to_numeric_lenient(series.astype(str)) - return numeric.round().astype("Int64") - - -def _round_to_int_half_tie_le(x: float) -> int: - """Fracc. > 0,5 hacia arriba; 0,5 o menos hacia abajo. Banda 1e-9 en 0,5 (float). - - Misma noción que en la descripción de la regla. Trabaja sobre |x| y - recompone el signo. - """ - sign = -1 if x < 0 else 1 - if x == 0.0 or x == -0.0: - return 0 - ax = abs(float(x)) - ip = math.trunc(ax) - frac = ax - ip - if frac < 0.5 - 1e-9: - out = ip - elif frac > 0.5 + 1e-9: - out = ip + 1 - else: - out = ip - return -out if sign < 0 else out - - -def _rule_round_decimals_custom(series: pd.Series) -> pd.Series: - """Mismo parser tolerante que convertir a entero; aprox. con fracc. >0,5 arriba.""" - numeric = _to_numeric_lenient(series.astype(str)) - - def _fn(x): - if _is_nullish_scalar(x) or (isinstance(x, (float, np.floating)) and (not np.isfinite(x) or np.isnan(x))): - return pd.NA - try: - f = float(x) - except (TypeError, ValueError, OverflowError): - return pd.NA - if not np.isfinite(f): - return pd.NA - if f == 0.0 or f == -0.0: - return 0 - return _round_to_int_half_tie_le(f) - - return numeric.map(_fn).astype("Int64") - - -def _rule_doc_type_homologate(series: pd.Series) -> pd.Series: - """Homologa una columna que contiene tipos de documento.""" - def _fn(x: Any): - if _is_nullish_scalar(x): - return x - tipo, _num = split_doc_tipo_y_numero_texto(x, homologate=True) - if tipo is not None: - return tipo - return homologate_doc_type(x) - - return series.map(_fn) - - -def _detect_compact_date_formats( - series: pd.Series, -) -> tuple[Optional[str], Optional[str]]: - cser = series.map(_normalize_messy_date_to_compact) - c8s = pd.Series( - [x for x in cser.tolist() if isinstance(x, str) and len(x) == 8] - ) - c6s = pd.Series( - [x for x in cser.tolist() if isinstance(x, str) and len(x) == 6] - ) - fmt8 = detect_date_format(c8s) if not c8s.empty else None - fmt6 = detect_date_format(c6s) if not c6s.empty else None - return fmt8, fmt6 - - -def _parse_any_date_to_tuple( - raw: Any, - *, - sep_fmt: Optional[str], - compact8_fmt: Optional[str], - compact6_fmt: Optional[str], -) -> Optional[tuple[int, int, int]]: - parsed_text = _parse_date_with_text_month(raw) - if parsed_text is not None: - return parsed_text - - if _is_nullish_scalar(raw): - return None - - raw_text = str(raw) - date_part, _ = _split_date_time(raw_text) - target = date_part if date_part else raw_text - if sep_fmt: - parsed = _parse_date_with_format(target, sep_fmt) - if parsed is not None: - return parsed - for fallback_fmt in ("DD/MM/YYYY", "MM/DD/YYYY", "YYYY/MM/DD"): - if fallback_fmt == sep_fmt: - continue - parsed = _parse_date_with_format(target, fallback_fmt) - if parsed is not None: - return parsed - - compact = _normalize_messy_date_to_compact(raw) - if not isinstance(compact, str): - return None - if len(compact) == 8: - for fmt in (compact8_fmt, "DDMMYYYY", "YYYYMMDD"): - if not fmt: - continue - parsed = _parse_date_with_format(compact, fmt) - if parsed is not None: - return parsed - if len(compact) == 6: - parsed = _parse_date_with_format(compact, compact6_fmt or "DDMMYY") - if parsed is not None: - return parsed - return None - - -def _rule_date_to_ddmmyyyy(series: pd.Series) -> pd.Series: - """Normaliza fechas numéricas, alfanuméricas, con meses en texto y con ruido.""" - stripped = series.astype(str).str.replace(r"[\s\/\-\.]", "", regex=True) - stripped = stripped.where(series.notna(), other=pd.NA) - sep_fmt = detect_date_format(series) or detect_date_format(stripped) - compact8_fmt, compact6_fmt = _detect_compact_date_formats(series) - - def _fn(x: Any): - if pd.isna(x) or str(x).strip() == "": - return x - parsed = _parse_any_date_to_tuple( - x, - sep_fmt=sep_fmt, - compact8_fmt=compact8_fmt, - compact6_fmt=compact6_fmt, - ) - if parsed is None: - return x - d, m, y = parsed - return f"{d:02d}/{m:02d}/{y:04d}" - - return series.map(_fn) - - -def _rule_date_numeric_to_ddmmyyyy(series: pd.Series) -> pd.Series: - """Entrada: fechas en formato numérico (ej. 20240315 o 15032024). - - Salida: "DD/MM/AAAA" tras detectar el formato predominante. - """ - # Para esta regla consideramos los valores "numéricos" (quitamos separadores - # comunes si los hay para detectar un YYYYMMDD/DDMMYYYY confiable). - stripped = series.astype(str).str.replace(r"[\s\/\-\.]", "", regex=True) - stripped = stripped.where(series.notna(), other=pd.NA) - fmt = detect_date_format(stripped) - if fmt is None: - # fallback: intentar con detector normal - fmt = detect_date_format(series) - return _format_dates_to_ddmmyyyy(series, fmt) - - -def _rule_date_alpha_to_ddmmyyyy(series: pd.Series) -> pd.Series: - """Entrada: fechas en formato alfanumérico con separadores. - - Salida: "DD/MM/AAAA". - """ - fmt = detect_date_format(series) - return _format_dates_to_ddmmyyyy(series, fmt) - - -def _format_dates_to_ddmmyyyy(series: pd.Series, fmt: Optional[str]) -> pd.Series: - """Convierte cada valor al formato DD/MM/AAAA usando `fmt` como layout de - entrada detectado. La parte de hora (si existe) se descarta. Mantiene el - valor original si no puede parsear la fecha. - """ - if fmt is None: - return series.copy() - - def _fn(x): - if pd.isna(x) or str(x).strip() == "": - return x - raw = str(x) - date_part, _time_part = _split_date_time(raw) - target = date_part if date_part else raw - parsed = _parse_date_with_format(target, fmt) - if parsed is None: - return x - d, m, y = parsed - return f"{d:02d}/{m:02d}/{y:04d}" - - return series.map(_fn) - - -# --------------------------------------------------------------------------- -# Fechas con ruido (texto sucio) → compacto numérico + DD/MM/AAAA -# --------------------------------------------------------------------------- - - -def _normalize_messy_date_to_compact(raw: Any) -> Optional[str]: - """Quita ruido (* + _ / espacios, etc.); deja 6 o 8 dígitos o None. - - Soporta separadores entremezclados; hora al final vía `_split_date_time`. - Con 9+ dígitos, heurística: tomar 8 iniciales (p. ej. 9.º = 0 colado) y - validar luego con `_parse_date_with_format`; con más de 8 sobrantes, - 8 iniciales. - """ - if _is_nullish_scalar(raw): - return None - s = str(raw).strip() - if s.lower() in ("nan", "none", "nat", ""): - return None - date_part, _ = _split_date_time(s) - body = date_part if date_part else s - d = re.sub(r"[^\d]", "", body) - if not d: - return None - n = len(d) - if n in (6, 8): - return d - if n < 6 or n == 7: - return None - # 9+ dígitos: 8 iniciales (p. ej. 9.º 0 sobrante, o ruido al final) - return d[:8] - - -def _rule_date_messy_text_to_ddmmyyyy(series: pd.Series) -> pd.Series: - """Fechas con símbolos y basura entre cifras → DD/MM/AAAA. - - Normaliza a solo 6/8 dígitos, vota el formato con `detect_date_format` por - separado en compact8 y compact6; si el detector no devuelve nada, usa - `DDMMYYYY` / `DDMMYY` (alineado con la preferencia hispana en fechas - numéricas). - """ - cser = series.map(_normalize_messy_date_to_compact) - c8s = pd.Series( - [x for x in cser.tolist() if isinstance(x, str) and len(x) == 8] - ) - c6s = pd.Series( - [x for x in cser.tolist() if isinstance(x, str) and len(x) == 6] - ) - if c8s.empty and c6s.empty: - return series.copy() - - fmt8: Optional[str] = None - fmt6: Optional[str] = None - if not c8s.empty: - fmt8 = detect_date_format(c8s) or "DDMMYYYY" - if not c6s.empty: - fmt6 = detect_date_format(c6s) or "DDMMYY" - - def _one(x: Any, c: Any) -> Any: - if _is_nullish_scalar(x): - return x - if not isinstance(c, str) or len(c) not in (6, 8): - return x - f = (fmt8 or "DDMMYYYY") if len(c) == 8 else (fmt6 or "DDMMYY") - t = _parse_date_with_format(c, f) - if t is None and len(c) == 8: - t = _parse_date_with_format(c, "DDMMYYYY") or _parse_date_with_format( - c, "YYYYMMDD" - ) - if t is None and len(c) == 6: - t = _parse_date_with_format(c, "DDMMYY") - if t is None: - return x - d, m, y = t - return f"{d:02d}/{m:02d}/{y:04d}" - - return pd.Series( - [ - _one(v, cser.iloc[i] if i < len(cser) else None) for i, v in enumerate(series) - ], - index=series.index, - dtype=object, - ) - - -# --------------------------------------------------------------------------- -# Tipo y número de documento (CC, NIT, CE, MS, …) → 2 columnas -# --------------------------------------------------------------------------- - -_DOC_TYPE_ALIASES: dict[str, tuple[str, ...]] = { - "CC": ("CC", "Cedula", "CD", "C", "C.C", "C C", "CCC"), - "TI": ("TI", "Tarjeta", "Tarjeta Identidad", "T1", "T", "TL"), - "CN": ("CN", "Certificado", "Nacido Vivo", "Certificado Nacido Vivo"), - "CE": ("CE", "Cedula de Extranjeria", "Extranjera"), - "ME": ("ME", "Menor sin Identificacion"), - "PS": ("PS", "Pasaporte"), - "PT": ("PT", "Permiso por proteccion Temporal"), - "RC": ("RC", "Registro Civil", "Registro", "TC"), - "SC": ("SC", "Salvo Conducto", "Salvoconducto"), - "PE": ("PE", "Permiso especial de permanencia", "Permiso"), -} - - -def _ascii_upper(value: Any) -> str: - text = unicodedata.normalize("NFKD", str(value or "")) - text = "".join(c for c in text if not unicodedata.combining(c)) - return text.upper() - - -def _normalize_doc_type_key(value: Any) -> str: - """Clave comparable: sin acentos, espacios, puntos ni símbolos.""" - return re.sub(r"[^A-Z0-9]", "", _ascii_upper(value)) - - -def _clean_doc_type_text(value: Any) -> Optional[str]: - cleaned = re.sub(r"[^A-Z0-9]+", " ", _ascii_upper(value)).strip() - cleaned = re.sub(r"\s+", " ", cleaned) - return cleaned or None - - -_DOC_TYPE_ALIAS_BY_KEY: dict[str, str] = { - _normalize_doc_type_key(alias): canonical - for canonical, aliases in _DOC_TYPE_ALIASES.items() - for alias in aliases -} - -_DOC_TYPE_ALIAS_DISPLAY_BY_KEY: dict[str, str] = {} -for _canonical, _aliases in _DOC_TYPE_ALIASES.items(): - for _alias in _aliases: - _alias_key = _normalize_doc_type_key(_alias) - _alias_display = _clean_doc_type_text(_alias) - if _alias_key and _alias_display: - _DOC_TYPE_ALIAS_DISPLAY_BY_KEY.setdefault(_alias_key, _alias_display) - -_DOC_TYPE_ALIAS_KEYS_SORTED: list[str] = sorted( - _DOC_TYPE_ALIAS_BY_KEY, - key=lambda k: (len(k), k), - reverse=True, -) - - -def homologate_doc_type(value: Any) -> Optional[str]: - """Homologa un tipo documental a sigla; si no existe, conserva texto limpio.""" - if _is_nullish_scalar(value): - return None - key = _normalize_doc_type_key(value) - if not key or key.lower() in ("nan", "none", "nat"): - return None - return _DOC_TYPE_ALIAS_BY_KEY.get(key) or _clean_doc_type_text(value) - - -def _compact_doc_text(value: Any) -> str: - return _normalize_doc_type_key(value) - - -def _looks_like_short_alias_prefix(compact: str, alias_key: str) -> bool: - """Evita que alias de 1-2 chars capturen palabras como CODIGO o TOTAL.""" - if not compact.startswith(alias_key): - return False - rest = compact[len(alias_key):] - if not rest: - return True - return bool(rest[0].isdigit()) - - -def _match_known_doc_type_prefix(compact: str) -> Optional[tuple[str, str]]: - for alias_key in _DOC_TYPE_ALIAS_KEYS_SORTED: - if len(alias_key) <= 2: - if not _looks_like_short_alias_prefix(compact, alias_key): - continue - elif not compact.startswith(alias_key): - continue - return _DOC_TYPE_ALIAS_BY_KEY[alias_key], alias_key - return None - - -def split_doc_tipo_y_numero_texto( - raw: Any, - *, - homologate: bool = True, -) -> tuple[Optional[str], Optional[str]]: - """Separa tipo documental y número, tolerando nombres largos y símbolos. - - Si `homologate` es True, tipos conocidos se devuelven como sigla (CC, TI, - etc.). Los tipos desconocidos se conservan limpios en mayúsculas. - """ - if _is_nullish_scalar(raw) and not (isinstance(raw, str) and not str(raw).strip()): - return (None, None) - t = str(raw).strip() - if t.lower() in ("nan", "none", "nat", ""): - return (None, None) - if t and t[0].isdigit(): - return (None, None) - - compact = _compact_doc_text(t) - if not compact: - return (None, None) - - match = _match_known_doc_type_prefix(compact) - if match is not None: - canonical, alias_key = match - rest = compact[len(alias_key):] - nums = re.sub(r"[^\d]", "", rest) - if homologate: - return (canonical, nums if nums else None) - raw_type = _DOC_TYPE_ALIAS_DISPLAY_BY_KEY.get(alias_key) or canonical - return (raw_type, nums if nums else None) - - m = re.match( - r"^([^\d]+?)\s*([0-9].*)\s*$", - t, - ) - if m: - to_t = _clean_doc_type_text(m.group(1)) - nums = re.sub(r"[^\d]", "", m.group(2)) - return (homologate_doc_type(to_t) if homologate else to_t, nums if nums else None) - - if not re.search(r"\d", t): - to_t = homologate_doc_type(t) if homologate else _clean_doc_type_text(t) - return (to_t, None) - return (None, None) - - -def _alloc_doc_split_column_names( - col_name: str, existing: set[str] -) -> tuple[str, str]: - """Nombres únicos para las dos columnas derivadas.""" - base = re.sub(r"[^\w]+", "_", str(col_name).strip(), flags=re.ASCII) or "col" - for i in range(0, 200): - suf = f"_{i}" if i else "" - t, n = f"{base}{suf}_doc_tipo", f"{base}{suf}_doc_numeros" - if t not in existing and n not in existing: - return t, n - raise ValueError("No se pudieron asignar nombres de columnas únicos.") - - -def _rule_passthrough_series(s: pd.Series) -> pd.Series: - return s.copy() - - -def _apply_doc_id_split_columns( - result: pd.DataFrame, col: str, log: list[str], *, homologate_type: bool = True -) -> None: - tcol, ncol = _alloc_doc_split_column_names(str(col), set(result.columns)) - st_tipo: list[Optional[str]] = [] - st_num: list[Any] = [] - for v in result[col].tolist(): - tt, nn = split_doc_tipo_y_numero_texto(v, homologate=homologate_type) - st_tipo.append(tt) - st_num.append(nn if nn is not None else pd.NA) - result[tcol] = st_tipo - result[ncol] = st_num - log.append( - f"[OK] '{str(col)}' -> columnas '{tcol}' y '{ncol}' <- Dividir tipo y número (documento)" - ) - - -# --- Texto tipo nota (Paciente / Autorización) -------------------------------- - -_PACIENTE_DOC_NOTA_RE = re.compile(r"(?i)Paciente:\s*(\d+)\s*-\s*") -_AUTORIZACION_NOTA_RE = re.compile(r"(?i)Autorizaci[oó]n\s*No\.?\s*(\d+)") - - -def _normalize_nota_text(raw: Any) -> str: - if _is_nullish_scalar(raw): - return "" - try: - if pd.isna(raw): - return "" - except TypeError: - pass - s = str(raw).strip() - if not s or s.lower() in ("nan", "none", "nat", ""): - return "" - return re.sub(r"\s+", " ", s) - - -def extract_paciente_documento_nota(raw: Any) -> Optional[str]: - """Número de documento tras ``Paciente:`` y antes del guión del nombre.""" - t = _normalize_nota_text(raw) - if not t: - return None - m = _PACIENTE_DOC_NOTA_RE.search(t) - return m.group(1) if m else None - - -def extract_autorizacion_nota(raw: Any) -> Optional[str]: - """Número tras ``Autorización No.`` (opcional por fila).""" - t = _normalize_nota_text(raw) - if not t: - return None - m = _AUTORIZACION_NOTA_RE.search(t) - return m.group(1) if m else None - - -def _alloc_nota_extract_column_names( - col_name: str, existing: set[str] -) -> tuple[str, str]: - """Nombres únicos para documento paciente y autorización.""" - base = re.sub(r"[^\w]+", "_", str(col_name).strip(), flags=re.ASCII) or "col" - for i in range(0, 200): - suf = f"_{i}" if i else "" - dcol, acol = f"{base}{suf}_nota_paciente_doc", f"{base}{suf}_nota_autorizacion" - if dcol not in existing and acol not in existing: - return dcol, acol - raise ValueError("No se pudieron asignar nombres de columnas únicos.") - - -def _apply_nota_extract_columns(result: pd.DataFrame, col: str, log: list[str]) -> None: - """Añade columnas con documento (Paciente:) y número de autorización si existe.""" - dcol, acol = _alloc_nota_extract_column_names(str(col), set(result.columns)) - docs: list[Any] = [] - auths: list[Any] = [] - for v in result[col].tolist(): - dd = extract_paciente_documento_nota(v) - aa = extract_autorizacion_nota(v) - docs.append(dd if dd is not None else pd.NA) - auths.append(aa if aa is not None else pd.NA) - result[dcol] = docs - result[acol] = auths - log.append( - f"[OK] '{str(col)}' -> columnas '{dcol}' y '{acol}' <- Extraer datos de notas" - ) - - -# Registro de reglas -CLEANING_RULES: list[dict[str, Any]] = [ - { - "id": "strip_special_spaces", - "name": "Eliminar caracteres especiales y espacios", - "description": ( - "Deja solo caracteres alfanuméricos (A-Z, a-z, 0-9). No usar antes de " - "«Normalizar fecha» en columnas con fecha y hora: pega cifras y puede impedir " - "interpretar el valor (aplique primero la normalización de fecha o use la regla " - "que conserva separadores)." - ), - "fn": _rule_strip_special_spaces, - }, - { - "id": "strip_special_keep_date_seps", - "name": "Quitar símbolos; conserva /, -, : y espacios (fechas o texto con fechas)", - "description": ( - "Quita comas, $, comillas, etc. y deja letras, números, /, -, : (hora) y espacios. " - "Adecuado p. ej. en columna 'Fecha Servicio'." - ), - "fn": _rule_strip_special_keep_date_seps, - }, - { - "id": "strip_spaces", - "name": "Eliminar únicamente espacios", - "description": "Elimina todos los caracteres de espacio en blanco.", - "fn": _rule_strip_spaces, - }, - { - "id": "to_integer", - "name": "Convertir a Entero", - "description": "Convierte a enteros con detección de miles/decimal (`,`/`.`) y moneda; redondea decimales.", - "fn": _rule_to_integer, - }, - { - "id": "date_to_ddmmyyyy", - "name": "Normalizar fecha → DD/MM/AAAA", - "description": ( - "Normaliza fechas numéricas, con separadores, con ruido o con meses en " - "texto (ej. 26-abril-2026) al formato DD/MM/AAAA." - ), - "fn": _rule_date_to_ddmmyyyy, - }, - { - "id": "date_numeric_to_ddmmyyyy", - "name": "Conversión de formato numérico (Fechas) → DD/MM/AAAA", - "description": "Detecta el formato numérico predominante (YYYYMMDD, DDMMYYYY, etc.) y normaliza a DD/MM/AAAA.", - "fn": _rule_date_numeric_to_ddmmyyyy, - }, - { - "id": "date_alpha_to_ddmmyyyy", - "name": "Conversión de formato alfanumérico (Fechas) → DD/MM/AAAA", - "description": "Detecta el formato con separadores predominante (DD/MM/YYYY, MM/DD/YYYY, YYYY/MM/DD, etc.) y normaliza a DD/MM/AAAA.", - "fn": _rule_date_alpha_to_ddmmyyyy, - }, - { - "id": "date_messy_text_to_ddmmyyyy", - "name": "Texto a fecha (ruido en fechas) → DD/MM/AAAA", - "description": "Quita ruido entre cifras (*, +, _, /, etc.), detecta el orden en la columna (DDMMYYYY, YYYYMMDD, DDMMYY) y unifica a DD/MM/AAAA.", - "fn": _rule_date_messy_text_to_ddmmyyyy, - }, - { - "id": "round_decimals_custom", - "name": "Aproximación Decimales (>0,5 sube, ≤0,5 baja)", - "description": "Usa el mismo análisis de miles, decimales y moneda que «Entero». Tras leer el número, si la parte fraccionaria de |valor| es mayor a 0,5 se aproxima al entero hacia lejos de cero; en caso contrario, hacia cero. El 0,5 exacto se baja.", - "fn": _rule_round_decimals_custom, - }, - { - "id": "doc_id_split_tipo_numero", - "name": "Dividir tipo y número (documento)", - "description": ( - "Añade dos columnas (nombres únicos p. ej. TipoID_doc_tipo, TipoID_doc_numeros): " - "prefijo con letras (NIT, CC, CE, …) y cifras sin separadores. No modifica la columna de origen." - ), - "fn": _rule_passthrough_series, - }, - { - "id": "doc_type_homologate", - "name": "Homologar tipo de documento", - "description": ( - "Convierte tipos de documento escritos como nombre largo, sigla o texto con " - "símbolos a su abreviatura estándar (CC, TI, CE, RC, etc.)." - ), - "fn": _rule_doc_type_homologate, - }, - { - "id": "cop_mod_split", - "name": "Separa COP/MOD (copago / moderadora)", - "description": ( - "Según el año en la columna Fecha Servicio que elija en la tarjeta, clasifica " - "el importe en cuota moderadora (valores por año en Configuración) o copago. " - "Se ejecuta al final del proceso de limpieza para usar fechas ya normalizadas." - ), - "fn": _rule_passthrough_series, - }, - { - "id": "nota_paciente_autorizacion", - "name": "Extraer datos de notas (documento y autorización)", - "description": ( - "Desde texto tipo nota hospitalaria, extrae el número tras «Paciente:» (antes del " - "guión del nombre) y, si existe, el de «Autorización No.». Añade dos columnas; " - "no modifica el texto original." - ), - "fn": _rule_passthrough_series, - }, -] - -_RULE_BY_ID: dict[str, Callable[[pd.Series], pd.Series]] = { - r["id"]: r["fn"] for r in CLEANING_RULES -} - -_RULE_NAME_BY_ID: dict[str, str] = {r["id"]: r["name"] for r in CLEANING_RULES} - - -def get_rule(rule_id: str) -> Optional[dict[str, Any]]: - """Devuelve el registro de una regla por id.""" - for r in CLEANING_RULES: - if r["id"] == rule_id: - return r - return None - - -# --------------------------------------------------------------------------- -# APLICACIÓN DE REGLAS -# --------------------------------------------------------------------------- - - -def apply_rules( - df: pd.DataFrame, - rules_per_column: dict[str, list[str]], - *, - cop_mod_fecha_column: Optional[dict[str, str]] = None, - cop_mod_config: Optional[dict[str, Any]] = None, -) -> tuple[pd.DataFrame, list[str]]: - """Aplica las reglas seleccionadas a cada columna. - - `rules_per_column` es un dict de `columna -> lista de IDs de reglas en el - orden a aplicar`. La regla `cop_mod_split` se difiere a una segunda pasada - tras el resto, para que la columna de fecha pueda normalizarse antes. - - `cop_mod_fecha_column`: mapa columna de importe -> nombre de columna Fecha Servicio. - `cop_mod_config`: opcional; puede incluir ``moderadora_by_year`` (dict año → lista). - """ - result = df.copy() - log: list[str] = [] - - moderadora_years = merge_moderadora_config(cop_mod_config) - - def _run_rules_for_column(col: str, rule_ids: list[str]) -> None: - if not rule_ids: - return - if col not in result.columns: - log.append(f"[WARN] Columna no encontrada: {col}") - return - split_has_homologation = ( - "doc_id_split_tipo_numero" in rule_ids and "doc_type_homologate" in rule_ids - ) - for rule_id in rule_ids: - if rule_id == "doc_id_split_tipo_numero": - try: - _apply_doc_id_split_columns( - result, col, log, homologate_type=split_has_homologation - ) - except Exception as exc: # noqa: BLE001 - logger.exception("Error aplicando regla %s a %s", rule_id, col) - log.append(f"[ERROR] '{col}' regla '{rule_id}': {exc}") - continue - if rule_id == "nota_paciente_autorizacion": - try: - _apply_nota_extract_columns(result, col, log) - except Exception as exc: # noqa: BLE001 - logger.exception("Error aplicando regla %s a %s", rule_id, col) - log.append(f"[ERROR] '{col}' regla '{rule_id}': {exc}") - continue - if rule_id == "doc_type_homologate" and split_has_homologation: - continue - fn = _RULE_BY_ID.get(rule_id) - if fn is None: - log.append(f"[WARN] Regla desconocida '{rule_id}' en columna '{col}'") - continue - try: - result[col] = fn(result[col]) - log.append(f"[OK] '{col}' <- {_RULE_NAME_BY_ID.get(rule_id, rule_id)}") - except Exception as exc: # noqa: BLE001 - logger.exception("Error aplicando regla %s a %s", rule_id, col) - log.append(f"[ERROR] '{col}' regla '{rule_id}': {exc}") - - # Fase 1: todas las reglas excepto cop_mod_split - for col, rule_ids in rules_per_column.items(): - filtered = [r for r in rule_ids if r != "cop_mod_split"] - _run_rules_for_column(col, filtered) - - # Fase 2: Separa COP/MOD sobre el DataFrame ya transformado - fecha_map = cop_mod_fecha_column or {} - for col, rule_ids in rules_per_column.items(): - if "cop_mod_split" not in rule_ids: - continue - fecha_col = fecha_map.get(col) - if not fecha_col: - log.append( - f"[WARN] COP/MOD en '{col}': indique la columna «Fecha Servicio» en la tarjeta; se omite." - ) - continue - try: - _apply_cop_mod_split( - result, - col, - fecha_col, - log, - moderadora_by_year=moderadora_years, - ) - except Exception as exc: # noqa: BLE001 - logger.exception("Error aplicando COP/MOD a %s", col) - log.append(f"[ERROR] '{col}' regla 'cop_mod_split': {exc}") - - return result, log - - -# --------------------------------------------------------------------------- -# CARGA ROBUSTA DE CSV -# --------------------------------------------------------------------------- - -_CANDIDATE_DELIMITERS = ["|", ";", "\t", ",", ":"] - - -def detect_delimiter(filepath: str) -> str: - """Detecta el delimitador del CSV de forma robusta. - - Estrategia: - 1. Intentar `csv.Sniffer` con un sample grande. - 2. Si falla o el resultado parece incorrecto, contar ocurrencias por línea - de cada delimitador candidato y elegir el que produce el mayor número - de columnas estable entre líneas. - """ - encodings = ("utf-8", "utf-8-sig", "latin-1", "cp1252") - sample = "" - for enc in encodings: - try: - with open(filepath, "r", encoding=enc, errors="replace") as f: - sample = f.read(65536) - break - except Exception: - continue - - if not sample: - return "," - - try: - dialect = _csv.Sniffer().sniff(sample, delimiters="".join(_CANDIDATE_DELIMITERS)) - sniffed = dialect.delimiter - except _csv.Error: - sniffed = None - - lines = [ln for ln in sample.splitlines() if ln.strip()] - if not lines: - return sniffed or "," - - # Para cada candidato, calcular el número promedio de columnas y la - # consistencia (cuántas líneas tienen el mismo conteo dominante). - scores: dict[str, tuple[int, int]] = {} - for delim in _CANDIDATE_DELIMITERS: - counts = [ln.count(delim) for ln in lines[:50]] - if max(counts) == 0: - continue - # Modo (valor más frecuente) - common = Counter(counts).most_common(1)[0] - mode_count, mode_freq = common - if mode_count == 0: - continue - scores[delim] = (mode_count, mode_freq) - - if scores: - # Elegir el delimitador con mayor número de columnas (mode_count) y, - # como desempate, mayor consistencia (mode_freq). - best = max(scores.items(), key=lambda kv: (kv[1][0], kv[1][1])) - best_delim = best[0] - # Si el sniffer dio algo razonable y coincide, prevalece; si difiere y - # nuestro candidato es claramente mejor, usamos el nuestro. - if sniffed and sniffed in scores: - sniffed_score = scores[sniffed] - if sniffed_score[0] >= best[1][0] - 1: - return sniffed - return best_delim - - return sniffed or "," - - -def load_csv(filepath: str) -> tuple[pd.DataFrame, str]: - """Carga un CSV con detección de delimitador y tolerancia a filas malas. - - Devuelve `(df, delimiter)`. Lanza excepción si todos los intentos fallan. - """ - delimiter = detect_delimiter(filepath) - - encodings_to_try = ["utf-8", "utf-8-sig", "latin-1", "cp1252"] - last_error: Optional[Exception] = None - - for enc in encodings_to_try: - try: - df = pd.read_csv( - filepath, - delimiter=delimiter, - dtype=str, - encoding=enc, - engine="python", - on_bad_lines="skip", - ) - return df, delimiter - except Exception as exc: # noqa: BLE001 - last_error = exc - continue - - # Último intento con quoting permisivo y sin separador detectado - try: - df = pd.read_csv( - filepath, - sep=None, - dtype=str, - engine="python", - on_bad_lines="skip", - ) - return df, "(auto)" - except Exception as exc: # noqa: BLE001 - last_error = exc - - raise RuntimeError(f"No se pudo leer el CSV: {last_error}") diff --git a/services/cleaning/domain/csv_cleaning/__init__.py b/services/cleaning/domain/csv_cleaning/__init__.py new file mode 100644 index 0000000..52edbe3 --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/__init__.py @@ -0,0 +1,64 @@ +"""Limpieza de CSV con análisis inteligente local y reglas configurables. + +API pública (estable): +- ``analyze_dataframe`` — resumen por columna +- ``CLEANING_RULES`` / ``register_rule`` / ``get_rule`` — catálogo de reglas +- ``apply_rules`` — aplica reglas por columna +- ``load_csv`` / ``load_csv_bytes`` / ``detect_delimiter`` — I/O + +Para agregar una regla nueva, ver ``rules/registry.py`` y ``rules/builtins.py``. +""" +from __future__ import annotations + +from .analyze import analyze_dataframe, sample_dataframe_rows +from .apply import apply_rules +from .cop_mod import ( + DEFAULT_MODERADORA_BY_YEAR, + merge_moderadora_config, + normalize_moderadora_by_year_dict, +) +from .documents import homologate_doc_type, split_doc_tipo_y_numero_texto +from .io import detect_delimiter, load_csv, load_csv_bytes +from .models import ( + TYPE_BOOLEAN, + TYPE_CATEGORICAL, + TYPE_DATE, + TYPE_DECIMAL, + TYPE_EMPTY, + TYPE_INTEGER, + TYPE_MIXED, + TYPE_TEXT, + ColumnSummary, +) +from .notas import extract_autorizacion_nota, extract_paciente_documento_nota +from .rules import CLEANING_RULES, get_rule, register_rule +from .type_inference import applicable_rule_ids_for_type + +__all__ = [ + "CLEANING_RULES", + "ColumnSummary", + "DEFAULT_MODERADORA_BY_YEAR", + "TYPE_BOOLEAN", + "TYPE_CATEGORICAL", + "TYPE_DATE", + "TYPE_DECIMAL", + "TYPE_EMPTY", + "TYPE_INTEGER", + "TYPE_MIXED", + "TYPE_TEXT", + "analyze_dataframe", + "applicable_rule_ids_for_type", + "apply_rules", + "detect_delimiter", + "extract_autorizacion_nota", + "extract_paciente_documento_nota", + "get_rule", + "homologate_doc_type", + "load_csv", + "load_csv_bytes", + "merge_moderadora_config", + "normalize_moderadora_by_year_dict", + "register_rule", + "sample_dataframe_rows", + "split_doc_tipo_y_numero_texto", +] diff --git a/services/cleaning/domain/csv_cleaning/analyze.py b/services/cleaning/domain/csv_cleaning/analyze.py new file mode 100644 index 0000000..ee051f4 --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/analyze.py @@ -0,0 +1,94 @@ +"""Análisis de DataFrames: resumen por columna y muestras.""" +from __future__ import annotations + +import json +import random +from typing import Any + +import pandas as pd + +from .models import ( + TYPE_CATEGORICAL, + TYPE_DATE, + TYPE_DECIMAL, + TYPE_INTEGER, + TYPE_TEXT, + ColumnSummary, +) +from .numbers import _to_numeric_lenient +from .type_inference import _applicable_rules_for_type, _infer_column_type + + +def analyze_dataframe(df: pd.DataFrame) -> list[ColumnSummary]: + """Genera un resumen por columna del DataFrame.""" + summaries: list[ColumnSummary] = [] + total_rows = len(df) + rng = random.Random(42) + + for col in df.columns: + series = df[col] + null_count = int(series.isna().sum() + (series.astype(str).str.strip() == "").sum()) + # Evitar doble conteo: considerar vacío solo si no-NaN + null_count = int((series.isna() | (series.astype(str).str.strip() == "")).sum()) + null_pct = (null_count / total_rows * 100) if total_rows else 0.0 + + non_null = series.dropna().astype(str) + non_null = non_null[non_null.str.strip() != ""] + + example = "" + samples: list[str] = [] + if not non_null.empty: + values = non_null.tolist() + k = min(5, len(values)) + picked = rng.sample(values, k) if len(values) >= k else list(values) + for raw in picked: + text = str(raw) + if len(text) > 80: + text = text[:77] + "..." + samples.append(text) + example = samples[0] if samples else "" + + unique_count = int(non_null.nunique()) + unique_pct = (unique_count / total_rows * 100) if total_rows else 0.0 + + inferred_type, date_fmt = _infer_column_type(series, str(col)) + + summary = ColumnSummary( + name=str(col), + inferred_type=inferred_type, + example=example, + samples=samples, + null_count=null_count, + null_pct=null_pct, + unique_count=unique_count, + unique_pct=unique_pct, + detected_date_format=date_fmt, + ) + + if inferred_type in (TYPE_INTEGER, TYPE_DECIMAL): + numeric = _to_numeric_lenient(non_null) + if numeric.notna().any(): + summary.min = float(numeric.min()) + summary.max = float(numeric.max()) + summary.mean = float(numeric.mean()) + elif inferred_type in (TYPE_TEXT, TYPE_CATEGORICAL): + if not non_null.empty: + summary.avg_length = float(non_null.str.len().mean()) + counts = non_null.value_counts().head(3) + summary.top_values = [(str(v), int(c)) for v, c in counts.items()] + + summary.applicable_rules = _applicable_rules_for_type(inferred_type) + summaries.append(summary) + + return summaries + + +def sample_dataframe_rows(df: pd.DataFrame, n: int = 8) -> list[dict[str, Any]]: + """Devuelve hasta `n` filas aleatorias como dicts JSON-serializables.""" + if df.empty or n <= 0: + return [] + take = min(n, len(df)) + rng = random.Random(42) + indices = rng.sample(list(range(len(df))), take) + subset = df.iloc[indices] + return json.loads(subset.to_json(orient="records", date_format="iso")) diff --git a/services/cleaning/domain/csv_cleaning/apply.py b/services/cleaning/domain/csv_cleaning/apply.py new file mode 100644 index 0000000..72d96dd --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/apply.py @@ -0,0 +1,105 @@ +"""Aplicación de reglas seleccionadas sobre un DataFrame.""" +from __future__ import annotations + +import logging +from typing import Any, Optional + +import pandas as pd + +from .cop_mod import _apply_cop_mod_split, merge_moderadora_config +from .documents import _apply_doc_id_split_columns +from .notas import _apply_nota_extract_columns +from .rules import get_rule_fn, get_rule_name + +logger = logging.getLogger(__name__) + + +def apply_rules( + df: pd.DataFrame, + rules_per_column: dict[str, list[str]], + *, + cop_mod_fecha_column: Optional[dict[str, str]] = None, + cop_mod_config: Optional[dict[str, Any]] = None, +) -> tuple[pd.DataFrame, list[str]]: + """Aplica las reglas seleccionadas a cada columna. + + `rules_per_column` es un dict de `columna -> lista de IDs de reglas en el + orden a aplicar`. La regla `cop_mod_split` se difiere a una segunda pasada + tras el resto, para que la columna de fecha pueda normalizarse antes. + + `cop_mod_fecha_column`: mapa columna de importe -> nombre de columna Fecha Servicio. + `cop_mod_config`: opcional; puede incluir ``moderadora_by_year`` (dict año → lista). + """ + result = df.copy() + log: list[str] = [] + + moderadora_years = merge_moderadora_config(cop_mod_config) + + def _run_rules_for_column(col: str, rule_ids: list[str]) -> None: + if not rule_ids: + return + if col not in result.columns: + log.append(f"[WARN] Columna no encontrada: {col}") + return + split_has_homologation = ( + "doc_id_split_tipo_numero" in rule_ids and "doc_type_homologate" in rule_ids + ) + for rule_id in rule_ids: + if rule_id == "doc_id_split_tipo_numero": + try: + _apply_doc_id_split_columns( + result, col, log, homologate_type=split_has_homologation + ) + except Exception as exc: # noqa: BLE001 + logger.exception("Error aplicando regla %s a %s", rule_id, col) + log.append(f"[ERROR] '{col}' regla '{rule_id}': {exc}") + continue + if rule_id == "nota_paciente_autorizacion": + try: + _apply_nota_extract_columns(result, col, log) + except Exception as exc: # noqa: BLE001 + logger.exception("Error aplicando regla %s a %s", rule_id, col) + log.append(f"[ERROR] '{col}' regla '{rule_id}': {exc}") + continue + if rule_id == "doc_type_homologate" and split_has_homologation: + continue + fn = get_rule_fn(rule_id) + if fn is None: + log.append(f"[WARN] Regla desconocida '{rule_id}' en columna '{col}'") + continue + try: + result[col] = fn(result[col]) + log.append(f"[OK] '{col}' <- {get_rule_name(rule_id)}") + except Exception as exc: # noqa: BLE001 + logger.exception("Error aplicando regla %s a %s", rule_id, col) + log.append(f"[ERROR] '{col}' regla '{rule_id}': {exc}") + + # Fase 1: todas las reglas excepto cop_mod_split + for col, rule_ids in rules_per_column.items(): + filtered = [r for r in rule_ids if r != "cop_mod_split"] + _run_rules_for_column(col, filtered) + + # Fase 2: Separa COP/MOD sobre el DataFrame ya transformado + fecha_map = cop_mod_fecha_column or {} + for col, rule_ids in rules_per_column.items(): + if "cop_mod_split" not in rule_ids: + continue + fecha_col = fecha_map.get(col) + if not fecha_col: + log.append( + f"[WARN] COP/MOD en '{col}': indique la columna «Fecha Servicio» en la tarjeta; se omite." + ) + continue + try: + _apply_cop_mod_split( + result, + col, + fecha_col, + log, + moderadora_by_year=moderadora_years, + ) + except Exception as exc: # noqa: BLE001 + logger.exception("Error aplicando COP/MOD a %s", col) + log.append(f"[ERROR] '{col}' regla 'cop_mod_split': {exc}") + + return result, log diff --git a/services/cleaning/domain/csv_cleaning/column_hints.py b/services/cleaning/domain/csv_cleaning/column_hints.py new file mode 100644 index 0000000..342b95b --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/column_hints.py @@ -0,0 +1,210 @@ +"""Heurísticas por nombre de columna y presentación del contenido.""" +from __future__ import annotations + +import re +from typing import Optional + +import pandas as pd + +from .nullish import _is_nullish_scalar + + +def _column_name_suggests_date_like(name: str) -> bool: + """Columnas cuyo nombre pide tratar el contenido como fecha a priori.""" + n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip() + if not n: + return False + if re.search( + r"\b(fecha|date|año|mes|dia|día|f[_-]?(alta|baja|proc|corte|pago|ini|fin|" + r"desde|hasta|vigenc|vence|venc|caduc|f\s?ec)\b|periodo|anio|ano|expir)", + n, + ) or n.startswith("fec_") or n.startswith("fecha"): + return True + return False + + +def _column_name_suggests_monetary_value(name: str) -> bool: + """importe, valor, neto, IVA, etc. — no interpretar cifras como fechas. + + Encabezados de tipo `fecha_…` no se tratan como moneda salvo mezcla con monto/valor. + """ + n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip() + if not n: + return False + if _column_name_suggests_date_like(name) and not re.search( + r"\b(monto|valor|importe|pago|total|bruto|neto|import|unit|precio|copago)\b", n + ): + return False + for key in ( + "monto", "importe", "import", "bruto", "neto", "subtotal", "sub total", + "total", "unitario", "unidad", "valor", "precio", "pago", "copago", "sald", "dcto", + "franq", "cuota", "iva", "tarif", "costo", "carg", "reten", "retencion", + ): + if key in n: + if key == "iva" and re.search(r"(priva|giva|diva|viva|cliva)\b", n): + continue + if key == "carg" and re.search(r"descarg|recarg|abord", n): + continue + return True + return False + + +def _column_name_suggests_code_id(name: str) -> bool: + """Código, ref., radicado, tipoynúm de documento — no forzar 6-8 cifras como fechas (DDMMYY, etc.).""" + n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip() + if not n or _column_name_suggests_date_like(name): + return False + for key in ( + "codigo", "código", "codig", "códig", "cód_", + "sku", "ean", "upc", "isbn", "upn", + "ref", "referen", "radic", "nro_fac", "nro fac", "nit", "cédul", "cedul", + "autoriz", "correlat", "secuenc", "línea de", "linea de", "cupo", + "tipy", "tipi", "tipynum", "typnum", "type_n", "n_doc", "n_docu", + "núm doc", "n_docu", "doc_id", "docum", "id_doc", + ): + if key in n: + return True + if re.search( + r"tipo[_\s\\-]*(&|/|u|e|y)[_\s\\-]*num|tipo[_\s\\-]+doc|tipo_y_n", + n, + re.IGNORECASE, + ) or n.startswith("tip_"): + return True + if n.startswith("cod_") or n.startswith("cód_") or n.startswith("ref_"): + return True + return bool(re.search(r"\b(nro[._]fac|#?\s*autor|#?\s*ref)\b", n, re.IGNORECASE)) + + +def _column_name_suggests_quantity_or_number_not_bool(name: str) -> bool: + """Nombres típicos de cantidad, importe, fechas: no inferir bool por 0/1.""" + n = re.sub(r"[_\s.]+", " ", (name or "")).lower().strip() + if not n: + return False + if _column_name_suggests_monetary_value(name) or _column_name_suggests_code_id(name): + return True + for key in ( + "fecha", + "monto", + "importe", + "bruto", + "neto", + "subtotal", + "unitario", + "unidad", + "valor", + "precio", + "pago", + "copago", + "cantid", + "quantity", + "qty", + "nro", + "núm", + "n°", + ): + if key in n: + return True + return False + + +def _raw_strings_suggest_monetary_presentation(non_null: pd.Series) -> bool: + """$ €, miles 1,234,12 o .00: presentación monetaria aunque el float sea "entero".""" + if non_null is None or len(non_null) == 0: + return False + n_hit = 0 + n_seen = 0 + for raw in non_null.head(500): + if _is_nullish_scalar(raw): + continue + t = str(raw) + s = t.strip() + n_seen += 1 + if re.search(r"[$\u00a3\u20ac\uffe5\uffe6\ufe69\uff04]", t): + n_hit += 1 + continue + if re.search(r"\b\d{1,3}(?:[.,\u00a0]\d{3})+\b", s) or re.search( + r"(?= 0.10 + + +def _cell_looks_cifra_con_moneda_iso_al_final(s: str) -> bool: + """Solo cifra/símbolo (no prefijo de documento) y código de moneda al final. + + Excluye filas al estilo `CC12…` (típico alfanumérico). + """ + t = str(s).strip() + u = t.upper() + if not re.search( + r"(?i)(USD|COP|EUR|CLP|ARS|GBP|MXN|PEN|DOP|UYU|VES|CRC|PYG|BRL|BOB|GTQ)\b\s*\Z", u + ): + return False + if re.match( + r"^(?i)(CC|CE|NIT|NUIP|TI|PA|MS|RC|DNI|NUIP|CN|SC|AS|PE|PT)\D*[\d0]", u + ): + return False + return re.match( + r"^[\$€0-9\s,.\-+%/]", t, + ) is not None + + +def _raw_strings_suggest_alnum_mixed_presentation(non_null: pd.Series) -> bool: + """Letras y dígitos juntos (p. ej. `CC32…`, `A1-2B`); no es puro 'entero/decimal' lógicamente. + + Excluido: columnas con apariencia monetaria, y celdas solo cifra + ISO al final (100 USD). + """ + if _raw_strings_suggest_monetary_presentation(non_null): + return False + n_m = 0 + n_s = 0 + for raw in non_null.head(500): + if _is_nullish_scalar(raw): + continue + t = str(raw).strip() + n_s += 1 + if not re.search(r"\d", t) or not re.search( + r"[A-Za-zÑñ\u00c0-\u024f]", t, re.IGNORECASE + ): + continue + if _cell_looks_cifra_con_moneda_iso_al_final(t): + continue + n_m += 1 + if n_s == 0: + return False + return n_m / n_s >= 0.20 + + +def _cell_looks_like_documento_id_tipo_nit(s: str) -> bool: + """Cadenas estilo `CC32324029`, `CE 1.234.567` — no son fechas (evita 8 cifras como DDMM). + + Al menos una letra al inicio y 5-16 dígitos en la celda (no `Jan`+`2020` = 4 dígitos). + """ + t = str(s).strip() + if not t or len(t) < 3: + return False + if not re.match(r"^[A-Za-z]{1,3}", t): + return False + dig = re.sub(r"[^\d]", "", t) + if len(dig) < 5 or len(dig) > 16: + return False + return True + + +def _content_looks_like_doc_id_letter_prefix_column(non_null: pd.Series) -> bool: + """Muestras con prefijo documental (CC, NIT, …) y ≥5 dígitos → no columna de fecha.""" + n_ok = 0 + n_seen = 0 + for raw in non_null.head(500): + if _is_nullish_scalar(raw): + continue + s = str(raw).strip() + n_seen += 1 + if _cell_looks_like_documento_id_tipo_nit(s): + n_ok += 1 + if n_seen == 0: + return False + return n_ok / n_seen >= 0.20 + diff --git a/services/cleaning/domain/csv_cleaning/cop_mod.py b/services/cleaning/domain/csv_cleaning/cop_mod.py new file mode 100644 index 0000000..0ffae04 --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/cop_mod.py @@ -0,0 +1,162 @@ +"""Separación de copago / cuota moderadora por año.""" +from __future__ import annotations + +import re +from typing import Any, Literal, Optional + +import numpy as np +import pandas as pd + +from .dates import ( + _detect_compact_date_formats, + _parse_any_date_to_tuple, + detect_date_format, +) +from .numbers import _infer_decimal_hint_from_column, _parse_lenient_one + + +DEFAULT_MODERADORA_BY_YEAR: dict[int, set[int]] = { + 2023: {4100, 16400, 43000}, + 2024: {4500, 18200, 47700}, + 2025: {4700, 19200, 50300}, + 2026: {5000, 20100, 52800}, +} + + +def normalize_moderadora_by_year_dict(raw: Any) -> dict[int, set[int]]: + """Normaliza un dict JSON año → lista de enteros (cuotas moderadora).""" + if not isinstance(raw, dict): + return {} + out: dict[int, set[int]] = {} + for k, v in raw.items(): + try: + yi = int(k) + except (TypeError, ValueError): + continue + if not isinstance(v, (list, tuple)): + continue + nums: set[int] = set() + for x in v: + try: + nums.add(int(round(float(x)))) + except (TypeError, ValueError): + continue + if nums: + out[yi] = nums + return out + + +def merge_moderadora_config( + cop_mod_config: Optional[dict[str, Any]], +) -> dict[int, set[int]]: + """Fusiona defaults con `moderadora_by_year` opcional del usuario.""" + merged = {y: set(vals) for y, vals in DEFAULT_MODERADORA_BY_YEAR.items()} + if cop_mod_config and isinstance(cop_mod_config.get("moderadora_by_year"), dict): + custom = normalize_moderadora_by_year_dict( + cop_mod_config["moderadora_by_year"] + ) + for y, vals in custom.items(): + merged[y] = set(vals) + return merged + + +def _alloc_cop_mod_column_names( + col_name: str, existing: set[str] +) -> tuple[str, str]: + """Nombres únicos para copago y moderadora.""" + base = re.sub(r"[^\w]+", "_", str(col_name).strip(), flags=re.ASCII) or "col" + for i in range(0, 200): + suf = f"_{i}" if i else "" + ccol, mcol = f"{base}{suf}_copago", f"{base}{suf}_moderadora" + if ccol not in existing and mcol not in existing: + return ccol, mcol + raise ValueError("No se pudieron asignar nombres de columnas únicos.") + + +def _amount_cell_to_optional_int( + val: Any, + decimal_hint: Optional[Literal[",", "."]], +) -> Optional[int]: + try: + f = _parse_lenient_one(val, decimal_hint) + except Exception: + return None + if f is None or (isinstance(f, float) and not np.isfinite(f)) or ( + isinstance(f, (float, np.floating)) and np.isnan(f) + ): + return None + try: + return int(round(float(f))) + except (TypeError, ValueError, OverflowError): + return None + + +def _apply_cop_mod_split( + result: pd.DataFrame, + amount_col: str, + fecha_column: str, + log: list[str], + *, + moderadora_by_year: dict[int, set[int]], +) -> None: + """Añade columnas copago y moderadora según cuotas por año de la fecha.""" + if fecha_column not in result.columns: + log.append( + f"[WARN] COP/MOD: columna de fecha '{fecha_column}' no existe; se omite." + ) + return + if amount_col not in result.columns: + log.append(f"[WARN] COP/MOD: columna de importe '{amount_col}' no existe; se omite.") + return + + fec_ser = result[fecha_column] + amt_ser = result[amount_col] + stripped = fec_ser.astype(str).str.replace(r"[\s\/\-\.]", "", regex=True) + stripped = stripped.where(fec_ser.notna(), other=pd.NA) + sep_fmt = detect_date_format(fec_ser) or detect_date_format(stripped) + compact8_fmt, compact6_fmt = _detect_compact_date_formats(fec_ser) + amt_hint = _infer_decimal_hint_from_column(amt_ser.astype(str)) + + ccol, mcol = _alloc_cop_mod_column_names(str(amount_col), set(result.columns)) + cop_vals: list[Any] = [] + mod_vals: list[Any] = [] + + for i in range(len(result)): + amt_raw = amt_ser.iloc[i] + fec_raw = fec_ser.iloc[i] + amt_i = _amount_cell_to_optional_int(amt_raw, amt_hint) + if amt_i is None: + cop_vals.append(0) + mod_vals.append(0) + continue + + parsed_d = _parse_any_date_to_tuple( + fec_raw, + sep_fmt=sep_fmt, + compact8_fmt=compact8_fmt, + compact6_fmt=compact6_fmt, + ) + if parsed_d is None: + cop_vals.append(amt_i) + mod_vals.append(0) + continue + _d, _m, year = parsed_d + tiers = moderadora_by_year.get(year) + if tiers is None: + cop_vals.append(amt_i) + mod_vals.append(0) + continue + + if amt_i in tiers: + cop_vals.append(0) + mod_vals.append(amt_i) + else: + cop_vals.append(amt_i) + mod_vals.append(0) + + result[ccol] = cop_vals + result[mcol] = mod_vals + log.append( + f"[OK] '{amount_col}' -> '{ccol}', '{mcol}' <- Separa COP/MOD " + f"(fecha: '{fecha_column}')" + ) diff --git a/services/cleaning/domain/csv_cleaning/dates.py b/services/cleaning/domain/csv_cleaning/dates.py new file mode 100644 index 0000000..2c50d39 --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/dates.py @@ -0,0 +1,459 @@ +"""Detección y normalización de fechas.""" +from __future__ import annotations + +import re +import unicodedata +from datetime import date +from typing import Any, Optional + +import pandas as pd + +from .nullish import _is_nullish_scalar + +_DATE_NUM_RE = re.compile(r"^\s*(\d{1,4})[\s\/\-\.](\d{1,2})[\s\/\-\.](\d{1,4})\s*$") +_DATE_COMPACT_RE = re.compile(r"^\s*(\d{6}|\d{8})\s*$") +# Hora opcional: HH[:MM[:SS]][ AM|PM] al final de la cadena +_TIME_TAIL_RE = re.compile( + r"[\sT]+(\d{1,2})(?::(\d{1,2})(?::(\d{1,2}))?)?\s*([APap]\.?\s*[Mm]\.?)?\s*$" +) + +_MONTH_NAME_TO_NUM = { + "enero": 1, + "ene": 1, + "febrero": 2, + "feb": 2, + "marzo": 3, + "mar": 3, + "abril": 4, + "abr": 4, + "mayo": 5, + "may": 5, + "junio": 6, + "jun": 6, + "julio": 7, + "jul": 7, + "agosto": 8, + "ago": 8, + "septiembre": 9, + "setiembre": 9, + "sep": 9, + "sept": 9, + "octubre": 10, + "oct": 10, + "noviembre": 11, + "nov": 11, + "diciembre": 12, + "dic": 12, +} + + +def _split_date_time(value: str) -> tuple[str, Optional[str]]: + """Separa la parte de fecha de la parte de hora si está presente. + + Devuelve `(date_part, time_part_normalized)`. `time_part_normalized` es + None si no hay hora. La hora se normaliza a `HH:MM[:SS][ AM|PM]`. + """ + if value is None: + return "", None + s = str(value).strip() + if not s: + return "", None + + m = _TIME_TAIL_RE.search(s) + if not m: + return s, None + + hh = int(m.group(1)) + mm = int(m.group(2)) if m.group(2) else 0 + ss = m.group(3) + ampm = m.group(4) + if not (0 <= hh <= 23 and 0 <= mm <= 59): + return s, None + if ss is not None: + try: + ssi = int(ss) + if not (0 <= ssi <= 59): + return s, None + except ValueError: + return s, None + # Validar contra AM/PM (con AM/PM, hora 1-12) + if ampm and not (1 <= hh <= 12): + return s, None + + # Quitar separadores habituales entre fecha y hora (p. ej. "12/27/24, 12:00 AM"). + date_part = s[: m.start()].rstrip(" \t,;T") + if not date_part: + return s, None + + parts: list[str] = [f"{hh:02d}:{mm:02d}"] + if ss is not None: + parts[0] += f":{int(ss):02d}" + if ampm: + norm_ampm = ampm.replace(".", "").replace(" ", "").upper() + if norm_ampm in ("AM", "PM"): + parts.append(norm_ampm) + time_norm = " ".join(parts) + return date_part, time_norm + + +def _extract_date_parts(value: str) -> Optional[tuple[int, int, int, str]]: + """Intenta extraer (a, b, c, layout) de una fecha en cualquier formato común. + + `layout` es uno de: 'sep' (con separadores), 'compact8' (YYYYMMDD o + DDMMYYYY), 'compact6' (DDMMYY o YYMMDD). No interpreta todavía qué + componente es día/mes/año; eso lo decide `detect_date_format` viendo la + columna completa. Acepta valores con hora al final (ignorada para los + componentes de fecha). + """ + if value is None: + return None + s = str(value).strip() + if not s or s.lower() in ("nan", "none", "nat"): + return None + + date_part, _ = _split_date_time(s) + if date_part: + s = date_part + + m = _DATE_NUM_RE.match(s) + if m: + try: + a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)) + return (a, b, c, "sep") + except ValueError: + return None + + m2 = _DATE_COMPACT_RE.match(s) + if m2: + digits = m2.group(1) + if len(digits) == 8: + return (int(digits[0:2]), int(digits[2:4]), int(digits[4:8]), "compact8") + if len(digits) == 6: + return (int(digits[0:2]), int(digits[2:4]), int(digits[4:6]), "compact6") + + return None + + +def detect_date_format(series: pd.Series, sample_size: int = 500) -> Optional[str]: + """Analiza la serie y determina el formato de fecha dominante. + + Devuelve un string con el layout detectado (p. ej. "DD/MM/YYYY", + "MM/DD/YYYY", "YYYYMMDD", "DDMMYYYY") o None si no parece fechas. + """ + if series is None or len(series) == 0: + return None + + non_null = series.dropna() + if non_null.empty: + return None + + if len(non_null) > sample_size: + non_null = non_null.sample(sample_size, random_state=42) + + layouts = {"sep": 0, "compact8": 0, "compact6": 0} + sep_first_gt12 = 0 # primer componente > 12 → probable día + sep_second_gt12 = 0 # segundo componente > 12 → probable mes solo posible si MM/DD + sep_third_4digits = 0 # tercer componente de 4 dígitos → año al final + sep_first_4digits = 0 # primer componente de 4 dígitos → año al inicio + compact8_year_first = 0 # primeros 4 dígitos > 1900 → YYYYMMDD + compact8_year_last = 0 # últimos 4 dígitos > 1900 → DDMMYYYY + total_parsed = 0 + + for val in non_null: + parsed = _extract_date_parts(val) + if parsed is None: + continue + total_parsed += 1 + a, b, c, layout = parsed + layouts[layout] = layouts.get(layout, 0) + 1 + + if layout == "sep": + # año puede estar al inicio o al final según cantidad de dígitos + if _digit_count(a) == 4: + sep_first_4digits += 1 + if _digit_count(c) == 4: + sep_third_4digits += 1 + if a > 12: + sep_first_gt12 += 1 + if b > 12: + sep_second_gt12 += 1 + elif layout == "compact8": + # a son los primeros 2 dígitos, c son los últimos 4 + if c >= 1900: + compact8_year_last += 1 + # a concatenado con b son los primeros 4 dígitos + first4 = a * 100 + b + if first4 >= 1900: + compact8_year_first += 1 + + if total_parsed == 0: + return None + + # Requerimos que >=70% de los no nulos parseen como fecha para considerarla + # columna de fechas. + if total_parsed / len(non_null) < 0.7: + return None + + dominant = max(layouts, key=lambda k: layouts[k]) + + if dominant == "compact8": + if compact8_year_first > compact8_year_last: + return "YYYYMMDD" + return "DDMMYYYY" + if dominant == "compact6": + return "DDMMYY" + + # Caso con separadores + if sep_first_4digits > sep_third_4digits: + return "YYYY/MM/DD" + # año al final + if sep_first_gt12 > 0 and sep_second_gt12 == 0: + return "DD/MM/YYYY" + if sep_second_gt12 > 0 and sep_first_gt12 == 0: + return "MM/DD/YYYY" + # ambiguo → preferencia hispana + return "DD/MM/YYYY" + + +def _digit_count(n: int) -> int: + return len(str(abs(n))) + + +def _parse_date_with_format(value: str, fmt: str) -> Optional[tuple[int, int, int]]: + """Convierte un valor de fecha a (día, mes, año) según el layout detectado.""" + parsed = _extract_date_parts(value) + if parsed is None: + return None + a, b, c, layout = parsed + + if fmt == "DD/MM/YYYY": + d, m, y = a, b, c + elif fmt == "MM/DD/YYYY": + m, d, y = a, b, c + elif fmt == "YYYY/MM/DD": + y, m, d = a, b, c + elif fmt == "YYYYMMDD" and layout == "compact8": + # reconstruir: primer4 = a*100 + b, luego mes y día + y = a * 100 + b + m = c // 100 + d = c % 100 + elif fmt == "DDMMYYYY" and layout == "compact8": + d = a + m = b + y = c + elif fmt == "DDMMYY" and layout == "compact6": + d = a + m = b + y = 2000 + c if c < 50 else 1900 + c + else: + # intentar como sep genérico + d, m, y = a, b, c + + if not (1 <= m <= 12 and 1 <= d <= 31 and 1000 <= y <= 9999): + # año en 2 dígitos + if y < 100: + y = 2000 + y if y < 50 else 1900 + y + if not (1 <= m <= 12 and 1 <= d <= 31 and 1000 <= y <= 9999): + return None + return (d, m, y) + + +def _normalize_date_text(value: Any) -> str: + text = unicodedata.normalize("NFKD", str(value or "")) + text = "".join(c for c in text if not unicodedata.combining(c)) + return text.lower() + + +def _valid_date_tuple(d: int, m: int, y: int) -> Optional[tuple[int, int, int]]: + if y < 100: + y = 2000 + y if y < 50 else 1900 + y + if not (1000 <= y <= 9999): + return None + try: + date(y, m, d) + except ValueError: + return None + return (d, m, y) + + +def _split_glued_year_suffix_hhmm(digits: str) -> Optional[tuple[int, int, int]]: + """Detecta `YYYY` + `HHMM` en una sola tira de 8 cifras (p. ej. `20241344`). + + Tras quitar separadores puede quedar pegado al mes (p. ej. `15may20241344`). + Devuelve (año, hora, minuto) si la parte horaria es válida; si no, None. + """ + if len(digits) != 8 or not digits.isdigit(): + return None + y = int(digits[:4]) + hh = int(digits[4:6]) + mm = int(digits[6:8]) + if not (1000 <= y <= 9999 and 0 <= hh <= 23 and 0 <= mm <= 59): + return None + return (y, hh, mm) + + +def _parse_text_month_candidate( + first: tuple[int, str], + month: int, + second: tuple[int, str], + *, + layout: Literal["around", "after", "before"], +) -> Optional[tuple[int, int, int]]: + a, astr = first + b, bstr = second + if layout == "around": + if len(astr) == 4: + return _valid_date_tuple(b, month, a) + if len(bstr) == 4: + return _valid_date_tuple(a, month, b) + return _valid_date_tuple(a, month, b) + if layout == "after": + if len(astr) == 4: + return _valid_date_tuple(b, month, a) + return _valid_date_tuple(a, month, b) + if len(astr) == 4: + return _valid_date_tuple(b, month, a) + return _valid_date_tuple(a, month, b) + + +def _parse_date_with_text_month(raw: Any) -> Optional[tuple[int, int, int]]: + if _is_nullish_scalar(raw): + return None + date_part, _ = _split_date_time(str(raw)) + normalized = _normalize_date_text(date_part or raw) + # `\d{8}` antes que `\d{1,4}` para no partir año+hora pegados tras strip agresivo. + tokens = re.findall(r"[a-z]+|\d{8}|\d{1,4}", normalized) + if not tokens: + return None + + for i, token in enumerate(tokens): + month = _MONTH_NAME_TO_NUM.get(token) + if month is None: + continue + nums_before = [ + (int(t), t) for t in tokens[:i] if t.isdigit() + ] + nums_after = [ + (int(t), t) for t in tokens[i + 1 :] if t.isdigit() + ] + candidates: list[Optional[tuple[int, int, int]]] = [] + if nums_before and len(nums_after) == 1: + glued = _split_glued_year_suffix_hhmm(nums_after[0][1]) + if glued is not None: + y, _hh, _mm = glued + candidates.append( + _parse_text_month_candidate( + nums_before[-1], month, (y, str(y)), layout="around" + ) + ) + if nums_before and nums_after: + candidates.append( + _parse_text_month_candidate( + nums_before[-1], month, nums_after[0], layout="around" + ) + ) + if len(nums_after) >= 2: + candidates.append( + _parse_text_month_candidate( + nums_after[0], month, nums_after[1], layout="after" + ) + ) + if len(nums_before) >= 2: + candidates.append( + _parse_text_month_candidate( + nums_before[-2], month, nums_before[-1], layout="before" + ) + ) + for parsed in candidates: + if parsed is not None: + return parsed + return None + + + +def _detect_compact_date_formats( + series: pd.Series, +) -> tuple[Optional[str], Optional[str]]: + cser = series.map(_normalize_messy_date_to_compact) + c8s = pd.Series( + [x for x in cser.tolist() if isinstance(x, str) and len(x) == 8] + ) + c6s = pd.Series( + [x for x in cser.tolist() if isinstance(x, str) and len(x) == 6] + ) + fmt8 = detect_date_format(c8s) if not c8s.empty else None + fmt6 = detect_date_format(c6s) if not c6s.empty else None + return fmt8, fmt6 + + +def _parse_any_date_to_tuple( + raw: Any, + *, + sep_fmt: Optional[str], + compact8_fmt: Optional[str], + compact6_fmt: Optional[str], +) -> Optional[tuple[int, int, int]]: + parsed_text = _parse_date_with_text_month(raw) + if parsed_text is not None: + return parsed_text + + if _is_nullish_scalar(raw): + return None + + raw_text = str(raw) + date_part, _ = _split_date_time(raw_text) + target = date_part if date_part else raw_text + if sep_fmt: + parsed = _parse_date_with_format(target, sep_fmt) + if parsed is not None: + return parsed + for fallback_fmt in ("DD/MM/YYYY", "MM/DD/YYYY", "YYYY/MM/DD"): + if fallback_fmt == sep_fmt: + continue + parsed = _parse_date_with_format(target, fallback_fmt) + if parsed is not None: + return parsed + + compact = _normalize_messy_date_to_compact(raw) + if not isinstance(compact, str): + return None + if len(compact) == 8: + for fmt in (compact8_fmt, "DDMMYYYY", "YYYYMMDD"): + if not fmt: + continue + parsed = _parse_date_with_format(compact, fmt) + if parsed is not None: + return parsed + if len(compact) == 6: + parsed = _parse_date_with_format(compact, compact6_fmt or "DDMMYY") + if parsed is not None: + return parsed + return None + + +def _normalize_messy_date_to_compact(raw: Any) -> Optional[str]: + """Quita ruido (* + _ / espacios, etc.); deja 6 o 8 dígitos o None. + + Soporta separadores entremezclados; hora al final vía `_split_date_time`. + Con 9+ dígitos, heurística: tomar 8 iniciales (p. ej. 9.º = 0 colado) y + validar luego con `_parse_date_with_format`; con más de 8 sobrantes, + 8 iniciales. + """ + if _is_nullish_scalar(raw): + return None + s = str(raw).strip() + if s.lower() in ("nan", "none", "nat", ""): + return None + date_part, _ = _split_date_time(s) + body = date_part if date_part else s + d = re.sub(r"[^\d]", "", body) + if not d: + return None + n = len(d) + if n in (6, 8): + return d + if n < 6 or n == 7: + return None + # 9+ dígitos: 8 iniciales (p. ej. 9.º 0 sobrante, o ruido al final) + return d[:8] + diff --git a/services/cleaning/domain/csv_cleaning/documents.py b/services/cleaning/domain/csv_cleaning/documents.py new file mode 100644 index 0000000..a32e3ce --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/documents.py @@ -0,0 +1,175 @@ +"""Homologación y división de tipo/número de documento.""" +from __future__ import annotations + +import re +import unicodedata +from typing import Any, Optional + +import pandas as pd + +from .nullish import _is_nullish_scalar + + +_DOC_TYPE_ALIASES: dict[str, tuple[str, ...]] = { + "CC": ("CC", "Cedula", "CD", "C", "C.C", "C C", "CCC"), + "TI": ("TI", "Tarjeta", "Tarjeta Identidad", "T1", "T", "TL"), + "CN": ("CN", "Certificado", "Nacido Vivo", "Certificado Nacido Vivo"), + "CE": ("CE", "Cedula de Extranjeria", "Extranjera"), + "ME": ("ME", "Menor sin Identificacion"), + "PS": ("PS", "Pasaporte"), + "PT": ("PT", "Permiso por proteccion Temporal"), + "RC": ("RC", "Registro Civil", "Registro", "TC"), + "SC": ("SC", "Salvo Conducto", "Salvoconducto"), + "PE": ("PE", "Permiso especial de permanencia", "Permiso"), +} + + +def _ascii_upper(value: Any) -> str: + text = unicodedata.normalize("NFKD", str(value or "")) + text = "".join(c for c in text if not unicodedata.combining(c)) + return text.upper() + + +def _normalize_doc_type_key(value: Any) -> str: + """Clave comparable: sin acentos, espacios, puntos ni símbolos.""" + return re.sub(r"[^A-Z0-9]", "", _ascii_upper(value)) + + +def _clean_doc_type_text(value: Any) -> Optional[str]: + cleaned = re.sub(r"[^A-Z0-9]+", " ", _ascii_upper(value)).strip() + cleaned = re.sub(r"\s+", " ", cleaned) + return cleaned or None + + +_DOC_TYPE_ALIAS_BY_KEY: dict[str, str] = { + _normalize_doc_type_key(alias): canonical + for canonical, aliases in _DOC_TYPE_ALIASES.items() + for alias in aliases +} + +_DOC_TYPE_ALIAS_DISPLAY_BY_KEY: dict[str, str] = {} +for _canonical, _aliases in _DOC_TYPE_ALIASES.items(): + for _alias in _aliases: + _alias_key = _normalize_doc_type_key(_alias) + _alias_display = _clean_doc_type_text(_alias) + if _alias_key and _alias_display: + _DOC_TYPE_ALIAS_DISPLAY_BY_KEY.setdefault(_alias_key, _alias_display) + +_DOC_TYPE_ALIAS_KEYS_SORTED: list[str] = sorted( + _DOC_TYPE_ALIAS_BY_KEY, + key=lambda k: (len(k), k), + reverse=True, +) + + +def homologate_doc_type(value: Any) -> Optional[str]: + """Homologa un tipo documental a sigla; si no existe, conserva texto limpio.""" + if _is_nullish_scalar(value): + return None + key = _normalize_doc_type_key(value) + if not key or key.lower() in ("nan", "none", "nat"): + return None + return _DOC_TYPE_ALIAS_BY_KEY.get(key) or _clean_doc_type_text(value) + + +def _compact_doc_text(value: Any) -> str: + return _normalize_doc_type_key(value) + + +def _looks_like_short_alias_prefix(compact: str, alias_key: str) -> bool: + """Evita que alias de 1-2 chars capturen palabras como CODIGO o TOTAL.""" + if not compact.startswith(alias_key): + return False + rest = compact[len(alias_key):] + if not rest: + return True + return bool(rest[0].isdigit()) + + +def _match_known_doc_type_prefix(compact: str) -> Optional[tuple[str, str]]: + for alias_key in _DOC_TYPE_ALIAS_KEYS_SORTED: + if len(alias_key) <= 2: + if not _looks_like_short_alias_prefix(compact, alias_key): + continue + elif not compact.startswith(alias_key): + continue + return _DOC_TYPE_ALIAS_BY_KEY[alias_key], alias_key + return None + + +def split_doc_tipo_y_numero_texto( + raw: Any, + *, + homologate: bool = True, +) -> tuple[Optional[str], Optional[str]]: + """Separa tipo documental y número, tolerando nombres largos y símbolos. + + Si `homologate` es True, tipos conocidos se devuelven como sigla (CC, TI, + etc.). Los tipos desconocidos se conservan limpios en mayúsculas. + """ + if _is_nullish_scalar(raw) and not (isinstance(raw, str) and not str(raw).strip()): + return (None, None) + t = str(raw).strip() + if t.lower() in ("nan", "none", "nat", ""): + return (None, None) + if t and t[0].isdigit(): + return (None, None) + + compact = _compact_doc_text(t) + if not compact: + return (None, None) + + match = _match_known_doc_type_prefix(compact) + if match is not None: + canonical, alias_key = match + rest = compact[len(alias_key):] + nums = re.sub(r"[^\d]", "", rest) + if homologate: + return (canonical, nums if nums else None) + raw_type = _DOC_TYPE_ALIAS_DISPLAY_BY_KEY.get(alias_key) or canonical + return (raw_type, nums if nums else None) + + m = re.match( + r"^([^\d]+?)\s*([0-9].*)\s*$", + t, + ) + if m: + to_t = _clean_doc_type_text(m.group(1)) + nums = re.sub(r"[^\d]", "", m.group(2)) + return (homologate_doc_type(to_t) if homologate else to_t, nums if nums else None) + + if not re.search(r"\d", t): + to_t = homologate_doc_type(t) if homologate else _clean_doc_type_text(t) + return (to_t, None) + return (None, None) + + +def _alloc_doc_split_column_names( + col_name: str, existing: set[str] +) -> tuple[str, str]: + """Nombres únicos para las dos columnas derivadas.""" + base = re.sub(r"[^\w]+", "_", str(col_name).strip(), flags=re.ASCII) or "col" + for i in range(0, 200): + suf = f"_{i}" if i else "" + t, n = f"{base}{suf}_doc_tipo", f"{base}{suf}_doc_numeros" + if t not in existing and n not in existing: + return t, n + raise ValueError("No se pudieron asignar nombres de columnas únicos.") + + +def _apply_doc_id_split_columns( + result: pd.DataFrame, col: str, log: list[str], *, homologate_type: bool = True +) -> None: + tcol, ncol = _alloc_doc_split_column_names(str(col), set(result.columns)) + st_tipo: list[Optional[str]] = [] + st_num: list[Any] = [] + for v in result[col].tolist(): + tt, nn = split_doc_tipo_y_numero_texto(v, homologate=homologate_type) + st_tipo.append(tt) + st_num.append(nn if nn is not None else pd.NA) + result[tcol] = st_tipo + result[ncol] = st_num + log.append( + f"[OK] '{str(col)}' -> columnas '{tcol}' y '{ncol}' <- Dividir tipo y número (documento)" + ) + diff --git a/services/cleaning/domain/csv_cleaning/io.py b/services/cleaning/domain/csv_cleaning/io.py new file mode 100644 index 0000000..a62380f --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/io.py @@ -0,0 +1,132 @@ +"""Carga robusta de CSV (delimitador, encoding).""" +from __future__ import annotations + +import csv as _csv +from collections import Counter +from typing import Optional + +import pandas as pd + + +_CANDIDATE_DELIMITERS = ["|", ";", "\t", ",", ":"] + + +def detect_delimiter(filepath: str) -> str: + """Detecta el delimitador del CSV de forma robusta. + + Estrategia: + 1. Intentar `csv.Sniffer` con un sample grande. + 2. Si falla o el resultado parece incorrecto, contar ocurrencias por línea + de cada delimitador candidato y elegir el que produce el mayor número + de columnas estable entre líneas. + """ + encodings = ("utf-8", "utf-8-sig", "latin-1", "cp1252") + sample = "" + for enc in encodings: + try: + with open(filepath, "r", encoding=enc, errors="replace") as f: + sample = f.read(65536) + break + except Exception: + continue + + if not sample: + return "," + + try: + dialect = _csv.Sniffer().sniff(sample, delimiters="".join(_CANDIDATE_DELIMITERS)) + sniffed = dialect.delimiter + except _csv.Error: + sniffed = None + + lines = [ln for ln in sample.splitlines() if ln.strip()] + if not lines: + return sniffed or "," + + # Para cada candidato, calcular el número promedio de columnas y la + # consistencia (cuántas líneas tienen el mismo conteo dominante). + scores: dict[str, tuple[int, int]] = {} + for delim in _CANDIDATE_DELIMITERS: + counts = [ln.count(delim) for ln in lines[:50]] + if max(counts) == 0: + continue + # Modo (valor más frecuente) + common = Counter(counts).most_common(1)[0] + mode_count, mode_freq = common + if mode_count == 0: + continue + scores[delim] = (mode_count, mode_freq) + + if scores: + # Elegir el delimitador con mayor número de columnas (mode_count) y, + # como desempate, mayor consistencia (mode_freq). + best = max(scores.items(), key=lambda kv: (kv[1][0], kv[1][1])) + best_delim = best[0] + # Si el sniffer dio algo razonable y coincide, prevalece; si difiere y + # nuestro candidato es claramente mejor, usamos el nuestro. + if sniffed and sniffed in scores: + sniffed_score = scores[sniffed] + if sniffed_score[0] >= best[1][0] - 1: + return sniffed + return best_delim + + return sniffed or "," + + +def load_csv(filepath: str) -> tuple[pd.DataFrame, str]: + """Carga un CSV con detección de delimitador y tolerancia a filas malas. + + Devuelve `(df, delimiter)`. Lanza excepción si todos los intentos fallan. + """ + delimiter = detect_delimiter(filepath) + + encodings_to_try = ["utf-8", "utf-8-sig", "latin-1", "cp1252"] + last_error: Optional[Exception] = None + + for enc in encodings_to_try: + try: + df = pd.read_csv( + filepath, + delimiter=delimiter, + dtype=str, + encoding=enc, + engine="python", + on_bad_lines="skip", + ) + return df, delimiter + except Exception as exc: # noqa: BLE001 + last_error = exc + continue + + # Último intento con quoting permisivo y sin separador detectado + try: + df = pd.read_csv( + filepath, + sep=None, + dtype=str, + engine="python", + on_bad_lines="skip", + ) + return df, "(auto)" + except Exception as exc: # noqa: BLE001 + last_error = exc + + raise RuntimeError(f"No se pudo leer el CSV: {last_error}") + + +def load_csv_bytes(content: bytes, filename: str = "upload.csv") -> tuple[pd.DataFrame, str]: + """Carga un CSV desde bytes (upload HTTP) vía archivo temporal.""" + import os + import tempfile + + suffix = os.path.splitext(filename)[1] or ".csv" + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: + tmp.write(content) + tmp_path = tmp.name + try: + return load_csv(tmp_path) + finally: + try: + os.unlink(tmp_path) + except OSError: + pass diff --git a/services/cleaning/domain/csv_cleaning/models.py b/services/cleaning/domain/csv_cleaning/models.py new file mode 100644 index 0000000..2940813 --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/models.py @@ -0,0 +1,67 @@ +"""Tipos de columna y resumen de análisis.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Optional + +TYPE_INTEGER = "integer" +TYPE_DECIMAL = "decimal" +TYPE_DATE = "date" +TYPE_BOOLEAN = "boolean" +TYPE_CATEGORICAL = "categorical" +TYPE_TEXT = "text" +TYPE_MIXED = "mixed" +TYPE_EMPTY = "empty" + + +@dataclass +class ColumnSummary: + """Resumen inteligente para una columna del DataFrame.""" + name: str + inferred_type: str + example: str = "" + samples: list[str] = field(default_factory=list) + null_count: int = 0 + null_pct: float = 0.0 + unique_count: int = 0 + unique_pct: float = 0.0 + min: Optional[float] = None + max: Optional[float] = None + mean: Optional[float] = None + avg_length: Optional[float] = None + top_values: list[tuple[str, int]] = field(default_factory=list) + detected_date_format: Optional[str] = None + applicable_rules: list[str] = field(default_factory=list) + + def info_text(self) -> str: + """Texto humano con información adicional (para subtítulo de la UI).""" + parts: list[str] = [] + parts.append(f"nulos: {self.null_count} ({self.null_pct:.0f}%)") + parts.append(f"únicos: {self.unique_count}") + if self.inferred_type in (TYPE_INTEGER, TYPE_DECIMAL): + if self.min is not None: + parts.append(f"min: {_fmt_num(self.min)}") + if self.max is not None: + parts.append(f"max: {_fmt_num(self.max)}") + if self.mean is not None: + parts.append(f"media: {_fmt_num(self.mean)}") + elif self.inferred_type == TYPE_DATE: + if self.detected_date_format: + parts.append(f"formato: {self.detected_date_format}") + elif self.inferred_type in (TYPE_TEXT, TYPE_CATEGORICAL): + if self.avg_length is not None: + parts.append(f"long. prom: {self.avg_length:.1f}") + if self.top_values: + tops = ", ".join(f'"{v}" ({c})' for v, c in self.top_values[:3]) + parts.append(f"top: {tops}") + return " · ".join(parts) + + +def _fmt_num(x: float) -> str: + if x is None: + return "" + if isinstance(x, float) and x.is_integer(): + return str(int(x)) + if isinstance(x, float): + return f"{x:.2f}" + return str(x) diff --git a/services/cleaning/domain/csv_cleaning/notas.py b/services/cleaning/domain/csv_cleaning/notas.py new file mode 100644 index 0000000..84ad67a --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/notas.py @@ -0,0 +1,73 @@ +"""Extracción de documento y autorización desde notas de texto.""" +from __future__ import annotations + +import re +from typing import Any, Optional + +import pandas as pd + +from .nullish import _is_nullish_scalar + + +def _normalize_nota_text(raw: Any) -> str: + if _is_nullish_scalar(raw): + return "" + try: + if pd.isna(raw): + return "" + except TypeError: + pass + s = str(raw).strip() + if not s or s.lower() in ("nan", "none", "nat", ""): + return "" + return re.sub(r"\s+", " ", s) + + +def extract_paciente_documento_nota(raw: Any) -> Optional[str]: + """Número de documento tras ``Paciente:`` y antes del guión del nombre.""" + t = _normalize_nota_text(raw) + if not t: + return None + m = _PACIENTE_DOC_NOTA_RE.search(t) + return m.group(1) if m else None + + +def extract_autorizacion_nota(raw: Any) -> Optional[str]: + """Número tras ``Autorización No.`` (opcional por fila).""" + t = _normalize_nota_text(raw) + if not t: + return None + m = _AUTORIZACION_NOTA_RE.search(t) + return m.group(1) if m else None + + +def _alloc_nota_extract_column_names( + col_name: str, existing: set[str] +) -> tuple[str, str]: + """Nombres únicos para documento paciente y autorización.""" + base = re.sub(r"[^\w]+", "_", str(col_name).strip(), flags=re.ASCII) or "col" + for i in range(0, 200): + suf = f"_{i}" if i else "" + dcol, acol = f"{base}{suf}_nota_paciente_doc", f"{base}{suf}_nota_autorizacion" + if dcol not in existing and acol not in existing: + return dcol, acol + raise ValueError("No se pudieron asignar nombres de columnas únicos.") + + +def _apply_nota_extract_columns(result: pd.DataFrame, col: str, log: list[str]) -> None: + """Añade columnas con documento (Paciente:) y número de autorización si existe.""" + dcol, acol = _alloc_nota_extract_column_names(str(col), set(result.columns)) + docs: list[Any] = [] + auths: list[Any] = [] + for v in result[col].tolist(): + dd = extract_paciente_documento_nota(v) + aa = extract_autorizacion_nota(v) + docs.append(dd if dd is not None else pd.NA) + auths.append(aa if aa is not None else pd.NA) + result[dcol] = docs + result[acol] = auths + log.append( + f"[OK] '{str(col)}' -> columnas '{dcol}' y '{acol}' <- Extraer datos de notas" + ) + + diff --git a/services/cleaning/domain/csv_cleaning/nullish.py b/services/cleaning/domain/csv_cleaning/nullish.py new file mode 100644 index 0000000..65638a8 --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/nullish.py @@ -0,0 +1,24 @@ +"""Detección de valores nulos / vacíos.""" +from __future__ import annotations + +from typing import Any + +import numpy as np +import pandas as pd + + +def _is_nullish_scalar(x: Any) -> bool: + if x is None or x is pd.NA: + return True + if isinstance(x, (bool, np.bool_)): + return False + if isinstance(x, (int, float, np.integer, np.floating)) and not isinstance( + x, (bool, np.bool_) + ): + if isinstance(x, float) and (np.isnan(x) or pd.isna(x)): + return True + return False + t = str(x).strip().lower() + if not t or t in ("nan", "none", "nat", "null", ""): + return True + return False diff --git a/services/cleaning/domain/csv_cleaning/numbers.py b/services/cleaning/domain/csv_cleaning/numbers.py new file mode 100644 index 0000000..e2051d7 --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/numbers.py @@ -0,0 +1,258 @@ +"""Parsing numérico tolerante (miles, decimales, moneda).""" +from __future__ import annotations + +import re +from typing import Any, Literal, Optional + +import numpy as np +import pandas as pd + +from .nullish import _is_nullish_scalar + +_HINT_MAJORITY = 0.6 +_MIN_HINT_VOTES = 1 + + +def _strip_currency(s: str) -> str: + t = s.replace("\u00a0", " ").replace("\u202f", " ") + t = re.sub(r"[\s]+", "", t) + t = t.strip() + if not t or t.lower() in ("nan", "none", "nat"): + return "" + neg = False + if t[0:1] == "-": + neg, t = not neg, t[1:].lstrip() + if t[0:1] == "(" and t.endswith(")"): + neg, t = not neg, t[1:-1] + t = re.sub( + r"^[\$€\u00a3\u0e3f\uffe5\uffe6\uff04\ufe69]+", + "", + t, + count=1, + ) + t = re.sub(r"[\$€%]+$", "", t) + if not t and neg: + return "-0" + return ("-" if neg else "") + t + + +def _extract_numeric_chunk(s: str) -> str: + t = s.strip() + t = t.replace("$", "").replace("€", "").replace("\u00a3", "") + t = t.replace(" ", "") + cands1 = re.findall( + r"[+\-]?(?:\d{1,3}(?:[.,]\d{3})*|\d+)(?:[.,]\d+)?", t + ) + cands2 = re.findall( + r"[+\-]?\d[.,\d ]*\d", t + ) + cands = cands1 + cands2 + if not cands: + m = re.search(r"[\d.,\-]+", t) + if m: + c = m.group(0).replace("(", "-").replace(")", "") + return c.lstrip("+") + return "" + best = max(cands, key=len) if cands else "" + best = best.replace(" ", "").replace("(", "-").replace(")", "") + if not re.search(r"\d", best): + return "" + return best.lstrip("+") + + +def _as_float_s(s: str) -> Optional[float]: + if not s: + return None + try: + f = float(s) + if not np.isfinite(f) or abs(f) > 1e100: + return None + return f + except (ValueError, TypeError, OverflowError): + return None + + +def _parse_both_separators(body: str) -> Optional[float]: + """El separador más a la DERECHA entre `,` y `.` es el decimal; el otro, miles.""" + if "," not in body or "." not in body: + return None + t = body.strip() + if re.search(r"[^0-9,.\-]", t): + t2 = re.sub(r"[^0-9,.\-]", "", t) + t = t2 + if not t: + return None + neg = t[0:1] == "-" + b = t[1:].lstrip() if neg else t + if not b: + return None + i_comma, i_dot = b.rfind(","), b.rfind(".") + if min(i_comma, i_dot) < 0: + return None + dpos = max(i_comma, i_dot) + th = "," if b[dpos] == "." else "." + intp, fr = b[:dpos], b[dpos + 1 :] + if not re.fullmatch(r"\d{1,4}$", fr): + return None + intc = intp.replace(th, "") + if not intc.isdigit(): + return None + v = _as_float_s(f"{intc}.{fr}") + if v is None: + return None + if neg and v and v is not None: + return -abs(v) + return v + + +def _only_commas(w: str, dec_hint: Optional[Literal[",", "."]]) -> Optional[float]: + neg = w[0:1] == "-" + wn = w[1:].lstrip() if neg else w + if wn and re.search(r"[^0-9,]", wn): + wn = re.sub(r"[^0-9,]", "", wn) + n = wn.count(",") + if n == 0 and wn.isdigit(): + v = float(wn) + return -v if neg else v + if n == 0 or "." in wn: + return None + if n == 1: + i = wn.rfind(",") + h, t = wn[:i], wn[i + 1 :] + if not t.isdigit() or not h.isdigit(): + return None + if len(t) in (1, 2): + v = _as_float_s(f"{h}.{t}") + if v is None: + return None + return -v if neg else v + if len(t) == 3: + if dec_hint is None or dec_hint == ",": + v = _as_float_s(f"{h}{t}") + if v is None: + return None + return -v if neg else v + if dec_hint == ".": + v = _as_float_s(wn.replace(",", "")) + if v is None: + return None + return -v if neg else v + v = _as_float_s(wn.replace(",", "")) + if v is None: + return None + return -v if neg else v + + +def _only_dots(w: str, dec_hint: Optional[Literal[",", "."]]) -> Optional[float]: + neg = w[0:1] == "-" + wn = w[1:].lstrip() if neg else w + if wn and re.search(r"[^0-9.]", wn): + wn = re.sub(r"[^0-9.]", "", wn) + n = wn.count(".") + if n == 0 and wn.isdigit(): + v = float(wn) + return -v if neg else v + if n == 0 or "," in wn: + return None + if n == 1: + i = wn.rfind(".") + h, t = wn[:i], wn[i + 1 :] + if not t.isdigit() or not h.isdigit(): + return None + if len(t) in (1, 2): + v = _as_float_s(f"{h}.{t}") + if v is None: + return None + return -v if neg else v + if len(t) == 3: + if dec_hint is None or dec_hint == ".": + v = _as_float_s(f"{h}{t}") + if v is None: + return None + return -v if neg else v + if dec_hint == ",": + v = _as_float_s(wn.replace(".", "")) + if v is None: + return None + return -v if neg else v + v = _as_float_s(wn.replace(".", "")) + if v is None: + return None + return -v if neg else v + + +def _parse_lenient_one( + raw: Any, + decimal_hint: Optional[Literal[",", "."]] = None, +) -> float: + if _is_nullish_scalar(raw): + return float("nan") + t = _strip_currency(str(raw)) + if not t or t == "-": + return float("nan") + c = _extract_numeric_chunk(t) or t + c = c.strip() + if not c or c == "-": + return float("nan") + n_comma, n_dot = c.count(","), c.count(".") + if n_comma and n_dot: + f = _parse_both_separators(c) + elif n_comma and not n_dot: + f = _only_commas(c, decimal_hint) + elif n_dot and not n_comma: + f = _only_dots(c, decimal_hint) + else: + c2 = re.sub(r"[^0-9.\-]", "", c) + f = _as_float_s(c2) if c2 and re.fullmatch(r"-?\d+$", c2) else None + if f is not None and np.isfinite(f): + return f + return float("nan") + + +def _infer_decimal_hint_from_column( + series: pd.Series, +) -> Optional[Literal[",", "."]]: + """Voto con celdas que traen `,` y `.`: gana el de más a la derecha = decimal.""" + votes: list[Literal[",", "."]] = [] + for raw in series.dropna(): + if _is_nullish_scalar(raw): + continue + t = _strip_currency(str(raw)) + c = _extract_numeric_chunk(t) or t + c = c.strip() + if "," not in c or "." not in c or _parse_both_separators(c) is None: + continue + if c.rfind(",") > c.rfind("."): + votes.append(",") + else: + votes.append(".") + n = len(votes) + if n < _MIN_HINT_VOTES: + return None + a = sum(1 for v in votes if v == ",") + b = sum(1 for v in votes if v == ".") + if a + b == 0: + return None + if a / n >= _HINT_MAJORITY and a > b: + return "," + if b / n >= _HINT_MAJORITY and b > a: + return "." + return None + + +def _to_numeric_lenient( + series: pd.Series, + decimal_hint: Optional[Literal[",", "."]] = None, +) -> pd.Series: + """Convierte a flotante; pista de columna por voto (si `decimal_hint` es None).""" + hint = decimal_hint if decimal_hint is not None else _infer_decimal_hint_from_column( + series + ) + + def _cell(x: Any) -> float: + if _is_nullish_scalar(x): + return float("nan") + return _parse_lenient_one(x, hint) + + return series.map(_cell) + diff --git a/services/cleaning/domain/csv_cleaning/rules/__init__.py b/services/cleaning/domain/csv_cleaning/rules/__init__.py new file mode 100644 index 0000000..071ae2e --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/rules/__init__.py @@ -0,0 +1,13 @@ +"""Reglas de limpieza registradas.""" +from __future__ import annotations + +from . import builtins as _builtins # noqa: F401 — registra reglas al importar +from .registry import CLEANING_RULES, get_rule, get_rule_fn, get_rule_name, register_rule + +__all__ = [ + "CLEANING_RULES", + "get_rule", + "get_rule_fn", + "get_rule_name", + "register_rule", +] diff --git a/services/cleaning/domain/csv_cleaning/rules/builtins.py b/services/cleaning/domain/csv_cleaning/rules/builtins.py new file mode 100644 index 0000000..2be0016 --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/rules/builtins.py @@ -0,0 +1,135 @@ +"""Registro de todas las reglas built-in. + +Añadir una regla nueva: implementa la función y llama a register_rule abajo. +""" +from __future__ import annotations + +from . import dates as date_rules +from . import documents as doc_rules +from . import numeric as num_rules +from . import special +from . import text as text_rules +from .registry import rebuild_cleaning_rules, register_rule + +register_rule( + "strip_special_spaces", + name="Eliminar caracteres especiales y espacios", + description=( + "Deja solo caracteres alfanuméricos (A-Z, a-z, 0-9). No usar antes de " + "«Normalizar fecha» en columnas con fecha y hora: pega cifras y puede impedir " + "interpretar el valor (aplique primero la normalización de fecha o use la regla " + "que conserva separadores)." + ), + fn=text_rules._rule_strip_special_spaces, +) +register_rule( + "strip_special_keep_date_seps", + name="Quitar símbolos; conserva /, -, : y espacios (fechas o texto con fechas)", + description=( + "Quita comas, $, comillas, etc. y deja letras, números, /, -, : (hora) y espacios. " + "Adecuado p. ej. en columna 'Fecha Servicio'." + ), + fn=text_rules._rule_strip_special_keep_date_seps, +) +register_rule( + "strip_spaces", + name="Eliminar únicamente espacios", + description="Elimina todos los caracteres de espacio en blanco.", + fn=text_rules._rule_strip_spaces, +) +register_rule( + "to_integer", + name="Convertir a Entero", + description=( + "Convierte a enteros con detección de miles/decimal (`,`/`.`) y moneda; " + "redondea decimales." + ), + fn=num_rules._rule_to_integer, +) +register_rule( + "date_to_ddmmyyyy", + name="Normalizar fecha → DD/MM/AAAA", + description=( + "Normaliza fechas numéricas, con separadores, con ruido o con meses en " + "texto (ej. 26-abril-2026) al formato DD/MM/AAAA." + ), + fn=date_rules._rule_date_to_ddmmyyyy, +) +register_rule( + "date_numeric_to_ddmmyyyy", + name="Conversión de formato numérico (Fechas) → DD/MM/AAAA", + description=( + "Detecta el formato numérico predominante (YYYYMMDD, DDMMYYYY, etc.) " + "y normaliza a DD/MM/AAAA." + ), + fn=date_rules._rule_date_numeric_to_ddmmyyyy, +) +register_rule( + "date_alpha_to_ddmmyyyy", + name="Conversión de formato alfanumérico (Fechas) → DD/MM/AAAA", + description=( + "Detecta el formato con separadores predominante " + "(DD/MM/YYYY, MM/DD/YYYY, YYYY/MM/DD, etc.) y normaliza a DD/MM/AAAA." + ), + fn=date_rules._rule_date_alpha_to_ddmmyyyy, +) +register_rule( + "date_messy_text_to_ddmmyyyy", + name="Texto a fecha (ruido en fechas) → DD/MM/AAAA", + description=( + "Quita ruido entre cifras (*, +, _, /, etc.), detecta el orden en la " + "columna (DDMMYYYY, YYYYMMDD, DDMMYY) y unifica a DD/MM/AAAA." + ), + fn=date_rules._rule_date_messy_text_to_ddmmyyyy, +) +register_rule( + "round_decimals_custom", + name="Aproximación Decimales (>0,5 sube, ≤0,5 baja)", + description=( + "Usa el mismo análisis de miles, decimales y moneda que «Entero». Tras leer " + "el número, si la parte fraccionaria de |valor| es mayor a 0,5 se aproxima al " + "entero hacia lejos de cero; en caso contrario, hacia cero. El 0,5 exacto se baja." + ), + fn=num_rules._rule_round_decimals_custom, +) +register_rule( + "doc_id_split_tipo_numero", + name="Dividir tipo y número (documento)", + description=( + "Añade dos columnas (nombres únicos p. ej. TipoID_doc_tipo, TipoID_doc_numeros): " + "prefijo con letras (NIT, CC, CE, …) y cifras sin separadores. No modifica la " + "columna de origen." + ), + fn=special._rule_passthrough_series, +) +register_rule( + "doc_type_homologate", + name="Homologar tipo de documento", + description=( + "Convierte tipos de documento escritos como nombre largo, sigla o texto con " + "símbolos a su abreviatura estándar (CC, TI, CE, RC, etc.)." + ), + fn=doc_rules._rule_doc_type_homologate, +) +register_rule( + "cop_mod_split", + name="Separa COP/MOD (copago / moderadora)", + description=( + "Según el año en la columna Fecha Servicio que elija en la tarjeta, clasifica " + "el importe en cuota moderadora (valores por año en Configuración) o copago. " + "Se ejecuta al final del proceso de limpieza para usar fechas ya normalizadas." + ), + fn=special._rule_passthrough_series, +) +register_rule( + "nota_paciente_autorizacion", + name="Extraer datos de notas (documento y autorización)", + description=( + "Desde texto tipo nota hospitalaria, extrae el número tras «Paciente:» (antes del " + "guión del nombre) y, si existe, el de «Autorización No.». Añade dos columnas; " + "no modifica el texto original." + ), + fn=special._rule_passthrough_series, +) + +rebuild_cleaning_rules() diff --git a/services/cleaning/domain/csv_cleaning/rules/dates.py b/services/cleaning/domain/csv_cleaning/rules/dates.py new file mode 100644 index 0000000..07d645c --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/rules/dates.py @@ -0,0 +1,141 @@ +"""Reglas de normalización de fechas a DD/MM/AAAA.""" +from __future__ import annotations + +from typing import Any, Optional + +import pandas as pd + +from ..dates import ( + _detect_compact_date_formats, + _normalize_messy_date_to_compact, + _parse_any_date_to_tuple, + _parse_date_with_format, + _parse_date_with_text_month, + detect_date_format, +) +from ..nullish import _is_nullish_scalar + + +def _rule_date_to_ddmmyyyy(series: pd.Series) -> pd.Series: + """Normaliza fechas numéricas, alfanuméricas, con meses en texto y con ruido.""" + stripped = series.astype(str).str.replace(r"[\s\/\-\.]", "", regex=True) + stripped = stripped.where(series.notna(), other=pd.NA) + sep_fmt = detect_date_format(series) or detect_date_format(stripped) + compact8_fmt, compact6_fmt = _detect_compact_date_formats(series) + + def _fn(x: Any): + if pd.isna(x) or str(x).strip() == "": + return x + parsed = _parse_any_date_to_tuple( + x, + sep_fmt=sep_fmt, + compact8_fmt=compact8_fmt, + compact6_fmt=compact6_fmt, + ) + if parsed is None: + return x + d, m, y = parsed + return f"{d:02d}/{m:02d}/{y:04d}" + + return series.map(_fn) + + +def _rule_date_numeric_to_ddmmyyyy(series: pd.Series) -> pd.Series: + """Entrada: fechas en formato numérico (ej. 20240315 o 15032024). + + Salida: "DD/MM/AAAA" tras detectar el formato predominante. + """ + # Para esta regla consideramos los valores "numéricos" (quitamos separadores + # comunes si los hay para detectar un YYYYMMDD/DDMMYYYY confiable). + stripped = series.astype(str).str.replace(r"[\s\/\-\.]", "", regex=True) + stripped = stripped.where(series.notna(), other=pd.NA) + fmt = detect_date_format(stripped) + if fmt is None: + # fallback: intentar con detector normal + fmt = detect_date_format(series) + return _format_dates_to_ddmmyyyy(series, fmt) + + +def _rule_date_alpha_to_ddmmyyyy(series: pd.Series) -> pd.Series: + """Entrada: fechas en formato alfanumérico con separadores. + + Salida: "DD/MM/AAAA". + """ + fmt = detect_date_format(series) + return _format_dates_to_ddmmyyyy(series, fmt) + + +def _format_dates_to_ddmmyyyy(series: pd.Series, fmt: Optional[str]) -> pd.Series: + """Convierte cada valor al formato DD/MM/AAAA usando `fmt` como layout de + entrada detectado. La parte de hora (si existe) se descarta. Mantiene el + valor original si no puede parsear la fecha. + """ + if fmt is None: + return series.copy() + + def _fn(x): + if pd.isna(x) or str(x).strip() == "": + return x + raw = str(x) + date_part, _time_part = _split_date_time(raw) + target = date_part if date_part else raw + parsed = _parse_date_with_format(target, fmt) + if parsed is None: + return x + d, m, y = parsed + return f"{d:02d}/{m:02d}/{y:04d}" + + return series.map(_fn) + + +def _rule_date_messy_text_to_ddmmyyyy(series: pd.Series) -> pd.Series: + """Fechas con símbolos y basura entre cifras → DD/MM/AAAA. + + Normaliza a solo 6/8 dígitos, vota el formato con `detect_date_format` por + separado en compact8 y compact6; si el detector no devuelve nada, usa + `DDMMYYYY` / `DDMMYY` (alineado con la preferencia hispana en fechas + numéricas). + """ + cser = series.map(_normalize_messy_date_to_compact) + c8s = pd.Series( + [x for x in cser.tolist() if isinstance(x, str) and len(x) == 8] + ) + c6s = pd.Series( + [x for x in cser.tolist() if isinstance(x, str) and len(x) == 6] + ) + if c8s.empty and c6s.empty: + return series.copy() + + fmt8: Optional[str] = None + fmt6: Optional[str] = None + if not c8s.empty: + fmt8 = detect_date_format(c8s) or "DDMMYYYY" + if not c6s.empty: + fmt6 = detect_date_format(c6s) or "DDMMYY" + + def _one(x: Any, c: Any) -> Any: + if _is_nullish_scalar(x): + return x + if not isinstance(c, str) or len(c) not in (6, 8): + return x + f = (fmt8 or "DDMMYYYY") if len(c) == 8 else (fmt6 or "DDMMYY") + t = _parse_date_with_format(c, f) + if t is None and len(c) == 8: + t = _parse_date_with_format(c, "DDMMYYYY") or _parse_date_with_format( + c, "YYYYMMDD" + ) + if t is None and len(c) == 6: + t = _parse_date_with_format(c, "DDMMYY") + if t is None: + return x + d, m, y = t + return f"{d:02d}/{m:02d}/{y:04d}" + + return pd.Series( + [ + _one(v, cser.iloc[i] if i < len(cser) else None) for i, v in enumerate(series) + ], + index=series.index, + dtype=object, + ) + diff --git a/services/cleaning/domain/csv_cleaning/rules/documents.py b/services/cleaning/domain/csv_cleaning/rules/documents.py new file mode 100644 index 0000000..0fa3fc9 --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/rules/documents.py @@ -0,0 +1,23 @@ +"""Reglas de documento (homologación; split se aplica en apply.py).""" +from __future__ import annotations + +from typing import Any + +import pandas as pd + +from ..documents import homologate_doc_type, split_doc_tipo_y_numero_texto +from ..nullish import _is_nullish_scalar + + +def _rule_doc_type_homologate(series: pd.Series) -> pd.Series: + """Homologa una columna que contiene tipos de documento.""" + + def _fn(x: Any): + if _is_nullish_scalar(x): + return x + tipo, _num = split_doc_tipo_y_numero_texto(x, homologate=True) + if tipo is not None: + return tipo + return homologate_doc_type(x) + + return series.map(_fn) diff --git a/services/cleaning/domain/csv_cleaning/rules/numeric.py b/services/cleaning/domain/csv_cleaning/rules/numeric.py new file mode 100644 index 0000000..be5047e --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/rules/numeric.py @@ -0,0 +1,57 @@ +"""Reglas numéricas (entero, redondeo).""" +from __future__ import annotations + +import math + +import numpy as np +import pandas as pd + +from ..nullish import _is_nullish_scalar +from ..numbers import _to_numeric_lenient + + +def _rule_to_integer(series: pd.Series) -> pd.Series: + """Convierte a entero (Int64 nullable) con parser tolerante.""" + numeric = _to_numeric_lenient(series.astype(str)) + return numeric.round().astype("Int64") + + +def _round_to_int_half_tie_le(x: float) -> int: + """Fracc. > 0,5 hacia arriba; 0,5 o menos hacia abajo. Banda 1e-9 en 0,5 (float). + + Misma noción que en la descripción de la regla. Trabaja sobre |x| y + recompone el signo. + """ + sign = -1 if x < 0 else 1 + if x == 0.0 or x == -0.0: + return 0 + ax = abs(float(x)) + ip = math.trunc(ax) + frac = ax - ip + if frac < 0.5 - 1e-9: + out = ip + elif frac > 0.5 + 1e-9: + out = ip + 1 + else: + out = ip + return -out if sign < 0 else out + + +def _rule_round_decimals_custom(series: pd.Series) -> pd.Series: + """Mismo parser tolerante que convertir a entero; aprox. con fracc. >0,5 arriba.""" + numeric = _to_numeric_lenient(series.astype(str)) + + def _fn(x): + if _is_nullish_scalar(x) or (isinstance(x, (float, np.floating)) and (not np.isfinite(x) or np.isnan(x))): + return pd.NA + try: + f = float(x) + except (TypeError, ValueError, OverflowError): + return pd.NA + if not np.isfinite(f): + return pd.NA + if f == 0.0 or f == -0.0: + return 0 + return _round_to_int_half_tie_le(f) + + return numeric.map(_fn).astype("Int64") diff --git a/services/cleaning/domain/csv_cleaning/rules/registry.py b/services/cleaning/domain/csv_cleaning/rules/registry.py new file mode 100644 index 0000000..d0ad72a --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/rules/registry.py @@ -0,0 +1,80 @@ +"""Registro de reglas de limpieza. + +Para agregar una regla nueva: +1. Implementa ``fn(series: pd.Series) -> pd.Series`` (o passthrough si es especial). +2. Llama a ``register_rule(...)`` en ``rules/builtins.py``. +3. Si la regla es especial (añade columnas, usa config), maneja el id en ``apply.py``. +4. Incluye el id en ``_applicable_rules_for_type`` (type_inference.py) si debe sugerirse. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Optional + +import pandas as pd + +RuleFn = Callable[[pd.Series], pd.Series] + + +@dataclass(frozen=True) +class RuleSpec: + id: str + name: str + description: str + fn: RuleFn + + def as_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "name": self.name, + "description": self.description, + "fn": self.fn, + } + + +_REGISTRY: dict[str, RuleSpec] = {} +_ORDER: list[str] = [] + + +def register_rule( + rule_id: str, + *, + name: str, + description: str, + fn: RuleFn, +) -> RuleSpec: + """Registra una regla. Si el id ya existe, lo reemplaza manteniendo el orden.""" + spec = RuleSpec(id=rule_id, name=name, description=description, fn=fn) + if rule_id not in _REGISTRY: + _ORDER.append(rule_id) + _REGISTRY[rule_id] = spec + return spec + + +def get_rule(rule_id: str) -> Optional[dict[str, Any]]: + spec = _REGISTRY.get(rule_id) + return spec.as_dict() if spec else None + + +def get_rule_fn(rule_id: str) -> Optional[RuleFn]: + spec = _REGISTRY.get(rule_id) + return spec.fn if spec else None + + +def get_rule_name(rule_id: str) -> str: + spec = _REGISTRY.get(rule_id) + return spec.name if spec else rule_id + + +def list_rules() -> list[dict[str, Any]]: + return [_REGISTRY[rid].as_dict() for rid in _ORDER if rid in _REGISTRY] + + +# Compat: lista mutable usada por la API (se rellena al importar builtins). +CLEANING_RULES: list[dict[str, Any]] = [] + + +def rebuild_cleaning_rules() -> list[dict[str, Any]]: + CLEANING_RULES.clear() + CLEANING_RULES.extend(list_rules()) + return CLEANING_RULES diff --git a/services/cleaning/domain/csv_cleaning/rules/special.py b/services/cleaning/domain/csv_cleaning/rules/special.py new file mode 100644 index 0000000..04c1521 --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/rules/special.py @@ -0,0 +1,8 @@ +"""Marcadores passthrough para reglas que no transforman la serie in-place.""" +from __future__ import annotations + +import pandas as pd + + +def _rule_passthrough_series(s: pd.Series) -> pd.Series: + return s.copy() diff --git a/services/cleaning/domain/csv_cleaning/rules/text.py b/services/cleaning/domain/csv_cleaning/rules/text.py new file mode 100644 index 0000000..66e6777 --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/rules/text.py @@ -0,0 +1,40 @@ +"""Reglas de limpieza de texto / espacios.""" +from __future__ import annotations + +import re + +import pandas as pd + + +def _rule_strip_special_spaces(series: pd.Series) -> pd.Series: + """Elimina todo lo que no sea alfanumérico (incluye espacios).""" + pattern = re.compile(r"[^A-Za-z0-9]", flags=re.UNICODE) + + def _fn(x): + if pd.isna(x): + return x + return pattern.sub("", str(x)) + + return series.map(_fn) + + +def _rule_strip_special_keep_date_seps(series: pd.Series) -> pd.Series: + """Conserva alfanuméricos, separadores de fecha '/', '-' y hora ':', y espacio.""" + pattern = re.compile(r"[^A-Za-z0-9\/\-: ]", flags=re.UNICODE) + + def _fn(x): + if pd.isna(x): + return x + return pattern.sub("", str(x)) + + return series.map(_fn) + + +def _rule_strip_spaces(series: pd.Series) -> pd.Series: + """Elimina únicamente espacios (incluidos tabs y saltos de línea).""" + def _fn(x): + if pd.isna(x): + return x + return re.sub(r"\s+", "", str(x)) + + return series.map(_fn) diff --git a/services/cleaning/domain/csv_cleaning/type_inference.py b/services/cleaning/domain/csv_cleaning/type_inference.py new file mode 100644 index 0000000..d85d219 --- /dev/null +++ b/services/cleaning/domain/csv_cleaning/type_inference.py @@ -0,0 +1,143 @@ +"""Inferencia de tipo de columna y reglas aplicables.""" +from __future__ import annotations + +from typing import Optional + +import pandas as pd + +from .column_hints import ( + _column_name_suggests_code_id, + _column_name_suggests_monetary_value, + _column_name_suggests_quantity_or_number_not_bool, + _content_looks_like_doc_id_letter_prefix_column, + _raw_strings_suggest_alnum_mixed_presentation, + _raw_strings_suggest_monetary_presentation, +) +from .dates import _normalize_messy_date_to_compact, detect_date_format +from .models import ( + TYPE_BOOLEAN, + TYPE_CATEGORICAL, + TYPE_DATE, + TYPE_DECIMAL, + TYPE_EMPTY, + TYPE_INTEGER, + TYPE_MIXED, + TYPE_TEXT, +) +from .numbers import _to_numeric_lenient + +_BOOLEAN_VALUES = {"true", "false", "si", "sí", "no", "1", "0", "yes"} + + +def _detect_messy_text_date_format_guess(non_null: pd.Series) -> Optional[str]: + """Fechas con ruido: compacta a dígitos; si hay consenso, reutiliza el detector. + (Depende de `_normalize_messy_date_to_compact`, definida más abajo en el módulo.) + """ + # resolución en tiempo de ejecución + norm = non_null.map(_normalize_messy_date_to_compact) # type: ignore[name-defined] + compacts: list[str] = [] + for c in norm: + if isinstance(c, str) and len(c) in (6, 8): + compacts.append(c) + n = max(len(non_null), 1) + if not compacts or len(compacts) < max(2, int(0.25 * n)): + return None + s_compact = pd.Series(compacts, dtype=object) + return detect_date_format(s_compact) + + +def _infer_column_type( + series: pd.Series, column_name: str = "" +) -> tuple[str, Optional[str]]: + """Devuelve (tipo, formato_fecha detectado; solo si aplica a tipo fecha).""" + non_null = series.dropna().astype(str).str.strip() + non_null = non_null[non_null != ""] + if non_null.empty: + return TYPE_EMPTY, None + + cname = str(column_name or "") + + # Boolean: no aplicar 0/1/ sí-no a columnas cuyo nombre pide importe, cantid., fecha… + if not _column_name_suggests_quantity_or_number_not_bool(cname): + lowered = non_null.str.lower() + if lowered.isin(_BOOLEAN_VALUES).all() and lowered.nunique() <= 3: + return TYPE_BOOLEAN, None + + is_money = _column_name_suggests_monetary_value(cname) or _raw_strings_suggest_monetary_presentation( + non_null + ) + is_code = _column_name_suggests_code_id(cname) + is_doc_pattern = _content_looks_like_doc_id_letter_prefix_column(non_null) + + # Fecha: no forzar en importe/código, ni con datos tipo `CC`+cifra (6–8 dígitos sueltos) + # ni cuando el contenido parece moneda (p. ej. 113,400.00 → no es DDMMYYYY). + if not is_money and not is_code and not is_doc_pattern: + date_fmt = detect_date_format(non_null) + if date_fmt: + return (TYPE_DATE, date_fmt) + messy_date_fmt = _detect_messy_text_date_format_guess(non_null) + if messy_date_fmt: + return (TYPE_DATE, messy_date_fmt) + + numeric_candidate = _to_numeric_lenient(non_null, decimal_hint=None) + numeric_ratio = numeric_candidate.notna().mean() + if numeric_ratio >= 0.9: + if _raw_strings_suggest_alnum_mixed_presentation(non_null): + return (TYPE_MIXED, None) + is_all_integer = (numeric_candidate.dropna() % 1 == 0).all() + if is_all_integer and not _raw_strings_suggest_monetary_presentation(non_null): + return (TYPE_INTEGER, None) + return (TYPE_DECIMAL, None) + + unique_ratio = non_null.nunique() / len(non_null) + if non_null.nunique() <= 15 and unique_ratio < 0.3: + return TYPE_CATEGORICAL, None + if 0.1 < numeric_ratio < 0.9: + return TYPE_MIXED, None + return TYPE_TEXT, None + + +def _applicable_rules_for_type(inferred_type: str) -> list[str]: + """Identificadores de reglas a mostrar según el tipo inferido (sin redundancias).""" + base = ["strip_special_spaces", "strip_special_keep_date_seps", "strip_spaces"] + if inferred_type == TYPE_INTEGER: + return base + ["cop_mod_split"] + if inferred_type == TYPE_DECIMAL: + return base + [ + "to_integer", + "round_decimals_custom", + "doc_id_split_tipo_numero", + "doc_type_homologate", + "cop_mod_split", + ] + if inferred_type == TYPE_DATE: + return base + [ + "doc_id_split_tipo_numero", + "doc_type_homologate", + "date_to_ddmmyyyy", + "nota_paciente_autorizacion", + ] + if inferred_type == TYPE_MIXED: + return base + [ + "to_integer", + "round_decimals_custom", + "doc_id_split_tipo_numero", + "doc_type_homologate", + "date_to_ddmmyyyy", + "cop_mod_split", + "nota_paciente_autorizacion", + ] + # Texto / categórico / boolean / empty → todas las de limpieza de texto + las + # conversoras (el usuario sabrá si aplica). + return base + [ + "to_integer", + "round_decimals_custom", + "doc_id_split_tipo_numero", + "doc_type_homologate", + "date_to_ddmmyyyy", + "cop_mod_split", + "nota_paciente_autorizacion", + ] + + +applicable_rule_ids_for_type = _applicable_rules_for_type diff --git a/services/cleaning/infrastructure/database.py b/services/cleaning/infrastructure/database.py index 697580e..e32fa0e 100644 --- a/services/cleaning/infrastructure/database.py +++ b/services/cleaning/infrastructure/database.py @@ -54,10 +54,26 @@ class ExtractionBatch(Base): updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) +class Job(Base): + """Contenedor de trabajo del usuario (lectura/validación de FK).""" + __tablename__ = "jobs" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + status: Mapped[str] = mapped_column(String(32), default="pending", index=True) + title: Mapped[str | None] = mapped_column(Text, nullable=True) + job_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) + + class Task(Base): + """Módulo/resultado dentro de un Job; un job puede tener N tasks (una por servicio).""" __tablename__ = "tasks" id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) + job_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("jobs.id"), nullable=True, index=True + ) type: Mapped[str] = mapped_column(String(64), index=True) status: Mapped[str] = mapped_column(String(32), default="pending", index=True) title: Mapped[str | None] = mapped_column(Text, nullable=True) @@ -88,7 +104,13 @@ class CleaningJob(Base): __tablename__ = "cleaning_jobs" id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) - extraction_batch_id: Mapped[str] = mapped_column(ForeignKey("extraction_batches.id"), index=True) + extraction_batch_id: Mapped[str | None] = mapped_column( + ForeignKey("extraction_batches.id"), nullable=True, index=True + ) + job_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("jobs.id"), nullable=True, index=True + ) + source: Mapped[str] = mapped_column(String(32), default="extraction", index=True) status: Mapped[str] = mapped_column(String(32), default="pending", index=True) request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) log: Mapped[list[str]] = mapped_column(JSON, default=list) @@ -102,7 +124,9 @@ class CleanedRow(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) cleaning_job_id: Mapped[str] = mapped_column(ForeignKey("cleaning_jobs.id"), index=True) - extraction_batch_id: Mapped[str] = mapped_column(ForeignKey("extraction_batches.id"), index=True) + extraction_batch_id: Mapped[str | None] = mapped_column( + ForeignKey("extraction_batches.id"), nullable=True, index=True + ) row_index: Mapped[int] = mapped_column(Integer, default=0) row_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) diff --git a/services/cleaning/requirements.txt b/services/cleaning/requirements.txt index 4c07d5e..e8835ed 100644 --- a/services/cleaning/requirements.txt +++ b/services/cleaning/requirements.txt @@ -3,5 +3,6 @@ numpy==2.2.6 pandas==2.2.3 psycopg[binary]==3.2.3 pydantic==2.11.9 +python-multipart==0.0.20 SQLAlchemy==2.0.36 uvicorn[standard]==0.32.1 diff --git a/services/db/alembic/versions/0003_cleaning_csv_and_job_id.py b/services/db/alembic/versions/0003_cleaning_csv_and_job_id.py new file mode 100644 index 0000000..d592c6c --- /dev/null +++ b/services/db/alembic/versions/0003_cleaning_csv_and_job_id.py @@ -0,0 +1,90 @@ +"""cleaning_jobs: batch_id nullable, job_id, source; cleaned_rows.batch_id nullable. + +Revision ID: 0003 +Revises: 0002 +Create Date: 2026-07-10 + +""" +from __future__ import annotations + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0003" +down_revision: Union[str, None] = "0002" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _col_exists(conn: sa.engine.Connection, table: str, column: str) -> bool: + inspector = sa.inspect(conn) + return any(c["name"] == column for c in inspector.get_columns(table)) + + +def upgrade() -> None: + conn = op.get_bind() + + if _col_exists(conn, "cleaning_jobs", "extraction_batch_id"): + op.alter_column( + "cleaning_jobs", + "extraction_batch_id", + existing_type=sa.String(36), + nullable=True, + ) + + if not _col_exists(conn, "cleaning_jobs", "job_id"): + op.add_column( + "cleaning_jobs", + sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True), + ) + op.create_index("ix_cleaning_jobs_job_id", "cleaning_jobs", ["job_id"]) + + if not _col_exists(conn, "cleaning_jobs", "source"): + op.add_column( + "cleaning_jobs", + sa.Column( + "source", + sa.String(32), + nullable=False, + server_default="extraction", + ), + ) + op.create_index("ix_cleaning_jobs_source", "cleaning_jobs", ["source"]) + + if _col_exists(conn, "cleaned_rows", "extraction_batch_id"): + op.alter_column( + "cleaned_rows", + "extraction_batch_id", + existing_type=sa.String(36), + nullable=True, + ) + + +def downgrade() -> None: + conn = op.get_bind() + + if _col_exists(conn, "cleaned_rows", "extraction_batch_id"): + op.alter_column( + "cleaned_rows", + "extraction_batch_id", + existing_type=sa.String(36), + nullable=False, + ) + + if _col_exists(conn, "cleaning_jobs", "source"): + op.drop_index("ix_cleaning_jobs_source", table_name="cleaning_jobs") + op.drop_column("cleaning_jobs", "source") + + if _col_exists(conn, "cleaning_jobs", "job_id"): + op.drop_index("ix_cleaning_jobs_job_id", table_name="cleaning_jobs") + op.drop_column("cleaning_jobs", "job_id") + + if _col_exists(conn, "cleaning_jobs", "extraction_batch_id"): + op.alter_column( + "cleaning_jobs", + "extraction_batch_id", + existing_type=sa.String(36), + nullable=False, + ) diff --git a/services/desktop_app/.env.minio.example b/services/desktop_app/.env.minio.example new file mode 100644 index 0000000..66b6d97 --- /dev/null +++ b/services/desktop_app/.env.minio.example @@ -0,0 +1,11 @@ +# Credenciales locales para publicar en MinIO (NO subir al git). +# Copia este archivo a .env.minio y rellena los valores. + +MINIO_ENDPOINT=https://s3.builker.xyz +MINIO_BUCKET=apps-versions +MINIO_PREFIX=IPSExtractor +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= + +# Opcional (DD/MM/YYYY). Si se omite, usa version.json o la fecha de hoy. +# RELEASE_DATE=06/07/2026 diff --git a/services/desktop_app/README.md b/services/desktop_app/README.md index c03ce34..32b5a8f 100644 --- a/services/desktop_app/README.md +++ b/services/desktop_app/README.md @@ -106,17 +106,59 @@ npm run dev ## Build para producción ```bash -npm run build # compila el renderer (TypeScript + Vite) -npm run dist # genera el instalador Windows NSIS +# 1) Sube la versión en package.json (ej. 1.0.1) y la fecha en src/shared/version.json +# 2) Configura credenciales una vez: copia .env.minio.example → .env.minio +# 3) Genera el instalador y súbelo a MinIO automáticamente: +npm run dist ``` -El instalador se crea en `release/IPSExtractor-Desktop-1.0.0-Setup.exe`. +El instalador queda en `release/` y se publica en: + +`s3.builker.xyz/apps-versions/IPSExtractor/{version}/` + +Si solo quieres el `.exe` local **sin** subir a MinIO: + +```bash +npm run dist:local +``` ### Versión de la app | Campo | Archivo | |-------|---------| | Número (`1.0.0`) | `package.json` → `version` | -| Fecha de release | `src/shared/version.json` → `releaseDate` | +| Fecha de release | `src/shared/version.json` → `releaseDate` (ISO) | -La versión se muestra en el menú lateral y en la esquina superior derecha. Al publicar una nueva versión, actualiza ambos archivos y ejecuta `npm run dist`. +La versión se muestra en el menú lateral y en la esquina superior derecha. + +### Auto-update (MinIO) + +Al iniciar la app **instalada**, consulta: + +`https://s3.builker.xyz/apps-versions/IPSExtractor/latest.json` + +Si hay una versión mayor, pregunta al usuario, descarga el Setup.exe, verifica `sha256` y lanza el instalador NSIS. + +Layout en el bucket `apps-versions`: + +```text +IPSExtractor/ + latest.json + 1.0.1/IPSExtractor-Desktop-1.0.1-Setup.exe + 1.0.1/DATA.txt ← fecha DD/MM/YYYY, ej. 06/07/2026 +``` + +**Requisito:** el prefijo `IPSExtractor/` debe permitir `GetObject` anónimo en la API (`s3.builker.xyz`). La política se configura en la consola (`minio.builker.xyz`). + +Credenciales de publicación (solo en tu PC): + +```bash +cp .env.minio.example .env.minio +# Edita MINIO_ACCESS_KEY y MINIO_SECRET_KEY +``` + +`npm run dist` las usa automáticamente al final. También puedes republicar sin rebuild: + +```bash +npm run publish:minio +``` diff --git a/services/desktop_app/electron/autoUpdate.cjs b/services/desktop_app/electron/autoUpdate.cjs new file mode 100644 index 0000000..bfc79f6 --- /dev/null +++ b/services/desktop_app/electron/autoUpdate.cjs @@ -0,0 +1,224 @@ +/** + * Auto-update desde MinIO (apps-versions/IPSExtractor). + * Sin credenciales: lee latest.json y el Setup.exe con GetObject público. + */ +const { app } = require("electron"); +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const fsp = require("node:fs/promises"); +const http = require("node:http"); +const https = require("node:https"); +const os = require("node:os"); +const path = require("node:path"); +const { spawn } = require("node:child_process"); +const { URL } = require("node:url"); + +const { version: CURRENT_VERSION } = require("../package.json"); + +const LATEST_JSON_URL = + process.env.IPS_UPDATE_LATEST_URL || + "https://s3.builker.xyz/apps-versions/IPSExtractor/latest.json"; + +const FETCH_TIMEOUT_MS = 12_000; + +/** + * @typedef {{ + * version: string; + * releaseDate?: string; + * releaseDateDisplay?: string; + * fileName: string; + * url: string; + * sha256: string; + * size?: number; + * }} UpdateManifest + */ + +/** + * @param {string} a + * @param {string} b + * @returns {number} -1 if ab + */ +function compareSemver(a, b) { + const pa = String(a).replace(/^v/i, "").split(".").map((x) => Number.parseInt(x, 10) || 0); + const pb = String(b).replace(/^v/i, "").split(".").map((x) => Number.parseInt(x, 10) || 0); + const len = Math.max(pa.length, pb.length); + for (let i = 0; i < len; i++) { + const x = pa[i] ?? 0; + const y = pb[i] ?? 0; + if (x < y) return -1; + if (x > y) return 1; + } + return 0; +} + +/** + * @param {string} urlStr + * @param {{ onProgress?: (received: number, total: number) => void, destPath?: string }} [opts] + * @returns {Promise<{ buffer?: Buffer, destPath?: string, headers: import('http').IncomingHttpHeaders }>} + */ +function download(urlStr, opts = {}) { + return new Promise((resolve, reject) => { + const url = new URL(urlStr); + const lib = url.protocol === "http:" ? http : https; + const req = lib.get( + url, + { + timeout: FETCH_TIMEOUT_MS, + headers: { "User-Agent": `IPSExtractor-Desktop/${CURRENT_VERSION}` }, + }, + (res) => { + if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + res.resume(); + download(res.headers.location, opts).then(resolve, reject); + return; + } + if (res.statusCode !== 200) { + res.resume(); + reject(new Error(`HTTP ${res.statusCode} al descargar ${urlStr}`)); + return; + } + const total = Number(res.headers["content-length"] || 0); + let received = 0; + + if (opts.destPath) { + const out = fs.createWriteStream(opts.destPath); + res.on("data", (chunk) => { + received += chunk.length; + opts.onProgress?.(received, total); + }); + res.pipe(out); + out.on("finish", () => resolve({ destPath: opts.destPath, headers: res.headers })); + out.on("error", reject); + res.on("error", reject); + return; + } + + const chunks = []; + res.on("data", (chunk) => { + chunks.push(chunk); + received += chunk.length; + opts.onProgress?.(received, total); + }); + res.on("end", () => resolve({ buffer: Buffer.concat(chunks), headers: res.headers })); + res.on("error", reject); + }, + ); + req.on("timeout", () => { + req.destroy(); + reject(new Error("Timeout al contactar el servidor de versiones")); + }); + req.on("error", reject); + }); +} + +/** + * @returns {Promise<{ updateAvailable: boolean, currentVersion: string, manifest?: UpdateManifest, error?: string }>} + */ +async function checkForUpdates() { + try { + const { buffer } = await download(LATEST_JSON_URL); + if (!buffer) { + return { updateAvailable: false, currentVersion: CURRENT_VERSION, error: "Respuesta vacía" }; + } + /** @type {UpdateManifest} */ + const manifest = JSON.parse(buffer.toString("utf8")); + if (!manifest?.version || !manifest?.url || !manifest?.fileName) { + return { + updateAvailable: false, + currentVersion: CURRENT_VERSION, + error: "Manifiesto inválido", + }; + } + + // Si falta la fecha de DATA.txt en el manifiesto, intentar leerla junto al exe + if (!manifest.releaseDateDisplay) { + try { + const dataUrl = manifest.url.replace(/[^/]+$/, "DATA.txt"); + const { buffer: dataBuf } = await download(dataUrl); + if (dataBuf) { + const display = dataBuf.toString("utf8").trim().split(/\r?\n/)[0]?.trim(); + if (display) { + manifest.releaseDateDisplay = display; + const iso = display.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/); + if (iso && !manifest.releaseDate) { + manifest.releaseDate = `${iso[3]}-${iso[2].padStart(2, "0")}-${iso[1].padStart(2, "0")}`; + } + } + } + } catch { + /* opcional */ + } + } + + const newer = compareSemver(CURRENT_VERSION, manifest.version) < 0; + return { + updateAvailable: newer, + currentVersion: CURRENT_VERSION, + manifest, + }; + } catch (err) { + return { + updateAvailable: false, + currentVersion: CURRENT_VERSION, + error: err instanceof Error ? err.message : String(err), + }; + } +} + +/** + * @param {UpdateManifest} manifest + * @param {(payload: { phase: string, received?: number, total?: number, percent?: number }) => void} [onProgress] + */ +async function downloadAndInstall(manifest, onProgress) { + if (!manifest?.url || !manifest?.fileName) { + throw new Error("Manifiesto incompleto"); + } + const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ipsextractor-update-")); + const destPath = path.join(tmpDir, path.basename(manifest.fileName)); + + onProgress?.({ phase: "downloading", received: 0, total: manifest.size || 0, percent: 0 }); + + await download(manifest.url, { + destPath, + onProgress: (received, total) => { + const t = total || manifest.size || 0; + const percent = t > 0 ? Math.min(100, Math.round((received / t) * 100)) : 0; + onProgress?.({ phase: "downloading", received, total: t, percent }); + }, + }); + + onProgress?.({ phase: "verifying" }); + if (manifest.sha256) { + const hash = crypto.createHash("sha256"); + const fileBuf = await fsp.readFile(destPath); + hash.update(fileBuf); + const digest = hash.digest("hex").toLowerCase(); + if (digest !== String(manifest.sha256).toLowerCase()) { + await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => undefined); + throw new Error("El instalador descargado no coincide con el hash esperado (sha256)."); + } + } + + onProgress?.({ phase: "installing" }); + // NSIS silent install; la app debe cerrarse para que el instalador reemplace archivos. + const child = spawn(destPath, ["/S"], { + detached: true, + stdio: "ignore", + windowsHide: false, + }); + child.unref(); + + setTimeout(() => { + app.quit(); + }, 400); + + return { ok: true, destPath }; +} + +module.exports = { + CURRENT_VERSION, + LATEST_JSON_URL, + compareSemver, + checkForUpdates, + downloadAndInstall, +}; diff --git a/services/desktop_app/electron/icon.png b/services/desktop_app/electron/icon.png new file mode 100644 index 0000000..54863e5 Binary files /dev/null and b/services/desktop_app/electron/icon.png differ diff --git a/services/desktop_app/electron/main.cjs b/services/desktop_app/electron/main.cjs index 5f91f21..0e870d6 100644 --- a/services/desktop_app/electron/main.cjs +++ b/services/desktop_app/electron/main.cjs @@ -2,10 +2,15 @@ const { app, BrowserWindow, dialog, ipcMain } = require("electron"); const fs = require("node:fs/promises"); const path = require("node:path"); const { indexFolder } = require("./folderIndex.cjs"); +const autoUpdate = require("./autoUpdate.cjs"); const { version: appVersion } = require("../package.json"); const DOCUMENT_EXTENSIONS = new Set([".pdf", ".tif", ".tiff", ".png", ".jpg", ".jpeg"]); +const APP_ICON = path.join(__dirname, "icon.png"); + +/** @type {BrowserWindow | null} */ +let mainWindow = null; function createWindow() { const win = new BrowserWindow({ @@ -14,6 +19,7 @@ function createWindow() { minWidth: 1080, minHeight: 720, title: `IPSExtractor Desktop v${appVersion}`, + icon: APP_ICON, backgroundColor: "#f6f7f9", webPreferences: { preload: path.join(__dirname, "preload.cjs"), @@ -21,6 +27,10 @@ function createWindow() { nodeIntegration: false, }, }); + mainWindow = win; + win.on("closed", () => { + if (mainWindow === win) mainWindow = null; + }); if (!app.isPackaged) { win.loadURL("http://127.0.0.1:5173"); @@ -30,7 +40,34 @@ function createWindow() { } } +function sendUpdateProgress(payload) { + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.webContents.send("updater:progress", payload); + } +} + +ipcMain.handle("updater:check", async () => { + if (!app.isPackaged) { + return { + updateAvailable: false, + currentVersion: appVersion, + error: "Auto-update desactivado en desarrollo", + }; + } + return autoUpdate.checkForUpdates(); +}); + +ipcMain.handle("updater:install", async (_event, manifest) => { + if (!app.isPackaged) { + throw new Error("Auto-update solo disponible en la app instalada"); + } + return autoUpdate.downloadAndInstall(manifest, sendUpdateProgress); +}); + app.whenReady().then(() => { + if (process.platform === "win32") { + app.setAppUserModelId("com.ipsextractor.desktop"); + } createWindow(); app.on("activate", () => { if (BrowserWindow.getAllWindows().length === 0) createWindow(); diff --git a/services/desktop_app/electron/preload.cjs b/services/desktop_app/electron/preload.cjs index 747e7a4..8f36881 100644 --- a/services/desktop_app/electron/preload.cjs +++ b/services/desktop_app/electron/preload.cjs @@ -9,4 +9,11 @@ contextBridge.exposeInMainWorld("desktopApi", { indexFolder: (rootPath, structureMode, docTypes, keywords) => ipcRenderer.invoke("fs:index-folder", rootPath, structureMode, docTypes, keywords), readFile: (filePath) => ipcRenderer.invoke("fs:read-file", filePath), + checkForUpdates: () => ipcRenderer.invoke("updater:check"), + installUpdate: (manifest) => ipcRenderer.invoke("updater:install", manifest), + onUpdateProgress: (callback) => { + const handler = (_event, payload) => callback(payload); + ipcRenderer.on("updater:progress", handler); + return () => ipcRenderer.removeListener("updater:progress", handler); + }, }); diff --git a/services/desktop_app/index.html b/services/desktop_app/index.html index 16c574b..c8bb5c3 100644 --- a/services/desktop_app/index.html +++ b/services/desktop_app/index.html @@ -3,6 +3,7 @@ + IPSExtractor Desktop diff --git a/services/desktop_app/package-lock.json b/services/desktop_app/package-lock.json index 9aed697..b992530 100644 --- a/services/desktop_app/package-lock.json +++ b/services/desktop_app/package-lock.json @@ -1,13 +1,14 @@ { "name": "ipsextractor-desktop", - "version": "0.1.0", + "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ipsextractor-desktop", - "version": "0.1.0", + "version": "1.0.0", "dependencies": { + "@aws-sdk/client-s3": "^3.1085.0", "lucide-react": "^0.468.0", "react": "^18.3.1", "react-dom": "^18.3.1" @@ -58,6 +59,314 @@ "dev": true, "license": "ISC" }, + "node_modules/@aws-sdk/checksums": { + "version": "3.1000.16", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.16.tgz", + "integrity": "sha512-EKnvkXSmz3IpA99tCNuI+dLFXyZyClSm8zns9sB/elvkU+MTuomAs6toJMPMBf98/fICG/urXDkzGz0/c3yyAQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.1085.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1085.0.tgz", + "integrity": "sha512-O0xe8sR50AYkwxlvRRsV0qytEO2dtXQTQ1CF3YBBdE5xtVkbu27H0vGa1mjQi1/+fbYM80AWEIPai5jZmXyubw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1000.16", + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/credential-provider-node": "^3.972.66", + "@aws-sdk/middleware-sdk-s3": "^3.972.62", + "@aws-sdk/signature-v4-multi-region": "^3.996.39", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/fetch-http-handler": "^5.6.4", + "@smithy/node-http-handler": "^4.9.4", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.975.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.975.1.tgz", + "integrity": "sha512-8qh/6EYb7hl/ZwVfQufhbMEZs1gQIc7GbdrIf4eprQJ7cv042+74nE6l3YDfyWNzb9iPXb8fRyYSHkNIk5eE6Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.0", + "@aws-sdk/xml-builder": "^3.972.34", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.29.2", + "@smithy/signature-v4": "^5.6.3", + "@smithy/types": "^4.16.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.57", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.57.tgz", + "integrity": "sha512-1RfJaF7SW1TOnvNGU7kaYjwUf5H3sfm+synGH1bHhRlqcnxCt3szebH3dmKEyY4tuGcbQ6ffzUT89cRitBV8OQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.59.tgz", + "integrity": "sha512-sRCkpTiFnCdQvuaRVjQ6SVoHu6i7RUpurVo1c4F81HWhPvUJ7Wdp5MNtSdX1O29CNXc8em3O5m52hCjVtAD9SA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/fetch-http-handler": "^5.6.4", + "@smithy/node-http-handler": "^4.9.4", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.1.tgz", + "integrity": "sha512-6d8H6ZAh3ZPKZ6fe1nG2OWeZEZPtt9ravoD1dezPdPtsSkJRoxGAnFSHwKT3E/Te6fHE30zRzjV6TD12rvF6yQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/credential-provider-env": "^3.972.57", + "@aws-sdk/credential-provider-http": "^3.972.59", + "@aws-sdk/credential-provider-login": "^3.972.63", + "@aws-sdk/credential-provider-process": "^3.972.57", + "@aws-sdk/credential-provider-sso": "^3.973.1", + "@aws-sdk/credential-provider-web-identity": "^3.972.63", + "@aws-sdk/nested-clients": "^3.997.31", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/credential-provider-imds": "^4.4.7", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.63", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.63.tgz", + "integrity": "sha512-GREWRrMj0XnNKMaVa/Mauoaui26qBEHu71WWqXbwZOu/jFQOnPZjTf7u0KtGKC8VGa6VUs9kDWGgocrKNLS9vw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/nested-clients": "^3.997.31", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.66", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.66.tgz", + "integrity": "sha512-f+qjRXZpz7sgzbc4QB+6nLKfyKFgRRXzWdXbsKPv/VhVRyHsDyq4yBWC/B75BAJpFIcUeI2XR/3gdWJ677zB4A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.57", + "@aws-sdk/credential-provider-http": "^3.972.59", + "@aws-sdk/credential-provider-ini": "^3.973.1", + "@aws-sdk/credential-provider-process": "^3.972.57", + "@aws-sdk/credential-provider-sso": "^3.973.1", + "@aws-sdk/credential-provider-web-identity": "^3.972.63", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/credential-provider-imds": "^4.4.7", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.57", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.57.tgz", + "integrity": "sha512-TiVQhuU0pbhIZAUZacbPHMyzrIdiH+lnx+PMY/Pu/b93dJrq3wdZwzUJ0TPpvNxaqbHsxJvQZW3/h/beLiKq7Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.1.tgz", + "integrity": "sha512-3foTZUJ4821Ij60X7K3NJroygiZLnbBmarN+T//O2cjkISan90zElN3NBmgSlDrTQ7Gs6z/yO8V7h60QNcDZHQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/nested-clients": "^3.997.31", + "@aws-sdk/token-providers": "3.1083.0", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.63", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.63.tgz", + "integrity": "sha512-8qZLFhM69eKcS37m459ctPR05Qimycm/74OPVioe6wNZabMT54GYhwBju0+J656RkMasNSawWQu+c8CmBe3TUQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/nested-clients": "^3.997.31", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.62", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.62.tgz", + "integrity": "sha512-k8JJwYXVYlOOjWnPZDThQS1xDFJgi5Dokt73qFlDtrZAbdcint5aIdjB9XgJAAQVP5OoqcefQmh1FYXiPpvsvw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/signature-v4-multi-region": "^3.996.39", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.31.tgz", + "integrity": "sha512-BDHTpwcsZHEBNEJzOg/B1BkFYJxAXY50dau/NyVWs3d51F0WgIUGSWZot/Os+N3KpDhXeaXnz37mWffAvduREw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/signature-v4-multi-region": "^3.996.39", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/fetch-http-handler": "^5.6.4", + "@smithy/node-http-handler": "^4.9.4", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.39", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.39.tgz", + "integrity": "sha512-8+srXqYIF8KYMLC4FxMLEM5Ek7kUNibJu1R4m8/fUhhNYIZZz26oGtKkCr8I/HiG2fFQxBvaGgQZT4/mqRCSnA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.0", + "@smithy/signature-v4": "^5.6.3", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1083.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1083.0.tgz", + "integrity": "sha512-s0woKnxuHrExLc5L2ArIH5BMkbonHPtt+5hSBM8oknp9M6QTuUmmAmJ2E0EdzCGONrO+8+ADPqvv6UX0nNcc7A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/nested-clients": "^3.997.31", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.0.tgz", + "integrity": "sha512-QIBrw90CDm4O0UaIIzkU6DrFdeJzEb2Va5EPEVpyldj6sHJxB6cshhStJuhZxk3wR3PmjJlYsjPmY1kNb+KGBg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.34", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.34.tgz", + "integrity": "sha512-wHhWL1y7sN3enBA8POrPpQM5jCcmu2ozyhbRei4c8OjVcEaEs6yLucLa/pla457ggS/ysuy7bosagz3HaJkZXA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -2008,6 +2317,87 @@ "url": "https://github.com/sindresorhus/is?sponsor=1" } }, + "node_modules/@smithy/core": { + "version": "3.29.3", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.3.tgz", + "integrity": "sha512-L+Ys6ecjk5vwPMAKHBpPKlJ3DkqwNcnfEISXBZIsVvWG/XKXfsAP8mwIYlTeLcd2ElHdesPI8OuOmJSFAPhm6A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.4.8", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.8.tgz", + "integrity": "sha512-q9J7JTiXrAhB8sDp4px97uEPT7CwKH61Co78grdNQvU8QZAdiuaSRhP0tUVf2ogy36RZTrlMU1rBmDEH+cnkiA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.3", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.6.5", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.5.tgz", + "integrity": "sha512-SuqeisTyPoiIPtIYru/sGxGyXzmZ+8nnFOhC+qRPglt06Ebd1yH//CDltZB2J/3WBNVhwfUaZ0EtHB3cm2X32g==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.3", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.9.5", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.5.tgz", + "integrity": "sha512-bNqdxTQTxmLbomSmlkZFz8L6B/feQ2HHzw4L2zY7Ecp2XffYAZq2uzdWDdxJHJFbEvqd+SRuluJso0P8+xPdbw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.3", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.6.4", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.4.tgz", + "integrity": "sha512-B89bpf2t/y/wia6LZ+4JfHXYQT9PnVftsH05rgJKKIStS7r/4XSs9HOjtPoLtgcA6HCW9jVqX5DBbq7E0PAkiQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.3", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.16.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz", + "integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -2957,6 +3347,12 @@ "license": "MIT", "optional": true }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "5.0.7", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", @@ -7751,7 +8147,6 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, "license": "0BSD" }, "node_modules/type-fest": { diff --git a/services/desktop_app/package.json b/services/desktop_app/package.json index 148a935..f8a0d7b 100644 --- a/services/desktop_app/package.json +++ b/services/desktop_app/package.json @@ -1,6 +1,6 @@ { "name": "ipsextractor-desktop", - "version": "1.0.0", + "version": "1.0.1", "private": true, "description": "Desktop extraction client for IPSExtractor microservices.", "main": "electron/main.cjs", @@ -10,22 +10,25 @@ "preview": "vite preview", "test": "vitest run", "test:watch": "vitest", - "dist": "npm run build && electron-builder --win", - "dist:win": "npm run dist" + "dist": "npm run build && electron-builder --win && npm run publish:minio", + "dist:win": "npm run dist", + "dist:local": "npm run build && electron-builder --win", + "publish:minio": "node scripts/publish-version.mjs" }, "dependencies": { + "@aws-sdk/client-s3": "^3.1085.0", "lucide-react": "^0.468.0", "react": "^18.3.1", "react-dom": "^18.3.1" }, "devDependencies": { - "@vitejs/plugin-react": "^4.3.4", "@testing-library/jest-dom": "^6.6.3", "@testing-library/react": "^16.1.0", "@testing-library/user-event": "^14.5.2", "@types/node": "^22.10.2", "@types/react": "^18.3.17", "@types/react-dom": "^18.3.5", + "@vitejs/plugin-react": "^4.3.4", "concurrently": "^9.1.0", "electron": "^33.2.1", "electron-builder": "^25.1.8", @@ -40,7 +43,8 @@ "productName": "IPSExtractor Desktop", "artifactName": "IPSExtractor-Desktop-${version}-Setup.${ext}", "directories": { - "output": "release" + "output": "release", + "buildResources": "resources" }, "files": [ "dist/**/*", @@ -48,9 +52,11 @@ "package.json" ], "mac": { - "category": "public.app-category.productivity" + "category": "public.app-category.productivity", + "icon": "resources/icon.png" }, "win": { + "icon": "resources/icon.ico", "target": [ { "target": "nsis", diff --git a/services/desktop_app/public/app-icon.png b/services/desktop_app/public/app-icon.png new file mode 100644 index 0000000..9b3b1f9 Binary files /dev/null and b/services/desktop_app/public/app-icon.png differ diff --git a/services/desktop_app/resources/icon-source.png b/services/desktop_app/resources/icon-source.png new file mode 100644 index 0000000..326f542 Binary files /dev/null and b/services/desktop_app/resources/icon-source.png differ diff --git a/services/desktop_app/resources/icon.ico b/services/desktop_app/resources/icon.ico new file mode 100644 index 0000000..249c451 Binary files /dev/null and b/services/desktop_app/resources/icon.ico differ diff --git a/services/desktop_app/resources/icon.png b/services/desktop_app/resources/icon.png new file mode 100644 index 0000000..9b3b1f9 Binary files /dev/null and b/services/desktop_app/resources/icon.png differ diff --git a/services/desktop_app/scripts/publish-version.mjs b/services/desktop_app/scripts/publish-version.mjs new file mode 100644 index 0000000..8436a49 --- /dev/null +++ b/services/desktop_app/scripts/publish-version.mjs @@ -0,0 +1,188 @@ +/** + * Publica el instalador NSIS en MinIO y actualiza latest.json + DATA.txt. + * + * Uso: + * set MINIO_ACCESS_KEY=... + * set MINIO_SECRET_KEY=... + * npm run publish:minio + * + * Opcional: + * MINIO_ENDPOINT=https://s3.builker.xyz + * MINIO_BUCKET=apps-versions + * MINIO_PREFIX=IPSExtractor + * RELEASE_DATE=06/07/2026 (formato DATA.txt DD/MM/YYYY; default: hoy o version.json) + */ +import { createHash } from "node:crypto"; +import { createReadStream, existsSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const root = join(__dirname, ".."); + +/** Carga KEY=VALUE desde un archivo local (sin dependencia dotenv). */ +function loadEnvFile(filePath) { + if (!existsSync(filePath)) return; + const text = readFileSync(filePath, "utf8"); + for (const line of text.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const eq = trimmed.indexOf("="); + if (eq <= 0) continue; + const key = trimmed.slice(0, eq).trim(); + let value = trimmed.slice(eq + 1).trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + if (!(key in process.env) || process.env[key] === "") { + process.env[key] = value; + } + } +} + +loadEnvFile(join(root, ".env.minio")); +loadEnvFile(join(root, ".env")); + +const pkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); +const version = pkg.version; +const endpoint = (process.env.MINIO_ENDPOINT || "https://s3.builker.xyz").replace(/\/$/, ""); +const bucket = process.env.MINIO_BUCKET || "apps-versions"; +const prefix = (process.env.MINIO_PREFIX || "IPSExtractor").replace(/^\/|\/$/g, ""); +const accessKey = process.env.MINIO_ACCESS_KEY || process.env.AWS_ACCESS_KEY_ID; +const secretKey = process.env.MINIO_SECRET_KEY || process.env.AWS_SECRET_ACCESS_KEY; + +if (!accessKey || !secretKey) { + console.error( + "Faltan credenciales. Define MINIO_ACCESS_KEY y MINIO_SECRET_KEY (o AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY).", + ); + process.exit(1); +} + +const setupName = `IPSExtractor-Desktop-${version}-Setup.exe`; +const setupPath = join(root, "release", setupName); +if (!existsSync(setupPath)) { + console.error(`No se encontró ${setupPath}. Ejecuta antes: npm run dist`); + process.exit(1); +} + +function todayDisplay() { + const d = new Date(); + const dd = String(d.getDate()).padStart(2, "0"); + const mm = String(d.getMonth() + 1).padStart(2, "0"); + const yyyy = d.getFullYear(); + return `${dd}/${mm}/${yyyy}`; +} + +function isoFromDisplay(display) { + // DD/MM/YYYY → YYYY-MM-DD + const m = String(display).trim().match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/); + if (!m) return null; + const [, dd, mm, yyyy] = m; + return `${yyyy}-${mm.padStart(2, "0")}-${dd.padStart(2, "0")}`; +} + +function displayFromIso(iso) { + const m = String(iso).trim().match(/^(\d{4})-(\d{2})-(\d{2})$/); + if (!m) return null; + const [, yyyy, mm, dd] = m; + return `${dd}/${mm}/${yyyy}`; +} + +function resolveReleaseDateDisplay() { + if (process.env.RELEASE_DATE) return String(process.env.RELEASE_DATE).trim(); + try { + const meta = JSON.parse(readFileSync(join(root, "src/shared/version.json"), "utf8")); + if (meta.releaseDate) { + return displayFromIso(meta.releaseDate) || todayDisplay(); + } + } catch { + /* ignore */ + } + return todayDisplay(); +} + +async function sha256File(filePath) { + return new Promise((resolve, reject) => { + const hash = createHash("sha256"); + const stream = createReadStream(filePath); + stream.on("data", (chunk) => hash.update(chunk)); + stream.on("error", reject); + stream.on("end", () => resolve(hash.digest("hex"))); + }); +} + +async function upload(client, key, body, contentType) { + await client.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + ContentType: contentType, + }), + ); + console.log(` ↑ s3://${bucket}/${key}`); +} + +async function main() { + const releaseDateDisplay = resolveReleaseDateDisplay(); + const releaseDateIso = isoFromDisplay(releaseDateDisplay) || new Date().toISOString().slice(0, 10); + const sha256 = await sha256File(setupPath); + const { size } = statSync(setupPath); + + const publicBase = `${endpoint}/${bucket}/${prefix}`; + const objectKeyExe = `${prefix}/${version}/${setupName}`; + const objectKeyData = `${prefix}/${version}/DATA.txt`; + const objectKeyLatest = `${prefix}/latest.json`; + const publicUrl = `${publicBase}/${version}/${setupName}`; + + const latest = { + version, + releaseDate: releaseDateIso, + releaseDateDisplay, + fileName: setupName, + url: publicUrl, + sha256, + size, + }; + + // Actualiza version.json local para alinear la UI con la fecha publicada + writeFileSync( + join(root, "src/shared/version.json"), + `${JSON.stringify({ releaseDate: releaseDateIso }, null, 2)}\n`, + "utf8", + ); + + const client = new S3Client({ + region: process.env.MINIO_REGION || "us-east-1", + endpoint, + forcePathStyle: true, + credentials: { + accessKeyId: accessKey, + secretAccessKey: secretKey, + }, + }); + + console.log(`Publicando v${version} → ${endpoint}/${bucket}/${prefix}/`); + await upload(client, objectKeyExe, createReadStream(setupPath), "application/octet-stream"); + await upload(client, objectKeyData, `${releaseDateDisplay}\n`, "text/plain; charset=utf-8"); + await upload( + client, + objectKeyLatest, + Buffer.from(`${JSON.stringify(latest, null, 2)}\n`, "utf8"), + "application/json", + ); + + console.log("\nListo."); + console.log(` latest.json: ${publicBase}/latest.json`); + console.log(` DATA.txt: ${publicBase}/${version}/DATA.txt → ${releaseDateDisplay}`); + console.log(` Setup: ${publicUrl}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/services/desktop_app/src/renderer/App.test.tsx b/services/desktop_app/src/renderer/App.test.tsx index cecd6f7..ec19b21 100644 --- a/services/desktop_app/src/renderer/App.test.tsx +++ b/services/desktop_app/src/renderer/App.test.tsx @@ -3,164 +3,49 @@ import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; import App from "./App"; -async function advanceToDiscover(user: ReturnType) { - await user.click(screen.getByRole("button", { name: /explorar/i })); - await user.click(screen.getByRole("button", { name: /siguiente/i })); -} - -async function discoverFields(user: ReturnType) { - await advanceToDiscover(user); - await user.click(screen.getByRole("button", { name: /descubrir campos/i })); -} - describe("App", () => { - it("renders the extraction wizard", () => { + it("renders home shell with jobs and configuration entry", () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue({ + ok: true, + text: async () => "[]", + } as Response); render(); expect(screen.getByText("IPSExtractor")).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /^extraccion$/i })).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /tareas/i })).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /configuracion/i })).toBeInTheDocument(); - expect(screen.getByText("Origen de los documentos")).toBeInTheDocument(); - expect(screen.queryByText("IPS_DOCUMENT_ROOT_LOCAL")).not.toBeInTheDocument(); - expect(screen.queryByText("IPS_DOCUMENT_ROOT_CONTAINER")).not.toBeInTheDocument(); - expect(screen.queryByText("Ruta enviada a Docker/API")).not.toBeInTheDocument(); - expect(screen.getByRole("button", { name: /siguiente/i })).toBeDisabled(); + expect(screen.getByRole("button", { name: /^trabajos$/i })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /configuraci[oó]n/i })).toBeInTheDocument(); }); - it("shows API URL settings in the configuration page", async () => { + it("opens configuration as a submenu with connection settings", async () => { const user = userEvent.setup(); + vi.spyOn(globalThis, "fetch").mockResolvedValue({ + ok: true, + text: async () => "[]", + } as Response); render(); - await user.click(screen.getByRole("button", { name: /configuracion/i })); - expect(screen.getByText("Conexion API")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: /configuraci[oó]n/i })); + expect( + screen.getByRole("navigation", { name: /secciones de configuraci[oó]n/i }), + ).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /^conexi[oó]n$/i })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /^limpieza$/i })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /^backend$/i })).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: /^conexi[oó]n$/i })).toBeInTheDocument(); const input = screen.getByLabelText("URL del API Gateway"); await user.clear(input); await user.type(input, "http://localhost:9000"); expect(input).toHaveValue("http://localhost:9000"); }); - it("adds and removes document types", async () => { + it("shows cleaning COP/MOD settings section", async () => { const user = userEvent.setup(); - render(); - await user.click(screen.getByRole("button", { name: /agregar tipo/i })); - expect(screen.getByLabelText("Nombre tipo 2")).toBeInTheDocument(); - const removeButtons = screen.getAllByLabelText("Eliminar tipo"); - await user.click(removeButtons[1]); - expect(screen.queryByLabelText("Nombre tipo 2")).not.toBeInTheDocument(); - }); - - it("does not show keywords in general folder mode", async () => { - const user = userEvent.setup(); - render(); - await user.click(screen.getByRole("button", { name: /carpeta general/i })); - expect(screen.queryByLabelText(/keywords/i)).not.toBeInTheDocument(); - expect(screen.queryByText(/^Keywords$/)).not.toBeInTheDocument(); - expect(screen.getByText(/se procesaran todos los pdf y tiff/i)).toBeInTheDocument(); - }); - - it("shows schema fields from a mocked API response", async () => { - const user = userEvent.setup(); - vi.spyOn(window.desktopApi!, "selectDirectory").mockResolvedValue("C:\\docs\\lote"); vi.spyOn(globalThis, "fetch").mockResolvedValue({ ok: true, - text: async () => JSON.stringify({ schema_by_type: { Factura: { numero: "str" } }, sample_by_type: { Factura: { numero: "A1" } } }), + text: async () => "[]", } as Response); render(); - await discoverFields(user); - await user.click(screen.getByRole("button", { name: /siguiente/i })); - expect(await screen.findByRole("checkbox", { name: "Seleccionar numero" })).toBeInTheDocument(); - }); - - it("blocks advancing while no fields are selected", async () => { - const user = userEvent.setup(); - vi.spyOn(window.desktopApi!, "selectDirectory").mockResolvedValue("C:\\docs\\lote"); - vi.spyOn(globalThis, "fetch").mockResolvedValue({ - ok: true, - text: async () => JSON.stringify({ schema_by_type: { Factura: { numero: "str" } }, sample_by_type: { Factura: { numero: "A1" } } }), - } as Response); - render(); - await discoverFields(user); - await user.click(screen.getByRole("button", { name: /siguiente/i })); - expect(screen.getByRole("button", { name: /siguiente/i })).toBeDisabled(); - }); - - it("creates an extraction task with mocked API responses", async () => { - const user = userEvent.setup(); - vi.spyOn(window.desktopApi!, "selectDirectory").mockResolvedValue("C:\\docs\\lote"); - vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { - const url = String(input); - if (url.includes("/extraction/schemas/upload")) { - return { - ok: true, - text: async () => JSON.stringify({ schema_by_type: { Factura: { numero: "str" } }, sample_by_type: { Factura: { numero: "A1" } } }), - } as Response; - } - if (url.includes("/tasks/extraction")) { - return { - ok: true, - text: async () => JSON.stringify({ - id: "task-1", - type: "extraction", - status: "running", - title: "Extraccion desktop", - request: { source: "desktop" }, - result: { extraction_batch_id: "batch-1", processed: 0, total: 1 }, - error: null, - owner_id: null, - created_at: "now", - updated_at: "now", - started_at: "now", - finished_at: null, - }), - } as Response; - } - if (url.includes("/tasks/task-1/ingest")) { - return { - ok: true, - text: async () => JSON.stringify({ processed: 1, total: 1, task: { id: "task-1", status: "completed", result: { extraction_batch_id: "batch-1" } } }), - } as Response; - } - if (url.includes("/tasks/task-1") && !url.includes("/rows") && !url.includes("/ingest")) { - return { - ok: true, - text: async () => JSON.stringify({ - id: "task-1", - type: "extraction", - status: "completed", - title: "Extraccion desktop", - request: { source: "desktop" }, - result: { extraction_batch_id: "batch-1", processed: 1, total: 1 }, - error: null, - owner_id: null, - created_at: "now", - updated_at: "now", - started_at: "now", - finished_at: "now", - }), - } as Response; - } - if (url.includes("/tasks?")) { - return { - ok: true, - text: async () => JSON.stringify([{ id: "task-1", type: "extraction", status: "completed", title: "Extraccion desktop", request: {}, result: { extraction_batch_id: "batch-1" }, error: null, owner_id: null, created_at: "now", updated_at: "now", started_at: "now", finished_at: "now" }]), - } as Response; - } - if (url.includes("/tasks/task-1/rows")) { - return { - ok: true, - text: async () => JSON.stringify({ task_id: "task-1", batch_id: "batch-1", rows: [{ radicado: "R1", numero: "A1" }], limit: 500, offset: 0 }), - } as Response; - } - return { ok: true, text: async () => "{}" } as Response; - }); - - render(); - await discoverFields(user); - await user.click(screen.getByRole("button", { name: /siguiente/i })); - await user.click(await screen.findByRole("checkbox", { name: "Seleccionar numero" })); - await user.click(screen.getByRole("button", { name: /siguiente/i })); - await user.click(screen.getByRole("button", { name: /ejecutar extraccion/i })); - expect(await screen.findByText("Tareas y resultados")).toBeInTheDocument(); - expect((await screen.findAllByText("completed")).length).toBeGreaterThan(0); - expect(await screen.findByText("A1")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: /configuraci[oó]n/i })); + await user.click(screen.getByRole("button", { name: /^limpieza$/i })); + expect(screen.getByRole("heading", { name: /limpieza.*cop/i })).toBeInTheDocument(); + expect(screen.getByLabelText("2024")).toBeInTheDocument(); }); }); diff --git a/services/desktop_app/src/renderer/App.tsx b/services/desktop_app/src/renderer/App.tsx index 14fd387..5bf3fd4 100644 --- a/services/desktop_app/src/renderer/App.tsx +++ b/services/desktop_app/src/renderer/App.tsx @@ -1,17 +1,14 @@ import { BarChart2, Briefcase, - CheckCircle2, Cpu, - Database, Hash, - Loader2, - Server, + Link2, Settings, - XCircle, + Sparkles, } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; -import type { ExtractionFormState, Job } from "../shared/types"; +import type { Job } from "../shared/types"; import { APP_RELEASE_DATE, APP_VERSION, @@ -19,14 +16,18 @@ import { formatVersionLabel, } from "../shared/appVersion"; import { ApiClient } from "./lib/apiClient"; -import { DEFAULT_API_BASE_URL, loadApiBaseUrl, migrateApiBaseUrlInStorage } from "./lib/apiDefaults"; +import { loadApiBaseUrl, migrateApiBaseUrlInStorage } from "./lib/apiDefaults"; import { HomeView } from "./views/HomeView"; import { JobView } from "./views/JobView"; +import { SettingsView, type SettingsSection } from "./views/SettingsView"; import type { JobSection } from "./types/jobSection"; +import appIcon from "./assets/app-icon.png"; +import { UpdatePrompt, useAutoUpdateCheck } from "./components/UpdatePrompt"; // Registrar módulos al cargar la app import "./modules/counting/CountingModule"; import "./modules/extraction/ExtractionModule"; +import "./modules/cleaning/CleaningModule"; type Status = "idle" | "ok" | "error" | "busy"; type View = "home" | "job" | "settings"; @@ -34,15 +35,19 @@ export type { JobSection } from "./types/jobSection"; function App() { const [view, setView] = useState("home"); + const [settingsReturnView, setSettingsReturnView] = useState<"home" | "job">("home"); + const [settingsSection, setSettingsSection] = useState("connection"); const [activeJob, setActiveJob] = useState(null); const [jobSection, setJobSection] = useState("counting"); const [jobSectionContext, setJobSectionContext] = useState>({}); const [apiBaseUrl, setApiBaseUrl] = useState(loadApiBaseUrl); const [connectionStatus, setConnectionStatus] = useState("idle"); - const [connectionMessage, setConnectionMessage] = useState("Sin verificar"); + const [connectionMessage, setConnectionMessage] = useState("Pulsa «Verificar conexión» para comprobar el gateway."); + const [healthJson, setHealthJson] = useState(null); const [configJson, setConfigJson] = useState(null); const [modelsJson, setModelsJson] = useState(null); const [busy, setBusy] = useState(false); + const update = useAutoUpdateCheck(); const client = useMemo(() => new ApiClient(apiBaseUrl), [apiBaseUrl]); @@ -54,21 +59,27 @@ function App() { } }, []); // eslint-disable-line react-hooks/exhaustive-deps - useEffect(() => { localStorage.setItem("apiBaseUrl", apiBaseUrl); }, [apiBaseUrl]); + useEffect(() => { + localStorage.setItem("apiBaseUrl", apiBaseUrl); + }, [apiBaseUrl]); async function verifyConnection() { setConnectionStatus("busy"); - setConnectionMessage("Verificando gateway..."); + setConnectionMessage("Comprobando gateway y servicios…"); setBusy(true); try { const [health, config, models] = await Promise.all([ - client.health(), client.config(), client.models(), + client.health(), + client.config(), + client.models(), ]); + setHealthJson(health); setConfigJson(config); setModelsJson(models); setConnectionStatus("ok"); - setConnectionMessage(`Conectado: ${JSON.stringify(health)}`); + setConnectionMessage("Gateway respondió correctamente."); } catch (err) { + setHealthJson(null); setConnectionStatus("error"); setConnectionMessage(err instanceof Error ? err.message : "No se pudo conectar"); } finally { @@ -88,27 +99,73 @@ function App() { setJobSectionContext(context); } + function openSettings(section: SettingsSection = "connection") { + setSettingsReturnView(view === "job" ? "job" : "home"); + setSettingsSection(section); + setView("settings"); + } + + function leaveSettings() { + if (settingsReturnView === "job" && activeJob) { + setView("job"); + } else { + setView("home"); + } + } + const isJobOpen = view === "job" && activeJob; + const isSettingsOpen = view === "settings"; return (
-
+
{formatVersionLabel()}
- {view === "home" && ( - - )} + {view === "home" && } {view === "job" && activeJob && ( )}
+ + {update.manifest && ( + void update.confirmInstall()} + installing={update.installing} + progress={update.progress} + error={update.error} + /> + )}
); } -type SettingsTab = "config" | "models"; - -function SettingsView({ - apiBaseUrl, - onApiBaseUrlChange, - connectionStatus, - connectionMessage, - verifyConnection, - busy, - configJson, - modelsJson, -}: { - apiBaseUrl: string; - onApiBaseUrlChange: (url: string) => void; - connectionStatus: Status; - connectionMessage: string; - verifyConnection: () => Promise; - busy: boolean; - configJson: unknown; - modelsJson: unknown; -}) { - const [backendTab, setBackendTab] = useState("config"); - - const connectionLabel = - connectionStatus === "ok" - ? "Conectado" - : connectionStatus === "busy" - ? "Verificando..." - : connectionStatus === "error" - ? "Sin conexión" - : "Sin verificar"; - - const backendData = backendTab === "config" ? configJson : modelsJson; - - return ( -
- - -
-
-
- -

Configuración del backend

-
-
- - -
-
-
- {backendData != null ? ( - 
{JSON.stringify(backendData, null, 2)}
- ) : ( -
- -

Sin datos del backend

- Verifica la conexión para cargar la {backendTab === "config" ? "configuración" : "lista de modelos"}. -
- )} -
-
-
- ); -} - export default App; diff --git a/services/desktop_app/src/renderer/assets/app-icon.png b/services/desktop_app/src/renderer/assets/app-icon.png new file mode 100644 index 0000000..54863e5 Binary files /dev/null and b/services/desktop_app/src/renderer/assets/app-icon.png differ diff --git a/services/desktop_app/src/renderer/components/BackgroundProgressBanner.tsx b/services/desktop_app/src/renderer/components/BackgroundProgressBanner.tsx index 75ff816..e91df2e 100644 --- a/services/desktop_app/src/renderer/components/BackgroundProgressBanner.tsx +++ b/services/desktop_app/src/renderer/components/BackgroundProgressBanner.tsx @@ -13,7 +13,10 @@ function pct(processed: number, total: number): number { export function BackgroundProgressBanner({ progress, compact = false }: Props) { const percent = pct(progress.processed, progress.total); - const moduleLabel = progress.module === "counting" ? "Conteo" : "Extracción"; + const moduleLabel = + progress.module === "counting" ? "Conteo" + : progress.module === "cleaning" ? "Limpieza" + : "Extracción"; return (
diff --git a/services/desktop_app/src/renderer/components/CleaningStepper.tsx b/services/desktop_app/src/renderer/components/CleaningStepper.tsx new file mode 100644 index 0000000..48d23aa --- /dev/null +++ b/services/desktop_app/src/renderer/components/CleaningStepper.tsx @@ -0,0 +1,44 @@ +import { Check } from "lucide-react"; + +export type CleaningStep = 1 | 2 | 3; + +const STEPS: Array<{ id: CleaningStep; label: string }> = [ + { id: 1, label: "Fuente" }, + { id: 2, label: "Reglas" }, + { id: 3, label: "Resultados" }, +]; + +interface CleaningStepperProps { + current: CleaningStep; + maxReached: CleaningStep; + onStepClick?: (step: CleaningStep) => void; +} + +export function CleaningStepper({ current, maxReached, onStepClick }: CleaningStepperProps) { + return ( + + ); +} diff --git a/services/desktop_app/src/renderer/components/CountingWizard.tsx b/services/desktop_app/src/renderer/components/CountingWizard.tsx index 894cdca..31169f4 100644 --- a/services/desktop_app/src/renderer/components/CountingWizard.tsx +++ b/services/desktop_app/src/renderer/components/CountingWizard.tsx @@ -24,6 +24,7 @@ import type { Task, } from "../../shared/types"; import { ApiClient } from "../lib/apiClient"; +import { downloadCsv, toCsv, buildExportCsvFilename } from "../lib/csv"; import { CountResultsTable } from "./CountResultsTable"; import { CountingStepper, type CountingStep } from "./CountingStepper"; @@ -377,25 +378,16 @@ export function CountingWizard({ client, localRoot, containerRoot, job, onStartE function handleExportCsv() { if (!savedFiles.length) return; const headers = ["filename", "radicado", "service_count", "auto_ajustado", "status", "is_valid", "error"]; - const rows = savedFiles.map((f) => - [ - `"${f.filename}"`, - `"${f.radicado ?? ""}"`, - f.service_count ?? "", - autoAdjustedFilenames.has(f.filename) ? "si" : "", - f.status, - f.is_valid === null ? "" : String(f.is_valid), - `"${(f.error ?? "").replace(/"/g, '""')}"`, - ].join(","), - ); - const csv = [headers.join(","), ...rows].join("\n"); - const blob = new Blob([csv], { type: "text/csv" }); - const url = URL.createObjectURL(blob); - const a = document.createElement("a"); - a.href = url; - a.download = `conteo-${new Date().toISOString().slice(0, 10)}.csv`; - a.click(); - URL.revokeObjectURL(url); + const rows = savedFiles.map((f) => [ + f.filename, + f.radicado ?? "", + f.service_count ?? "", + autoAdjustedFilenames.has(f.filename) ? "si" : "", + f.status, + f.is_valid === null ? "" : String(f.is_valid), + f.error ?? "", + ]); + downloadCsv(buildExportCsvFilename("conteo", job?.title), toCsv(headers, rows)); } function resetWizard() { diff --git a/services/desktop_app/src/renderer/components/ExtractionResultsTable.tsx b/services/desktop_app/src/renderer/components/ExtractionResultsTable.tsx new file mode 100644 index 0000000..3bbeb20 --- /dev/null +++ b/services/desktop_app/src/renderer/components/ExtractionResultsTable.tsx @@ -0,0 +1,147 @@ +import { Search } from "lucide-react"; +import { useMemo, useState } from "react"; + +type Row = Record; +type FilterKey = "all" | "complete" | "incomplete"; + +interface Props { + rows: Row[]; + columns: string[]; +} + +function cellValue(row: Row, col: string): string { + const v = row[col]; + if (v === null || v === undefined) return ""; + return String(v); +} + +function isEmpty(row: Row, col: string): boolean { + return cellValue(row, col).trim() === ""; +} + +function isComplete(row: Row, columns: string[]): boolean { + return columns.every((c) => !isEmpty(row, c)); +} + +export function ExtractionResultsTable({ rows, columns }: Props) { + const [filter, setFilter] = useState("all"); + const [search, setSearch] = useState(""); + + const summary = useMemo(() => { + const complete = rows.filter((r) => isComplete(r, columns)).length; + return { + total: rows.length, + complete, + incomplete: rows.length - complete, + }; + }, [rows, columns]); + + const filtered = useMemo(() => { + let list = rows; + if (filter === "complete") list = list.filter((r) => isComplete(r, columns)); + else if (filter === "incomplete") list = list.filter((r) => !isComplete(r, columns)); + + const q = search.trim().toLowerCase(); + if (q) { + list = list.filter((r) => + columns.some((c) => cellValue(r, c).toLowerCase().includes(q)), + ); + } + return list; + }, [rows, columns, filter, search]); + + const FILTERS: { key: FilterKey; label: string; count: number }[] = [ + { key: "all", label: "Todos", count: summary.total }, + { key: "complete", label: "Completos", count: summary.complete }, + { key: "incomplete", label: "Incompletos", count: summary.incomplete }, + ]; + + return ( +
+
+
+ {summary.total} + Total +
+
+ {summary.complete} + Completos +
+
+ {summary.incomplete} + Incompletos +
+
+ +
+
+ {FILTERS.map(({ key, label, count }) => ( + + ))} +
+ +
+
+ + setSearch(e.target.value)} + /> +
+
+
+ +
+ + + + {columns.map((c) => ( + + ))} + + + + {filtered.length === 0 ? ( + + + + ) : ( + filtered.map((row, i) => ( + + {columns.map((c) => ( + + ))} + + )) + )} + +
{c}
+ {search + ? "No hay filas que coincidan con la búsqueda." + : "No hay filas en esta categoría."} +
+ {isEmpty(row, c) ? ( + — + ) : ( + cellValue(row, c) + )} +
+
+ + {filtered.length !== rows.length && ( +

+ Mostrando {filtered.length} de {rows.length} filas +

+ )} +
+ ); +} diff --git a/services/desktop_app/src/renderer/components/ExtractionWizard.tsx b/services/desktop_app/src/renderer/components/ExtractionWizard.tsx index 056dee9..c2e2773 100644 --- a/services/desktop_app/src/renderer/components/ExtractionWizard.tsx +++ b/services/desktop_app/src/renderer/components/ExtractionWizard.tsx @@ -15,6 +15,7 @@ import { RefreshCw, Search, Settings2, + Shuffle, Trash2, Zap, } from "lucide-react"; @@ -36,6 +37,9 @@ interface ExtractionWizardProps { statusText: string; selectRoot: () => Promise; discoverSchema: () => Promise; + rediscoverWithOtherSample?: () => Promise; + discoverySampleLabels?: string[]; + canRediscoverWithOtherSample?: boolean; estimate: () => Promise; runExtraction: () => Promise; selection: FieldSelection; @@ -104,6 +108,9 @@ export function ExtractionWizard({ statusText, selectRoot, discoverSchema, + rediscoverWithOtherSample, + discoverySampleLabels = [], + canRediscoverWithOtherSample = false, estimate, runExtraction, selection, @@ -373,7 +380,39 @@ export function ExtractionWizard({ {fieldCount > 0 && (
- {fieldCount} campos detectados. Continua al siguiente paso para elegir cuales exportar. + {fieldCount} campos detectados + {discoverySampleLabels.length > 0 + ? ` con «${discoverySampleLabels.join(", ")}».` + : "."}{" "} + Continúa al siguiente paso para elegir cuáles exportar. +
+ )} + + {schemaResponse && rediscoverWithOtherSample && ( +
+
+ ¿Algo no salió como esperabas? + + Vuelve a descubrir los campos usando otro archivo aleatorio de la carpeta. + {discoverySampleLabels.length > 0 && ( + <> Muestra actual: {discoverySampleLabels.join(", ")}. + )} + +
+
)}
@@ -424,6 +463,34 @@ export function ExtractionWizard({ />
+ {schemaResponse && rediscoverWithOtherSample && ( +
+
+ ¿Algo no salió como esperabas? + + Reintenta el descubrimiento con otro archivo aleatorio + {discoverySampleLabels.length > 0 && ( + <> (ahora: {discoverySampleLabels.join(", ")}) + )} + . + +
+ +
+ )}

Vista previa {selection.fields.length > 0 && ( diff --git a/services/desktop_app/src/renderer/components/UpdatePrompt.tsx b/services/desktop_app/src/renderer/components/UpdatePrompt.tsx new file mode 100644 index 0000000..77e4da7 --- /dev/null +++ b/services/desktop_app/src/renderer/components/UpdatePrompt.tsx @@ -0,0 +1,172 @@ +import { Download, Loader2, X } from "lucide-react"; +import { useEffect, useState } from "react"; +import type { UpdateManifest, UpdateProgress } from "../../shared/types"; + +type Props = { + open: boolean; + currentVersion: string; + manifest: UpdateManifest; + onClose: () => void; + onConfirm: () => void; + installing: boolean; + progress: UpdateProgress | null; + error: string | null; +}; + +function formatBytes(n: number | undefined): string { + if (!n || n <= 0) return ""; + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; + return `${(n / (1024 * 1024)).toFixed(1)} MB`; +} + +export function UpdatePrompt({ + open, + currentVersion, + manifest, + onClose, + onConfirm, + installing, + progress, + error, +}: Props) { + if (!open) return null; + + const dateLabel = manifest.releaseDateDisplay || manifest.releaseDate || ""; + const percent = progress?.percent ?? 0; + const phaseLabel = + progress?.phase === "verifying" + ? "Verificando archivo…" + : progress?.phase === "installing" + ? "Iniciando instalador…" + : progress?.phase === "downloading" + ? `Descargando… ${percent}%` + : null; + + return ( +
+
+
+

Nueva versión disponible

+ {!installing && ( + + )} +
+ +

+ Hay una versión nueva: v{manifest.version} + {dateLabel ? <> (lanzamiento {dateLabel}) : null}. +
+ Tienes instalada la v{currentVersion}. +

+ + {manifest.size != null && manifest.size > 0 && ( +

Tamaño aproximado: {formatBytes(manifest.size)}

+ )} + + {installing && ( +
+
+
+
+ + {phaseLabel || "Preparando…"} + {progress?.phase === "downloading" && progress.total + ? ` · ${formatBytes(progress.received)} / ${formatBytes(progress.total)}` + : ""} + +
+ )} + + {error && ( +
+ {error} +
+ )} + +
+ + +
+
+
+ ); +} + +/** Comprueba actualizaciones al montar (solo si existe desktopApi). */ +export function useAutoUpdateCheck() { + const [open, setOpen] = useState(false); + const [currentVersion, setCurrentVersion] = useState(""); + const [manifest, setManifest] = useState(null); + const [installing, setInstalling] = useState(false); + const [progress, setProgress] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + const api = window.desktopApi; + if (!api?.checkForUpdates) return; + + let cancelled = false; + const timer = window.setTimeout(() => { + void (async () => { + try { + const result = await api.checkForUpdates(); + if (cancelled || !result.updateAvailable || !result.manifest) return; + setCurrentVersion(result.currentVersion); + setManifest(result.manifest); + setOpen(true); + } catch { + /* silencioso */ + } + })(); + }, 2000); + + return () => { + cancelled = true; + window.clearTimeout(timer); + }; + }, []); + + useEffect(() => { + const api = window.desktopApi; + if (!api?.onUpdateProgress || !installing) return; + return api.onUpdateProgress((p) => setProgress(p)); + }, [installing]); + + async function confirmInstall() { + if (!manifest || !window.desktopApi?.installUpdate) return; + setInstalling(true); + setError(null); + setProgress({ phase: "downloading", percent: 0 }); + try { + await window.desktopApi.installUpdate(manifest); + // La app debería cerrarse desde main; si no, mostramos mensaje. + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + setInstalling(false); + } + } + + return { + open, + currentVersion, + manifest, + installing, + progress, + error, + close: () => { + if (!installing) setOpen(false); + }, + confirmInstall, + }; +} diff --git a/services/desktop_app/src/renderer/lib/apiClient.ts b/services/desktop_app/src/renderer/lib/apiClient.ts index 8e0a43f..b5184e1 100644 --- a/services/desktop_app/src/renderer/lib/apiClient.ts +++ b/services/desktop_app/src/renderer/lib/apiClient.ts @@ -1,4 +1,10 @@ import type { + CleaningAnalysisResponse, + CleaningJob, + CleaningPreviewPayload, + CleaningPreviewResponse, + CleaningRowsResponse, + CleaningRuleInfo, CountJobFile, CountSyncResponse, CreateCountJobPayload, @@ -7,6 +13,7 @@ import type { ExtractionFromCountPayload, Job, RowsResponse, + RunCleaningPayload, SchemaResponse, Task, UpdateJobPayload, @@ -51,7 +58,9 @@ export class ApiClient { return ( path.startsWith("/count/") || path.includes("/ingest") || - path.includes("/upload") + path.includes("/upload") || + path.startsWith("/cleaning/analyze-csv") || + path.startsWith("/cleaning/jobs/from-csv") ); } @@ -218,6 +227,51 @@ export class ApiClient { return this.postJson("/tasks/extraction/from-count", payload); } + // ── Cleaning ─────────────────────────────────────────────────────────────── + + async cleaningRules(): Promise { + return this.getJson("/cleaning/rules"); + } + + async analyzeCleaningBatch(batchId: string): Promise { + return this.getJson( + `/cleaning/batches/${encodeURIComponent(batchId)}/analysis`, + ); + } + + async analyzeCleaningCsv(file: File | Blob, filename = "upload.csv"): Promise { + const form = new FormData(); + form.append("file", file, filename); + return this.postForm("/cleaning/analyze-csv", form); + } + + async runCleaning(payload: RunCleaningPayload): Promise { + return this.postJson("/cleaning/jobs", payload); + } + + async previewCleaning(payload: CleaningPreviewPayload): Promise { + return this.postJson("/cleaning/preview", payload); + } + + async runCleaningFromCsv(form: FormData): Promise { + return this.postForm("/cleaning/jobs/from-csv", form); + } + + async cleaningJobs(jobId?: string): Promise { + const qs = jobId ? `?job_id=${encodeURIComponent(jobId)}` : ""; + return this.getJson(`/cleaning/jobs${qs}`); + } + + async cleaningJob(cleaningJobId: string): Promise { + return this.getJson(`/cleaning/jobs/${encodeURIComponent(cleaningJobId)}`); + } + + async cleaningJobRows(cleaningJobId: string, limit = 500, offset = 0): Promise { + return this.getJson( + `/cleaning/jobs/${encodeURIComponent(cleaningJobId)}/rows?limit=${limit}&offset=${offset}`, + ); + } + private async handle(response: Response): Promise { const text = await response.text(); let data: unknown = null; diff --git a/services/desktop_app/src/renderer/lib/cleaningSettings.test.ts b/services/desktop_app/src/renderer/lib/cleaningSettings.test.ts new file mode 100644 index 0000000..7c04599 --- /dev/null +++ b/services/desktop_app/src/renderer/lib/cleaningSettings.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it, beforeEach } from "vitest"; +import { + defaultCopModConfig, + loadCopModConfig, + saveCopModConfig, + CLEANING_COPMOD_STORAGE_KEY, +} from "./cleaningSettings"; + +describe("cleaningSettings", () => { + beforeEach(() => { + localStorage.removeItem(CLEANING_COPMOD_STORAGE_KEY); + }); + + it("returns defaults when empty", () => { + const cfg = loadCopModConfig(); + expect(cfg.moderadora_by_year["2024"]).toEqual([4500, 18200, 47700]); + }); + + it("persists and reloads custom years", () => { + saveCopModConfig({ + moderadora_by_year: { "2030": [1000, 2000] }, + }); + const cfg = loadCopModConfig(); + expect(cfg.moderadora_by_year).toEqual({ "2030": [1000, 2000] }); + }); + + it("defaultCopModConfig is a deep copy", () => { + const a = defaultCopModConfig(); + a.moderadora_by_year["2024"].push(999); + expect(defaultCopModConfig().moderadora_by_year["2024"]).not.toContain(999); + }); +}); diff --git a/services/desktop_app/src/renderer/lib/cleaningSettings.ts b/services/desktop_app/src/renderer/lib/cleaningSettings.ts new file mode 100644 index 0000000..4101d36 --- /dev/null +++ b/services/desktop_app/src/renderer/lib/cleaningSettings.ts @@ -0,0 +1,66 @@ +/** Preferencias locales de limpieza (COP/MOD). */ + +export const CLEANING_COPMOD_STORAGE_KEY = "cleaningCopModConfig"; + +/** Valores por defecto alineados con el backend (`DEFAULT_MODERADORA_BY_YEAR`). */ +export const DEFAULT_MODERADORA_BY_YEAR: Record = { + "2023": [4100, 16400, 43000], + "2024": [4500, 18200, 47700], + "2025": [4700, 19200, 50300], + "2026": [5000, 20100, 52800], +}; + +export type CopModConfig = { + moderadora_by_year: Record; +}; + +export function defaultCopModConfig(): CopModConfig { + return { + moderadora_by_year: Object.fromEntries( + Object.entries(DEFAULT_MODERADORA_BY_YEAR).map(([y, vals]) => [y, [...vals]]), + ), + }; +} + +function normalizeYearMap(raw: unknown): Record { + if (!raw || typeof raw !== "object") return {}; + const out: Record = {}; + for (const [k, v] of Object.entries(raw as Record)) { + const year = String(Number.parseInt(k, 10)); + if (!Number.isFinite(Number(year)) || year === "NaN") continue; + if (!Array.isArray(v)) continue; + const nums = v + .map((x) => Math.round(Number(x))) + .filter((n) => Number.isFinite(n) && n >= 0); + if (nums.length > 0) { + out[year] = [...new Set(nums)].sort((a, b) => a - b); + } + } + return out; +} + +export function loadCopModConfig(): CopModConfig { + try { + const raw = localStorage.getItem(CLEANING_COPMOD_STORAGE_KEY); + if (!raw) return defaultCopModConfig(); + const parsed = JSON.parse(raw) as { moderadora_by_year?: unknown }; + const years = normalizeYearMap(parsed?.moderadora_by_year); + if (Object.keys(years).length === 0) return defaultCopModConfig(); + return { moderadora_by_year: years }; + } catch { + return defaultCopModConfig(); + } +} + +export function saveCopModConfig(config: CopModConfig): void { + const years = normalizeYearMap(config.moderadora_by_year); + localStorage.setItem( + CLEANING_COPMOD_STORAGE_KEY, + JSON.stringify({ moderadora_by_year: years }), + ); +} + +/** Payload listo para enviar al API de cleaning. */ +export function copModConfigForApi(config?: CopModConfig | null): CopModConfig { + return config ?? loadCopModConfig(); +} diff --git a/services/desktop_app/src/renderer/lib/csv.test.ts b/services/desktop_app/src/renderer/lib/csv.test.ts index 8992a45..b3fa69f 100644 --- a/services/desktop_app/src/renderer/lib/csv.test.ts +++ b/services/desktop_app/src/renderer/lib/csv.test.ts @@ -1,18 +1,40 @@ import { describe, expect, it } from "vitest"; -import { rowsToCsv } from "./csv"; +import { + buildExportCsvFilename, + csvEscape, + rowsToCsv, + sanitizeFilenamePart, + toCsv, +} from "./csv"; describe("csv export", () => { - it("escapes commas, quotes and newlines", () => { + it("usa pipe como separador", () => { + const csv = toCsv(["a", "b"], [["1", "2"], ["3", "4"]]); + expect(csv).toBe("a|b\n1|2\n3|4"); + }); + + it("escapa pipes, comillas y saltos de línea", () => { const csv = rowsToCsv( [ { radicado: "R1", - nombre: 'Ana, "IPS"', + nombre: 'Ana| "IPS"', nota: "linea 1\nlinea 2", }, ], { nombre_original: "nombre" }, ); - expect(csv).toContain('R1,"Ana, ""IPS""","linea 1\nlinea 2"'); + expect(csv).toContain('R1|"Ana| ""IPS"""|"linea 1\nlinea 2"'); + }); + + it("no escapa comas (no son el delimitador)", () => { + expect(csvEscape("Ana, IPS")).toBe("Ana, IPS"); + }); + + it("incluye el título del trabajo en el nombre de descarga", () => { + expect(sanitizeFilenamePart("Lote Abril / 2026")).toBe("Lote-Abril-2026"); + expect(buildExportCsvFilename("limpieza", "Lote Abril", "validos")).toMatch( + /^limpieza-Lote-Abril-validos-\d{4}-\d{2}-\d{2}\.csv$/, + ); }); }); diff --git a/services/desktop_app/src/renderer/lib/csv.ts b/services/desktop_app/src/renderer/lib/csv.ts index c6f32b2..44c9bbc 100644 --- a/services/desktop_app/src/renderer/lib/csv.ts +++ b/services/desktop_app/src/renderer/lib/csv.ts @@ -1,4 +1,6 @@ -const NEEDS_QUOTES = /[",\r\n]/; +export const CSV_DELIMITER = "|"; + +const NEEDS_QUOTES = /["|\r\n]/; export function csvEscape(value: unknown): string { if (value === null || value === undefined) return ""; @@ -7,13 +9,21 @@ export function csvEscape(value: unknown): string { return `"${text.replace(/"/g, '""')}"`; } +/** Serializa encabezado + filas con separador pipe (`|`). */ +export function toCsv(headers: string[], rows: Array>): string { + const lines = [ + headers.map(csvEscape).join(CSV_DELIMITER), + ...rows.map((row) => row.map(csvEscape).join(CSV_DELIMITER)), + ]; + return lines.join("\n"); +} + export function rowsToCsv(rows: Array>, aliasMap: Record = {}): string { const header = inferHeader(rows, aliasMap); - const lines = [header.map(csvEscape).join(",")]; - for (const row of rows) { - lines.push(header.map((column) => csvEscape(row[column])).join(",")); - } - return `${lines.join("\r\n")}\r\n`; + return toCsv( + header, + rows.map((row) => header.map((column) => row[column])), + ); } export function inferHeader(rows: Array>, aliasMap: Record = {}): string[] { @@ -38,3 +48,43 @@ export function inferHeader(rows: Array>, aliasMap: Reco } return header; } + +export function downloadCsv(filename: string, csv: string): void { + const blob = new Blob([csv], { type: "text/csv;charset=utf-8" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = filename; + a.click(); + URL.revokeObjectURL(url); +} + +/** Normaliza el título del trabajo para usarlo en nombres de archivo. */ +export function sanitizeFilenamePart(value: string | null | undefined, fallback = "sin-titulo"): string { + const cleaned = String(value ?? "") + .normalize("NFKD") + .replace(/[\u0300-\u036f]/g, "") + .replace(/[^a-zA-Z0-9._-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^-|-$/g, "") + .slice(0, 80); + return cleaned || fallback; +} + +/** + * Nombre de descarga: `{servicio}-{titulo-trabajo}[-{extra}]-{YYYY-MM-DD}.csv` + * Ej.: `limpieza-lote-abril-2026-07-10.csv` + */ +export function buildExportCsvFilename( + service: string, + jobTitle?: string | null, + extra?: string | null, +): string { + const parts = [ + sanitizeFilenamePart(service, "export"), + sanitizeFilenamePart(jobTitle, "sin-titulo"), + ]; + if (extra) parts.push(sanitizeFilenamePart(extra)); + parts.push(new Date().toISOString().slice(0, 10)); + return `${parts.join("-")}.csv`; +} diff --git a/services/desktop_app/src/renderer/modules/cleaning/CleaningModule.ts b/services/desktop_app/src/renderer/modules/cleaning/CleaningModule.ts new file mode 100644 index 0000000..1570534 --- /dev/null +++ b/services/desktop_app/src/renderer/modules/cleaning/CleaningModule.ts @@ -0,0 +1,18 @@ +import { CleaningWizardModule } from "./CleaningWizardModule"; +import { CleaningResults } from "./CleaningResults"; +import type { ModuleDescriptor } from "../registry"; +import { registerModule } from "../registry"; +import type { Task } from "../../../shared/types"; + +const CleaningModule: ModuleDescriptor = { + type: "cleaning", + label: "Limpieza", + iconName: "Sparkles", + hasTask: (tasks: Task[]) => tasks.some((t) => t.type === "cleaning"), + Wizard: CleaningWizardModule, + Results: CleaningResults, +}; + +registerModule(CleaningModule); + +export { CleaningModule }; diff --git a/services/desktop_app/src/renderer/modules/cleaning/CleaningResults.tsx b/services/desktop_app/src/renderer/modules/cleaning/CleaningResults.tsx new file mode 100644 index 0000000..2a356d1 --- /dev/null +++ b/services/desktop_app/src/renderer/modules/cleaning/CleaningResults.tsx @@ -0,0 +1,125 @@ +import { Download, Loader2, RefreshCw, Sparkles } from "lucide-react"; +import { useEffect, useState } from "react"; +import type { CleaningJob } from "../../../shared/types"; +import { ExtractionResultsTable } from "../../components/ExtractionResultsTable"; +import { buildExportCsvFilename, downloadCsv, toCsv } from "../../lib/csv"; +import type { ModuleResultsProps } from "../registry"; + +type Row = Record; + +export function CleaningResults({ client, task, job }: ModuleResultsProps) { + const [cleaningJob, setCleaningJob] = useState(null); + const [rows, setRows] = useState([]); + const [columns, setColumns] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + async function load() { + try { + const cleaningJobId = String( + (task.result as Record)?.cleaning_job_id ?? "", + ); + if (!cleaningJobId) { + setError("La tarea no tiene cleaning_job_id en result"); + return; + } + const jobData = await client.cleaningJob(cleaningJobId); + setCleaningJob(jobData); + if (jobData.status === "completed") { + const response = await client.cleaningJobRows(cleaningJobId, 500, 0); + const data = response.rows ?? []; + setRows(data); + setColumns(Array.from(new Set(data.flatMap((r) => Object.keys(r))))); + } + } catch (e) { + setError(String(e instanceof Error ? e.message : e)); + } finally { + setLoading(false); + } + } + + useEffect(() => { + void load(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [task.id]); + + function exportCsv() { + const csv = toCsv( + columns, + rows.map((r) => columns.map((h) => r[h])), + ); + downloadCsv(buildExportCsvFilename("limpieza", job.title), csv); + } + + if (loading) { + return ( +
+ + Cargando resultados de limpieza… +
+ ); + } + + if (error) { + return ( +
+ {error} + +
+ ); + } + + return ( +
+
+
+

+ + {task.title ?? "Limpieza"} +

+ + {cleaningJob?.status ?? task.status} + + {cleaningJob?.source && ( + + · {cleaningJob.source === "csv" ? "CSV" : "extracción"} + + )} +
+ {rows.length > 0 && ( + + )} +
+ + {cleaningJob?.log && cleaningJob.log.length > 0 && ( +
+ Log ({cleaningJob.log.length}) +
    + {cleaningJob.log.map((line, i) => ( +
  • {line}
    • + ))} +
+
+ )} + + {rows.length === 0 ? ( +
+

No se encontraron filas limpiadas.

+
+ ) : ( + + )} +
+ ); +} diff --git a/services/desktop_app/src/renderer/modules/cleaning/CleaningWizard.tsx b/services/desktop_app/src/renderer/modules/cleaning/CleaningWizard.tsx new file mode 100644 index 0000000..fb20cb0 --- /dev/null +++ b/services/desktop_app/src/renderer/modules/cleaning/CleaningWizard.tsx @@ -0,0 +1,967 @@ +import { + AlertCircle, + ArrowRight, + CheckCircle2, + Cpu, + Download, + FileSpreadsheet, + Loader2, + Play, + Sparkles, + Upload, +} from "lucide-react"; +import { useEffect, useRef, useState } from "react"; +import type { + BackgroundProgress, + CleaningAnalysisResponse, + CleaningColumnAnalysis, + CleaningJob, + CleaningPreviewColumn, + CleaningPreviewResponse, + CleaningRuleInfo, + Job, + Task, +} from "../../../shared/types"; +import { CleaningStepper, type CleaningStep } from "../../components/CleaningStepper"; +import { ExtractionResultsTable } from "../../components/ExtractionResultsTable"; +import type { ApiClient } from "../../lib/apiClient"; +import { loadCopModConfig } from "../../lib/cleaningSettings"; +import { downloadCsv, toCsv, buildExportCsvFilename } from "../../lib/csv"; + +type SourceMode = "extraction" | "csv"; + +interface Props { + client: ApiClient; + job: Job; + extractionBatchId?: string; + extractionTaskId?: string; + onTaskCreated?: (task: Task) => void; + onProgress?: (progress: BackgroundProgress) => void; +} + +function batchIdFromTask(task: Task): string { + return String((task.result as Record)?.extraction_batch_id ?? ""); +} + +function formatTaskDate(iso: string): string { + try { + return new Date(iso).toLocaleString("es-CO", { + day: "2-digit", + month: "short", + year: "numeric", + hour: "2-digit", + minute: "2-digit", + }); + } catch { + return iso; + } +} + +export function CleaningWizard({ + client, + job, + extractionBatchId: initialBatchId, + extractionTaskId: initialTaskId, + onTaskCreated, + onProgress, +}: Props) { + const hasHandoff = Boolean(initialBatchId || initialTaskId); + const [step, setStep] = useState(1); + const [maxReached, setMaxReached] = useState(1); + const [source, setSource] = useState(hasHandoff ? "extraction" : "extraction"); + const [extractionTasks, setExtractionTasks] = useState([]); + const [loadingTasks, setLoadingTasks] = useState(true); + const [selectedTaskId, setSelectedTaskId] = useState(initialTaskId ?? null); + const [batchId, setBatchId] = useState(initialBatchId ?? ""); + const [csvFile, setCsvFile] = useState(null); + const [rulesCatalog, setRulesCatalog] = useState([]); + const [analysis, setAnalysis] = useState(null); + const [rulesPerColumn, setRulesPerColumn] = useState>({}); + const [copModFechaColumn, setCopModFechaColumn] = useState>({}); + const [loading, setLoading] = useState(false); + const [running, setRunning] = useState(false); + const [error, setError] = useState(null); + const [resultJob, setResultJob] = useState(null); + const [resultRows, setResultRows] = useState[]>([]); + const [handoffAnalyzed, setHandoffAnalyzed] = useState(false); + const [preview, setPreview] = useState(null); + const [previewLoading, setPreviewLoading] = useState(false); + const [previewError, setPreviewError] = useState(null); + const [activeColumn, setActiveColumn] = useState(null); + const [sampleMode, setSampleMode] = useState<"before" | "after">("before"); + const fileInputRef = useRef(null); + const previewTimerRef = useRef | null>(null); + + function goToStep(next: CleaningStep) { + setStep(next); + setMaxReached((m) => (next > m ? next : m)); + } + + useEffect(() => { + client.cleaningRules().then(setRulesCatalog).catch(() => setRulesCatalog([])); + }, [client]); + + useEffect(() => { + let cancelled = false; + async function loadExtractions() { + setLoadingTasks(true); + try { + const tasks = await client.jobTasks(job.id); + if (cancelled) return; + const extractions = tasks + .filter((t) => t.type === "extraction" && t.status === "completed") + .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); + setExtractionTasks(extractions); + + // Preselección por handoff o la más reciente + if (initialTaskId && extractions.some((t) => t.id === initialTaskId)) { + const t = extractions.find((x) => x.id === initialTaskId)!; + setSelectedTaskId(t.id); + setBatchId(batchIdFromTask(t) || initialBatchId || ""); + setSource("extraction"); + } else if (initialBatchId) { + const match = extractions.find((t) => batchIdFromTask(t) === initialBatchId); + if (match) { + setSelectedTaskId(match.id); + setBatchId(initialBatchId); + } else { + setBatchId(initialBatchId); + } + setSource("extraction"); + } else if (extractions.length > 0 && !selectedTaskId) { + const newest = extractions[0]; + setSelectedTaskId(newest.id); + setBatchId(batchIdFromTask(newest)); + } + } catch { + if (!cancelled) setExtractionTasks([]); + } finally { + if (!cancelled) setLoadingTasks(false); + } + } + void loadExtractions(); + return () => { cancelled = true; }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [client, job.id, initialBatchId, initialTaskId]); + + // Handoff: analizar automáticamente al llegar desde Resultados + useEffect(() => { + if (!hasHandoff || handoffAnalyzed || loadingTasks) return; + const id = batchId || initialBatchId; + if (!id) return; + setHandoffAnalyzed(true); + void loadAnalysis("extraction", id, null); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [hasHandoff, handoffAnalyzed, loadingTasks, batchId, initialBatchId]); + + function selectExtraction(task: Task) { + setSelectedTaskId(task.id); + setBatchId(batchIdFromTask(task)); + setSource("extraction"); + setError(null); + } + + function ruleName(id: string): string { + return rulesCatalog.find((r) => r.id === id)?.name ?? id; + } + + function guessFechaColumn(columns: CleaningColumnAnalysis[], amountCol: string): string { + const candidates = columns.filter( + (c) => + c.name !== amountCol && + (c.inferred_type === "date" || + /fecha|date|servicio|fec[_ ]/i.test(c.name)), + ); + const preferred = + candidates.find((c) => /fecha\s*servicio|fec[_ ]?serv/i.test(c.name)) ?? + candidates.find((c) => c.inferred_type === "date") ?? + candidates[0]; + return preferred?.name ?? ""; + } + + function toggleRule(column: string, ruleId: string) { + setRulesPerColumn((prev) => { + const current = prev[column] ?? []; + const turningOn = !current.includes(ruleId); + const next = turningOn + ? [...current, ruleId] + : current.filter((r) => r !== ruleId); + return { ...prev, [column]: next }; + }); + if (ruleId === "cop_mod_split") { + setCopModFechaColumn((prev) => { + const current = rulesPerColumn[column] ?? []; + const turningOn = !current.includes(ruleId); + if (!turningOn) { + const next = { ...prev }; + delete next[column]; + return next; + } + if (prev[column]) return prev; + const guess = analysis ? guessFechaColumn(analysis.columns, column) : ""; + return guess ? { ...prev, [column]: guess } : prev; + }); + } + setSampleMode("after"); + } + + function suggestRules(columns: CleaningColumnAnalysis[]) { + const suggested: Record = {}; + for (const col of columns) { + const applicable = col.applicable_rules ?? []; + suggested[col.name] = applicable.includes("strip_spaces") ? ["strip_spaces"] : []; + } + setRulesPerColumn(suggested); + setCopModFechaColumn({}); + } + + function cleaningExtraPayload(): { + cop_mod_fecha_column?: Record; + cop_mod_config: ReturnType; + } { + const fechaEntries = Object.entries(copModFechaColumn).filter( + ([col, fecha]) => + Boolean(fecha) && (rulesPerColumn[col] ?? []).includes("cop_mod_split"), + ); + return { + ...(fechaEntries.length > 0 + ? { cop_mod_fecha_column: Object.fromEntries(fechaEntries) } + : {}), + cop_mod_config: loadCopModConfig(), + }; + } + + function previewColumn(name: string): CleaningPreviewColumn | undefined { + return preview?.columns.find((c) => c.name === name); + } + + function displaySamples(col: CleaningColumnAnalysis): string[] { + if (col.samples && col.samples.length > 0) return col.samples; + if (col.example) return [String(col.example)]; + return []; + } + + /** Columnas nuevas respecto al análisis (p. ej. *_doc_tipo / *_doc_numeros). */ + function derivedColumnNames(sourceCol: string): string[] { + if (!preview) return []; + const original = new Set(analysis?.columns.map((c) => c.name) ?? []); + return preview.columns + .map((c) => c.name) + .filter( + (name) => + !original.has(name) && + name.toLowerCase().startsWith(`${sourceCol.toLowerCase()}_`), + ); + } + + /** Orden: columna activa → columnas derivadas → resto (sin cortar). */ + function orderedSampleColumnNames(mode: "before" | "after"): string[] { + const analysisNames = analysis?.columns.map((c) => c.name) ?? []; + const names = + mode === "after" && preview + ? preview.columns.map((c) => c.name) + : analysisNames; + const focus = activeColumn; + if (!focus || !names.includes(focus)) return names; + const derived = names.filter( + (n) => n !== focus && n.toLowerCase().startsWith(`${focus.toLowerCase()}_`), + ); + const rest = names.filter((n) => n !== focus && !derived.includes(n)); + return [focus, ...derived, ...rest]; + } + + // Preview en vivo al cambiar reglas (debounce 300ms) + useEffect(() => { + if (step !== 2 || !analysis) return; + const sampleRows = analysis.sample_rows ?? []; + if (previewTimerRef.current) clearTimeout(previewTimerRef.current); + + previewTimerRef.current = setTimeout(() => { + void (async () => { + setPreviewLoading(true); + setPreviewError(null); + try { + const extra = cleaningExtraPayload(); + const payload = + sampleRows.length > 0 + ? { rows: sampleRows, rules_per_column: rulesPerColumn, ...extra } + : batchId + ? { + extraction_batch_id: batchId, + rules_per_column: rulesPerColumn, + ...extra, + } + : null; + if (!payload) { + setPreview(null); + return; + } + const data = await client.previewCleaning(payload); + setPreview(data); + } catch (e) { + setPreviewError(String(e instanceof Error ? e.message : e)); + } finally { + setPreviewLoading(false); + } + })(); + }, 300); + + return () => { + if (previewTimerRef.current) clearTimeout(previewTimerRef.current); + }; + }, [step, analysis, rulesPerColumn, copModFechaColumn, batchId, client]); + + async function loadAnalysis(mode: SourceMode, batch?: string, file?: File | null) { + setLoading(true); + setError(null); + setAnalysis(null); + onProgress?.({ + module: "cleaning", + total: 1, + processed: 0, + failed: 0, + currentFile: mode === "csv" ? (file?.name ?? "CSV") : "extracción", + running: true, + label: "Analizando columnas…", + }); + try { + let data: CleaningAnalysisResponse; + if (mode === "extraction") { + if (!batch) throw new Error("Selecciona una extracción de la lista"); + data = await client.analyzeCleaningBatch(batch); + } else { + if (!file) throw new Error("Selecciona un archivo CSV"); + data = await client.analyzeCleaningCsv(file, file.name); + } + setAnalysis(data); + suggestRules(data.columns); + setActiveColumn(data.columns[0]?.name ?? null); + setSampleMode("before"); + setPreview(null); + goToStep(2); + } catch (e) { + setError(String(e instanceof Error ? e.message : e)); + } finally { + setLoading(false); + onProgress?.({ + module: "cleaning", + total: 1, + processed: 1, + failed: 0, + currentFile: "", + running: false, + label: "", + }); + } + } + + async function handleAnalyze() { + await loadAnalysis(source, batchId, csvFile); + } + + async function handleRun() { + setRunning(true); + setError(null); + setResultJob(null); + setResultRows([]); + onProgress?.({ + module: "cleaning", + total: 1, + processed: 0, + failed: 0, + currentFile: source === "csv" ? (csvFile?.name ?? "CSV") : "extracción", + running: true, + label: "Aplicando reglas de limpieza…", + }); + try { + let cleaningJob: CleaningJob; + if (source === "extraction") { + if (!batchId) throw new Error("Selecciona una extracción"); + cleaningJob = await client.runCleaning({ + extraction_batch_id: batchId, + job_id: job.id, + rules_per_column: rulesPerColumn, + ...cleaningExtraPayload(), + }); + } else { + if (!csvFile) throw new Error("Selecciona un archivo CSV"); + const form = new FormData(); + const extra = cleaningExtraPayload(); + form.append("file", csvFile, csvFile.name); + form.append("job_id", job.id); + form.append("rules_per_column", JSON.stringify(rulesPerColumn)); + if (extra.cop_mod_fecha_column) { + form.append("cop_mod_fecha_column", JSON.stringify(extra.cop_mod_fecha_column)); + } + form.append("cop_mod_config", JSON.stringify(extra.cop_mod_config)); + cleaningJob = await client.runCleaningFromCsv(form); + } + setResultJob(cleaningJob); + if (cleaningJob.status === "completed") { + const rowsResp = await client.cleaningJobRows(cleaningJob.id, 500, 0); + setResultRows(rowsResp.rows ?? []); + } + try { + const tasks = await client.jobTasks(job.id); + const latest = tasks + .filter((t) => t.type === "cleaning") + .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())[0]; + if (latest) onTaskCreated?.(latest); + } catch { /* silent */ } + goToStep(3); + } catch (e) { + setError(String(e instanceof Error ? e.message : e)); + } finally { + setRunning(false); + onProgress?.({ + module: "cleaning", + total: 1, + processed: 1, + failed: 0, + currentFile: "", + running: false, + label: "", + }); + } + } + + function exportCsv() { + if (resultRows.length === 0) return; + const columns = Array.from(new Set(resultRows.flatMap((r) => Object.keys(r)))); + const csv = toCsv( + columns, + resultRows.map((r) => columns.map((h) => r[h])), + ); + downloadCsv(buildExportCsvFilename("limpieza", job.title), csv); + } + + function resetWizard() { + setStep(1); + setMaxReached(1); + setResultJob(null); + setResultRows([]); + setAnalysis(null); + setHandoffAnalyzed(true); // no re-auto-analizar + setError(null); + } + + const resultColumns = + resultRows.length > 0 + ? Array.from(new Set(resultRows.flatMap((r) => Object.keys(r)))) + : []; + + const selectedExtraction = extractionTasks.find((t) => t.id === selectedTaskId) ?? null; + const canAnalyze = + source === "extraction" ? Boolean(batchId) : Boolean(csvFile); + + return ( +
+ + +
+ {error && ( +
+ {error} +
+ )} + + {/* ── PASO 1: Fuente ── */} + {step === 1 && ( +
+
+

Origen de los datos

+

+ Elige una extracción completada de este trabajo o sube un CSV para limpiar. +

+
+ +
+
+ + +
+
+ + {source === "extraction" ? ( +
+ + {loadingTasks ? ( +
+ Cargando extracciones… +
+ ) : extractionTasks.length === 0 ? ( +
+ No hay extracciones completadas en este trabajo. + +
+ ) : ( +
+ {extractionTasks.map((task, index) => { + const selected = task.id === selectedTaskId; + return ( + + ); + })} +
+ )} + {selectedExtraction && ( +

+ Seleccionada: {selectedExtraction.title ?? "Extracción"} +

+ )} +
+ ) : ( +
+ + setCsvFile(e.target.files?.[0] ?? null)} + /> +
+
+ {csvFile ? ( + <> + + {csvFile.name} + + ) : ( + Ningún archivo seleccionado + )} +
+ +
+
+ )} + +
+ +
+
+ )} + + {/* ── PASO 2: Reglas ── */} + {step === 2 && ( +
+
+
+

Reglas por columna

+

+ Elige una columna, aplica reglas y mira el cambio al instante. + {analysis?.row_count != null ? ` · ${analysis.row_count} filas` : ""} + {previewLoading ? " · actualizando…" : ""} +

+
+ {previewLoading && } +
+ + {loading && !analysis ? ( +
+ Analizando… +
+ ) : analysis ? ( +
+ {/* Lista compacta de columnas */} + + + {/* Detalle de la columna activa */} +
+ {previewError && ( +
+ {previewError} +
+ )} + + {(() => { + const col = analysis.columns.find((c) => c.name === activeColumn) ?? analysis.columns[0]; + if (!col) return

Sin columnas.

; + const applicable = col.applicable_rules ?? []; + const selected = rulesPerColumn[col.name] ?? []; + const beforeFromSamples = displaySamples(col); + const pv = previewColumn(col.name); + const beforeVals = pv?.before?.length ? pv.before.slice(0, 4) : beforeFromSamples.slice(0, 4); + const derived = derivedColumnNames(col.name); + const afterVals = pv?.after?.length + ? pv.after.slice(0, 4) + : selected.length === 0 + ? beforeVals + : []; + const sourceUnchanged = + beforeVals.length > 0 && + afterVals.length > 0 && + beforeVals.every((v, i) => String(v) === String(afterVals[i] ?? "")); + const changed = Boolean(pv?.changed) || derived.length > 0; + + return ( + <> +
+
+

{col.name}

+
+ {col.inferred_type} + {col.null_count != null && ( + nulos {col.null_count}{col.null_pct != null ? ` (${Math.round(col.null_pct)}%)` : ""} + )} + {col.unique_count != null && únicos {col.unique_count}} +
+
+
+ +
+
+
Antes
+
+ {beforeVals.length === 0 ? ( + Sin valores + ) : ( + beforeVals.map((v, i) => ( + {v || "—"} + )) + )} +
+
+ +
+
+ Después +
+
+ {derived.length > 0 ? ( + derived.map((dName) => { + const dPv = previewColumn(dName); + const sample = (dPv?.after ?? []).find((v) => v) ?? (dPv?.after?.[0] ?? ""); + return ( + + {dName.replace(/^.*_doc_/, "_doc_")} + {sample || "—"} + + ); + }) + ) : afterVals.length === 0 ? ( + + {previewLoading ? "…" : "Activa una regla"} + + ) : sourceUnchanged ? ( + Sin cambio en origen + ) : ( + afterVals.map((v, i) => { + const before = beforeVals[i] ?? ""; + const isDiff = String(v) !== String(before); + return ( + + {v || "—"} + + ); + }) + )} +
+
+
+ +
+
Reglas
+ {applicable.length === 0 ? ( + Sin reglas para este tipo + ) : ( +
+ {applicable.map((ruleId) => { + const on = selected.includes(ruleId); + const info = rulesCatalog.find((r) => r.id === ruleId); + return ( + + ); + })} +
+ )} + {selected.length > 0 && ( +

+ {selected + .map((id) => rulesCatalog.find((r) => r.id === id)?.description || id) + .join(" · ")} +

+ )} + {selected.includes("cop_mod_split") && ( +
+ + +

+ Las cuotas por año se editan en Configuración → Limpieza. +

+
+ )} +
+ + ); + })()} + + {/* Una sola tabla de muestra: Antes | Después */} + {((analysis.sample_rows?.length ?? 0) > 0 || (preview?.rows_after.length ?? 0) > 0) && ( +
+
+ Muestra +
+ + +
+
+
+ + + + {orderedSampleColumnNames(sampleMode).map((name) => { + const isNew = !(analysis.columns.some((c) => c.name === name)); + const changed = preview?.columns.find((x) => x.name === name)?.changed; + return ( + + ); + })} + + + + {(sampleMode === "after" && preview + ? preview.rows_after + : analysis.sample_rows ?? preview?.rows_before ?? [] + ).slice(0, 4).map((row, i) => ( + + {orderedSampleColumnNames(sampleMode).map((name) => { + const val = row[name]; + const before = preview?.rows_before[i]?.[name]; + const isNew = !(analysis.columns.some((c) => c.name === name)); + const diff = + sampleMode === "after" && + (isNew || String(before ?? "") !== String(val ?? "")); + return ( + + ); + })} + + ))} + +
+ {name} +
+ {val == null || val === "" + ? — + : String(val).slice(0, 36)} +
+
+
+ )} +
+
+ ) : ( +

No hay análisis. Vuelve al paso Fuente.

+ )} + +
+ + +
+
+ )} + + {/* ── PASO 3: Resultados ── */} + {step === 3 && ( +
+
+

Resultado de la limpieza

+
+ + {resultJob?.status === "completed" ? ( + <> +
+ + + Limpieza completada + {resultJob.log?.length ? ` · ${resultJob.log.length} operaciones` : null} + {resultRows.length > 0 ? ` · ${resultRows.length} filas` : null} + +
+ + {resultJob.log && resultJob.log.length > 0 && ( +
+ Ver log de operaciones +
+
    + {resultJob.log.map((line, i) => ( +
  • {line}
    • + ))} +
+
+
+ )} + +
+ + Vista previa{resultRows.length > 100 ? " (primeras 100)" : ""} + + {resultRows.length > 0 && ( + + )} +
+ + {resultRows.length > 0 ? ( + + ) : ( +

Sin filas en el resultado.

+ )} + + ) : ( +
+ + {resultJob?.error ?? "La limpieza no se completó"} +
+ )} + +
+ + +
+
+ )} +
+
+ ); +} diff --git a/services/desktop_app/src/renderer/modules/cleaning/CleaningWizardModule.tsx b/services/desktop_app/src/renderer/modules/cleaning/CleaningWizardModule.tsx new file mode 100644 index 0000000..41cb367 --- /dev/null +++ b/services/desktop_app/src/renderer/modules/cleaning/CleaningWizardModule.tsx @@ -0,0 +1,26 @@ +import type { BackgroundProgress } from "../../../shared/types"; +import type { ModuleWizardProps } from "../registry"; +import { CleaningWizard } from "./CleaningWizard"; + +interface Props extends ModuleWizardProps { + onProgress?: (p: BackgroundProgress) => void; +} + +export function CleaningWizardModule({ + client, + job, + onTaskCreated, + onProgress, + context, +}: Props) { + return ( + + ); +} diff --git a/services/desktop_app/src/renderer/modules/counting/CountingResults.tsx b/services/desktop_app/src/renderer/modules/counting/CountingResults.tsx index 1e5d848..b0becd7 100644 --- a/services/desktop_app/src/renderer/modules/counting/CountingResults.tsx +++ b/services/desktop_app/src/renderer/modules/counting/CountingResults.tsx @@ -2,6 +2,7 @@ import { ArrowRight, Download, Loader2, RefreshCw } from "lucide-react"; import { useEffect, useState } from "react"; import type { CountJobFile } from "../../../shared/types"; import { CountResultsTable } from "../../components/CountResultsTable"; +import { downloadCsv, toCsv, buildExportCsvFilename } from "../../lib/csv"; import type { ModuleResultsProps } from "../registry"; export function CountingResults({ @@ -47,25 +48,16 @@ export function CountingResults({ : files; const headers = ["filename", "radicado", "service_count", "status", "is_valid", "error"]; - const rows = subset_files.map((f) => - [ - `"${f.filename}"`, - `"${f.radicado ?? ""}"`, - f.service_count ?? "", - f.status, - f.is_valid === null ? "" : String(f.is_valid), - `"${(f.error ?? "").replace(/"/g, '""')}"`, - ].join(","), - ); - const csv = [headers.join(","), ...rows].join("\n"); - const blob = new Blob([csv], { type: "text/csv" }); - const url = URL.createObjectURL(blob); - const a = document.createElement("a"); - a.href = url; + const rows = subset_files.map((f) => [ + f.filename, + f.radicado ?? "", + f.service_count ?? "", + f.status, + f.is_valid === null ? "" : String(f.is_valid), + f.error ?? "", + ]); const label = subset === "valid" ? "validos" : subset === "invalid" ? "invalidos" : "todos"; - a.download = `conteo-${label}-${new Date().toISOString().slice(0, 10)}.csv`; - a.click(); - URL.revokeObjectURL(url); + downloadCsv(buildExportCsvFilename("conteo", job.title, label), toCsv(headers, rows)); } const validCount = files.filter(isFileValid).length; diff --git a/services/desktop_app/src/renderer/modules/extraction/ExtractionResults.tsx b/services/desktop_app/src/renderer/modules/extraction/ExtractionResults.tsx index 25e6e26..de07aea 100644 --- a/services/desktop_app/src/renderer/modules/extraction/ExtractionResults.tsx +++ b/services/desktop_app/src/renderer/modules/extraction/ExtractionResults.tsx @@ -1,11 +1,13 @@ -import { Download, Loader2, RefreshCw } from "lucide-react"; +import { ArrowRight, Download, Loader2, RefreshCw, Sparkles } from "lucide-react"; import { useEffect, useState } from "react"; import type { ExtractionBatch } from "../../../shared/types"; +import { ExtractionResultsTable } from "../../components/ExtractionResultsTable"; +import { downloadCsv, toCsv, buildExportCsvFilename } from "../../lib/csv"; import type { ModuleResultsProps } from "../registry"; type Row = Record; -export function ExtractionResults({ client, task }: ModuleResultsProps) { +export function ExtractionResults({ client, task, job, onActivateModule }: ModuleResultsProps) { const [batch, setBatch] = useState(null); const [rows, setRows] = useState([]); const [columns, setColumns] = useState([]); @@ -54,25 +56,24 @@ export function ExtractionResults({ client, task }: ModuleResultsProps) { }, [task.id]); // eslint-disable-line react-hooks/exhaustive-deps function exportCsv() { - const headers = columns; - const csvRows = rows.map((r) => - headers - .map((h) => { - const v = r[h]; - if (v === null || v === undefined) return ""; - const s = String(v); - return `"${s.replace(/"/g, '""')}"`; - }) - .join(","), + const csv = toCsv( + columns, + rows.map((r) => columns.map((h) => r[h])), ); - const csv = [headers.join(","), ...csvRows].join("\n"); - const blob = new Blob([csv], { type: "text/csv" }); - const url = URL.createObjectURL(blob); - const a = document.createElement("a"); - a.href = url; - a.download = `extraccion-${new Date().toISOString().slice(0, 10)}.csv`; - a.click(); - URL.revokeObjectURL(url); + downloadCsv(buildExportCsvFilename("extraccion", job.title), csv); + } + + function startCleaning() { + const batchId = String( + (task.result as Record)?.extraction_batch_id + ?? batch?.id + ?? "", + ); + onActivateModule?.("cleaning", { + extractionBatchId: batchId || undefined, + extractionTaskId: task.id, + jobId: job.id, + }); } if (loading && !batch) { @@ -109,15 +110,22 @@ export function ExtractionResults({ client, task }: ModuleResultsProps) { {batch?.status ?? "pendiente"}
- {rows.length > 0 && ( - - )} +
+ {rows.length > 0 && ( + + )} + {rows.length > 0 && !isRunning && onActivateModule && ( + + )} +

{isRunning && ( -
+
Extracción en progreso… @@ -132,32 +140,9 @@ export function ExtractionResults({ client, task }: ModuleResultsProps) {

No se encontraron filas extraídas.

- ) : ( -
- - - - {columns.map((c) => ( - - ))} - - - - {rows.map((row, i) => ( - - {columns.map((c) => ( - - ))} - - ))} - -
{c}
- {row[c] === null || row[c] === undefined - ? — - : String(row[c])} -
-
- )} + ) : rows.length > 0 ? ( + + ) : null}
); } diff --git a/services/desktop_app/src/renderer/modules/extraction/JobExtractionWizard.tsx b/services/desktop_app/src/renderer/modules/extraction/JobExtractionWizard.tsx index 779948b..3a58981 100644 --- a/services/desktop_app/src/renderer/modules/extraction/JobExtractionWizard.tsx +++ b/services/desktop_app/src/renderer/modules/extraction/JobExtractionWizard.tsx @@ -30,6 +30,7 @@ import { defaultAlias } from "../../lib/schemaDisplay"; import { loadApiBaseUrl } from "../../lib/apiDefaults"; import { splitRadicados } from "../../lib/pathMapping"; import type { ModuleWizardProps } from "../registry"; +import { ExtractionResults } from "./ExtractionResults"; declare const desktopApi: import("../../../shared/types").DesktopBridge; @@ -71,6 +72,7 @@ export function JobExtractionWizard({ client, job, onTaskCreated, + onActivateModule, context, countingTaskId: countingTaskIdProp, validCounts: validCountsProp, @@ -95,6 +97,9 @@ export function JobExtractionWizard({ const [extractionStep, setExtractionStep] = useState(() => countingTaskId ? 2 : 1); const [maxReachedStep, setMaxReachedStep] = useState(() => countingTaskId ? 2 : 1); const [taskCreated, setTaskCreated] = useState(null); + /** Rutas de muestras ya usadas en descubrimiento (para no repetir al reintentar). */ + const [usedDiscoveryPaths, setUsedDiscoveryPaths] = useState([]); + const [discoverySampleLabels, setDiscoverySampleLabels] = useState([]); const flattened = useMemo( () => (schemaResponse ? flattenSchema(schemaResponse) : { schema: {}, sample: null }), @@ -151,6 +156,8 @@ export function JobExtractionWizard({ setSchemaResponse(null); setSelection({ fields: [], aliasMap: {} }); setEstimateJson(null); + setUsedDiscoveryPaths([]); + setDiscoverySampleLabels([]); setMaxReachedStep(1); if (extractionStep > 1) setExtractionStep(1); } @@ -167,7 +174,6 @@ export function JobExtractionWizard({ resetSchemaState(); setMaxReachedStep(1); setStatusText(`${indexed.length} documento(s) listos`); - // Actualizar carpeta del job try { await client.updateJob(job.id, { folder_path: selected }); } catch { /* ignore */ } } catch (err) { setManifest([]); @@ -190,41 +196,91 @@ export function JobExtractionWizard({ } } - async function discoverSchema() { - await runTask("Descubriendo campos...", async () => { - if (!window.desktopApi) throw new Error("Requiere Electron."); - if (manifest.length === 0) throw new Error("Selecciona una carpeta con documentos."); + function pickSampleEntry( + candidates: ManifestEntry[], + exclude: Set, + preferRandom: boolean, + ): ManifestEntry { + if (candidates.length === 0) { + throw new Error("No hay documentos candidatos para la muestra."); + } + const unused = candidates.filter((e) => !exclude.has(e.localPath)); + const pool = unused.length > 0 ? unused : candidates; + if (!preferRandom && unused.length === candidates.length) { + return candidates[0]; + } + return pool[Math.floor(Math.random() * pool.length)]!; + } - const responses: SchemaResponse[] = []; - if (form.mode === "general") { - const entry = manifest[0]; - const fileData = await window.desktopApi.readFile(entry.localPath); - const blob = new Blob([fileData.data]); - responses.push(await client.discoverSchemaUpload(buildSchemaUploadForm(form, blob, fileData.filename))); - } else { - for (const docType of validDocTypes(form.docTypes)) { - const entry = manifest.find((item) => item.docType === docType.name); - if (!entry) throw new Error(`No se encontró muestra para ${docType.name}.`); + async function discoverSchema(options?: { randomOther?: boolean }) { + const randomOther = Boolean(options?.randomOther); + await runTask( + randomOther ? "Probando con otro archivo…" : "Descubriendo campos...", + async () => { + if (!window.desktopApi) throw new Error("Requiere Electron."); + if (manifest.length === 0) throw new Error("Selecciona una carpeta con documentos."); + + const exclude = new Set(randomOther ? usedDiscoveryPaths : []); + const responses: SchemaResponse[] = []; + const picked: ManifestEntry[] = []; + + if (form.mode === "general") { + const entry = pickSampleEntry(manifest, exclude, randomOther); + picked.push(entry); const fileData = await window.desktopApi.readFile(entry.localPath); const blob = new Blob([fileData.data]); responses.push( - await client.discoverSchemaUpload(buildSchemaUploadForm(form, blob, fileData.filename, docType.name)), + await client.discoverSchemaUpload( + buildSchemaUploadForm(form, blob, fileData.filename), + ), ); + } else { + for (const docType of validDocTypes(form.docTypes)) { + const typed = manifest.filter((item) => item.docType === docType.name); + if (typed.length === 0) { + throw new Error(`No se encontró muestra para ${docType.name}.`); + } + const entry = pickSampleEntry(typed, exclude, randomOther); + picked.push(entry); + const fileData = await window.desktopApi.readFile(entry.localPath); + const blob = new Blob([fileData.data]); + responses.push( + await client.discoverSchemaUpload( + buildSchemaUploadForm(form, blob, fileData.filename, docType.name), + ), + ); + } } - } - const schema = mergeSchemaResponses(responses); - setSchemaResponse(schema); - const flat = flattenSchema(schema); - setSelection({ - fields: [], - aliasMap: Object.fromEntries(Object.keys(flat.schema).map((p) => [p, defaultAlias(p)])), - }); - setEstimateJson(null); - setStatusText(`${Object.keys(flat.schema).length} campo(s) descubiertos`); - setMaxReachedStep((cur) => (cur < 3 ? 3 : cur)); - setExtractionStep(3); - }); + const schema = mergeSchemaResponses(responses); + setSchemaResponse(schema); + const flat = flattenSchema(schema); + setSelection({ + fields: [], + aliasMap: Object.fromEntries(Object.keys(flat.schema).map((p) => [p, defaultAlias(p)])), + }); + setEstimateJson(null); + const labels = picked.map((e) => e.filename); + setDiscoverySampleLabels(labels); + setUsedDiscoveryPaths((prev) => { + const next = new Set(randomOther ? prev : []); + for (const e of picked) next.add(e.localPath); + return [...next]; + }); + setStatusText( + `${Object.keys(flat.schema).length} campo(s) descubiertos` + + (labels.length ? ` · muestra: ${labels.join(", ")}` : ""), + ); + setMaxReachedStep((cur) => (cur < 3 ? 3 : cur)); + if (!randomOther) { + setExtractionStep(3); + } + }, + ); + } + + async function rediscoverWithOtherSample() { + await discoverSchema({ randomOther: true }); } async function estimate() { @@ -419,14 +475,26 @@ export function JobExtractionWizard({ }); } - // Si ya se creó tarea, muestra mensaje y el módulo Results tomará el relevo + // Tras crear la tarea: mostrar resultados (con polling) en esta misma pestaña if (taskCreated) { return ( -
-

✓ Extracción iniciada

-

- Los resultados aparecerán automáticamente en esta pestaña. -

+
+ {(ingestProgress || busy) && ( +
+ + {statusText || "Extracción en curso…"} +
+ )} + { + setTaskCreated(t); + onTaskCreated(t); + }} + onActivateModule={onActivateModule} + />
); } @@ -461,7 +529,16 @@ export function JobExtractionWizard({ error={error} statusText={statusText} selectRoot={selectRoot} - discoverSchema={discoverSchema} + discoverSchema={() => void discoverSchema()} + rediscoverWithOtherSample={() => void rediscoverWithOtherSample()} + discoverySampleLabels={discoverySampleLabels} + canRediscoverWithOtherSample={ + form.mode === "general" + ? manifest.length > 1 + : validDocTypes(form.docTypes).some( + (dt) => manifest.filter((m) => m.docType === dt.name).length > 1, + ) + } estimate={estimate} runExtraction={runExtraction} selection={selection} diff --git a/services/desktop_app/src/renderer/styles.css b/services/desktop_app/src/renderer/styles.css index ab82bae..c727d36 100644 --- a/services/desktop_app/src/renderer/styles.css +++ b/services/desktop_app/src/renderer/styles.css @@ -73,15 +73,12 @@ button:disabled { } .brand-mark { - align-items: center; - background: linear-gradient(135deg, #2e7db2, #3b9ad9); + background: #0b0b0b; border-radius: 10px; - color: #ffffff; - display: flex; - font-size: 13px; - font-weight: 800; + display: block; + flex-shrink: 0; height: 34px; - justify-content: center; + object-fit: contain; width: 34px; } @@ -289,6 +286,212 @@ button:disabled { flex: 1; } +.settings-page { + display: flex; + flex: 1; + flex-direction: column; + min-height: 0; + overflow: hidden; +} + +.settings-page-header { + border-bottom: 1px solid #e8edf4; + flex-shrink: 0; + padding: 20px 28px 16px; +} + +.settings-page-header h2 { + font-size: 18px; + font-weight: 700; + margin: 0 0 6px; +} + +.settings-page-header p { + color: #64748b; + font-size: 13px; + line-height: 1.45; + margin: 0; + max-width: 640px; +} + +.settings-page-header-row { + align-items: flex-start; + display: flex; + gap: 16px; + justify-content: space-between; +} + +.settings-page-actions { + align-items: center; + display: flex; + flex-shrink: 0; + gap: 8px; +} + +.settings-saved-badge { + background: #dcfce7; + border-radius: 999px; + color: #15803d; + font-size: 11px; + font-weight: 600; + padding: 4px 10px; +} + +.settings-page-body { + display: flex; + flex: 1; + flex-direction: column; + gap: 16px; + min-height: 0; + overflow-y: auto; + padding: 20px 28px 28px; +} + +.settings-page-body--fill { + overflow: hidden; +} + +.settings-panel { + background: #ffffff; + border: 1px solid #e3e8ef; + border-radius: 12px; + display: flex; + flex-direction: column; + gap: 14px; + max-width: 720px; + padding: 18px 20px; +} + +.settings-panel-title { + font-size: 14px; + font-weight: 700; + margin: 0; +} + +.copmod-year-list { + display: flex; + flex-direction: column; + gap: 10px; +} + +.copmod-year-row { + align-items: center; + display: grid; + gap: 10px; + grid-template-columns: 64px minmax(0, 1fr) auto; +} + +.copmod-year-label { + color: #334155; + font-size: 13px; + font-weight: 700; +} + +.copmod-add-row { + align-items: center; + display: flex; + gap: 10px; + margin-top: 4px; +} + +.copmod-add-year { + max-width: 120px; +} + +.cleaning-copmod-fecha { + display: flex; + flex-direction: column; + gap: 6px; + margin-top: 12px; + max-width: 420px; +} + +.settings-status-summary { + color: #15803d; + font-size: 12px; + font-weight: 600; +} + +.settings-status-card--error .settings-status-summary { + color: #b91c1c; +} + +.settings-service-grid { + display: grid; + gap: 10px; + grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); + max-width: 720px; +} + +.settings-service-chip { + align-items: center; + background: #ffffff; + border: 1px solid #e3e8ef; + border-radius: 10px; + display: flex; + gap: 10px; + padding: 12px 14px; +} + +.settings-service-chip--ok { + background: #f0fdf6; + border-color: #bbf7d0; +} + +.settings-service-chip--err { + background: #fff5f5; + border-color: #fecaca; +} + +.settings-service-chip-dot { + background: #94a3b8; + border-radius: 50%; + flex-shrink: 0; + height: 8px; + width: 8px; +} + +.settings-service-chip--ok .settings-service-chip-dot { + background: #16a34a; +} + +.settings-service-chip--err .settings-service-chip-dot { + background: #dc2626; +} + +.settings-service-chip-text { + display: flex; + flex-direction: column; + gap: 2px; + min-width: 0; +} + +.settings-service-chip-text strong { + color: #18212f; + font-size: 13px; + font-weight: 700; +} + +.settings-service-chip-text span { + color: #64748b; + font-size: 11.5px; +} + +.settings-url-row { + align-items: stretch; + display: flex; + gap: 10px; +} + +.settings-url-row .form-input { + flex: 1; + min-width: 0; +} + +.settings-url-row .btn { + flex-shrink: 0; +} + .topbar-inline { align-items: center; display: flex; @@ -1520,7 +1723,7 @@ th { align-items: center; display: flex; flex-shrink: 0; - padding: 0 24px 24px; + padding: 20px 24px 24px; } .stepper-item { @@ -1620,6 +1823,15 @@ th { padding: 0 2px 8px; } +.job-extraction-results-pane { + display: flex; + flex: 1; + flex-direction: column; + min-height: 0; + overflow: auto; + padding: 8px 4px 16px; +} + .wiz-step { display: flex; flex-direction: column; @@ -1995,6 +2207,52 @@ th { padding: 10px 16px; } +.wiz-rediscover-card { + align-items: flex-start; + background: #f8fafc; + border: 1px solid #e3e8ef; + border-radius: 10px; + display: flex; + flex-wrap: wrap; + gap: 12px; + justify-content: space-between; + margin-top: 14px; + max-width: 560px; + padding: 14px 16px; +} + +.wiz-rediscover-card--compact { + margin-bottom: 12px; + margin-top: 0; + max-width: none; +} + +.wiz-rediscover-text { + display: flex; + flex: 1; + flex-direction: column; + gap: 4px; + min-width: 180px; +} + +.wiz-rediscover-text strong { + color: #18212f; + font-size: 13px; + font-weight: 700; +} + +.wiz-rediscover-text span { + color: #64748b; + font-size: 12px; + line-height: 1.45; +} + +.wiz-rediscover-text em { + color: #334155; + font-style: normal; + font-weight: 600; +} + /* ═══════════════════════════════════════════ Review grid (step 4) ═══════════════════════════════════════════ */ @@ -2496,9 +2754,13 @@ th { .count-results { display: flex; + flex: 1 1 auto; flex-direction: column; gap: 16px; margin-top: 12px; + min-height: 0; + overflow: hidden; + padding: 0 16px 16px; } /* Summary stat overrides */ @@ -2626,7 +2888,9 @@ th { .count-table-wrap { border: 1px solid #e8edf4; border-radius: 10px; - overflow: hidden; + flex: 1 1 auto; + min-height: 0; + overflow: auto; } .count-table { @@ -3332,7 +3596,7 @@ th { background: none; border: none; border-radius: 6px; - color: var(--text-muted); + color: #8a9ab4; cursor: pointer; display: flex; font-size: 12.5px; @@ -3344,27 +3608,33 @@ th { } .side-job-back:hover { - background: var(--border); - color: var(--text); + background: rgba(255, 255, 255, 0.05); + color: #c8d5e4; } .side-job-name { - color: var(--text); - font-size: 13px; - font-weight: 700; - margin: 4px 12px 6px; + color: #c8d5e4; + font-size: 12.5px; + font-weight: 600; + line-height: 1.35; + margin: 2px 12px 10px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .side-section-divider { - background: var(--border); + background: rgba(255, 255, 255, 0.06); + flex-shrink: 0; height: 1px; - margin: 4px 10px; + margin: 8px 10px; } .side-nav--bottom { + display: flex; + flex-direction: column; + flex-shrink: 0; + gap: 2px; margin-top: auto; } @@ -3700,8 +3970,49 @@ th { .task-badge--failed { background: #fee2e2; color: #b91c1c; } /* ── Results table (extracción) ── */ +.extraction-results { + display: flex; + flex: 1 1 auto; + flex-direction: column; + gap: 16px; + min-height: 0; + overflow: hidden; + padding: 12px 16px 16px; +} + +.extraction-table-wrap { + border: 1px solid #e8edf4; + border-radius: 10px; + flex: 1 1 auto; + min-height: 0; + overflow: auto; + padding: 0; +} + +.extraction-empty { + color: var(--text-muted); + padding: 28px 12px !important; + text-align: center; +} + +.extraction-empty-cell { + color: var(--text-muted); +} + +.extraction-row--incomplete td { + background: #fefce8; +} + +.extraction-filter-hint { + color: var(--text-muted); + flex-shrink: 0; + font-size: 12px; + margin: 0; +} + .results-table-wrapper { flex: 1; + min-height: 0; overflow: auto; padding: 0 16px 16px; } @@ -3953,3 +4264,445 @@ th { margin-left: auto; padding: 3px 10px; } + +/* ── Cleaning wizard: lista de extracciones / reglas ── */ +.cleaning-source-list { + display: flex; + flex-direction: column; + gap: 8px; + max-height: min(280px, 36vh); + overflow-y: auto; +} + +.cleaning-source-item { + align-items: center; + background: #fff; + border: 1px solid #e2e8f0; + border-radius: 10px; + cursor: pointer; + display: flex; + gap: 12px; + justify-content: space-between; + padding: 12px 14px; + text-align: left; + transition: border-color 0.15s, background 0.15s; + width: 100%; +} + +.cleaning-source-item:hover { + border-color: #94a3b8; + background: #f8fafc; +} + +.cleaning-source-item--selected { + border-color: #2563eb; + background: #eff6ff; + box-shadow: 0 0 0 1px #2563eb; +} + +.cleaning-source-item-main { + align-items: flex-start; + display: flex; + gap: 10px; + min-width: 0; +} + +.cleaning-source-item-title { + color: #0f172a; + font-size: 13px; + font-weight: 600; +} + +.cleaning-source-item-meta { + color: #64748b; + font-size: 12px; + margin-top: 2px; +} + +/* ── Paso Reglas: lista fija + detalle con scroll ── */ +.wiz-step--cleaning-rules { + display: flex; + flex: 1; + flex-direction: column; + gap: 12px; + min-height: 0; +} + +.cleaning-rules-toolbar { + align-items: flex-start; + display: flex; + flex-shrink: 0; + gap: 12px; + justify-content: space-between; +} + +.cleaning-rules-workspace { + display: grid; + flex: 1; + gap: 12px; + grid-template-columns: minmax(160px, 220px) minmax(0, 1fr); + min-height: 0; + overflow: hidden; +} + +.cleaning-col-nav { + background: #f8fafc; + border: 1px solid #e2e8f0; + border-radius: 10px; + display: flex; + flex-direction: column; + gap: 2px; + min-height: 0; + overflow-y: auto; + padding: 6px; +} + +.cleaning-col-nav-item { + align-items: center; + background: transparent; + border: none; + border-radius: 8px; + color: #334155; + cursor: pointer; + display: flex; + gap: 8px; + justify-content: space-between; + padding: 8px 10px; + text-align: left; + width: 100%; +} + +.cleaning-col-nav-item:hover { + background: #eef2f7; +} + +.cleaning-col-nav-item--active { + background: #fff; + box-shadow: 0 0 0 1px #2563eb; + color: #0f172a; +} + +.cleaning-col-nav-item--changed:not(.cleaning-col-nav-item--active) { + background: #eff6ff; +} + +.cleaning-col-nav-name { + font-size: 12px; + font-weight: 600; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.cleaning-col-nav-meta { + align-items: center; + display: flex; + flex-shrink: 0; + gap: 4px; +} + +.cleaning-col-nav-count { + background: #2563eb; + border-radius: 999px; + color: #fff; + font-size: 10px; + font-weight: 700; + min-width: 18px; + padding: 1px 6px; + text-align: center; +} + +.cleaning-col-detail { + background: #fff; + border: 1px solid #e2e8f0; + border-radius: 10px; + display: flex; + flex-direction: column; + gap: 12px; + min-height: 0; + min-width: 0; + overflow-x: hidden; + overflow-y: auto; + padding: 12px 14px; +} + +.cleaning-col-detail-header, +.cleaning-preview-compare--compact, +.cleaning-rules-block, +.cleaning-copmod-fecha { + flex-shrink: 0; +} + +.cleaning-col-detail-title { + font-size: 15px; + font-weight: 700; + margin: 0; +} + +.cleaning-rules-block { + display: flex; + flex-direction: column; + gap: 6px; +} + +.cleaning-rule-desc { + margin: 0; +} + +.cleaning-preview-compare--compact { + background: #f8fafc; + border: 1px solid #e8edf4; + border-radius: 8px; + margin: 0; + padding: 10px; +} + +.cleaning-preview-compare--changed { + background: #f0f7ff; + border-color: #bfdbfe; +} + +.cleaning-sample-dock { + border-top: 1px solid #e8edf4; + display: flex; + flex: 0 0 auto; + flex-direction: column; + gap: 8px; + min-height: 140px; + padding-top: 10px; +} + +.cleaning-sample-dock-header { + align-items: center; + display: flex; + flex-shrink: 0; + gap: 12px; + justify-content: space-between; +} + +.cleaning-sample-tabs { + background: #eef2f7; + border-radius: 8px; + display: inline-flex; + padding: 2px; +} + +.cleaning-sample-tab { + background: transparent; + border: none; + border-radius: 6px; + color: #64748b; + cursor: pointer; + font-size: 12px; + font-weight: 600; + padding: 4px 10px; +} + +.cleaning-sample-tab:disabled { + cursor: not-allowed; + opacity: 0.45; +} + +.cleaning-sample-tab--active { + background: #fff; + box-shadow: 0 1px 2px rgba(15, 23, 42, 0.08); + color: #0f172a; +} + +.cleaning-sample-table-scroll { + max-height: 220px; + min-height: 0; + overflow: auto; +} + +.cleaning-rules-nav { + flex-shrink: 0; + gap: 8px; + margin-top: 0; +} + +.cleaning-type-badge { + background: #e2e8f0; + border-radius: 999px; + color: #334155; + font-size: 10px; + font-weight: 600; + padding: 2px 7px; +} + +.cleaning-rule-card-meta { + align-items: center; + color: #64748b; + display: flex; + flex-wrap: wrap; + font-size: 12px; + gap: 8px; + margin-top: 4px; +} + +.cleaning-rule-chips { + display: flex; + flex-wrap: wrap; + gap: 6px; +} + +.cleaning-preview-compare { + align-items: center; + display: grid; + gap: 8px; + grid-template-columns: 1fr auto 1fr; +} + +.cleaning-preview-side { + min-width: 0; +} + +.cleaning-preview-label { + color: #64748b; + font-size: 11px; + font-weight: 700; + letter-spacing: 0.02em; + margin-bottom: 6px; + text-transform: uppercase; +} + +.cleaning-preview-label--after { + color: #1d4ed8; +} + +.cleaning-preview-arrow { + color: #94a3b8; + flex-shrink: 0; +} + +.cleaning-preview-values { + display: flex; + flex-wrap: wrap; + gap: 4px; +} + +.cleaning-preview-chip { + background: #f1f5f9; + border: 1px solid #e2e8f0; + border-radius: 6px; + color: #0f172a; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 11px; + max-width: 100%; + overflow: hidden; + padding: 3px 7px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.cleaning-preview-chip--changed { + background: #dbeafe; + border-color: #93c5fd; + color: #1e3a8a; + font-weight: 600; +} + +.cleaning-preview-chip-col { + color: #64748b; + font-weight: 500; + margin-right: 6px; +} + +.cleaning-preview-empty { + color: #94a3b8; + font-size: 12px; +} + +.cleaning-preview-table { + font-size: 12px; +} + +.cleaning-th-changed, +.cleaning-td-changed { + background: #eff6ff; +} + +@media (max-width: 900px) { + .cleaning-rules-workspace { + grid-template-columns: 1fr; + overflow: auto; + } + + .cleaning-col-nav { + display: grid; + grid-auto-flow: column; + grid-auto-columns: max-content; + max-height: none; + overflow-x: auto; + overflow-y: hidden; + } +} + +/* ── Auto-update modal ── */ +.update-modal-backdrop { + align-items: center; + background: rgba(15, 23, 42, 0.45); + display: flex; + inset: 0; + justify-content: center; + position: fixed; + z-index: 1000; +} + +.update-modal { + background: #ffffff; + border-radius: 12px; + box-shadow: 0 20px 50px rgba(15, 23, 42, 0.25); + display: flex; + flex-direction: column; + gap: 12px; + max-width: 440px; + padding: 20px 22px; + width: calc(100% - 32px); +} + +.update-modal-header { + align-items: center; + display: flex; + justify-content: space-between; + gap: 12px; +} + +.update-modal-header h2 { + font-size: 16px; + font-weight: 700; + margin: 0; +} + +.update-modal-body { + color: #334155; + font-size: 13.5px; + line-height: 1.5; + margin: 0; +} + +.update-modal-actions { + display: flex; + gap: 8px; + justify-content: flex-end; + margin-top: 8px; +} + +.update-progress { + display: flex; + flex-direction: column; + gap: 6px; +} + +.update-progress-bar { + background: #e2e8f0; + border-radius: 999px; + height: 8px; + overflow: hidden; +} + +.update-progress-fill { + background: #2563eb; + height: 100%; + transition: width 0.2s ease; +} diff --git a/services/desktop_app/src/renderer/types/jobSection.ts b/services/desktop_app/src/renderer/types/jobSection.ts index fb2469b..c37a28a 100644 --- a/services/desktop_app/src/renderer/types/jobSection.ts +++ b/services/desktop_app/src/renderer/types/jobSection.ts @@ -1 +1 @@ -export type JobSection = "counting" | "extraction" | "results"; +export type JobSection = "counting" | "extraction" | "cleaning" | "results"; diff --git a/services/desktop_app/src/renderer/views/JobResultsView.tsx b/services/desktop_app/src/renderer/views/JobResultsView.tsx index e4e3eed..7a2dd16 100644 --- a/services/desktop_app/src/renderer/views/JobResultsView.tsx +++ b/services/desktop_app/src/renderer/views/JobResultsView.tsx @@ -8,14 +8,17 @@ import { Loader2, Plus, RefreshCw, + Sparkles, } from "lucide-react"; import { useEffect, useState } from "react"; -import type { BackgroundProgress, CountJobFile, Job, Task } from "../../shared/types"; +import type { BackgroundProgress, CleaningJob, CountJobFile, Job, Task } from "../../shared/types"; import { CountResultsTable } from "../components/CountResultsTable"; +import { ExtractionResultsTable } from "../components/ExtractionResultsTable"; import { BackgroundProgressBanner } from "../components/BackgroundProgressBanner"; import type { ApiClient } from "../lib/apiClient"; +import { downloadCsv, toCsv, buildExportCsvFilename } from "../lib/csv"; -type ResultTab = "counting" | "extraction"; +type ResultTab = "counting" | "extraction" | "cleaning"; interface Props { client: ApiClient; @@ -23,9 +26,11 @@ interface Props { tasks: Task[]; countingProgress?: BackgroundProgress | null; extractionProgress?: BackgroundProgress | null; + cleaningProgress?: BackgroundProgress | null; onRefresh: () => Promise; onStartCounting: () => void; onStartExtraction: (ctx?: Record) => void; + onStartCleaning: (ctx?: Record) => void; } export function JobResultsView({ @@ -34,9 +39,11 @@ export function JobResultsView({ tasks, countingProgress, extractionProgress, + cleaningProgress, onRefresh, onStartCounting, onStartExtraction, + onStartCleaning, }: Props) { const [activeTab, setActiveTab] = useState("counting"); const [refreshing, setRefreshing] = useState(false); @@ -45,6 +52,8 @@ export function JobResultsView({ .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); const extractionTasks = tasks.filter((t) => t.type === "extraction") .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); + const cleaningTasks = tasks.filter((t) => t.type === "cleaning") + .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); async function refresh() { setRefreshing(true); @@ -54,7 +63,7 @@ export function JobResultsView({ return (
- {/* ── Sub-tabs: Conteo | Extracción ── */} + {/* ── Sub-tabs: Conteo | Extracción | Limpieza ── */}
+
@@ -129,12 +165,14 @@ export function JobResultsView({ ══════════════════════════════════════════ */ function CountingResultsSection({ client, + job, tasks, liveProgress, onStartCounting, onExtractFromCount, }: { client: ApiClient; + job: Job; tasks: Task[]; liveProgress?: BackgroundProgress | null; onStartCounting: () => void; @@ -175,17 +213,15 @@ function CountingResultsSection({ function exportCsv(subset: "valid" | "invalid") { const subset_files = subset === "valid" ? files.filter(isFileValid) : files.filter((f) => !isFileValid(f)); const headers = ["filename", "radicado", "service_count", "status", "is_valid", "error"]; - const rows = subset_files.map((f) => - [`"${f.filename}"`, `"${f.radicado ?? ""}"`, f.service_count ?? "", f.status, f.is_valid === null ? "" : String(f.is_valid), `"${(f.error ?? "").replace(/"/g, '""')}"`].join(","), - ); - const csv = [headers.join(","), ...rows].join("\n"); - const blob = new Blob([csv], { type: "text/csv" }); - const url = URL.createObjectURL(blob); - const a = document.createElement("a"); - a.href = url; - a.download = `conteo-${subset}-${new Date().toISOString().slice(0, 10)}.csv`; - a.click(); - URL.revokeObjectURL(url); + const rows = subset_files.map((f) => [ + f.filename, + f.radicado ?? "", + f.service_count ?? "", + f.status, + f.is_valid === null ? "" : String(f.is_valid), + f.error ?? "", + ]); + downloadCsv(buildExportCsvFilename("conteo", job.title, subset), toCsv(headers, rows)); } if (tasks.length === 0 && !liveProgress?.running) { @@ -291,14 +327,18 @@ type Row = Record; function ExtractionResultsSection({ client, + job, tasks, liveProgress, onStartExtraction, + onStartCleaning, }: { client: ApiClient; + job: Job; tasks: Task[]; liveProgress?: BackgroundProgress | null; onStartExtraction: () => void; + onStartCleaning: (ctx?: Record) => void; }) { const [selectedTask, setSelectedTask] = useState(tasks[0] ?? null); const [rows, setRows] = useState([]); @@ -344,20 +384,11 @@ function ExtractionResultsSection({ }, [selectedTask?.id]); // eslint-disable-line react-hooks/exhaustive-deps function exportCsv() { - const csv = [columns.join(","), ...rows.map((r) => - columns.map((c) => { - const v = r[c]; - if (v === null || v === undefined) return ""; - return `"${String(v).replace(/"/g, '""')}"`; - }).join(","), - )].join("\n"); - const blob = new Blob([csv], { type: "text/csv" }); - const url = URL.createObjectURL(blob); - const a = document.createElement("a"); - a.href = url; - a.download = `extraccion-${new Date().toISOString().slice(0, 10)}.csv`; - a.click(); - URL.revokeObjectURL(url); + const csv = toCsv( + columns, + rows.map((r) => columns.map((c) => r[c])), + ); + downloadCsv(buildExportCsvFilename("extraccion", job.title), csv); } if (tasks.length === 0 && !liveProgress?.running) { @@ -422,6 +453,24 @@ function ExtractionResultsSection({ CSV ({rows.length} filas) )} + {rows.length > 0 && selectedTask?.status === "completed" && ( + + )} @@ -468,26 +517,162 @@ function ExtractionResultsSection({

No hay filas extraídas.

) : ( -
- - - {columns.map((c) => )} - - - {rows.map((row, i) => ( - - {columns.map((c) => ( - - ))} - + + )} + + ); +} + +/* ══════════════════════════════════════════ + Sección de resultados de limpieza +══════════════════════════════════════════ */ +function CleaningResultsSection({ + client, + job, + tasks, + liveProgress, + onStartCleaning, +}: { + client: ApiClient; + job: Job; + tasks: Task[]; + liveProgress?: BackgroundProgress | null; + onStartCleaning: () => void; +}) { + const [selectedTask, setSelectedTask] = useState(tasks[0] ?? null); + const [cleaningJob, setCleaningJob] = useState(null); + const [rows, setRows] = useState([]); + const [columns, setColumns] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + setSelectedTask(tasks[0] ?? null); + }, [tasks.map((t) => t.id).join(",")]); // eslint-disable-line react-hooks/exhaustive-deps + + useEffect(() => { + if (!selectedTask) { + setRows([]); + setColumns([]); + setCleaningJob(null); + return; + } + let cancelled = false; + async function load() { + setLoading(true); + setError(null); + try { + const cleaningJobId = String( + (selectedTask!.result as Record)?.cleaning_job_id ?? "", + ); + if (!cleaningJobId) { + setError("La tarea no tiene cleaning_job_id"); + return; + } + const jobData = await client.cleaningJob(cleaningJobId); + if (cancelled) return; + setCleaningJob(jobData); + if (jobData.status === "completed") { + const resp = await client.cleaningJobRows(cleaningJobId, 500, 0); + if (cancelled) return; + const data = resp.rows ?? []; + setRows(data); + setColumns(Array.from(new Set(data.flatMap((r) => Object.keys(r))))); + } + } catch (e) { + if (!cancelled) setError(String(e instanceof Error ? e.message : e)); + } finally { + if (!cancelled) setLoading(false); + } + } + void load(); + return () => { cancelled = true; }; + }, [selectedTask?.id, client]); // eslint-disable-line react-hooks/exhaustive-deps + + function exportCsv() { + const csv = toCsv( + columns, + rows.map((r) => columns.map((h) => r[h])), + ); + downloadCsv(buildExportCsvFilename("limpieza", job.title), csv); + } + + if (tasks.length === 0 && !liveProgress?.running) { + return ( +
+ +

No hay resultados de limpieza todavía.

+ +
+ ); + } + + if (tasks.length === 0 && liveProgress?.running) { + return ( +
+ +
+ ); + } + + return ( +
+ {liveProgress?.running && ( +
+ +
+ )} +
+
+ {tasks.length > 1 && ( +
-
{c}
- {row[c] === null || row[c] === undefined - ? — - : String(row[c])} -
+ + )} + {selectedTask && ( + + {cleaningJob?.status ?? selectedTask.status} + + )} +
+ +
+ {rows.length > 0 && ( + + )} + +
+
+ + {error && ( +
+ {error} +
+ )} + + {loading && rows.length === 0 ? ( +
+ ) : rows.length === 0 ? ( +
+

No hay filas limpiadas.

+ ) : ( + )}
); diff --git a/services/desktop_app/src/renderer/views/JobView.tsx b/services/desktop_app/src/renderer/views/JobView.tsx index a1072fc..75b4f26 100644 --- a/services/desktop_app/src/renderer/views/JobView.tsx +++ b/services/desktop_app/src/renderer/views/JobView.tsx @@ -5,6 +5,7 @@ import type { BackgroundProgress, Job, Task } from "../../shared/types"; import type { JobSection } from "../types/jobSection"; import { BackgroundProgressBanner } from "../components/BackgroundProgressBanner"; import { CountingWizardModule } from "../modules/counting/CountingWizardModule"; +import { CleaningWizardModule } from "../modules/cleaning/CleaningWizardModule"; import { JobExtractionWizard } from "../modules/extraction/JobExtractionWizard"; import { JobResultsView } from "./JobResultsView"; @@ -24,10 +25,13 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio const [savingTitle, setSavingTitle] = useState(false); const [countingProgress, setCountingProgress] = useState(null); const [extractionProgress, setExtractionProgress] = useState(null); + const [cleaningProgress, setCleaningProgress] = useState(null); const pollingRef = useRef | null>(null); const backgroundRunning = - countingProgress?.running === true || extractionProgress?.running === true; + countingProgress?.running === true || + extractionProgress?.running === true || + cleaningProgress?.running === true; const loadTasks = useCallback(async () => { try { @@ -74,6 +78,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio const activeBackground = countingProgress?.running ? countingProgress : extractionProgress?.running ? extractionProgress + : cleaningProgress?.running ? cleaningProgress : null; return ( @@ -114,6 +119,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
{section === "counting" && "Conteo de servicios"} {section === "extraction" && "Extracción de datos"} + {section === "cleaning" && "Limpieza de datos"} {section === "results" && "Resultados del trabajo"}
@@ -161,6 +167,18 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio />
+
+ setCleaningProgress(p.running ? p : null)} + onActivateModule={(type, ctx) => onActivateSection(type as JobSection, ctx)} + context={sectionContext} + /> +
+
onActivateSection("counting")} onStartExtraction={(ctx?: Record) => onActivateSection("extraction", ctx)} + onStartCleaning={(ctx?: Record) => onActivateSection("cleaning", ctx)} />
diff --git a/services/desktop_app/src/renderer/views/SettingsView.tsx b/services/desktop_app/src/renderer/views/SettingsView.tsx new file mode 100644 index 0000000..251c3c4 --- /dev/null +++ b/services/desktop_app/src/renderer/views/SettingsView.tsx @@ -0,0 +1,471 @@ +import { + CheckCircle2, + Database, + Loader2, + Plus, + RotateCcw, + Server, + Trash2, + XCircle, +} from "lucide-react"; +import { useEffect, useState } from "react"; +import { DEFAULT_API_BASE_URL } from "../lib/apiDefaults"; +import { + type CopModConfig, + defaultCopModConfig, + loadCopModConfig, + saveCopModConfig, +} from "../lib/cleaningSettings"; + +export type SettingsSection = "connection" | "cleaning" | "backend"; +type Status = "idle" | "ok" | "error" | "busy"; +type BackendTab = "config" | "models"; + +type SettingsViewProps = { + section: SettingsSection; + apiBaseUrl: string; + onApiBaseUrlChange: (url: string) => void; + connectionStatus: Status; + connectionMessage: string; + healthJson?: unknown; + verifyConnection: () => Promise; + busy: boolean; + configJson: unknown; + modelsJson: unknown; +}; + +const SERVICE_LABELS: Record = { + extraction: "Extracción", + cleaning: "Limpieza", + counting: "Conteo", +}; + +type ServiceRow = { id: string; label: string; ok: boolean; detail: string }; + +function parseHealthServices(health: unknown): ServiceRow[] { + if (!health || typeof health !== "object") return []; + const services = (health as { services?: unknown }).services; + if (!services || typeof services !== "object") return []; + return Object.entries(services as Record).map(([id, raw]) => { + const info = raw && typeof raw === "object" ? (raw as Record) : {}; + const ok = Boolean(info.ok); + const code = info.status_code != null ? String(info.status_code) : ""; + const err = typeof info.error === "string" ? info.error : ""; + return { + id, + label: SERVICE_LABELS[id] ?? id, + ok, + detail: ok ? (code ? `HTTP ${code}` : "OK") : err || (code ? `HTTP ${code}` : "Error"), + }; + }); +} + +export function SettingsView({ + section, + apiBaseUrl, + onApiBaseUrlChange, + connectionStatus, + connectionMessage, + healthJson, + verifyConnection, + busy, + configJson, + modelsJson, +}: SettingsViewProps) { + if (section === "connection") { + return ( + + ); + } + if (section === "cleaning") { + return ; + } + return ( + + ); +} + +function ConnectionSettings({ + apiBaseUrl, + onApiBaseUrlChange, + connectionStatus, + connectionMessage, + healthJson, + verifyConnection, + busy, +}: { + apiBaseUrl: string; + onApiBaseUrlChange: (url: string) => void; + connectionStatus: Status; + connectionMessage: string; + healthJson?: unknown; + verifyConnection: () => Promise; + busy: boolean; +}) { + const connectionLabel = + connectionStatus === "ok" + ? "Conectado" + : connectionStatus === "busy" + ? "Verificando…" + : connectionStatus === "error" + ? "Sin conexión" + : "Sin verificar"; + + const services = parseHealthServices(healthJson); + const allOk = services.length > 0 && services.every((s) => s.ok); + + return ( +
+
+

Conexión

+

URL del API Gateway y estado de los servicios del backend.

+
+ +
+
+
+ {connectionStatus === "ok" ? ( + + ) : connectionStatus === "error" ? ( + + ) : connectionStatus === "busy" ? ( + + ) : ( + + )} +
+
+ {connectionLabel} + {connectionMessage} + {connectionStatus === "ok" && services.length > 0 && ( + + {allOk + ? `${services.length} servicios operativos` + : `${services.filter((s) => s.ok).length} de ${services.length} servicios OK`} + + )} +
+
+ + {services.length > 0 && connectionStatus === "ok" && ( +
+ {services.map((svc) => ( +
+ +
+ {svc.label} + {svc.detail} +
+
+ ))} +
+ )} + +
+
+ +
+ onApiBaseUrlChange(e.target.value)} + placeholder={DEFAULT_API_BASE_URL} + spellCheck={false} + /> + +
+

Los cambios se guardan automáticamente en este equipo.

+
+
+
+
+ ); +} + +function parseAmountList(text: string): number[] { + return [ + ...new Set( + text + .split(/[,;\s]+/) + .map((s) => s.trim()) + .filter(Boolean) + .map((s) => Math.round(Number(s.replace(/\./g, "").replace(",", ".")))) + .filter((n) => Number.isFinite(n) && n >= 0), + ), + ].sort((a, b) => a - b); +} + +function CleaningSettings() { + const [config, setConfig] = useState(() => loadCopModConfig()); + const [drafts, setDrafts] = useState>(() => { + const initial = loadCopModConfig(); + return Object.fromEntries( + Object.entries(initial.moderadora_by_year).map(([y, vals]) => [y, vals.join(", ")]), + ); + }); + const [savedFlash, setSavedFlash] = useState(false); + const [newYear, setNewYear] = useState(""); + const [hydrated, setHydrated] = useState(false); + + useEffect(() => { + setHydrated(true); + }, []); + + useEffect(() => { + if (!hydrated) return; + saveCopModConfig(config); + setSavedFlash(true); + const t = window.setTimeout(() => setSavedFlash(false), 1200); + return () => window.clearTimeout(t); + }, [config, hydrated]); + + const years = Object.keys(config.moderadora_by_year).sort( + (a, b) => Number(a) - Number(b), + ); + + function commitYear(year: string, text: string) { + const nums = parseAmountList(text); + setDrafts((prev) => ({ ...prev, [year]: nums.join(", ") })); + setConfig((prev) => ({ + moderadora_by_year: { + ...prev.moderadora_by_year, + [year]: nums, + }, + })); + } + + function removeYear(year: string) { + setConfig((prev) => { + const next = { ...prev.moderadora_by_year }; + delete next[year]; + return { moderadora_by_year: next }; + }); + setDrafts((prev) => { + const next = { ...prev }; + delete next[year]; + return next; + }); + } + + function addYear() { + const y = String(Number.parseInt(newYear, 10)); + if (!Number.isFinite(Number(y)) || y === "NaN") return; + if (config.moderadora_by_year[y]) { + setNewYear(""); + return; + } + setConfig((prev) => ({ + moderadora_by_year: { ...prev.moderadora_by_year, [y]: [] }, + })); + setDrafts((prev) => ({ ...prev, [y]: "" })); + setNewYear(""); + } + + function resetDefaults() { + const defaults = defaultCopModConfig(); + setConfig(defaults); + setDrafts( + Object.fromEntries( + Object.entries(defaults.moderadora_by_year).map(([y, vals]) => [y, vals.join(", ")]), + ), + ); + } + + return ( +
+
+
+
+

Limpieza · COP / MOD

+

+ Cuotas moderadora por año. La regla «Separa COP/MOD» clasifica el importe + según el año de la fecha de servicio que elijas en el wizard. +

+
+
+ {savedFlash && Guardado} + +
+
+
+ +
+
+

Cuotas moderadora por año

+

+ Separa los valores con coma o espacio. Si el importe coincide con uno de + estos montos en el año de la fecha, se marca como moderadora; si no, como + copago. +

+ +
+ {years.length === 0 && ( +

No hay años configurados. Añade uno o restaura los defaults.

+ )} + {years.map((year) => ( +
+ + + setDrafts((prev) => ({ ...prev, [year]: e.target.value })) + } + onBlur={(e) => commitYear(year, e.target.value)} + placeholder="ej. 5000, 20100, 52800" + /> + +
+ ))} +
+ +
+ setNewYear(e.target.value)} + placeholder="Año" + inputMode="numeric" + aria-label="Nuevo año" + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + addYear(); + } + }} + /> + +
+
+
+
+ ); +} + +function BackendSettings({ + connectionStatus, + configJson, + modelsJson, + verifyConnection, + busy, +}: { + connectionStatus: Status; + configJson: unknown; + modelsJson: unknown; + verifyConnection: () => Promise; + busy: boolean; +}) { + const [backendTab, setBackendTab] = useState("config"); + const backendData = backendTab === "config" ? configJson : modelsJson; + + return ( +
+
+
+
+

Backend

+

Respuesta en vivo de /config y /models del gateway.

+
+ +
+
+ +
+
+
+ +

Datos del servidor

+
+
+ + +
+
+ +
+ {backendData != null ? ( + 
{JSON.stringify(backendData, null, 2)}
+ ) : ( +
+ +

Sin datos del backend

+ + {connectionStatus === "ok" + ? "Pulsa Actualizar para cargar los datos." + : "Verifica la conexión en la sección Conexión."} + +
+ )} +
+
+
+ ); +} diff --git a/services/desktop_app/src/shared/types.ts b/services/desktop_app/src/shared/types.ts index bbc3d67..5873b12 100644 --- a/services/desktop_app/src/shared/types.ts +++ b/services/desktop_app/src/shared/types.ts @@ -164,6 +164,83 @@ export interface CountSyncResponse { failed: number; } +export interface CleaningRuleInfo { + id: string; + name: string; + description: string; +} + +export interface CleaningColumnAnalysis { + name: string; + inferred_type: string; + example?: string | null; + samples?: string[]; + null_count?: number; + null_pct?: number; + unique_count?: number; + applicable_rules?: string[]; + [key: string]: unknown; +} + +export interface CleaningAnalysisResponse { + batch_id?: string; + filename?: string; + delimiter?: string; + row_count?: number; + columns: CleaningColumnAnalysis[]; + sample_rows?: Array>; +} + +export interface CleaningPreviewColumn { + name: string; + before: string[]; + after: string[]; + changed?: boolean; +} + +export interface CleaningPreviewResponse { + rows_before: Array>; + rows_after: Array>; + columns: CleaningPreviewColumn[]; +} + +export interface CleaningPreviewPayload { + rows?: Array>; + extraction_batch_id?: string; + rules_per_column: Record; + cop_mod_fecha_column?: Record; + cop_mod_config?: Record | null; + sample_size?: number; +} + +export interface CleaningJob { + id: string; + extraction_batch_id: string | null; + job_id: string | null; + source: "extraction" | "csv" | string; + status: "pending" | "running" | "completed" | "failed" | string; + request: Record; + log: string[]; + error: string | null; + created_at: string; + updated_at: string; +} + +export interface RunCleaningPayload { + extraction_batch_id: string; + job_id?: string; + rules_per_column: Record; + cop_mod_fecha_column?: Record; + cop_mod_config?: Record | null; +} + +export interface CleaningRowsResponse { + job_id: string; + rows: Array>; + limit: number; + offset: number; +} + export interface CountingFormState { localRoot: string; containerRoot: string; @@ -174,7 +251,7 @@ export interface CountingFormState { /** Estado de progreso de un módulo ejecutándose en segundo plano. */ export interface BackgroundProgress { - module: "counting" | "extraction"; + module: "counting" | "extraction" | "cleaning"; total: number; processed: number; failed: number; @@ -189,6 +266,31 @@ export interface RuntimeConfig { documentRootContainer: string; } +export interface UpdateManifest { + version: string; + releaseDate?: string; + /** Fecha de lanzamiento como en DATA.txt, ej. "06/07/2026" */ + releaseDateDisplay?: string; + fileName: string; + url: string; + sha256: string; + size?: number; +} + +export interface UpdateCheckResult { + updateAvailable: boolean; + currentVersion: string; + manifest?: UpdateManifest; + error?: string; +} + +export interface UpdateProgress { + phase: "downloading" | "verifying" | "installing" | string; + received?: number; + total?: number; + percent?: number; +} + export interface DesktopBridge { getRuntimeConfig(): Promise; selectDirectory(): Promise; @@ -202,4 +304,7 @@ export interface DesktopBridge { keywords: string[], ): Promise; readFile(filePath: string): Promise; + checkForUpdates(): Promise; + installUpdate(manifest: UpdateManifest): Promise<{ ok: boolean; destPath?: string }>; + onUpdateProgress(callback: (progress: UpdateProgress) => void): () => void; } diff --git a/services/extraction/api/main.py b/services/extraction/api/main.py index 7395b7f..5d3eb7c 100644 --- a/services/extraction/api/main.py +++ b/services/extraction/api/main.py @@ -1126,8 +1126,7 @@ def create_count_job(request: CreateCountJobRequest) -> dict[str, Any]: job = session.get(Job, request.job_id) if job is None: raise HTTPException(status_code=404, detail="Job no encontrado") - if request.title: - job.title = request.title + # No sobrescribir el nombre que el usuario dio al trabajo. job.status = "running" job.updated_at = utcnow() else: diff --git a/services/extraction/infrastructure/database.py b/services/extraction/infrastructure/database.py index f521832..dfbc08a 100644 --- a/services/extraction/infrastructure/database.py +++ b/services/extraction/infrastructure/database.py @@ -136,7 +136,13 @@ class CleaningJob(Base): __tablename__ = "cleaning_jobs" id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id) - extraction_batch_id: Mapped[str] = mapped_column(ForeignKey("extraction_batches.id"), index=True) + extraction_batch_id: Mapped[str | None] = mapped_column( + ForeignKey("extraction_batches.id"), nullable=True, index=True + ) + job_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("jobs.id"), nullable=True, index=True + ) + source: Mapped[str] = mapped_column(String(32), default="extraction", index=True) status: Mapped[str] = mapped_column(String(32), default="pending", index=True) request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) log: Mapped[list[str]] = mapped_column(JSON, default=list) @@ -150,7 +156,9 @@ class CleanedRow(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) cleaning_job_id: Mapped[str] = mapped_column(ForeignKey("cleaning_jobs.id"), index=True) - extraction_batch_id: Mapped[str] = mapped_column(ForeignKey("extraction_batches.id"), index=True) + extraction_batch_id: Mapped[str | None] = mapped_column( + ForeignKey("extraction_batches.id"), nullable=True, index=True + ) row_index: Mapped[int] = mapped_column(Integer, default=0) row_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)