feat: Add poblacion_apta service to docker-compose, enhance API gateway for new service integration, and update README with service details. Introduce new database tables for poblacion_apta_jobs and poblacion_apta_rows, and implement functionality for handling cop/mod calculations in the cleaning service.
This commit is contained in:
@@ -12,6 +12,7 @@ Arquitectura de microservicios: cada servicio tiene su propia API, dependencias,
|
||||
| `api_gateway` | **8189** | 8000 | Python/FastAPI | Única puerta de entrada expuesta al host |
|
||||
| `extraction` | — | 8001 | Python/FastAPI | Extracción LLM; solo red Docker |
|
||||
| `cleaning` | — | 8002 | Python/FastAPI | Limpieza; solo red Docker |
|
||||
| `poblacion_apta` | — | 8003 | Python/FastAPI | Población apta (Verificador ST); solo red Docker |
|
||||
| `count-api` | — | 8080 | Go | Conteo OCR+ONNX; solo red Docker |
|
||||
| `database` | — | 5432 | PostgreSQL 16 | Solo red Docker |
|
||||
| `redis` | — | 6379 | Redis 7 | Solo red Docker |
|
||||
@@ -122,6 +123,7 @@ services/
|
||||
api_gateway/ — proxy HTTP (Python) → README
|
||||
extraction/ — LLM + jobs + conteo BD (Python) → README
|
||||
cleaning/ — limpieza de datos (Python) → README
|
||||
poblacion_apta/ — población apta / Verificador ST → README
|
||||
bot_descarga/ — API control remoto de bots → README
|
||||
count-api/ — OCR + ONNX stateless (Go) → README
|
||||
db/ — migraciones Alembic → README
|
||||
@@ -138,12 +140,14 @@ docker-compose.yml — stack completo
|
||||
|
||||
```
|
||||
jobs — contenedor de un trabajo del usuario
|
||||
tasks — módulo ejecutado (counting / extraction / cleaning)
|
||||
tasks — módulo ejecutado (counting / extraction / cleaning / poblacion_apta)
|
||||
count_job_files — resultado de conteo por PDF
|
||||
extraction_batches — lote de extracción
|
||||
extraction_rows — fila extraída por documento
|
||||
cleaning_jobs — job de limpieza
|
||||
cleaned_rows — fila transformada
|
||||
poblacion_apta_jobs — consolidado validado (PASO_BOT)
|
||||
poblacion_apta_rows
|
||||
```
|
||||
|
||||
Esquema completo: [`docs/database.dbml`](docs/database.dbml)
|
||||
@@ -156,6 +160,7 @@ Esquema completo: [`docs/database.dbml`](docs/database.dbml)
|
||||
- [`services/api_gateway/README.md`](services/api_gateway/README.md)
|
||||
- [`services/extraction/README.md`](services/extraction/README.md)
|
||||
- [`services/cleaning/README.md`](services/cleaning/README.md)
|
||||
- [`services/poblacion_apta/README.md`](services/poblacion_apta/README.md)
|
||||
- [`services/services-count-facturas-medicas-serving/README.md`](services/services-count-facturas-medicas-serving/README.md)
|
||||
- [`services/db/README.md`](services/db/README.md)
|
||||
- [`services/desktop_app/README.md`](services/desktop_app/README.md)
|
||||
|
||||
+1847
File diff suppressed because it is too large
Load Diff
@@ -126,6 +126,19 @@ services:
|
||||
environment:
|
||||
DATABASE_URL: postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor
|
||||
|
||||
poblacion_apta:
|
||||
build:
|
||||
context: ./services/poblacion_apta
|
||||
dockerfile: Dockerfile
|
||||
container_name: ipsextractor-poblacion-apta
|
||||
depends_on:
|
||||
database:
|
||||
condition: service_healthy
|
||||
db-migrations:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
DATABASE_URL: postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor
|
||||
|
||||
count-api:
|
||||
build:
|
||||
context: ./services/services-count-facturas-medicas-serving
|
||||
@@ -196,6 +209,8 @@ services:
|
||||
condition: service_healthy
|
||||
cleaning:
|
||||
condition: service_started
|
||||
poblacion_apta:
|
||||
condition: service_started
|
||||
count-api:
|
||||
condition: service_healthy
|
||||
bot-descarga:
|
||||
@@ -203,6 +218,7 @@ services:
|
||||
environment:
|
||||
EXTRACTION_URL: http://extraction:8001
|
||||
CLEANING_URL: http://cleaning:8002
|
||||
POBLACION_APTA_URL: http://poblacion_apta:8003
|
||||
COUNT_URL: http://count-api:8080
|
||||
BOT_DESCARGA_URL: http://bot-descarga:8200
|
||||
ports:
|
||||
|
||||
@@ -29,6 +29,8 @@ TableGroup pipeline {
|
||||
extraction_rows
|
||||
cleaning_jobs
|
||||
cleaned_rows
|
||||
poblacion_apta_jobs
|
||||
poblacion_apta_rows
|
||||
}
|
||||
|
||||
TableGroup counting {
|
||||
@@ -274,3 +276,38 @@ Table cleaned_rows {
|
||||
|
||||
Note: 'Salida del servicio de limpieza.'
|
||||
}
|
||||
|
||||
Table poblacion_apta_jobs {
|
||||
id varchar(36) [pk, note: 'UUID v4']
|
||||
job_id varchar(36) [null, ref: > jobs.id, note: 'Trabajo Desktop al que pertenece']
|
||||
status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed']
|
||||
request jsonb [not null, default: '{}', note: 'cleaning_job_ids, job_id']
|
||||
log jsonb [not null, default: '[]']
|
||||
summary jsonb [not null, default: '{}', note: 'totales paso_bot / errores / por fuente']
|
||||
error text
|
||||
created_at timestamptz [not null]
|
||||
updated_at timestamptz [not null]
|
||||
|
||||
indexes {
|
||||
job_id
|
||||
status
|
||||
}
|
||||
|
||||
Note: 'Consolidación y validación Verificador ST sobre una o más limpiezas.'
|
||||
}
|
||||
|
||||
Table poblacion_apta_rows {
|
||||
id integer [pk, increment]
|
||||
poblacion_apta_job_id varchar(36) [not null, ref: > poblacion_apta_jobs.id]
|
||||
cleaning_job_id varchar(36) [null, note: 'Limpieza de origen de la fila']
|
||||
row_index integer [not null, default: 0]
|
||||
row_json jsonb [not null, default: '{}', note: 'Schema FINAL_IT + OBSERVACION']
|
||||
created_at timestamptz [not null]
|
||||
|
||||
indexes {
|
||||
poblacion_apta_job_id
|
||||
cleaning_job_id
|
||||
}
|
||||
|
||||
Note: 'Filas del consolidado Población apta (PASO_BOT y observaciones).'
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name
|
||||
|
||||
EXTRACTION_URL = os.getenv("EXTRACTION_URL", "http://extraction:8001").rstrip("/")
|
||||
CLEANING_URL = os.getenv("CLEANING_URL", "http://cleaning:8002").rstrip("/")
|
||||
POBLACION_APTA_URL = os.getenv("POBLACION_APTA_URL", "http://poblacion_apta:8003").rstrip("/")
|
||||
COUNT_URL = os.getenv("COUNT_URL", "http://count-api:8080").rstrip("/")
|
||||
BOT_DESCARGA_URL = os.getenv("BOT_DESCARGA_URL", "http://bot-descarga:8200").rstrip("/")
|
||||
|
||||
@@ -35,9 +36,10 @@ async def lifespan(app: FastAPI):
|
||||
limits=httpx.Limits(max_connections=100, max_keepalive_connections=20),
|
||||
)
|
||||
_log.info(
|
||||
"Gateway iniciado — extraction=%s cleaning=%s count=%s bot_descarga=%s",
|
||||
"Gateway iniciado — extraction=%s cleaning=%s poblacion_apta=%s count=%s bot_descarga=%s",
|
||||
EXTRACTION_URL,
|
||||
CLEANING_URL,
|
||||
POBLACION_APTA_URL,
|
||||
COUNT_URL,
|
||||
BOT_DESCARGA_URL,
|
||||
)
|
||||
@@ -79,6 +81,8 @@ def _is_upload_path(path: str) -> bool:
|
||||
or "/upload" in path
|
||||
or path.startswith("/cleaning/analyze-csv")
|
||||
or path.startswith("/cleaning/jobs/from-csv")
|
||||
or path.startswith("/poblacion-apta/jobs/from-csv")
|
||||
or path.startswith("/poblacion-apta/jobs/from-cleaning")
|
||||
or path.startswith("/bot-descarga/jobs")
|
||||
)
|
||||
|
||||
@@ -98,13 +102,15 @@ async def health() -> dict[str, Any]:
|
||||
client = _client()
|
||||
extraction = await _safe_health(client, EXTRACTION_URL)
|
||||
cleaning = await _safe_health(client, CLEANING_URL)
|
||||
poblacion_apta = await _safe_health(client, POBLACION_APTA_URL)
|
||||
counting = await _safe_health(client, COUNT_URL)
|
||||
bot_descarga = await _safe_health(client, BOT_DESCARGA_URL)
|
||||
return {
|
||||
"status": "ok" if extraction["ok"] and cleaning["ok"] else "degraded",
|
||||
"status": "ok" if extraction["ok"] and cleaning["ok"] and poblacion_apta["ok"] else "degraded",
|
||||
"services": {
|
||||
"extraction": extraction,
|
||||
"cleaning": cleaning,
|
||||
"poblacion_apta": poblacion_apta,
|
||||
"counting": counting,
|
||||
"bot_descarga": bot_descarga,
|
||||
},
|
||||
@@ -116,6 +122,11 @@ async def models() -> Response:
|
||||
return await _proxy("GET", f"{EXTRACTION_URL}/models")
|
||||
|
||||
|
||||
@app.get("/openai/balance")
|
||||
async def openai_balance() -> Response:
|
||||
return await _proxy("GET", f"{EXTRACTION_URL}/openai/balance")
|
||||
|
||||
|
||||
@app.get("/config")
|
||||
async def config() -> dict[str, Any]:
|
||||
client = _client()
|
||||
@@ -147,6 +158,11 @@ async def cleaning_proxy(path: str, request: Request) -> Response:
|
||||
return await _proxy_request(request, f"{CLEANING_URL}/{path}")
|
||||
|
||||
|
||||
@app.api_route("/poblacion-apta/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
|
||||
async def poblacion_apta_proxy(path: str, request: Request) -> Response:
|
||||
return await _proxy_request(request, f"{POBLACION_APTA_URL}/{path}")
|
||||
|
||||
|
||||
@app.api_route("/count/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
|
||||
async def count_proxy(path: str, request: Request) -> Response:
|
||||
return await _proxy_request(request, f"{COUNT_URL}/{path}")
|
||||
|
||||
@@ -48,6 +48,8 @@ class CleaningRequest(BaseModel):
|
||||
job_id: str | None = None
|
||||
rules_per_column: dict[str, list[str]] = Field(default_factory=dict)
|
||||
cop_mod_fecha_column: dict[str, str] = Field(default_factory=dict)
|
||||
cop_mod_bruto_column: dict[str, str] = Field(default_factory=dict)
|
||||
cop_mod_neto_column: dict[str, str] = Field(default_factory=dict)
|
||||
cop_mod_config: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@@ -56,6 +58,8 @@ class PreviewRequest(BaseModel):
|
||||
extraction_batch_id: str | None = None
|
||||
rules_per_column: dict[str, list[str]] = Field(default_factory=dict)
|
||||
cop_mod_fecha_column: dict[str, str] = Field(default_factory=dict)
|
||||
cop_mod_bruto_column: dict[str, str] = Field(default_factory=dict)
|
||||
cop_mod_neto_column: dict[str, str] = Field(default_factory=dict)
|
||||
cop_mod_config: dict[str, Any] | None = None
|
||||
sample_size: int = 8
|
||||
|
||||
@@ -65,6 +69,12 @@ def dataframe_to_records(df: pd.DataFrame) -> list[dict[str, Any]]:
|
||||
|
||||
|
||||
def serialize_job(job: CleaningJob) -> dict[str, Any]:
|
||||
# started/finished derivados: created_at = inicio; updated_at = fin si ya terminó
|
||||
finished = (
|
||||
job.updated_at.isoformat()
|
||||
if job.status in {"completed", "failed"}
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"id": job.id,
|
||||
"extraction_batch_id": job.extraction_batch_id,
|
||||
@@ -76,6 +86,8 @@ def serialize_job(job: CleaningJob) -> dict[str, Any]:
|
||||
"error": job.error,
|
||||
"created_at": job.created_at.isoformat(),
|
||||
"updated_at": job.updated_at.isoformat(),
|
||||
"started_at": job.created_at.isoformat(),
|
||||
"finished_at": finished,
|
||||
}
|
||||
|
||||
|
||||
@@ -96,6 +108,7 @@ def _create_cleaning_task(
|
||||
request: dict[str, Any],
|
||||
error: str | None = None,
|
||||
title: str | None = None,
|
||||
started_at: Any | None = None,
|
||||
) -> str | None:
|
||||
"""Crea Task(type=cleaning) vinculada al job Desktop. Devuelve task.id o None."""
|
||||
if not job_id:
|
||||
@@ -109,7 +122,7 @@ def _create_cleaning_task(
|
||||
request=request,
|
||||
result={"cleaning_job_id": cleaning_job_id},
|
||||
error=error,
|
||||
started_at=now,
|
||||
started_at=started_at or now,
|
||||
finished_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -142,12 +155,16 @@ def _run_rules_on_dataframe(
|
||||
rules_per_column: dict[str, list[str]],
|
||||
cop_mod_fecha_column: dict[str, str],
|
||||
cop_mod_config: dict[str, Any] | None,
|
||||
cop_mod_bruto_column: dict[str, str] | None = None,
|
||||
cop_mod_neto_column: dict[str, str] | None = None,
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
result_df, log = apply_rules(
|
||||
df,
|
||||
rules_per_column,
|
||||
cop_mod_fecha_column=cop_mod_fecha_column,
|
||||
cop_mod_config=cop_mod_config,
|
||||
cop_mod_bruto_column=cop_mod_bruto_column,
|
||||
cop_mod_neto_column=cop_mod_neto_column,
|
||||
)
|
||||
return dataframe_to_records(result_df), log
|
||||
|
||||
@@ -186,6 +203,8 @@ def _preview_from_dataframe(
|
||||
rules_per_column: dict[str, list[str]],
|
||||
cop_mod_fecha_column: dict[str, str],
|
||||
cop_mod_config: dict[str, Any] | None,
|
||||
cop_mod_bruto_column: dict[str, str] | None = None,
|
||||
cop_mod_neto_column: dict[str, str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
rows_before = dataframe_to_records(df)
|
||||
result_df, _log = apply_rules(
|
||||
@@ -193,6 +212,8 @@ def _preview_from_dataframe(
|
||||
rules_per_column,
|
||||
cop_mod_fecha_column=cop_mod_fecha_column,
|
||||
cop_mod_config=cop_mod_config,
|
||||
cop_mod_bruto_column=cop_mod_bruto_column,
|
||||
cop_mod_neto_column=cop_mod_neto_column,
|
||||
)
|
||||
rows_after = dataframe_to_records(result_df)
|
||||
|
||||
@@ -299,6 +320,8 @@ def preview_cleaning(request: PreviewRequest) -> dict[str, Any]:
|
||||
request.rules_per_column,
|
||||
request.cop_mod_fecha_column,
|
||||
request.cop_mod_config,
|
||||
request.cop_mod_bruto_column,
|
||||
request.cop_mod_neto_column,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
@@ -349,6 +372,8 @@ def run_cleaning(request: CleaningRequest) -> dict[str, Any]:
|
||||
request.rules_per_column,
|
||||
request.cop_mod_fecha_column,
|
||||
request.cop_mod_config,
|
||||
request.cop_mod_bruto_column,
|
||||
request.cop_mod_neto_column,
|
||||
)
|
||||
|
||||
with session_scope() as session:
|
||||
@@ -372,6 +397,7 @@ def run_cleaning(request: CleaningRequest) -> dict[str, Any]:
|
||||
status="completed",
|
||||
request=request_dump,
|
||||
title="Limpieza desde extracción",
|
||||
started_at=job.created_at,
|
||||
)
|
||||
session.flush()
|
||||
return serialize_job(job)
|
||||
@@ -380,6 +406,7 @@ def run_cleaning(request: CleaningRequest) -> dict[str, Any]:
|
||||
except Exception as exc:
|
||||
with session_scope() as session:
|
||||
job = session.get(CleaningJob, job_id)
|
||||
started = job.created_at if job is not None else None
|
||||
if job is not None:
|
||||
job.status = "failed"
|
||||
job.error = str(exc)
|
||||
@@ -393,6 +420,7 @@ def run_cleaning(request: CleaningRequest) -> dict[str, Any]:
|
||||
request=request_dump,
|
||||
error=str(exc),
|
||||
title="Limpieza desde extracción",
|
||||
started_at=started,
|
||||
)
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
|
||||
@@ -403,6 +431,8 @@ async def run_cleaning_from_csv(
|
||||
job_id: str | None = Form(None),
|
||||
rules_per_column: str | None = Form(None),
|
||||
cop_mod_fecha_column: str | None = Form(None),
|
||||
cop_mod_bruto_column: str | None = Form(None),
|
||||
cop_mod_neto_column: str | None = Form(None),
|
||||
cop_mod_config: str | None = Form(None),
|
||||
) -> dict[str, Any]:
|
||||
content = await file.read()
|
||||
@@ -411,11 +441,17 @@ async def run_cleaning_from_csv(
|
||||
|
||||
rules = _parse_json_form(rules_per_column, {})
|
||||
fecha_map = _parse_json_form(cop_mod_fecha_column, {})
|
||||
bruto_map = _parse_json_form(cop_mod_bruto_column, {})
|
||||
neto_map = _parse_json_form(cop_mod_neto_column, {})
|
||||
cop_cfg = _parse_json_form(cop_mod_config, None)
|
||||
if not isinstance(rules, dict):
|
||||
raise HTTPException(status_code=400, detail="rules_per_column debe ser un objeto JSON")
|
||||
if not isinstance(fecha_map, dict):
|
||||
raise HTTPException(status_code=400, detail="cop_mod_fecha_column debe ser un objeto JSON")
|
||||
if not isinstance(bruto_map, dict):
|
||||
raise HTTPException(status_code=400, detail="cop_mod_bruto_column debe ser un objeto JSON")
|
||||
if not isinstance(neto_map, dict):
|
||||
raise HTTPException(status_code=400, detail="cop_mod_neto_column debe ser un objeto JSON")
|
||||
|
||||
try:
|
||||
df, delimiter = load_csv_bytes(content, file.filename or "upload.csv")
|
||||
@@ -431,6 +467,8 @@ async def run_cleaning_from_csv(
|
||||
"job_id": job_id,
|
||||
"rules_per_column": rules,
|
||||
"cop_mod_fecha_column": fecha_map,
|
||||
"cop_mod_bruto_column": bruto_map,
|
||||
"cop_mod_neto_column": neto_map,
|
||||
"cop_mod_config": cop_cfg,
|
||||
}
|
||||
|
||||
@@ -450,7 +488,9 @@ async def run_cleaning_from_csv(
|
||||
desktop_job_id = job_id
|
||||
|
||||
try:
|
||||
output_rows, log = _run_rules_on_dataframe(df, rules, fecha_map, cop_cfg)
|
||||
output_rows, log = _run_rules_on_dataframe(
|
||||
df, rules, fecha_map, cop_cfg, bruto_map, neto_map
|
||||
)
|
||||
|
||||
with session_scope() as session:
|
||||
_persist_cleaned_rows(
|
||||
@@ -473,6 +513,7 @@ async def run_cleaning_from_csv(
|
||||
status="completed",
|
||||
request=request_dump,
|
||||
title=f"Limpieza CSV — {file.filename or 'upload.csv'}",
|
||||
started_at=job.created_at,
|
||||
)
|
||||
session.flush()
|
||||
return serialize_job(job)
|
||||
@@ -481,6 +522,7 @@ async def run_cleaning_from_csv(
|
||||
except Exception as exc:
|
||||
with session_scope() as session:
|
||||
job = session.get(CleaningJob, cleaning_job_id)
|
||||
started = job.created_at if job is not None else None
|
||||
if job is not None:
|
||||
job.status = "failed"
|
||||
job.error = str(exc)
|
||||
@@ -494,6 +536,7 @@ async def run_cleaning_from_csv(
|
||||
request=request_dump,
|
||||
error=str(exc),
|
||||
title=f"Limpieza CSV — {file.filename or 'upload.csv'}",
|
||||
started_at=started,
|
||||
)
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@@ -6,7 +6,11 @@ from typing import Any, Optional
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from .cop_mod import _apply_cop_mod_split, merge_moderadora_config
|
||||
from .cop_mod import (
|
||||
_apply_cop_mod_bruto_neto,
|
||||
_apply_cop_mod_split,
|
||||
merge_moderadora_config,
|
||||
)
|
||||
from .documents import _apply_doc_id_split_columns
|
||||
from .notas import _apply_nota_extract_columns
|
||||
from .rules import get_rule_fn, get_rule_name
|
||||
@@ -20,20 +24,24 @@ def apply_rules(
|
||||
*,
|
||||
cop_mod_fecha_column: Optional[dict[str, str]] = None,
|
||||
cop_mod_config: Optional[dict[str, Any]] = None,
|
||||
cop_mod_bruto_column: Optional[dict[str, str]] = None,
|
||||
cop_mod_neto_column: Optional[dict[str, str]] = None,
|
||||
) -> tuple[pd.DataFrame, list[str]]:
|
||||
"""Aplica las reglas seleccionadas a cada columna.
|
||||
|
||||
`rules_per_column` es un dict de `columna -> lista de IDs de reglas en el
|
||||
orden a aplicar`. La regla `cop_mod_split` se difiere a una segunda pasada
|
||||
tras el resto, para que la columna de fecha pueda normalizarse antes.
|
||||
orden a aplicar`. Las reglas `cop_mod_split` y `cop_mod_bruto_neto` se
|
||||
difieren a una segunda pasada tras el resto.
|
||||
|
||||
`cop_mod_fecha_column`: mapa columna de importe -> nombre de columna Fecha Servicio.
|
||||
`cop_mod_config`: opcional; puede incluir ``moderadora_by_year`` (dict año → lista).
|
||||
`cop_mod_fecha_column`: mapa columna ancla -> Fecha Servicio (para split).
|
||||
`cop_mod_bruto_column` / `cop_mod_neto_column`: mapas ancla -> bruto / neto.
|
||||
`cop_mod_config`: opcional; puede incluir ``moderadora_by_year``.
|
||||
"""
|
||||
result = df.copy()
|
||||
log: list[str] = []
|
||||
|
||||
moderadora_years = merge_moderadora_config(cop_mod_config)
|
||||
deferred = {"cop_mod_split", "cop_mod_bruto_neto"}
|
||||
|
||||
def _run_rules_for_column(col: str, rule_ids: list[str]) -> None:
|
||||
if not rule_ids:
|
||||
@@ -74,12 +82,12 @@ def apply_rules(
|
||||
logger.exception("Error aplicando regla %s a %s", rule_id, col)
|
||||
log.append(f"[ERROR] '{col}' regla '{rule_id}': {exc}")
|
||||
|
||||
# Fase 1: todas las reglas excepto cop_mod_split
|
||||
# Fase 1: todas las reglas excepto COP/MOD diferidas
|
||||
for col, rule_ids in rules_per_column.items():
|
||||
filtered = [r for r in rule_ids if r != "cop_mod_split"]
|
||||
filtered = [r for r in rule_ids if r not in deferred]
|
||||
_run_rules_for_column(col, filtered)
|
||||
|
||||
# Fase 2: Separa COP/MOD sobre el DataFrame ya transformado
|
||||
# Fase 2a: Separa COP/MOD (cuotas por año)
|
||||
fecha_map = cop_mod_fecha_column or {}
|
||||
for col, rule_ids in rules_per_column.items():
|
||||
if "cop_mod_split" not in rule_ids:
|
||||
@@ -102,4 +110,23 @@ def apply_rules(
|
||||
logger.exception("Error aplicando COP/MOD a %s", col)
|
||||
log.append(f"[ERROR] '{col}' regla 'cop_mod_split': {exc}")
|
||||
|
||||
# Fase 2b: COP/MOD = bruto − neto
|
||||
bruto_map = cop_mod_bruto_column or {}
|
||||
neto_map = cop_mod_neto_column or {}
|
||||
for col, rule_ids in rules_per_column.items():
|
||||
if "cop_mod_bruto_neto" not in rule_ids:
|
||||
continue
|
||||
bruto_col = (bruto_map.get(col) or col).strip()
|
||||
neto_col = (neto_map.get(col) or "").strip()
|
||||
if not neto_col:
|
||||
log.append(
|
||||
f"[WARN] COP/MOD (bruto−neto) en '{col}': indique la columna «Valor neto»; se omite."
|
||||
)
|
||||
continue
|
||||
try:
|
||||
_apply_cop_mod_bruto_neto(result, bruto_col, neto_col, log)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Error aplicando COP/MOD bruto−neto a %s", col)
|
||||
log.append(f"[ERROR] '{col}' regla 'cop_mod_bruto_neto': {exc}")
|
||||
|
||||
return result, log
|
||||
|
||||
@@ -160,3 +160,59 @@ def _apply_cop_mod_split(
|
||||
f"[OK] '{amount_col}' -> '{ccol}', '{mcol}' <- Separa COP/MOD "
|
||||
f"(fecha: '{fecha_column}')"
|
||||
)
|
||||
|
||||
|
||||
def _alloc_cop_mod_diff_column_name(existing: set[str]) -> str:
|
||||
"""Nombre único para la columna resultado (preferido: cop/mod)."""
|
||||
preferred = "cop/mod"
|
||||
if preferred not in existing:
|
||||
return preferred
|
||||
for i in range(2, 200):
|
||||
name = f"cop/mod_{i}"
|
||||
if name not in existing:
|
||||
return name
|
||||
raise ValueError("No se pudo asignar nombre de columna único para cop/mod.")
|
||||
|
||||
|
||||
def _apply_cop_mod_bruto_neto(
|
||||
result: pd.DataFrame,
|
||||
bruto_col: str,
|
||||
neto_col: str,
|
||||
log: list[str],
|
||||
) -> None:
|
||||
"""Crea columna ``cop/mod`` = valor bruto − valor neto."""
|
||||
if bruto_col not in result.columns:
|
||||
log.append(
|
||||
f"[WARN] COP/MOD (bruto−neto): columna bruto '{bruto_col}' no existe; se omite."
|
||||
)
|
||||
return
|
||||
if neto_col not in result.columns:
|
||||
log.append(
|
||||
f"[WARN] COP/MOD (bruto−neto): columna neto '{neto_col}' no existe; se omite."
|
||||
)
|
||||
return
|
||||
if bruto_col == neto_col:
|
||||
log.append(
|
||||
f"[WARN] COP/MOD (bruto−neto): bruto y neto son la misma columna '{bruto_col}'; se omite."
|
||||
)
|
||||
return
|
||||
|
||||
bruto_ser = result[bruto_col]
|
||||
neto_ser = result[neto_col]
|
||||
bruto_hint = _infer_decimal_hint_from_column(bruto_ser.astype(str))
|
||||
neto_hint = _infer_decimal_hint_from_column(neto_ser.astype(str))
|
||||
|
||||
out_col = _alloc_cop_mod_diff_column_name(set(result.columns))
|
||||
values: list[Any] = []
|
||||
for i in range(len(result)):
|
||||
bruto_i = _amount_cell_to_optional_int(bruto_ser.iloc[i], bruto_hint)
|
||||
neto_i = _amount_cell_to_optional_int(neto_ser.iloc[i], neto_hint)
|
||||
if bruto_i is None or neto_i is None:
|
||||
values.append("")
|
||||
continue
|
||||
values.append(bruto_i - neto_i)
|
||||
|
||||
result[out_col] = values
|
||||
log.append(
|
||||
f"[OK] '{out_col}' = '{bruto_col}' − '{neto_col}' <- COP/MOD (bruto − neto)"
|
||||
)
|
||||
|
||||
@@ -121,6 +121,15 @@ register_rule(
|
||||
),
|
||||
fn=special._rule_passthrough_series,
|
||||
)
|
||||
register_rule(
|
||||
"cop_mod_bruto_neto",
|
||||
name="COP/MOD (bruto − neto)",
|
||||
description=(
|
||||
"Calcula la columna «cop/mod» como Valor bruto − Valor neto. "
|
||||
"Elija ambas columnas en la tarjeta (se detectan automáticamente si existen)."
|
||||
),
|
||||
fn=special._rule_passthrough_series,
|
||||
)
|
||||
register_rule(
|
||||
"nota_paciente_autorizacion",
|
||||
name="Extraer datos de notas (documento y autorización)",
|
||||
|
||||
@@ -101,7 +101,7 @@ def _applicable_rules_for_type(inferred_type: str) -> list[str]:
|
||||
"""Identificadores de reglas a mostrar según el tipo inferido (sin redundancias)."""
|
||||
base = ["strip_special_spaces", "strip_special_keep_date_seps", "strip_spaces"]
|
||||
if inferred_type == TYPE_INTEGER:
|
||||
return base + ["cop_mod_split"]
|
||||
return base + ["cop_mod_split", "cop_mod_bruto_neto"]
|
||||
if inferred_type == TYPE_DECIMAL:
|
||||
return base + [
|
||||
"to_integer",
|
||||
@@ -109,6 +109,7 @@ def _applicable_rules_for_type(inferred_type: str) -> list[str]:
|
||||
"doc_id_split_tipo_numero",
|
||||
"doc_type_homologate",
|
||||
"cop_mod_split",
|
||||
"cop_mod_bruto_neto",
|
||||
]
|
||||
if inferred_type == TYPE_DATE:
|
||||
return base + [
|
||||
@@ -125,6 +126,7 @@ def _applicable_rules_for_type(inferred_type: str) -> list[str]:
|
||||
"doc_type_homologate",
|
||||
"date_to_ddmmyyyy",
|
||||
"cop_mod_split",
|
||||
"cop_mod_bruto_neto",
|
||||
"nota_paciente_autorizacion",
|
||||
]
|
||||
# Texto / categórico / boolean / empty → todas las de limpieza de texto + las
|
||||
@@ -136,6 +138,7 @@ def _applicable_rules_for_type(inferred_type: str) -> list[str]:
|
||||
"doc_type_homologate",
|
||||
"date_to_ddmmyyyy",
|
||||
"cop_mod_split",
|
||||
"cop_mod_bruto_neto",
|
||||
"nota_paciente_autorizacion",
|
||||
]
|
||||
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Add poblacion_apta_jobs and poblacion_apta_rows tables.
|
||||
|
||||
Revision ID: 0006
|
||||
Revises: 0005
|
||||
Create Date: 2026-07-28
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0006"
|
||||
down_revision: Union[str, None] = "0005"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
existing = set(inspector.get_table_names())
|
||||
|
||||
if "poblacion_apta_jobs" not in existing:
|
||||
op.create_table(
|
||||
"poblacion_apta_jobs",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("job_id", sa.String(36), sa.ForeignKey("jobs.id"), nullable=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="pending"),
|
||||
sa.Column("request", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("log", sa.JSON, nullable=False, server_default="[]"),
|
||||
sa.Column("summary", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("error", sa.Text, nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_poblacion_apta_jobs_job_id", "poblacion_apta_jobs", ["job_id"])
|
||||
op.create_index("ix_poblacion_apta_jobs_status", "poblacion_apta_jobs", ["status"])
|
||||
|
||||
if "poblacion_apta_rows" not in existing:
|
||||
op.create_table(
|
||||
"poblacion_apta_rows",
|
||||
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
|
||||
sa.Column(
|
||||
"poblacion_apta_job_id",
|
||||
sa.String(36),
|
||||
sa.ForeignKey("poblacion_apta_jobs.id"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("cleaning_job_id", sa.String(36), nullable=True),
|
||||
sa.Column("row_index", sa.Integer, nullable=False, server_default="0"),
|
||||
sa.Column("row_json", sa.JSON, nullable=False, server_default="{}"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_poblacion_apta_rows_poblacion_apta_job_id",
|
||||
"poblacion_apta_rows",
|
||||
["poblacion_apta_job_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_poblacion_apta_rows_cleaning_job_id",
|
||||
"poblacion_apta_rows",
|
||||
["cleaning_job_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
existing = set(inspector.get_table_names())
|
||||
if "poblacion_apta_rows" in existing:
|
||||
op.drop_index("ix_poblacion_apta_rows_cleaning_job_id", table_name="poblacion_apta_rows")
|
||||
op.drop_index(
|
||||
"ix_poblacion_apta_rows_poblacion_apta_job_id", table_name="poblacion_apta_rows"
|
||||
)
|
||||
op.drop_table("poblacion_apta_rows")
|
||||
if "poblacion_apta_jobs" in existing:
|
||||
op.drop_index("ix_poblacion_apta_jobs_status", table_name="poblacion_apta_jobs")
|
||||
op.drop_index("ix_poblacion_apta_jobs_job_id", table_name="poblacion_apta_jobs")
|
||||
op.drop_table("poblacion_apta_jobs")
|
||||
@@ -65,6 +65,25 @@ ipcMain.handle("updater:install", async (_event, manifest) => {
|
||||
return autoUpdate.downloadAndInstall(manifest, sendUpdateProgress);
|
||||
});
|
||||
|
||||
ipcMain.handle("net:download-url", async (event, urlStr) => {
|
||||
const raw = String(urlStr || "").trim();
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
throw new Error("URL de descarga inválida");
|
||||
}
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new Error("Solo se permiten descargas http(s)");
|
||||
}
|
||||
const win = BrowserWindow.fromWebContents(event.sender) || mainWindow;
|
||||
if (!win || win.isDestroyed()) {
|
||||
throw new Error("Ventana no disponible para iniciar la descarga");
|
||||
}
|
||||
win.webContents.downloadURL(parsed.toString());
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
app.whenReady().then(() => {
|
||||
if (process.platform === "win32") {
|
||||
app.setAppUserModelId("com.ipsextractor.desktop");
|
||||
|
||||
@@ -21,4 +21,5 @@ contextBridge.exposeInMainWorld("desktopApi", {
|
||||
ipcRenderer.on("updater:progress", handler);
|
||||
return () => ipcRenderer.removeListener("updater:progress", handler);
|
||||
},
|
||||
downloadUrl: (url) => ipcRenderer.invoke("net:download-url", url),
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "ipsextractor-desktop",
|
||||
"version": "1.1.1",
|
||||
"version": "1.2.0",
|
||||
"private": true,
|
||||
"description": "Desktop extraction client for IPSExtractor microservices.",
|
||||
"main": "electron/main.cjs",
|
||||
|
||||
@@ -103,14 +103,32 @@ async function sha256File(filePath) {
|
||||
}
|
||||
|
||||
async function upload(client, key, body, contentType) {
|
||||
await client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Body: body,
|
||||
ContentType: contentType,
|
||||
}),
|
||||
);
|
||||
const payload = Buffer.isBuffer(body)
|
||||
? body
|
||||
: typeof body === "string"
|
||||
? Buffer.from(body, "utf8")
|
||||
: body;
|
||||
|
||||
const params = {
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Body: payload,
|
||||
ContentType: contentType,
|
||||
};
|
||||
|
||||
// MinIO exige Content-Length; con Body string/stream el SDK a veces no lo envía.
|
||||
if (Buffer.isBuffer(payload)) {
|
||||
params.ContentLength = payload.byteLength;
|
||||
} else if (typeof payload === "object" && payload && "path" in payload) {
|
||||
// ReadStream desde createReadStream(path)
|
||||
try {
|
||||
params.ContentLength = statSync(payload.path).size;
|
||||
} catch {
|
||||
/* ignore — el stream del .exe suele ir bien */
|
||||
}
|
||||
}
|
||||
|
||||
await client.send(new PutObjectCommand(params));
|
||||
console.log(` ↑ s3://${bucket}/${key}`);
|
||||
}
|
||||
|
||||
@@ -201,7 +219,13 @@ async function main() {
|
||||
|
||||
console.log(`Publicando v${version} → ${endpoint}/${bucket}/${prefix}/`);
|
||||
await upload(client, objectKeyExe, createReadStream(setupPath), "application/octet-stream");
|
||||
await upload(client, objectKeyData, `${releaseDateDisplay}\n`, "text/plain; charset=utf-8");
|
||||
// Re-upload metadata even if exe already exists (reintento tras fallo parcial).
|
||||
await upload(
|
||||
client,
|
||||
objectKeyData,
|
||||
Buffer.from(`${releaseDateDisplay}\n`, "utf8"),
|
||||
"text/plain; charset=utf-8",
|
||||
);
|
||||
await upload(
|
||||
client,
|
||||
objectKeyLatest,
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
import {
|
||||
BarChart2,
|
||||
Briefcase,
|
||||
ChevronDown,
|
||||
Cpu,
|
||||
Download,
|
||||
Hash,
|
||||
Link2,
|
||||
Package,
|
||||
Settings,
|
||||
Sparkles,
|
||||
Users,
|
||||
Wallet,
|
||||
} from "lucide-react";
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import type { Job } from "../shared/types";
|
||||
@@ -29,12 +33,49 @@ import { UpdatePrompt, CheckUpdatesButton, useAutoUpdateCheck } from "./componen
|
||||
import "./modules/counting/CountingModule";
|
||||
import "./modules/extraction/ExtractionModule";
|
||||
import "./modules/cleaning/CleaningModule";
|
||||
import "./modules/poblacion_apta/PoblacionAptaModule";
|
||||
import "./modules/downloads/DownloadsModule";
|
||||
|
||||
const BOT_PACKAGE_URL =
|
||||
"http://s3.builker.xyz/apps-versions/BotDescargaPYSCuentasMedicas/latest/bot_descarga.zip";
|
||||
|
||||
type Status = "idle" | "ok" | "error" | "busy";
|
||||
type View = "home" | "job" | "settings";
|
||||
type OpenAiBalanceState = {
|
||||
status: Status;
|
||||
available: number | null;
|
||||
currency: string;
|
||||
error: string | null;
|
||||
};
|
||||
|
||||
export type { JobSection } from "./types/jobSection";
|
||||
|
||||
function formatUsd(amount: number): string {
|
||||
return new Intl.NumberFormat("en-US", {
|
||||
style: "currency",
|
||||
currency: "USD",
|
||||
minimumFractionDigits: 2,
|
||||
maximumFractionDigits: 2,
|
||||
}).format(amount);
|
||||
}
|
||||
|
||||
function formatOpenaiBalanceError(error: string | null): string {
|
||||
const msg = (error || "").toLowerCase();
|
||||
if (!msg) return "No disponible";
|
||||
if (msg.includes("mock-key") || msg.includes("incorrect api key")) return "Key inválida";
|
||||
if (msg.includes("no configurada") || msg.includes("api key de openai no")) return "Sin API key";
|
||||
if (
|
||||
msg.includes("session key") ||
|
||||
msg.includes("navegador") ||
|
||||
msg.includes("panel web") ||
|
||||
msg.includes("no permite consultar")
|
||||
) {
|
||||
return "Solo en web";
|
||||
}
|
||||
if (msg.includes("sin conexión") || msg.includes("connection refused")) return "Sin red";
|
||||
return "No disponible";
|
||||
}
|
||||
|
||||
function App() {
|
||||
const [view, setView] = useState<View>("home");
|
||||
const [settingsReturnView, setSettingsReturnView] = useState<"home" | "job">("home");
|
||||
@@ -48,8 +89,17 @@ function App() {
|
||||
const [healthJson, setHealthJson] = useState<unknown>(null);
|
||||
const [configJson, setConfigJson] = useState<unknown>(null);
|
||||
const [modelsJson, setModelsJson] = useState<unknown>(null);
|
||||
const [openaiBalance, setOpenaiBalance] = useState<OpenAiBalanceState>({
|
||||
status: "idle",
|
||||
available: null,
|
||||
currency: "USD",
|
||||
error: null,
|
||||
});
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [releaseDate, setReleaseDate] = useState(APP_RELEASE_DATE);
|
||||
const [downloadsMenuOpen, setDownloadsMenuOpen] = useState(true);
|
||||
const [botDownloadBusy, setBotDownloadBusy] = useState(false);
|
||||
const [botDownloadError, setBotDownloadError] = useState<string | null>(null);
|
||||
const update = useAutoUpdateCheck();
|
||||
|
||||
useEffect(() => {
|
||||
@@ -80,6 +130,35 @@ function App() {
|
||||
localStorage.setItem("apiBaseUrl", apiBaseUrl);
|
||||
}, [apiBaseUrl]);
|
||||
|
||||
const refreshOpenaiBalance = useCallback(async () => {
|
||||
setOpenaiBalance((prev) => ({ ...prev, status: "busy", error: null }));
|
||||
try {
|
||||
const balance = await client.openaiBalance();
|
||||
if (balance.ok && typeof balance.available === "number") {
|
||||
setOpenaiBalance({
|
||||
status: "ok",
|
||||
available: balance.available,
|
||||
currency: balance.currency || "USD",
|
||||
error: null,
|
||||
});
|
||||
} else {
|
||||
setOpenaiBalance({
|
||||
status: "error",
|
||||
available: null,
|
||||
currency: balance.currency || "USD",
|
||||
error: balance.error || "No se pudo obtener el crédito de OpenAI.",
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
setOpenaiBalance({
|
||||
status: "error",
|
||||
available: null,
|
||||
currency: "USD",
|
||||
error: err instanceof Error ? err.message : "No se pudo obtener el crédito de OpenAI.",
|
||||
});
|
||||
}
|
||||
}, [client]);
|
||||
|
||||
const verifyConnection = useCallback(async () => {
|
||||
setConnectionStatus("busy");
|
||||
setConnectionMessage("Comprobando gateway y servicios…");
|
||||
@@ -95,22 +174,38 @@ function App() {
|
||||
setModelsJson(models);
|
||||
setConnectionStatus("ok");
|
||||
setConnectionMessage("Gateway respondió correctamente.");
|
||||
void refreshOpenaiBalance();
|
||||
} catch (err) {
|
||||
setHealthJson(null);
|
||||
setConnectionStatus("error");
|
||||
setConnectionMessage(err instanceof Error ? err.message : "No se pudo conectar");
|
||||
setOpenaiBalance({
|
||||
status: "error",
|
||||
available: null,
|
||||
currency: "USD",
|
||||
error: "Sin conexión al gateway.",
|
||||
});
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}, [client]);
|
||||
}, [client, refreshOpenaiBalance]);
|
||||
|
||||
useEffect(() => {
|
||||
void verifyConnection();
|
||||
}, [verifyConnection]);
|
||||
|
||||
useEffect(() => {
|
||||
if (connectionStatus !== "ok") return;
|
||||
const timer = window.setInterval(() => {
|
||||
void refreshOpenaiBalance();
|
||||
}, 5 * 60_000);
|
||||
return () => window.clearInterval(timer);
|
||||
}, [connectionStatus, refreshOpenaiBalance]);
|
||||
|
||||
function openJob(job: Job) {
|
||||
setActiveJob(job);
|
||||
setJobSection("downloads");
|
||||
setDownloadsMenuOpen(true);
|
||||
setJobSectionContext({});
|
||||
setView("job");
|
||||
}
|
||||
@@ -118,6 +213,31 @@ function App() {
|
||||
function handleActivateSection(section: JobSection, context: Record<string, unknown> = {}) {
|
||||
setJobSection(section);
|
||||
setJobSectionContext(context);
|
||||
setDownloadsMenuOpen(section === "downloads");
|
||||
}
|
||||
|
||||
async function downloadBotPackage() {
|
||||
setBotDownloadBusy(true);
|
||||
setBotDownloadError(null);
|
||||
try {
|
||||
const api = window.desktopApi;
|
||||
if (typeof api?.downloadUrl === "function") {
|
||||
await api.downloadUrl(BOT_PACKAGE_URL);
|
||||
return;
|
||||
}
|
||||
const a = document.createElement("a");
|
||||
a.href = BOT_PACKAGE_URL;
|
||||
a.download = "bot_descarga.zip";
|
||||
a.rel = "noopener noreferrer";
|
||||
a.target = "_blank";
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
a.remove();
|
||||
} catch (err) {
|
||||
setBotDownloadError(err instanceof Error ? err.message : "No se pudo iniciar la descarga del bot.");
|
||||
} finally {
|
||||
setBotDownloadBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
function openSettings(section: SettingsSection = "connection") {
|
||||
@@ -137,6 +257,20 @@ function App() {
|
||||
const isJobOpen = view === "job" && activeJob;
|
||||
const isSettingsOpen = view === "settings";
|
||||
|
||||
const openaiBalanceLabel =
|
||||
openaiBalance.status === "busy"
|
||||
? "Consultando…"
|
||||
: openaiBalance.status === "ok" && openaiBalance.available != null
|
||||
? formatUsd(openaiBalance.available)
|
||||
: openaiBalance.status === "error"
|
||||
? formatOpenaiBalanceError(openaiBalance.error)
|
||||
: "—";
|
||||
|
||||
const openaiBalanceTitle =
|
||||
openaiBalance.status === "ok" && openaiBalance.available != null
|
||||
? `Crédito OpenAI disponible: ${formatUsd(openaiBalance.available)}`
|
||||
: openaiBalance.error || "Crédito disponible de la API de OpenAI";
|
||||
|
||||
return (
|
||||
<main className="app-frame">
|
||||
<aside className="sidebar">
|
||||
@@ -202,14 +336,41 @@ function App() {
|
||||
</div>
|
||||
<div className="side-section-divider" />
|
||||
<nav className="side-nav" aria-label="Módulos del trabajo">
|
||||
<button
|
||||
type="button"
|
||||
className={jobSection === "downloads" ? "active" : ""}
|
||||
onClick={() => handleActivateSection("downloads")}
|
||||
>
|
||||
<Download size={17} />
|
||||
<span>Bot de descarga</span>
|
||||
</button>
|
||||
<div className="side-nav-group">
|
||||
<button
|
||||
type="button"
|
||||
className={jobSection === "downloads" ? "active" : ""}
|
||||
aria-expanded={downloadsMenuOpen}
|
||||
onClick={() => {
|
||||
if (jobSection === "downloads") {
|
||||
setDownloadsMenuOpen((open) => !open);
|
||||
} else {
|
||||
handleActivateSection("downloads");
|
||||
}
|
||||
}}
|
||||
>
|
||||
<Download size={17} />
|
||||
<span>Bot de descarga</span>
|
||||
<ChevronDown
|
||||
size={14}
|
||||
className={`side-nav-chevron${downloadsMenuOpen ? " side-nav-chevron--open" : ""}`}
|
||||
aria-hidden
|
||||
/>
|
||||
</button>
|
||||
{downloadsMenuOpen && (
|
||||
<div className="side-nav-sub" role="group" aria-label="Opciones del bot de descarga">
|
||||
<button
|
||||
type="button"
|
||||
disabled={botDownloadBusy}
|
||||
title={botDownloadError || "Descargar el paquete del bot"}
|
||||
onClick={() => void downloadBotPackage()}
|
||||
>
|
||||
<Package size={15} />
|
||||
<span>{botDownloadBusy ? "Descargando…" : "Descargar bot"}</span>
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
className={jobSection === "counting" ? "active" : ""}
|
||||
@@ -234,6 +395,14 @@ function App() {
|
||||
<Sparkles size={17} />
|
||||
<span>Limpieza</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={jobSection === "poblacion_apta" ? "active" : ""}
|
||||
onClick={() => handleActivateSection("poblacion_apta")}
|
||||
>
|
||||
<Users size={17} />
|
||||
<span>Población apta</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={jobSection === "results" ? "active" : ""}
|
||||
@@ -270,6 +439,19 @@ function App() {
|
||||
)}
|
||||
|
||||
<div className="sidebar-footer">
|
||||
<button
|
||||
type="button"
|
||||
className={`sidebar-openai ${openaiBalance.status}`}
|
||||
title={openaiBalanceTitle}
|
||||
onClick={() => void refreshOpenaiBalance()}
|
||||
disabled={openaiBalance.status === "busy"}
|
||||
>
|
||||
<Wallet size={14} />
|
||||
<div className="sidebar-openai-text">
|
||||
<span className="sidebar-openai-label">OpenAI</span>
|
||||
<span className="sidebar-openai-value">{openaiBalanceLabel}</span>
|
||||
</div>
|
||||
</button>
|
||||
<div className={`sidebar-gateway ${connectionStatus}`}>
|
||||
<div className={`gateway-dot ${connectionStatus}`} />
|
||||
<span className="gateway-label">
|
||||
|
||||
@@ -16,6 +16,7 @@ export function BackgroundProgressBanner({ progress, compact = false }: Props) {
|
||||
const moduleLabel =
|
||||
progress.module === "counting" ? "Conteo"
|
||||
: progress.module === "cleaning" ? "Limpieza"
|
||||
: progress.module === "poblacion_apta" ? "Población apta"
|
||||
: "Extracción";
|
||||
|
||||
return (
|
||||
|
||||
@@ -27,6 +27,7 @@ import type {
|
||||
import { ApiClient } from "../lib/apiClient";
|
||||
import { downloadCsv, toCsv, buildExportCsvFilename } from "../lib/csv";
|
||||
import { countFlatFolders, downloadFilesToManifest } from "../lib/downloadManifest";
|
||||
import { taskTimingLabel, timingDuration } from "../lib/time";
|
||||
import { CountResultsTable } from "./CountResultsTable";
|
||||
import { CountingStepper, type CountingStep } from "./CountingStepper";
|
||||
import {
|
||||
@@ -425,6 +426,7 @@ export function CountingWizard({
|
||||
job_id: job?.id,
|
||||
download_job_id: downloadJobIdRef.current ?? null,
|
||||
source: fromDownload ? "from-download" : "desktop",
|
||||
started_at: new Date(startedAt).toISOString(),
|
||||
results: finalResults.map((r, i) => {
|
||||
const entry = manifest[i];
|
||||
return {
|
||||
@@ -727,15 +729,22 @@ export function CountingWizard({
|
||||
</div>
|
||||
|
||||
{savedTask ? (
|
||||
<CountResultsTable
|
||||
files={savedFiles}
|
||||
validCounts={form.validCounts}
|
||||
onValidCountsChange={(v) => setForm((f) => ({ ...f, validCounts: v }))}
|
||||
autoAdjustedFilenames={autoAdjustedFilenames}
|
||||
onToggleValid={handleToggleValid}
|
||||
onExportCsv={handleExportCsv}
|
||||
loading={running}
|
||||
/>
|
||||
<>
|
||||
<p className="wiz-hint" style={{ marginBottom: 10 }}>
|
||||
Duración: <strong>{timingDuration(savedTask)}</strong>
|
||||
{" · "}
|
||||
{taskTimingLabel(savedTask)}
|
||||
</p>
|
||||
<CountResultsTable
|
||||
files={savedFiles}
|
||||
validCounts={form.validCounts}
|
||||
onValidCountsChange={(v) => setForm((f) => ({ ...f, validCounts: v }))}
|
||||
autoAdjustedFilenames={autoAdjustedFilenames}
|
||||
onToggleValid={handleToggleValid}
|
||||
onExportCsv={handleExportCsv}
|
||||
loading={running}
|
||||
/>
|
||||
</>
|
||||
) : (
|
||||
!error && (
|
||||
<div className="count-no-results">
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
import { Search } from "lucide-react";
|
||||
import { useMemo, useState } from "react";
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
type FilterKey = "all" | "aptos" | "no_aptos";
|
||||
|
||||
interface Props {
|
||||
rows: Row[];
|
||||
/** Resumen opcional del backend; si falta se calcula desde filas. */
|
||||
summary?: {
|
||||
total?: number;
|
||||
paso_bot?: number;
|
||||
con_error?: number;
|
||||
duplicados?: number;
|
||||
};
|
||||
}
|
||||
|
||||
const PREFERRED_COLUMNS = [
|
||||
"radicado",
|
||||
"total_cargos",
|
||||
"autorizacion",
|
||||
"ingreso",
|
||||
"tipo_documento",
|
||||
"documento",
|
||||
"tipo_servicio",
|
||||
"copago",
|
||||
"cuota_moderadora",
|
||||
"OBSERVACION",
|
||||
"archivo_origen",
|
||||
];
|
||||
|
||||
const COLUMN_LABELS: Record<string, string> = {
|
||||
radicado: "Radicado",
|
||||
total_cargos: "Bruto",
|
||||
autorizacion: "Autorización",
|
||||
ingreso: "Fecha",
|
||||
tipo_documento: "Tipo ID",
|
||||
documento: "Documento",
|
||||
tipo_servicio: "Servicio",
|
||||
copago: "Copago",
|
||||
cuota_moderadora: "Moderadora",
|
||||
OBSERVACION: "Observación",
|
||||
archivo_origen: "Origen",
|
||||
tipo_archivo: "Tipo archivo",
|
||||
};
|
||||
|
||||
const HIDDEN_COLUMNS = new Set(["cleaning_job_id", "tipo_archivo"]);
|
||||
|
||||
function cellValue(row: Row, col: string): string {
|
||||
const v = row[col];
|
||||
if (v === null || v === undefined) return "";
|
||||
return String(v);
|
||||
}
|
||||
|
||||
function observacionOf(row: Row): string {
|
||||
return cellValue(row, "OBSERVACION").trim();
|
||||
}
|
||||
|
||||
function isApto(row: Row): boolean {
|
||||
return observacionOf(row) === "PASO_BOT";
|
||||
}
|
||||
|
||||
function isDuplicado(row: Row): boolean {
|
||||
return observacionOf(row).includes("RADICADO_DUPLICADO");
|
||||
}
|
||||
|
||||
function orderColumns(rows: Row[]): string[] {
|
||||
const present = new Set(rows.flatMap((r) => Object.keys(r)));
|
||||
const ordered: string[] = [];
|
||||
for (const c of PREFERRED_COLUMNS) {
|
||||
if (present.has(c) && !HIDDEN_COLUMNS.has(c)) ordered.push(c);
|
||||
}
|
||||
for (const c of present) {
|
||||
if (!ordered.includes(c) && !HIDDEN_COLUMNS.has(c)) ordered.push(c);
|
||||
}
|
||||
return ordered;
|
||||
}
|
||||
|
||||
function ObsBadge({ value }: { value: string }) {
|
||||
if (!value) return <span className="extraction-empty-cell">—</span>;
|
||||
const apto = value === "PASO_BOT";
|
||||
const dup = value.includes("RADICADO_DUPLICADO");
|
||||
const cls = apto
|
||||
? "pa-obs-badge pa-obs-badge--apto"
|
||||
: dup
|
||||
? "pa-obs-badge pa-obs-badge--dup"
|
||||
: "pa-obs-badge pa-obs-badge--error";
|
||||
return <span className={cls} title={value}>{value}</span>;
|
||||
}
|
||||
|
||||
export function PoblacionAptaResultsTable({ rows, summary: summaryProp }: Props) {
|
||||
const [filter, setFilter] = useState<FilterKey>("all");
|
||||
const [search, setSearch] = useState("");
|
||||
|
||||
const columns = useMemo(() => orderColumns(rows), [rows]);
|
||||
|
||||
const summary = useMemo(() => {
|
||||
const aptos = rows.filter(isApto).length;
|
||||
const dups = rows.filter(isDuplicado).length;
|
||||
return {
|
||||
total: summaryProp?.total ?? rows.length,
|
||||
aptos: summaryProp?.paso_bot ?? aptos,
|
||||
noAptos: summaryProp?.con_error ?? rows.length - aptos,
|
||||
duplicados: summaryProp?.duplicados ?? dups,
|
||||
};
|
||||
}, [rows, summaryProp]);
|
||||
|
||||
const filtered = useMemo(() => {
|
||||
let list = rows;
|
||||
if (filter === "aptos") list = list.filter(isApto);
|
||||
else if (filter === "no_aptos") list = list.filter((r) => !isApto(r));
|
||||
|
||||
const q = search.trim().toLowerCase();
|
||||
if (q) {
|
||||
list = list.filter((r) =>
|
||||
columns.some((c) => cellValue(r, c).toLowerCase().includes(q)),
|
||||
);
|
||||
}
|
||||
return list;
|
||||
}, [rows, columns, filter, search]);
|
||||
|
||||
const FILTERS: { key: FilterKey; label: string; count: number }[] = [
|
||||
{ key: "all", label: "Todos", count: summary.total },
|
||||
{ key: "aptos", label: "Aptos", count: summary.aptos },
|
||||
{ key: "no_aptos", label: "No aptos", count: summary.noAptos },
|
||||
];
|
||||
|
||||
return (
|
||||
<div className="extraction-results pa-results">
|
||||
<div className="wiz-stats-row count-results-summary pa-stats-row">
|
||||
<div className="wiz-stat">
|
||||
<span className="wiz-stat-value">{summary.total}</span>
|
||||
<span className="wiz-stat-label">Total</span>
|
||||
</div>
|
||||
<div className="wiz-stat count-stat--valid">
|
||||
<span className="wiz-stat-value">{summary.aptos}</span>
|
||||
<span className="wiz-stat-label">Aptos</span>
|
||||
</div>
|
||||
<div className="wiz-stat count-stat--invalid">
|
||||
<span className="wiz-stat-value">{summary.noAptos}</span>
|
||||
<span className="wiz-stat-label">No aptos</span>
|
||||
</div>
|
||||
{summary.duplicados > 0 && (
|
||||
<div className="wiz-stat pa-stat--dup">
|
||||
<span className="wiz-stat-value">{summary.duplicados}</span>
|
||||
<span className="wiz-stat-label">Duplicados</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="count-toolbar">
|
||||
<div className="count-filter-tabs">
|
||||
{FILTERS.map(({ key, label, count }) => (
|
||||
<button
|
||||
key={key}
|
||||
type="button"
|
||||
className={`count-filter-tab ${filter === key ? "active" : ""}`}
|
||||
onClick={() => setFilter(key)}
|
||||
>
|
||||
{label}
|
||||
<span className="count-filter-count">{count}</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="count-toolbar-right">
|
||||
<div className="wiz-search-box">
|
||||
<Search size={14} />
|
||||
<input
|
||||
placeholder="Buscar radicado, documento…"
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.target.value)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="results-table-wrapper extraction-table-wrap">
|
||||
<table className="results-table pa-table">
|
||||
<thead>
|
||||
<tr>
|
||||
{columns.map((c) => (
|
||||
<th key={c} className={c === "OBSERVACION" ? "pa-th-obs" : undefined}>
|
||||
{COLUMN_LABELS[c] ?? c}
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{filtered.length === 0 ? (
|
||||
<tr>
|
||||
<td colSpan={Math.max(columns.length, 1)} className="extraction-empty">
|
||||
{search
|
||||
? "No hay filas que coincidan con la búsqueda."
|
||||
: "No hay filas en esta categoría."}
|
||||
</td>
|
||||
</tr>
|
||||
) : (
|
||||
filtered.map((row, i) => {
|
||||
const apto = isApto(row);
|
||||
return (
|
||||
<tr
|
||||
key={i}
|
||||
className={apto ? "pa-row--apto" : "pa-row--no-apto"}
|
||||
>
|
||||
{columns.map((c) => (
|
||||
<td key={c} className={c === "OBSERVACION" ? "pa-td-obs" : undefined}>
|
||||
{c === "OBSERVACION" ? (
|
||||
<ObsBadge value={observacionOf(row)} />
|
||||
) : cellValue(row, c).trim() === "" ? (
|
||||
<span className="extraction-empty-cell">—</span>
|
||||
) : (
|
||||
cellValue(row, c)
|
||||
)}
|
||||
</td>
|
||||
))}
|
||||
</tr>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{filtered.length !== rows.length && (
|
||||
<p className="extraction-filter-hint">
|
||||
Mostrando {filtered.length} de {rows.length} filas
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -16,8 +16,11 @@ import type {
|
||||
ExtractionBatch,
|
||||
ExtractionFromCountPayload,
|
||||
Job,
|
||||
PoblacionAptaJob,
|
||||
PoblacionAptaRowsResponse,
|
||||
RowsResponse,
|
||||
RunCleaningPayload,
|
||||
RunPoblacionAptaPayload,
|
||||
SchemaResponse,
|
||||
Task,
|
||||
UpdateJobPayload,
|
||||
@@ -82,6 +85,8 @@ export class ApiClient {
|
||||
path.includes("/upload") ||
|
||||
path.startsWith("/cleaning/analyze-csv") ||
|
||||
path.startsWith("/cleaning/jobs/from-csv") ||
|
||||
path.startsWith("/poblacion-apta/jobs/from-csv") ||
|
||||
path.startsWith("/poblacion-apta/jobs/from-cleaning") ||
|
||||
path === "/bot-descarga/jobs"
|
||||
);
|
||||
}
|
||||
@@ -129,6 +134,18 @@ export class ApiClient {
|
||||
return this.getJson("/models");
|
||||
}
|
||||
|
||||
async openaiBalance(): Promise<{
|
||||
ok: boolean;
|
||||
available: number | null;
|
||||
total_granted?: number | null;
|
||||
total_used?: number | null;
|
||||
currency?: string;
|
||||
source?: string | null;
|
||||
error?: string | null;
|
||||
}> {
|
||||
return this.getJson("/openai/balance");
|
||||
}
|
||||
|
||||
async discoverSchema(payload: Record<string, unknown>): Promise<SchemaResponse> {
|
||||
return this.postJson<SchemaResponse>("/extraction/schemas", payload);
|
||||
}
|
||||
@@ -380,6 +397,57 @@ export class ApiClient {
|
||||
);
|
||||
}
|
||||
|
||||
async runPoblacionApta(payload: RunPoblacionAptaPayload): Promise<PoblacionAptaJob> {
|
||||
return this.postJson<PoblacionAptaJob>("/poblacion-apta/jobs", payload);
|
||||
}
|
||||
|
||||
async runPoblacionAptaFromCleaning(form: FormData): Promise<PoblacionAptaJob> {
|
||||
return this.postForm<PoblacionAptaJob>("/poblacion-apta/jobs/from-cleaning", form);
|
||||
}
|
||||
|
||||
async runPoblacionAptaFromCsv(form: FormData): Promise<PoblacionAptaJob> {
|
||||
return this.postForm<PoblacionAptaJob>("/poblacion-apta/jobs/from-csv", form);
|
||||
}
|
||||
|
||||
async poblacionAptaJobs(jobId?: string): Promise<PoblacionAptaJob[]> {
|
||||
const qs = jobId ? `?job_id=${encodeURIComponent(jobId)}` : "";
|
||||
return this.getJson<PoblacionAptaJob[]>(`/poblacion-apta/jobs${qs}`);
|
||||
}
|
||||
|
||||
async poblacionAptaJob(jobId: string): Promise<PoblacionAptaJob> {
|
||||
return this.getJson<PoblacionAptaJob>(`/poblacion-apta/jobs/${encodeURIComponent(jobId)}`);
|
||||
}
|
||||
|
||||
async poblacionAptaJobRows(
|
||||
jobId: string,
|
||||
limit = 500,
|
||||
offset = 0,
|
||||
observacion?: string,
|
||||
): Promise<PoblacionAptaRowsResponse> {
|
||||
const params = new URLSearchParams({
|
||||
limit: String(limit),
|
||||
offset: String(offset),
|
||||
});
|
||||
if (observacion) params.set("observacion", observacion);
|
||||
return this.getJson<PoblacionAptaRowsResponse>(
|
||||
`/poblacion-apta/jobs/${encodeURIComponent(jobId)}/rows?${params}`,
|
||||
);
|
||||
}
|
||||
|
||||
async poblacionAptaExportCsv(
|
||||
jobId: string,
|
||||
subset: "aptos" | "no_aptos" = "aptos",
|
||||
): Promise<Blob> {
|
||||
const params = new URLSearchParams({ subset });
|
||||
const path = `/poblacion-apta/jobs/${encodeURIComponent(jobId)}/export.csv?${params}`;
|
||||
const response = await fetchWithTimeout(this.url(path), {}, this.timeoutFor(path));
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
throw new Error(text || `Error ${response.status}`);
|
||||
}
|
||||
return response.blob();
|
||||
}
|
||||
|
||||
private async handle<T>(response: Response): Promise<T> {
|
||||
const text = await response.text();
|
||||
let data: unknown = null;
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
/** Formato de fechas y duraciones para tareas/procesos. */
|
||||
|
||||
export function formatDateTime(iso: string | null | undefined): string {
|
||||
if (!iso) return "—";
|
||||
try {
|
||||
return new Date(iso).toLocaleString("es-CO", {
|
||||
dateStyle: "short",
|
||||
timeStyle: "short",
|
||||
});
|
||||
} catch {
|
||||
return iso;
|
||||
}
|
||||
}
|
||||
|
||||
/** Duración legible entre dos ISO (o ms). */
|
||||
export function formatDuration(
|
||||
started: string | number | null | undefined,
|
||||
finished?: string | number | null,
|
||||
): string {
|
||||
if (started == null) return "—";
|
||||
const startMs = typeof started === "number" ? started : Date.parse(started);
|
||||
if (Number.isNaN(startMs)) return "—";
|
||||
const endMs =
|
||||
finished == null
|
||||
? Date.now()
|
||||
: typeof finished === "number"
|
||||
? finished
|
||||
: Date.parse(finished);
|
||||
if (Number.isNaN(endMs) || endMs < startMs) return "—";
|
||||
|
||||
let secs = Math.round((endMs - startMs) / 1000);
|
||||
if (secs < 1) return "<1s";
|
||||
const hours = Math.floor(secs / 3600);
|
||||
secs %= 3600;
|
||||
const mins = Math.floor(secs / 60);
|
||||
secs %= 60;
|
||||
if (hours > 0) return `${hours}h ${mins}m ${secs}s`;
|
||||
if (mins > 0) return `${mins}m ${secs}s`;
|
||||
return `${secs}s`;
|
||||
}
|
||||
|
||||
export interface TimingFields {
|
||||
started_at?: string | null;
|
||||
finished_at?: string | null;
|
||||
created_at?: string;
|
||||
updated_at?: string;
|
||||
status?: string;
|
||||
}
|
||||
|
||||
/** Inicio · fin · duración (o “en curso”). */
|
||||
export function taskTimingLabel(task: TimingFields): string {
|
||||
const start = task.started_at || task.created_at || null;
|
||||
const end =
|
||||
task.finished_at ||
|
||||
(task.status && ["completed", "failed"].includes(task.status)
|
||||
? task.updated_at
|
||||
: null) ||
|
||||
null;
|
||||
if (!start) return "—";
|
||||
if (!end) {
|
||||
return `${formatDateTime(start)} · en curso (${formatDuration(start)})`;
|
||||
}
|
||||
return `${formatDateTime(start)} · ${formatDuration(start, end)}`;
|
||||
}
|
||||
|
||||
/** Solo la duración, con respaldo created/updated. */
|
||||
export function timingDuration(task: TimingFields): string {
|
||||
const start = task.started_at || task.created_at || null;
|
||||
const end =
|
||||
task.finished_at ||
|
||||
(task.status && ["completed", "failed"].includes(task.status)
|
||||
? task.updated_at
|
||||
: null) ||
|
||||
null;
|
||||
return formatDuration(start, end);
|
||||
}
|
||||
@@ -27,6 +27,7 @@ import { ExtractionResultsTable } from "../../components/ExtractionResultsTable"
|
||||
import type { ApiClient } from "../../lib/apiClient";
|
||||
import { loadCopModConfig } from "../../lib/cleaningSettings";
|
||||
import { downloadCsv, toCsv, buildExportCsvFilename } from "../../lib/csv";
|
||||
import { taskTimingLabel, timingDuration } from "../../lib/time";
|
||||
|
||||
type SourceMode = "extraction" | "csv";
|
||||
|
||||
@@ -43,20 +44,6 @@ function batchIdFromTask(task: Task): string {
|
||||
return String((task.result as Record<string, unknown>)?.extraction_batch_id ?? "");
|
||||
}
|
||||
|
||||
function formatTaskDate(iso: string): string {
|
||||
try {
|
||||
return new Date(iso).toLocaleString("es-CO", {
|
||||
day: "2-digit",
|
||||
month: "short",
|
||||
year: "numeric",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
});
|
||||
} catch {
|
||||
return iso;
|
||||
}
|
||||
}
|
||||
|
||||
export function CleaningWizard({
|
||||
client,
|
||||
job,
|
||||
@@ -78,6 +65,8 @@ export function CleaningWizard({
|
||||
const [analysis, setAnalysis] = useState<CleaningAnalysisResponse | null>(null);
|
||||
const [rulesPerColumn, setRulesPerColumn] = useState<Record<string, string[]>>({});
|
||||
const [copModFechaColumn, setCopModFechaColumn] = useState<Record<string, string>>({});
|
||||
const [copModBrutoColumn, setCopModBrutoColumn] = useState<Record<string, string>>({});
|
||||
const [copModNetoColumn, setCopModNetoColumn] = useState<Record<string, string>>({});
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [running, setRunning] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
@@ -179,6 +168,26 @@ export function CleaningWizard({
|
||||
return preferred?.name ?? "";
|
||||
}
|
||||
|
||||
function guessBrutoColumn(columns: CleaningColumnAnalysis[], preferCol?: string): string {
|
||||
if (preferCol && /bruto|total[_\s]?cargos|valor[_\s]?bruto|valor[_\s]?factura/i.test(preferCol)) {
|
||||
return preferCol;
|
||||
}
|
||||
const preferred =
|
||||
columns.find((c) => /valor[_\s]?bruto|^bruto$|total[_\s]?cargos/i.test(c.name)) ??
|
||||
columns.find((c) => /bruto/i.test(c.name)) ??
|
||||
(preferCol ? columns.find((c) => c.name === preferCol) : undefined);
|
||||
return preferred?.name ?? preferCol ?? "";
|
||||
}
|
||||
|
||||
function guessNetoColumn(columns: CleaningColumnAnalysis[], brutoCol: string): string {
|
||||
const preferred =
|
||||
columns.find(
|
||||
(c) => c.name !== brutoCol && /valor[_\s]?neto|^neto$|valor[_\s]?pagado/i.test(c.name),
|
||||
) ??
|
||||
columns.find((c) => c.name !== brutoCol && /neto/i.test(c.name));
|
||||
return preferred?.name ?? "";
|
||||
}
|
||||
|
||||
function toggleRule(column: string, ruleId: string) {
|
||||
setRulesPerColumn((prev) => {
|
||||
const current = prev[column] ?? [];
|
||||
@@ -202,6 +211,27 @@ export function CleaningWizard({
|
||||
return guess ? { ...prev, [column]: guess } : prev;
|
||||
});
|
||||
}
|
||||
if (ruleId === "cop_mod_bruto_neto") {
|
||||
const current = rulesPerColumn[column] ?? [];
|
||||
const turningOn = !current.includes(ruleId);
|
||||
if (!turningOn) {
|
||||
setCopModBrutoColumn((prev) => {
|
||||
const next = { ...prev };
|
||||
delete next[column];
|
||||
return next;
|
||||
});
|
||||
setCopModNetoColumn((prev) => {
|
||||
const next = { ...prev };
|
||||
delete next[column];
|
||||
return next;
|
||||
});
|
||||
} else if (analysis) {
|
||||
const bruto = guessBrutoColumn(analysis.columns, column);
|
||||
const neto = guessNetoColumn(analysis.columns, bruto || column);
|
||||
setCopModBrutoColumn((prev) => (prev[column] ? prev : { ...prev, [column]: bruto || column }));
|
||||
setCopModNetoColumn((prev) => (prev[column] ? prev : { ...prev, [column]: neto }));
|
||||
}
|
||||
}
|
||||
setSampleMode("after");
|
||||
}
|
||||
|
||||
@@ -213,20 +243,38 @@ export function CleaningWizard({
|
||||
}
|
||||
setRulesPerColumn(suggested);
|
||||
setCopModFechaColumn({});
|
||||
setCopModBrutoColumn({});
|
||||
setCopModNetoColumn({});
|
||||
}
|
||||
|
||||
function cleaningExtraPayload(): {
|
||||
cop_mod_fecha_column?: Record<string, string>;
|
||||
cop_mod_bruto_column?: Record<string, string>;
|
||||
cop_mod_neto_column?: Record<string, string>;
|
||||
cop_mod_config: ReturnType<typeof loadCopModConfig>;
|
||||
} {
|
||||
const fechaEntries = Object.entries(copModFechaColumn).filter(
|
||||
([col, fecha]) =>
|
||||
Boolean(fecha) && (rulesPerColumn[col] ?? []).includes("cop_mod_split"),
|
||||
);
|
||||
const brutoEntries = Object.entries(copModBrutoColumn).filter(
|
||||
([col, bruto]) =>
|
||||
Boolean(bruto) && (rulesPerColumn[col] ?? []).includes("cop_mod_bruto_neto"),
|
||||
);
|
||||
const netoEntries = Object.entries(copModNetoColumn).filter(
|
||||
([col, neto]) =>
|
||||
Boolean(neto) && (rulesPerColumn[col] ?? []).includes("cop_mod_bruto_neto"),
|
||||
);
|
||||
return {
|
||||
...(fechaEntries.length > 0
|
||||
? { cop_mod_fecha_column: Object.fromEntries(fechaEntries) }
|
||||
: {}),
|
||||
...(brutoEntries.length > 0
|
||||
? { cop_mod_bruto_column: Object.fromEntries(brutoEntries) }
|
||||
: {}),
|
||||
...(netoEntries.length > 0
|
||||
? { cop_mod_neto_column: Object.fromEntries(netoEntries) }
|
||||
: {}),
|
||||
cop_mod_config: loadCopModConfig(),
|
||||
};
|
||||
}
|
||||
@@ -241,17 +289,24 @@ export function CleaningWizard({
|
||||
return [];
|
||||
}
|
||||
|
||||
/** Columnas nuevas respecto al análisis (p. ej. *_doc_tipo / *_doc_numeros). */
|
||||
/** Columnas nuevas respecto al análisis (p. ej. *_doc_tipo / *_doc_numeros / cop/mod). */
|
||||
function derivedColumnNames(sourceCol: string): string[] {
|
||||
if (!preview) return [];
|
||||
const original = new Set(analysis?.columns.map((c) => c.name) ?? []);
|
||||
const selected = rulesPerColumn[sourceCol] ?? [];
|
||||
return preview.columns
|
||||
.map((c) => c.name)
|
||||
.filter(
|
||||
(name) =>
|
||||
!original.has(name) &&
|
||||
name.toLowerCase().startsWith(`${sourceCol.toLowerCase()}_`),
|
||||
);
|
||||
.filter((name) => {
|
||||
if (original.has(name)) return false;
|
||||
if (name.toLowerCase().startsWith(`${sourceCol.toLowerCase()}_`)) return true;
|
||||
if (
|
||||
selected.includes("cop_mod_bruto_neto") &&
|
||||
(name === "cop/mod" || name.startsWith("cop/mod_"))
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
/** Orden: columna activa → columnas derivadas → resto (sin cortar). */
|
||||
@@ -309,7 +364,7 @@ export function CleaningWizard({
|
||||
return () => {
|
||||
if (previewTimerRef.current) clearTimeout(previewTimerRef.current);
|
||||
};
|
||||
}, [step, analysis, rulesPerColumn, copModFechaColumn, batchId, client]);
|
||||
}, [step, analysis, rulesPerColumn, copModFechaColumn, copModBrutoColumn, copModNetoColumn, batchId, client]);
|
||||
|
||||
async function loadAnalysis(mode: SourceMode, batch?: string, file?: File | null) {
|
||||
setLoading(true);
|
||||
@@ -393,6 +448,12 @@ export function CleaningWizard({
|
||||
if (extra.cop_mod_fecha_column) {
|
||||
form.append("cop_mod_fecha_column", JSON.stringify(extra.cop_mod_fecha_column));
|
||||
}
|
||||
if (extra.cop_mod_bruto_column) {
|
||||
form.append("cop_mod_bruto_column", JSON.stringify(extra.cop_mod_bruto_column));
|
||||
}
|
||||
if (extra.cop_mod_neto_column) {
|
||||
form.append("cop_mod_neto_column", JSON.stringify(extra.cop_mod_neto_column));
|
||||
}
|
||||
form.append("cop_mod_config", JSON.stringify(extra.cop_mod_config));
|
||||
cleaningJob = await client.runCleaningFromCsv(form);
|
||||
}
|
||||
@@ -528,7 +589,7 @@ export function CleaningWizard({
|
||||
{task.title ?? `Extracción ${extractionTasks.length - index}`}
|
||||
</div>
|
||||
<div className="cleaning-source-item-meta">
|
||||
{formatTaskDate(task.created_at)}
|
||||
{taskTimingLabel(task)}
|
||||
{index === 0 ? " · más reciente" : ""}
|
||||
</div>
|
||||
</div>
|
||||
@@ -791,6 +852,63 @@ export function CleaningWizard({
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
{selected.includes("cop_mod_bruto_neto") && (
|
||||
<div className="cleaning-copmod-fecha">
|
||||
<label className="wiz-field-label" htmlFor={`copmod-bruto-${col.name}`}>
|
||||
Valor bruto
|
||||
</label>
|
||||
<select
|
||||
id={`copmod-bruto-${col.name}`}
|
||||
className="form-input"
|
||||
value={copModBrutoColumn[col.name] ?? col.name}
|
||||
onChange={(e) =>
|
||||
setCopModBrutoColumn((prev) => ({
|
||||
...prev,
|
||||
[col.name]: e.target.value,
|
||||
}))
|
||||
}
|
||||
>
|
||||
{analysis.columns.map((c) => (
|
||||
<option key={c.name} value={c.name}>
|
||||
{c.name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<label
|
||||
className="wiz-field-label"
|
||||
htmlFor={`copmod-neto-${col.name}`}
|
||||
style={{ marginTop: 10 }}
|
||||
>
|
||||
Valor neto
|
||||
</label>
|
||||
<select
|
||||
id={`copmod-neto-${col.name}`}
|
||||
className="form-input"
|
||||
value={copModNetoColumn[col.name] ?? ""}
|
||||
onChange={(e) =>
|
||||
setCopModNetoColumn((prev) => ({
|
||||
...prev,
|
||||
[col.name]: e.target.value,
|
||||
}))
|
||||
}
|
||||
>
|
||||
<option value="">Selecciona columna neto…</option>
|
||||
{analysis.columns
|
||||
.filter(
|
||||
(c) =>
|
||||
c.name !== (copModBrutoColumn[col.name] ?? col.name),
|
||||
)
|
||||
.map((c) => (
|
||||
<option key={c.name} value={c.name}>
|
||||
{c.name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<p className="form-hint">
|
||||
Se crea la columna <strong>cop/mod</strong> = bruto − neto.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
@@ -911,6 +1029,7 @@ export function CleaningWizard({
|
||||
Limpieza completada
|
||||
{resultJob.log?.length ? ` · ${resultJob.log.length} operaciones` : null}
|
||||
{resultRows.length > 0 ? ` · ${resultRows.length} filas` : null}
|
||||
{` · duración ${timingDuration(resultJob)}`}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { PoblacionAptaWizardModule } from "./PoblacionAptaWizardModule";
|
||||
import { PoblacionAptaResults } from "./PoblacionAptaResults";
|
||||
import type { ModuleDescriptor } from "../registry";
|
||||
import { registerModule } from "../registry";
|
||||
import type { Task } from "../../../shared/types";
|
||||
|
||||
const PoblacionAptaModule: ModuleDescriptor = {
|
||||
type: "poblacion_apta",
|
||||
label: "Población apta",
|
||||
iconName: "Users",
|
||||
hasTask: (tasks: Task[]) => tasks.some((t) => t.type === "poblacion_apta"),
|
||||
Wizard: PoblacionAptaWizardModule,
|
||||
Results: PoblacionAptaResults,
|
||||
};
|
||||
|
||||
registerModule(PoblacionAptaModule);
|
||||
|
||||
export { PoblacionAptaModule };
|
||||
@@ -0,0 +1,134 @@
|
||||
import { Download, Loader2, RefreshCw, Users } from "lucide-react";
|
||||
import { useEffect, useState } from "react";
|
||||
import type { PoblacionAptaJob } from "../../../shared/types";
|
||||
import { PoblacionAptaResultsTable } from "../../components/PoblacionAptaResultsTable";
|
||||
import { buildExportCsvFilename, downloadCsv } from "../../lib/csv";
|
||||
import type { ModuleResultsProps } from "../registry";
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
export function PoblacionAptaResults({ client, task, job }: ModuleResultsProps) {
|
||||
const [paJob, setPaJob] = useState<PoblacionAptaJob | null>(null);
|
||||
const [rows, setRows] = useState<Row[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
async function load() {
|
||||
try {
|
||||
const paJobId = String(
|
||||
(task.result as Record<string, unknown>)?.poblacion_apta_job_id ?? "",
|
||||
);
|
||||
if (!paJobId) {
|
||||
setError("La tarea no tiene poblacion_apta_job_id en result");
|
||||
return;
|
||||
}
|
||||
const jobData = await client.poblacionAptaJob(paJobId);
|
||||
setPaJob(jobData);
|
||||
if (jobData.status === "completed") {
|
||||
const response = await client.poblacionAptaJobRows(paJobId, 500, 0);
|
||||
setRows(response.rows ?? []);
|
||||
}
|
||||
} catch (e) {
|
||||
setError(String(e instanceof Error ? e.message : e));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
void load();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [task.id]);
|
||||
|
||||
async function exportAptos() {
|
||||
const paJobId = String(
|
||||
(task.result as Record<string, unknown>)?.poblacion_apta_job_id ?? "",
|
||||
);
|
||||
if (!paJobId) return;
|
||||
const blob = await client.poblacionAptaExportCsv(paJobId, "aptos");
|
||||
const text = await blob.text();
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_aptos", job.title), text);
|
||||
}
|
||||
|
||||
async function exportNoAptos() {
|
||||
const paJobId = String(
|
||||
(task.result as Record<string, unknown>)?.poblacion_apta_job_id ?? "",
|
||||
);
|
||||
if (!paJobId) return;
|
||||
const blob = await client.poblacionAptaExportCsv(paJobId, "no_aptos");
|
||||
const text = await blob.text();
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text);
|
||||
}
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="module-loading">
|
||||
<Loader2 size={22} className="spin" />
|
||||
<span>Cargando población apta…</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div className="wiz-toast wiz-toast--error" style={{ margin: 16 }}>
|
||||
{error}
|
||||
<button
|
||||
className="btn btn-sm"
|
||||
onClick={() => {
|
||||
setError(null);
|
||||
setLoading(true);
|
||||
void load();
|
||||
}}
|
||||
>
|
||||
<RefreshCw size={13} /> Reintentar
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const summary = paJob?.summary ?? {};
|
||||
|
||||
return (
|
||||
<div className="module-results">
|
||||
<div className="pa-results-header">
|
||||
<div className="module-results-title">
|
||||
<h3>
|
||||
<Users size={14} style={{ marginRight: 6, verticalAlign: "middle" }} />
|
||||
{task.title ?? "Población apta"}
|
||||
</h3>
|
||||
<span className={`task-badge task-badge--${paJob?.status ?? task.status}`}>
|
||||
{paJob?.status ?? task.status}
|
||||
</span>
|
||||
</div>
|
||||
<div className="pa-results-header-actions">
|
||||
<button type="button" className="btn btn-sm btn-primary" onClick={() => void exportAptos()}>
|
||||
<Download size={13} /> CSV aptos ({summary.paso_bot ?? 0})
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportNoAptos()}>
|
||||
<Download size={13} /> CSV no aptos ({summary.con_error ?? 0})
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{paJob?.log && paJob.log.length > 0 && (
|
||||
<details className="pa-results-log">
|
||||
<summary>Log ({paJob.log.length})</summary>
|
||||
<ul>
|
||||
{paJob.log.map((line, i) => (
|
||||
<li key={i}>{line}</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
)}
|
||||
|
||||
{rows.length === 0 ? (
|
||||
<div className="wiz-step wiz-step-centered" style={{ padding: 32 }}>
|
||||
<p className="wiz-desc">No se encontraron filas.</p>
|
||||
</div>
|
||||
) : (
|
||||
<PoblacionAptaResultsTable rows={rows} summary={summary} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,523 @@
|
||||
import {
|
||||
AlertCircle,
|
||||
CheckSquare,
|
||||
Download,
|
||||
FileSpreadsheet,
|
||||
Loader2,
|
||||
Square,
|
||||
Upload,
|
||||
Users,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import type { BackgroundProgress, Job, PoblacionAptaJob, Task } from "../../../shared/types";
|
||||
import { PoblacionAptaResultsTable } from "../../components/PoblacionAptaResultsTable";
|
||||
import type { ApiClient } from "../../lib/apiClient";
|
||||
import { buildExportCsvFilename, downloadCsv } from "../../lib/csv";
|
||||
import { taskTimingLabel, timingDuration } from "../../lib/time";
|
||||
|
||||
interface Props {
|
||||
client: ApiClient;
|
||||
job: Job;
|
||||
onTaskCreated: (task: Task) => void;
|
||||
onProgress?: (p: BackgroundProgress) => void;
|
||||
onActivateModule?: (type: string, context?: Record<string, unknown>) => void;
|
||||
}
|
||||
|
||||
type SourceMode = "cleaning" | "csv";
|
||||
|
||||
function cleaningJobIdFromTask(task: Task): string {
|
||||
return String((task.result as Record<string, unknown>)?.cleaning_job_id ?? "");
|
||||
}
|
||||
|
||||
function formatBytes(n: number): string {
|
||||
if (n < 1024) return `${n} B`;
|
||||
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
|
||||
return `${(n / (1024 * 1024)).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
export function PoblacionAptaWizard({
|
||||
client,
|
||||
job,
|
||||
onTaskCreated,
|
||||
onProgress,
|
||||
onActivateModule,
|
||||
}: Props) {
|
||||
const [source, setSource] = useState<SourceMode>("cleaning");
|
||||
const [cleaningTasks, setCleaningTasks] = useState<Task[]>([]);
|
||||
const [selectedIds, setSelectedIds] = useState<Set<string>>(new Set());
|
||||
const [csvFiles, setCsvFiles] = useState<File[]>([]);
|
||||
const [origenServicio, setOrigenServicio] = useState<File | null>(null);
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
const origenInputRef = useRef<HTMLInputElement>(null);
|
||||
const [loadingTasks, setLoadingTasks] = useState(true);
|
||||
const [running, setRunning] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [resultJob, setResultJob] = useState<PoblacionAptaJob | null>(null);
|
||||
const [resultRows, setResultRows] = useState<Record<string, unknown>[]>([]);
|
||||
const [done, setDone] = useState(false);
|
||||
|
||||
const loadCleaningTasks = useCallback(async () => {
|
||||
setLoadingTasks(true);
|
||||
try {
|
||||
const tasks = await client.jobTasks(job.id);
|
||||
const cleaning = tasks
|
||||
.filter((t) => t.type === "cleaning" && t.status === "completed" && cleaningJobIdFromTask(t))
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
setCleaningTasks(cleaning);
|
||||
} catch {
|
||||
setCleaningTasks([]);
|
||||
} finally {
|
||||
setLoadingTasks(false);
|
||||
}
|
||||
}, [client, job.id]);
|
||||
|
||||
useEffect(() => {
|
||||
void loadCleaningTasks();
|
||||
}, [loadCleaningTasks]);
|
||||
|
||||
function toggle(taskId: string) {
|
||||
setSelectedIds((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(taskId)) next.delete(taskId);
|
||||
else next.add(taskId);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
function toggleAll() {
|
||||
if (selectedIds.size === cleaningTasks.length) {
|
||||
setSelectedIds(new Set());
|
||||
} else {
|
||||
setSelectedIds(new Set(cleaningTasks.map((t) => t.id)));
|
||||
}
|
||||
}
|
||||
|
||||
function onCsvPicked(list: FileList | null) {
|
||||
if (!list || list.length === 0) return;
|
||||
const next = Array.from(list).filter((f) =>
|
||||
/\.csv$/i.test(f.name) || f.type === "text/csv" || f.type === "application/vnd.ms-excel",
|
||||
);
|
||||
setCsvFiles((prev) => {
|
||||
const names = new Set(prev.map((f) => `${f.name}:${f.size}`));
|
||||
const merged = [...prev];
|
||||
for (const f of next) {
|
||||
const key = `${f.name}:${f.size}`;
|
||||
if (!names.has(key)) {
|
||||
merged.push(f);
|
||||
names.add(key);
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
});
|
||||
if (fileInputRef.current) fileInputRef.current.value = "";
|
||||
}
|
||||
|
||||
function removeCsv(index: number) {
|
||||
setCsvFiles((prev) => prev.filter((_, i) => i !== index));
|
||||
}
|
||||
|
||||
function onOrigenPicked(list: FileList | null) {
|
||||
if (!list || list.length === 0) return;
|
||||
const file = list[0];
|
||||
const ok =
|
||||
/\.(csv|xlsx|xlsm|xls)$/i.test(file.name) ||
|
||||
file.type === "text/csv" ||
|
||||
file.type.includes("spreadsheet") ||
|
||||
file.type.includes("excel");
|
||||
if (!ok) {
|
||||
setError("Origen Servicio debe ser CSV o Excel (.xlsx / .xls)");
|
||||
return;
|
||||
}
|
||||
setOrigenServicio(file);
|
||||
setError(null);
|
||||
if (origenInputRef.current) origenInputRef.current.value = "";
|
||||
}
|
||||
|
||||
const canRun =
|
||||
source === "cleaning" ? selectedIds.size > 0 : csvFiles.length > 0;
|
||||
|
||||
async function handleRun() {
|
||||
setRunning(true);
|
||||
setError(null);
|
||||
setResultJob(null);
|
||||
setResultRows([]);
|
||||
|
||||
const total =
|
||||
source === "cleaning" ? selectedIds.size : csvFiles.length;
|
||||
const label =
|
||||
source === "cleaning"
|
||||
? `${selectedIds.size} limpiezas`
|
||||
: `${csvFiles.length} CSV`;
|
||||
|
||||
onProgress?.({
|
||||
module: "poblacion_apta",
|
||||
total,
|
||||
processed: 0,
|
||||
failed: 0,
|
||||
currentFile: label,
|
||||
running: true,
|
||||
label: "Validando población apta…",
|
||||
});
|
||||
|
||||
try {
|
||||
let paJob: PoblacionAptaJob;
|
||||
if (source === "cleaning") {
|
||||
const cleaningJobIds = cleaningTasks
|
||||
.filter((t) => selectedIds.has(t.id))
|
||||
.map(cleaningJobIdFromTask)
|
||||
.filter(Boolean);
|
||||
if (cleaningJobIds.length === 0) {
|
||||
throw new Error("Selecciona al menos una limpieza completada");
|
||||
}
|
||||
const form = new FormData();
|
||||
form.append("job_id", job.id);
|
||||
form.append("cleaning_job_ids", JSON.stringify(cleaningJobIds));
|
||||
if (origenServicio) {
|
||||
form.append("origen_servicio", origenServicio, origenServicio.name);
|
||||
}
|
||||
paJob = await client.runPoblacionAptaFromCleaning(form);
|
||||
} else {
|
||||
if (csvFiles.length === 0) {
|
||||
throw new Error("Selecciona al menos un archivo CSV");
|
||||
}
|
||||
const form = new FormData();
|
||||
form.append("job_id", job.id);
|
||||
for (const file of csvFiles) {
|
||||
form.append("files", file, file.name);
|
||||
}
|
||||
if (origenServicio) {
|
||||
form.append("origen_servicio", origenServicio, origenServicio.name);
|
||||
}
|
||||
paJob = await client.runPoblacionAptaFromCsv(form);
|
||||
}
|
||||
|
||||
setResultJob(paJob);
|
||||
if (paJob.status === "completed") {
|
||||
const resp = await client.poblacionAptaJobRows(paJob.id, 500, 0);
|
||||
setResultRows(resp.rows ?? []);
|
||||
}
|
||||
try {
|
||||
const tasks = await client.jobTasks(job.id);
|
||||
const latest = tasks
|
||||
.filter((t) => t.type === "poblacion_apta")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())[0];
|
||||
if (latest) onTaskCreated(latest);
|
||||
} catch { /* silent */ }
|
||||
setDone(true);
|
||||
} catch (e) {
|
||||
setError(String(e instanceof Error ? e.message : e));
|
||||
} finally {
|
||||
setRunning(false);
|
||||
onProgress?.({
|
||||
module: "poblacion_apta",
|
||||
total,
|
||||
processed: total,
|
||||
failed: 0,
|
||||
currentFile: "",
|
||||
running: false,
|
||||
label: "",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function exportAptos() {
|
||||
if (!resultJob) return;
|
||||
const blob = await client.poblacionAptaExportCsv(resultJob.id, "aptos");
|
||||
const text = await blob.text();
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_aptos", job.title), text);
|
||||
}
|
||||
|
||||
async function exportNoAptos() {
|
||||
if (!resultJob) return;
|
||||
const blob = await client.poblacionAptaExportCsv(resultJob.id, "no_aptos");
|
||||
const text = await blob.text();
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text);
|
||||
}
|
||||
|
||||
function reset() {
|
||||
setDone(false);
|
||||
setResultJob(null);
|
||||
setResultRows([]);
|
||||
setError(null);
|
||||
setSelectedIds(new Set());
|
||||
setCsvFiles([]);
|
||||
setOrigenServicio(null);
|
||||
void loadCleaningTasks();
|
||||
}
|
||||
|
||||
if (done && resultJob) {
|
||||
const summary = resultJob.summary ?? {};
|
||||
return (
|
||||
<div className="wiz-shell">
|
||||
<div className="pa-results-header">
|
||||
<div className="wiz-header" style={{ marginBottom: 0 }}>
|
||||
<h2>
|
||||
<Users size={18} style={{ marginRight: 8, verticalAlign: "middle" }} />
|
||||
Población apta
|
||||
</h2>
|
||||
<p className="wiz-desc">
|
||||
Consolidado validado: aptos para bot (PASO_BOT) y registros con observación.
|
||||
{" · "}Duración: {timingDuration(resultJob)}
|
||||
{resultJob.started_at || resultJob.created_at
|
||||
? ` (${taskTimingLabel(resultJob)})`
|
||||
: ""}
|
||||
</p>
|
||||
</div>
|
||||
<div className="pa-results-header-actions">
|
||||
<button type="button" className="btn btn-sm btn-primary" onClick={() => void exportAptos()}>
|
||||
<Download size={13} /> CSV aptos ({summary.paso_bot ?? 0})
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportNoAptos()}>
|
||||
<Download size={13} /> CSV no aptos ({summary.con_error ?? 0})
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm" onClick={reset}>
|
||||
Nueva corrida
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{resultJob.log?.length > 0 && (
|
||||
<details className="pa-results-log">
|
||||
<summary>Log ({resultJob.log.length})</summary>
|
||||
<ul>
|
||||
{resultJob.log.map((line, i) => (
|
||||
<li key={i}>{line}</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
)}
|
||||
|
||||
{resultRows.length === 0 ? (
|
||||
<p className="wiz-desc">No hay filas en el consolidado.</p>
|
||||
) : (
|
||||
<PoblacionAptaResultsTable rows={resultRows} summary={summary} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="wiz-shell">
|
||||
<div className="wiz-header">
|
||||
<h2>
|
||||
<Users size={18} style={{ marginRight: 8, verticalAlign: "middle" }} />
|
||||
Población apta
|
||||
</h2>
|
||||
<p className="wiz-desc">
|
||||
Elige limpiezas completadas o sube uno o varios CSV. Opcionalmente adjunta Origen Servicio
|
||||
(Excel/CSV) para cruzar radicado → tipo de servicio. Se valida y consolida en un único
|
||||
resultado (PASO_BOT / observaciones).
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="wiz-toast wiz-toast--error" style={{ marginBottom: 12 }}>
|
||||
<AlertCircle size={14} /> {error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="wiz-section">
|
||||
<label className="wiz-field-label">Origen</label>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap" }}>
|
||||
<button
|
||||
type="button"
|
||||
className={`btn${source === "cleaning" ? " btn-primary" : ""}`}
|
||||
onClick={() => setSource("cleaning")}
|
||||
>
|
||||
<FileSpreadsheet size={14} /> Desde limpiezas
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`btn${source === "csv" ? " btn-primary" : ""}`}
|
||||
onClick={() => setSource("csv")}
|
||||
>
|
||||
<Upload size={14} /> Subir CSV
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{source === "cleaning" ? (
|
||||
<div className="wiz-section">
|
||||
<div style={{ display: "flex", justifyContent: "space-between", alignItems: "center", marginBottom: 8 }}>
|
||||
<label className="wiz-field-label">Limpiezas de este trabajo</label>
|
||||
{cleaningTasks.length > 0 && (
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={toggleAll}>
|
||||
{selectedIds.size === cleaningTasks.length ? (
|
||||
<><CheckSquare size={13} /> Quitar todas</>
|
||||
) : (
|
||||
<><Square size={13} /> Seleccionar todas</>
|
||||
)}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{loadingTasks ? (
|
||||
<div className="module-loading" style={{ padding: 24 }}>
|
||||
<Loader2 size={18} className="spin" /> Cargando limpiezas…
|
||||
</div>
|
||||
) : cleaningTasks.length === 0 ? (
|
||||
<div className="wiz-found-badge" style={{ flexDirection: "column", alignItems: "flex-start", gap: 8 }}>
|
||||
<span>No hay limpiezas completadas en este trabajo.</span>
|
||||
<div style={{ display: "flex", gap: 8 }}>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm"
|
||||
onClick={() => onActivateModule?.("cleaning")}
|
||||
>
|
||||
Ir a Limpieza
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm" onClick={() => setSource("csv")}>
|
||||
<Upload size={13} /> Subir CSV
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="cleaning-source-list" role="listbox" aria-multiselectable="true" aria-label="Limpiezas">
|
||||
{cleaningTasks.map((task, index) => {
|
||||
const selected = selectedIds.has(task.id);
|
||||
return (
|
||||
<button
|
||||
key={task.id}
|
||||
type="button"
|
||||
role="option"
|
||||
aria-selected={selected}
|
||||
className={`cleaning-source-item${selected ? " cleaning-source-item--selected" : ""}`}
|
||||
onClick={() => toggle(task.id)}
|
||||
>
|
||||
<div className="cleaning-source-item-main">
|
||||
{selected ? <CheckSquare size={16} /> : <Square size={16} />}
|
||||
<div>
|
||||
<div className="cleaning-source-item-title">
|
||||
{task.title ?? `Limpieza ${cleaningTasks.length - index}`}
|
||||
</div>
|
||||
<div className="cleaning-source-item-meta">
|
||||
{taskTimingLabel(task)}
|
||||
{index === 0 ? " · más reciente" : ""}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<span className={`task-badge task-badge--${task.status}`}>{task.status}</span>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{selectedIds.size > 0 && (
|
||||
<p className="wiz-hint" style={{ marginTop: 8 }}>
|
||||
Seleccionadas: <strong>{selectedIds.size}</strong>
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<div className="wiz-section">
|
||||
<label className="wiz-field-label">Archivos CSV</label>
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept=".csv,text/csv"
|
||||
multiple
|
||||
style={{ display: "none" }}
|
||||
onChange={(e) => onCsvPicked(e.target.files)}
|
||||
/>
|
||||
<div className="folder-picker">
|
||||
<div className="folder-picker-display">
|
||||
{csvFiles.length === 0
|
||||
? "Ningún archivo seleccionado"
|
||||
: `${csvFiles.length} archivo${csvFiles.length === 1 ? "" : "s"}`}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm"
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
>
|
||||
<Upload size={13} /> Elegir CSV…
|
||||
</button>
|
||||
</div>
|
||||
{csvFiles.length > 0 && (
|
||||
<ul className="cleaning-source-list" style={{ marginTop: 10 }}>
|
||||
{csvFiles.map((file, index) => (
|
||||
<li key={`${file.name}-${file.size}-${index}`} className="cleaning-source-item">
|
||||
<div className="cleaning-source-item-main">
|
||||
<FileSpreadsheet size={16} />
|
||||
<div>
|
||||
<div className="cleaning-source-item-title">{file.name}</div>
|
||||
<div className="cleaning-source-item-meta">{formatBytes(file.size)}</div>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-ghost"
|
||||
aria-label={`Quitar ${file.name}`}
|
||||
onClick={() => removeCsv(index)}
|
||||
>
|
||||
<X size={13} />
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
<p className="wiz-hint" style={{ marginTop: 8 }}>
|
||||
Puedes seleccionar varios CSV a la vez; cada uno se valida y se consolida en un solo resultado.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="wiz-section">
|
||||
<label className="wiz-field-label">Origen Servicio (opcional)</label>
|
||||
<input
|
||||
ref={origenInputRef}
|
||||
type="file"
|
||||
accept=".csv,.xlsx,.xlsm,.xls,text/csv,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/vnd.ms-excel"
|
||||
style={{ display: "none" }}
|
||||
onChange={(e) => onOrigenPicked(e.target.files)}
|
||||
/>
|
||||
<div className="folder-picker">
|
||||
<div className="folder-picker-display">
|
||||
{origenServicio
|
||||
? `${origenServicio.name} (${formatBytes(origenServicio.size)})`
|
||||
: "Sin archivo — se usa tipo_servicio de la fuente si existe"}
|
||||
</div>
|
||||
<div style={{ display: "flex", gap: 6 }}>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm"
|
||||
onClick={() => origenInputRef.current?.click()}
|
||||
>
|
||||
<Upload size={13} /> Elegir…
|
||||
</button>
|
||||
{origenServicio && (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-ghost"
|
||||
aria-label="Quitar Origen Servicio"
|
||||
onClick={() => setOrigenServicio(null)}
|
||||
>
|
||||
<X size={13} />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<p className="wiz-hint" style={{ marginTop: 8 }}>
|
||||
Archivo Excel o CSV con el número de radicado y el tipo de servicio (por ejemplo
|
||||
ambulatorio u hospitalario). Se usa para completar el servicio de cada registro.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="wiz-actions">
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-primary"
|
||||
disabled={running || !canRun}
|
||||
onClick={() => void handleRun()}
|
||||
>
|
||||
{running ? (
|
||||
<><Loader2 size={14} className="spin" /> Procesando…</>
|
||||
) : (
|
||||
<><Users size={14} /> Generar población apta</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
import type { BackgroundProgress } from "../../../shared/types";
|
||||
import type { ModuleWizardProps } from "../registry";
|
||||
import { PoblacionAptaWizard } from "./PoblacionAptaWizard";
|
||||
|
||||
interface Props extends ModuleWizardProps {
|
||||
onProgress?: (p: BackgroundProgress) => void;
|
||||
}
|
||||
|
||||
export function PoblacionAptaWizardModule({
|
||||
client,
|
||||
job,
|
||||
onTaskCreated,
|
||||
onProgress,
|
||||
onActivateModule,
|
||||
}: Props) {
|
||||
return (
|
||||
<PoblacionAptaWizard
|
||||
client={client}
|
||||
job={job}
|
||||
onTaskCreated={onTaskCreated}
|
||||
onProgress={onProgress}
|
||||
onActivateModule={onActivateModule}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -148,6 +148,42 @@ button:disabled {
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.side-nav-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.side-nav-group > button .side-nav-chevron {
|
||||
flex-shrink: 0;
|
||||
margin-left: auto;
|
||||
opacity: 0.65;
|
||||
transition: transform 0.15s ease;
|
||||
}
|
||||
|
||||
.side-nav-group > button .side-nav-chevron--open {
|
||||
transform: rotate(180deg);
|
||||
}
|
||||
|
||||
.side-nav-sub {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1px;
|
||||
margin: 0 0 2px;
|
||||
padding: 0 0 0 12px;
|
||||
}
|
||||
|
||||
.side-nav-sub button {
|
||||
font-size: 12.5px;
|
||||
min-height: 32px;
|
||||
padding-left: 14px;
|
||||
}
|
||||
|
||||
.side-nav-sub button:disabled {
|
||||
cursor: wait;
|
||||
opacity: 0.65;
|
||||
}
|
||||
|
||||
.sidebar-footer {
|
||||
border-top: 1px solid rgba(255, 255, 255, 0.06);
|
||||
flex-shrink: 0;
|
||||
@@ -155,6 +191,75 @@ button:disabled {
|
||||
padding: 14px 8px 0;
|
||||
}
|
||||
|
||||
.sidebar-openai {
|
||||
align-items: center;
|
||||
background: transparent;
|
||||
border: 0;
|
||||
border-radius: 8px;
|
||||
color: inherit;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
margin-bottom: 10px;
|
||||
padding: 6px 4px;
|
||||
text-align: left;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.sidebar-openai:hover:not(:disabled) {
|
||||
background: rgba(255, 255, 255, 0.04);
|
||||
}
|
||||
|
||||
.sidebar-openai:disabled {
|
||||
cursor: default;
|
||||
opacity: 0.85;
|
||||
}
|
||||
|
||||
.sidebar-openai svg {
|
||||
color: #6b7f96;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.sidebar-openai.ok svg {
|
||||
color: #34d399;
|
||||
}
|
||||
|
||||
.sidebar-openai.error svg {
|
||||
color: #f87171;
|
||||
}
|
||||
|
||||
.sidebar-openai.busy svg {
|
||||
color: #f59e0b;
|
||||
}
|
||||
|
||||
.sidebar-openai-text {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.sidebar-openai-label {
|
||||
color: #6b7f96;
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
letter-spacing: 0.02em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.sidebar-openai-value {
|
||||
color: #c7d2e0;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.sidebar-openai.error .sidebar-openai-value {
|
||||
color: #f87171;
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.sidebar-gateway {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
@@ -4267,6 +4372,13 @@ th {
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.results-timing {
|
||||
color: var(--muted, #64748b);
|
||||
font-size: 12px;
|
||||
font-variant-numeric: tabular-nums;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.results-section-actions {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
@@ -5111,3 +5223,132 @@ th {
|
||||
font-weight: 600;
|
||||
padding: 1px 6px;
|
||||
}
|
||||
|
||||
/* ── Población apta results ── */
|
||||
.wiz-shell {
|
||||
box-sizing: border-box;
|
||||
display: flex;
|
||||
flex: 1;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
min-height: 0;
|
||||
overflow: auto;
|
||||
padding: 20px 28px 28px;
|
||||
}
|
||||
|
||||
.pa-results {
|
||||
padding: 8px 0 16px;
|
||||
}
|
||||
|
||||
.pa-stats-row {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.pa-stat--dup .wiz-stat-value {
|
||||
color: #b45309;
|
||||
}
|
||||
|
||||
.pa-obs-badge {
|
||||
border-radius: 6px;
|
||||
display: inline-block;
|
||||
font-size: 11.5px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.02em;
|
||||
line-height: 1.3;
|
||||
max-width: 280px;
|
||||
overflow: hidden;
|
||||
padding: 3px 8px;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.pa-obs-badge--apto {
|
||||
background: #dcfce7;
|
||||
color: #15803d;
|
||||
}
|
||||
|
||||
.pa-obs-badge--error {
|
||||
background: #fff7ed;
|
||||
color: #c2410c;
|
||||
}
|
||||
|
||||
.pa-obs-badge--dup {
|
||||
background: #fef3c7;
|
||||
color: #a16207;
|
||||
}
|
||||
|
||||
.pa-row--apto td {
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
.pa-row--no-apto td {
|
||||
background: #fffbeb;
|
||||
}
|
||||
|
||||
.pa-th-obs,
|
||||
.pa-td-obs {
|
||||
min-width: 140px;
|
||||
}
|
||||
|
||||
.pa-table th {
|
||||
text-transform: none;
|
||||
letter-spacing: 0.01em;
|
||||
}
|
||||
|
||||
.pa-results-header {
|
||||
align-items: flex-start;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 12px 16px;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 4px;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.pa-results-header-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.pa-results-log {
|
||||
background: var(--surface, #f8fafc);
|
||||
border: 1px solid var(--border, #e2e8f0);
|
||||
border-radius: 8px;
|
||||
font-size: 12px;
|
||||
margin: 0 0 4px;
|
||||
padding: 10px 14px;
|
||||
}
|
||||
|
||||
.pa-results-log summary {
|
||||
cursor: pointer;
|
||||
font-weight: 550;
|
||||
}
|
||||
|
||||
.pa-results-log ul {
|
||||
color: var(--text-muted);
|
||||
margin: 8px 0 0;
|
||||
padding-left: 18px;
|
||||
}
|
||||
|
||||
/* Módulo / pestaña Resultados: mismo aire lateral */
|
||||
.module-results:has(.pa-results),
|
||||
.module-results:has(.pa-results-header) {
|
||||
box-sizing: border-box;
|
||||
padding: 16px 28px 24px;
|
||||
}
|
||||
|
||||
.results-section:has(.pa-results) .results-section-header {
|
||||
padding-left: 28px;
|
||||
padding-right: 28px;
|
||||
}
|
||||
|
||||
.results-section:has(.pa-results) .pa-results {
|
||||
padding-left: 28px;
|
||||
padding-right: 28px;
|
||||
}
|
||||
|
||||
.results-section:has(.pa-results) .wiz-toast {
|
||||
margin-left: 28px !important;
|
||||
margin-right: 28px !important;
|
||||
}
|
||||
|
||||
@@ -24,5 +24,6 @@ beforeEach(() => {
|
||||
listVersions: vi.fn().mockResolvedValue({ currentVersion: "1.0.0", versions: [] }),
|
||||
installUpdate: vi.fn().mockResolvedValue({ ok: true }),
|
||||
onUpdateProgress: vi.fn().mockReturnValue(() => undefined),
|
||||
downloadUrl: vi.fn().mockResolvedValue({ ok: true }),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -1,2 +1,7 @@
|
||||
export type JobSection = "counting" | "extraction" | "cleaning" | "downloads" | "results";
|
||||
|
||||
export type JobSection =
|
||||
| "counting"
|
||||
| "extraction"
|
||||
| "cleaning"
|
||||
| "poblacion_apta"
|
||||
| "downloads"
|
||||
| "results";
|
||||
|
||||
@@ -9,17 +9,28 @@ import {
|
||||
Plus,
|
||||
RefreshCw,
|
||||
Sparkles,
|
||||
Users,
|
||||
} from "lucide-react";
|
||||
import { useEffect, useState } from "react";
|
||||
import type { BackgroundProgress, CleaningJob, CountJobFile, DownloadJobFile, Job, Task } from "../../shared/types";
|
||||
import type {
|
||||
BackgroundProgress,
|
||||
CleaningJob,
|
||||
CountJobFile,
|
||||
DownloadJobFile,
|
||||
Job,
|
||||
PoblacionAptaJob,
|
||||
Task,
|
||||
} from "../../shared/types";
|
||||
import { CountResultsTable } from "../components/CountResultsTable";
|
||||
import { ExtractionResultsTable } from "../components/ExtractionResultsTable";
|
||||
import { PoblacionAptaResultsTable } from "../components/PoblacionAptaResultsTable";
|
||||
import { BackgroundProgressBanner } from "../components/BackgroundProgressBanner";
|
||||
import type { ApiClient } from "../lib/apiClient";
|
||||
import { downloadCsv, toCsv, buildExportCsvFilename } from "../lib/csv";
|
||||
import { taskTimingLabel, timingDuration } from "../lib/time";
|
||||
import { DownloadsResultsPanel } from "../modules/downloads/DownloadsResultsPanel";
|
||||
|
||||
type ResultTab = "counting" | "extraction" | "cleaning" | "downloads";
|
||||
type ResultTab = "counting" | "extraction" | "cleaning" | "poblacion_apta" | "downloads";
|
||||
|
||||
interface Props {
|
||||
client: ApiClient;
|
||||
@@ -28,10 +39,12 @@ interface Props {
|
||||
countingProgress?: BackgroundProgress | null;
|
||||
extractionProgress?: BackgroundProgress | null;
|
||||
cleaningProgress?: BackgroundProgress | null;
|
||||
poblacionAptaProgress?: BackgroundProgress | null;
|
||||
onRefresh: () => Promise<void>;
|
||||
onStartCounting: () => void;
|
||||
onStartExtraction: (ctx?: Record<string, unknown>) => void;
|
||||
onStartCleaning: (ctx?: Record<string, unknown>) => void;
|
||||
onStartPoblacionApta: () => void;
|
||||
onStartDownload: () => void;
|
||||
onContinueToCounting?: (ctx: {
|
||||
downloadJobId: string;
|
||||
@@ -47,10 +60,12 @@ export function JobResultsView({
|
||||
countingProgress,
|
||||
extractionProgress,
|
||||
cleaningProgress,
|
||||
poblacionAptaProgress,
|
||||
onRefresh,
|
||||
onStartCounting,
|
||||
onStartExtraction,
|
||||
onStartCleaning,
|
||||
onStartPoblacionApta,
|
||||
onStartDownload,
|
||||
onContinueToCounting,
|
||||
}: Props) {
|
||||
@@ -63,6 +78,8 @@ export function JobResultsView({
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
const cleaningTasks = tasks.filter((t) => t.type === "cleaning")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
const poblacionAptaTasks = tasks.filter((t) => t.type === "poblacion_apta")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
const downloadTasks = tasks.filter((t) => t.type === "downloads")
|
||||
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
||||
|
||||
@@ -121,6 +138,21 @@ export function JobResultsView({
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`job-results-tab${activeTab === "poblacion_apta" ? " job-results-tab--active" : ""}`}
|
||||
onClick={() => setActiveTab("poblacion_apta")}
|
||||
>
|
||||
<Users size={15} />
|
||||
Población apta
|
||||
{(poblacionAptaTasks.length > 0 || poblacionAptaProgress?.running) && (
|
||||
<span className={`job-results-tab-count${poblacionAptaProgress?.running ? " job-results-tab-count--live" : ""}`}>
|
||||
{poblacionAptaProgress?.running
|
||||
? `${poblacionAptaProgress.processed}/${poblacionAptaProgress.total}`
|
||||
: poblacionAptaTasks.length}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`job-results-tab${activeTab === "downloads" ? " job-results-tab--active" : ""}`}
|
||||
@@ -177,6 +209,15 @@ export function JobResultsView({
|
||||
onStartCleaning={() => onStartCleaning()}
|
||||
/>
|
||||
)}
|
||||
{activeTab === "poblacion_apta" && (
|
||||
<PoblacionAptaResultsSection
|
||||
client={client}
|
||||
job={job}
|
||||
tasks={poblacionAptaTasks}
|
||||
liveProgress={poblacionAptaProgress}
|
||||
onStartPoblacionApta={onStartPoblacionApta}
|
||||
/>
|
||||
)}
|
||||
{activeTab === "downloads" && (
|
||||
<DownloadsResultsPanel
|
||||
client={client}
|
||||
@@ -309,13 +350,18 @@ function CountingResultsSection({
|
||||
>
|
||||
{tasks.map((t, i) => (
|
||||
<option key={t.id} value={t.id}>
|
||||
{t.title ?? `Conteo ${tasks.length - i}`} — {new Date(t.created_at).toLocaleDateString("es-CO")}
|
||||
{t.title ?? `Conteo ${tasks.length - i}`} — {taskTimingLabel(t)}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
{selectedTask && (
|
||||
<span className={`task-badge task-badge--${selectedTask.status}`}>{selectedTask.status}</span>
|
||||
<>
|
||||
<span className={`task-badge task-badge--${selectedTask.status}`}>{selectedTask.status}</span>
|
||||
<span className="results-timing" title="Duración del proceso">
|
||||
{timingDuration(selectedTask)}
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -471,16 +517,21 @@ function ExtractionResultsSection({
|
||||
>
|
||||
{tasks.map((t, i) => (
|
||||
<option key={t.id} value={t.id}>
|
||||
{t.title ?? `Extracción ${tasks.length - i}`} — {new Date(t.created_at).toLocaleDateString("es-CO")}
|
||||
{t.title ?? `Extracción ${tasks.length - i}`} — {taskTimingLabel(t)}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
{selectedTask && (
|
||||
<span className={`task-badge task-badge--${selectedTask.status}`}>
|
||||
{isRunning && <Loader2 size={11} className="spin" />}
|
||||
{taskStatus}
|
||||
</span>
|
||||
<>
|
||||
<span className={`task-badge task-badge--${selectedTask.status}`}>
|
||||
{isRunning && <Loader2 size={11} className="spin" />}
|
||||
{taskStatus}
|
||||
</span>
|
||||
<span className="results-timing" title="Duración del proceso">
|
||||
{timingDuration(selectedTask)}
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -672,15 +723,20 @@ function CleaningResultsSection({
|
||||
>
|
||||
{tasks.map((t, i) => (
|
||||
<option key={t.id} value={t.id}>
|
||||
{t.title ?? `Limpieza ${tasks.length - i}`} — {new Date(t.created_at).toLocaleDateString("es-CO")}
|
||||
{t.title ?? `Limpieza ${tasks.length - i}`} — {taskTimingLabel(t)}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
{selectedTask && (
|
||||
<span className={`task-badge task-badge--${cleaningJob?.status ?? selectedTask.status}`}>
|
||||
{cleaningJob?.status ?? selectedTask.status}
|
||||
</span>
|
||||
<>
|
||||
<span className={`task-badge task-badge--${cleaningJob?.status ?? selectedTask.status}`}>
|
||||
{cleaningJob?.status ?? selectedTask.status}
|
||||
</span>
|
||||
<span className="results-timing" title="Duración del proceso">
|
||||
{timingDuration(cleaningJob ?? selectedTask)}
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -714,3 +770,174 @@ function CleaningResultsSection({
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════
|
||||
Sección de resultados de población apta
|
||||
══════════════════════════════════════════ */
|
||||
function PoblacionAptaResultsSection({
|
||||
client,
|
||||
job,
|
||||
tasks,
|
||||
liveProgress,
|
||||
onStartPoblacionApta,
|
||||
}: {
|
||||
client: ApiClient;
|
||||
job: Job;
|
||||
tasks: Task[];
|
||||
liveProgress?: BackgroundProgress | null;
|
||||
onStartPoblacionApta: () => void;
|
||||
}) {
|
||||
const [selectedTask, setSelectedTask] = useState<Task | null>(tasks[0] ?? null);
|
||||
const [paJob, setPaJob] = useState<PoblacionAptaJob | null>(null);
|
||||
const [rows, setRows] = useState<Row[]>([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
setSelectedTask(tasks[0] ?? null);
|
||||
}, [tasks.map((t) => t.id).join(",")]); // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedTask) {
|
||||
setRows([]);
|
||||
setPaJob(null);
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
async function load() {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const paJobId = String(
|
||||
(selectedTask!.result as Record<string, unknown>)?.poblacion_apta_job_id ?? "",
|
||||
);
|
||||
if (!paJobId) {
|
||||
setError("La tarea no tiene poblacion_apta_job_id");
|
||||
return;
|
||||
}
|
||||
const jobData = await client.poblacionAptaJob(paJobId);
|
||||
if (cancelled) return;
|
||||
setPaJob(jobData);
|
||||
if (jobData.status === "completed") {
|
||||
const resp = await client.poblacionAptaJobRows(paJobId, 500, 0);
|
||||
if (cancelled) return;
|
||||
setRows(resp.rows ?? []);
|
||||
}
|
||||
} catch (e) {
|
||||
if (!cancelled) setError(String(e instanceof Error ? e.message : e));
|
||||
} finally {
|
||||
if (!cancelled) setLoading(false);
|
||||
}
|
||||
}
|
||||
void load();
|
||||
return () => { cancelled = true; };
|
||||
}, [selectedTask?.id, client]); // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
async function exportAptos() {
|
||||
const paJobId = String(
|
||||
(selectedTask?.result as Record<string, unknown>)?.poblacion_apta_job_id ?? "",
|
||||
);
|
||||
if (!paJobId) return;
|
||||
const blob = await client.poblacionAptaExportCsv(paJobId, "aptos");
|
||||
const text = await blob.text();
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_aptos", job.title), text);
|
||||
}
|
||||
|
||||
async function exportNoAptos() {
|
||||
const paJobId = String(
|
||||
(selectedTask?.result as Record<string, unknown>)?.poblacion_apta_job_id ?? "",
|
||||
);
|
||||
if (!paJobId) return;
|
||||
const blob = await client.poblacionAptaExportCsv(paJobId, "no_aptos");
|
||||
const text = await blob.text();
|
||||
downloadCsv(buildExportCsvFilename("poblacion_apta_no_aptos", job.title), text);
|
||||
}
|
||||
|
||||
if (tasks.length === 0 && !liveProgress?.running) {
|
||||
return (
|
||||
<div className="job-results-empty">
|
||||
<Users size={36} style={{ opacity: 0.3 }} />
|
||||
<p>No hay resultados de población apta todavía.</p>
|
||||
<button type="button" className="btn btn-primary" onClick={onStartPoblacionApta}>
|
||||
<Plus size={14} /> Generar población apta
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (tasks.length === 0 && liveProgress?.running) {
|
||||
return (
|
||||
<div className="results-section results-section--live-only">
|
||||
<BackgroundProgressBanner progress={liveProgress} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const summary = paJob?.summary ?? {};
|
||||
|
||||
return (
|
||||
<div className="results-section">
|
||||
{liveProgress?.running && (
|
||||
<div className="results-live-progress">
|
||||
<BackgroundProgressBanner progress={liveProgress} />
|
||||
</div>
|
||||
)}
|
||||
<div className="results-section-header">
|
||||
<div className="results-task-selector">
|
||||
{tasks.length > 1 && (
|
||||
<select
|
||||
className="form-input"
|
||||
style={{ width: "auto", fontSize: 13 }}
|
||||
value={selectedTask?.id ?? ""}
|
||||
onChange={(e) => setSelectedTask(tasks.find((t) => t.id === e.target.value) ?? null)}
|
||||
>
|
||||
{tasks.map((t, i) => (
|
||||
<option key={t.id} value={t.id}>
|
||||
{t.title ?? `Población apta ${tasks.length - i}`} — {taskTimingLabel(t)}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
{selectedTask && (
|
||||
<>
|
||||
<span className={`task-badge task-badge--${paJob?.status ?? selectedTask.status}`}>
|
||||
{paJob?.status ?? selectedTask.status}
|
||||
</span>
|
||||
<span className="results-timing" title="Duración del proceso">
|
||||
{timingDuration(paJob ?? selectedTask)}
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="results-section-actions">
|
||||
<button type="button" className="btn btn-sm btn-primary" onClick={() => void exportAptos()}>
|
||||
<Download size={13} /> CSV aptos ({summary.paso_bot ?? 0})
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm" onClick={() => void exportNoAptos()}>
|
||||
<Download size={13} /> CSV no aptos ({summary.con_error ?? 0})
|
||||
</button>
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={onStartPoblacionApta}>
|
||||
<Plus size={13} /> Nueva corrida
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="wiz-toast wiz-toast--error" style={{ margin: "12px 16px" }}>
|
||||
<AlertCircle size={14} /> {error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{loading && rows.length === 0 ? (
|
||||
<div className="module-loading"><Loader2 size={20} className="spin" /></div>
|
||||
) : rows.length === 0 ? (
|
||||
<div className="job-results-empty" style={{ padding: 40 }}>
|
||||
<p>No hay filas en el consolidado.</p>
|
||||
</div>
|
||||
) : (
|
||||
<PoblacionAptaResultsTable rows={rows} summary={summary} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import type { JobSection } from "../types/jobSection";
|
||||
import { BackgroundProgressBanner } from "../components/BackgroundProgressBanner";
|
||||
import { CountingWizardModule } from "../modules/counting/CountingWizardModule";
|
||||
import { CleaningWizardModule } from "../modules/cleaning/CleaningWizardModule";
|
||||
import { PoblacionAptaWizardModule } from "../modules/poblacion_apta/PoblacionAptaWizardModule";
|
||||
import { JobExtractionWizard } from "../modules/extraction/JobExtractionWizard";
|
||||
import { DownloadsWizardModule } from "../modules/downloads/DownloadsWizardModule";
|
||||
import { JobResultsView } from "./JobResultsView";
|
||||
@@ -27,12 +28,14 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
const [countingProgress, setCountingProgress] = useState<BackgroundProgress | null>(null);
|
||||
const [extractionProgress, setExtractionProgress] = useState<BackgroundProgress | null>(null);
|
||||
const [cleaningProgress, setCleaningProgress] = useState<BackgroundProgress | null>(null);
|
||||
const [poblacionAptaProgress, setPoblacionAptaProgress] = useState<BackgroundProgress | null>(null);
|
||||
const pollingRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
|
||||
const backgroundRunning =
|
||||
countingProgress?.running === true ||
|
||||
extractionProgress?.running === true ||
|
||||
cleaningProgress?.running === true;
|
||||
cleaningProgress?.running === true ||
|
||||
poblacionAptaProgress?.running === true;
|
||||
|
||||
const loadTasks = useCallback(async () => {
|
||||
try {
|
||||
@@ -80,6 +83,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
countingProgress?.running ? countingProgress
|
||||
: extractionProgress?.running ? extractionProgress
|
||||
: cleaningProgress?.running ? cleaningProgress
|
||||
: poblacionAptaProgress?.running ? poblacionAptaProgress
|
||||
: null;
|
||||
|
||||
return (
|
||||
@@ -121,6 +125,7 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
{section === "counting" && "Conteo de servicios"}
|
||||
{section === "extraction" && "Extracción de datos"}
|
||||
{section === "cleaning" && "Limpieza de datos"}
|
||||
{section === "poblacion_apta" && "Población apta"}
|
||||
{section === "downloads" && "Bot de descarga"}
|
||||
{section === "results" && "Resultados del trabajo"}
|
||||
</div>
|
||||
@@ -182,6 +187,16 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className={`job-section-pane${section === "poblacion_apta" ? "" : " job-section-pane--hidden"}`}>
|
||||
<PoblacionAptaWizardModule
|
||||
client={client}
|
||||
job={job}
|
||||
onTaskCreated={handleTaskCreated}
|
||||
onProgress={(p) => setPoblacionAptaProgress(p.running ? p : null)}
|
||||
onActivateModule={(type, ctx) => onActivateSection(type as JobSection, ctx)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className={`job-section-pane${section === "downloads" ? "" : " job-section-pane--hidden"}`}>
|
||||
<DownloadsWizardModule client={client} job={job} />
|
||||
</div>
|
||||
@@ -194,10 +209,12 @@ export function JobView({ client, job, section, sectionContext, onActivateSectio
|
||||
countingProgress={countingProgress}
|
||||
extractionProgress={extractionProgress}
|
||||
cleaningProgress={cleaningProgress}
|
||||
poblacionAptaProgress={poblacionAptaProgress}
|
||||
onRefresh={loadTasks}
|
||||
onStartCounting={() => onActivateSection("counting")}
|
||||
onStartExtraction={(ctx?: Record<string, unknown>) => onActivateSection("extraction", ctx)}
|
||||
onStartCleaning={(ctx?: Record<string, unknown>) => onActivateSection("cleaning", ctx)}
|
||||
onStartPoblacionApta={() => onActivateSection("poblacion_apta")}
|
||||
onStartDownload={() => onActivateSection("downloads")}
|
||||
onContinueToCounting={(ctx) => onActivateSection("counting", ctx)}
|
||||
/>
|
||||
|
||||
@@ -144,6 +144,8 @@ export interface CreateCountJobPayload {
|
||||
results: CountJobFileCreateEntry[];
|
||||
download_job_id?: string | null;
|
||||
source?: string | null;
|
||||
/** ISO del inicio real del conteo en el cliente */
|
||||
started_at?: string | null;
|
||||
}
|
||||
|
||||
export interface ExtractionFromCountPayload {
|
||||
@@ -219,6 +221,8 @@ export interface CleaningPreviewPayload {
|
||||
extraction_batch_id?: string;
|
||||
rules_per_column: Record<string, string[]>;
|
||||
cop_mod_fecha_column?: Record<string, string>;
|
||||
cop_mod_bruto_column?: Record<string, string>;
|
||||
cop_mod_neto_column?: Record<string, string>;
|
||||
cop_mod_config?: Record<string, unknown> | null;
|
||||
sample_size?: number;
|
||||
}
|
||||
@@ -234,6 +238,8 @@ export interface CleaningJob {
|
||||
error: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
started_at?: string | null;
|
||||
finished_at?: string | null;
|
||||
}
|
||||
|
||||
export interface RunCleaningPayload {
|
||||
@@ -241,6 +247,8 @@ export interface RunCleaningPayload {
|
||||
job_id?: string;
|
||||
rules_per_column: Record<string, string[]>;
|
||||
cop_mod_fecha_column?: Record<string, string>;
|
||||
cop_mod_bruto_column?: Record<string, string>;
|
||||
cop_mod_neto_column?: Record<string, string>;
|
||||
cop_mod_config?: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
@@ -251,6 +259,40 @@ export interface CleaningRowsResponse {
|
||||
offset: number;
|
||||
}
|
||||
|
||||
export interface PoblacionAptaJob {
|
||||
id: string;
|
||||
job_id: string | null;
|
||||
status: "pending" | "running" | "completed" | "failed" | string;
|
||||
request: Record<string, unknown>;
|
||||
log: string[];
|
||||
summary: {
|
||||
total?: number;
|
||||
paso_bot?: number;
|
||||
con_error?: number;
|
||||
duplicados?: number;
|
||||
por_fuente?: Array<Record<string, unknown>>;
|
||||
por_observacion?: Array<Record<string, unknown>>;
|
||||
};
|
||||
error: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
started_at?: string | null;
|
||||
finished_at?: string | null;
|
||||
}
|
||||
|
||||
export interface RunPoblacionAptaPayload {
|
||||
job_id?: string;
|
||||
cleaning_job_ids: string[];
|
||||
}
|
||||
|
||||
export interface PoblacionAptaRowsResponse {
|
||||
job_id: string;
|
||||
rows: Array<Record<string, unknown>>;
|
||||
total: number;
|
||||
limit: number;
|
||||
offset: number;
|
||||
}
|
||||
|
||||
export interface CountingFormState {
|
||||
localRoot: string;
|
||||
containerRoot: string;
|
||||
@@ -261,7 +303,7 @@ export interface CountingFormState {
|
||||
|
||||
/** Estado de progreso de un módulo ejecutándose en segundo plano. */
|
||||
export interface BackgroundProgress {
|
||||
module: "counting" | "extraction" | "cleaning";
|
||||
module: "counting" | "extraction" | "cleaning" | "poblacion_apta";
|
||||
total: number;
|
||||
processed: number;
|
||||
failed: number;
|
||||
@@ -396,4 +438,6 @@ export interface DesktopBridge {
|
||||
listVersions(): Promise<VersionsListResult>;
|
||||
installUpdate(manifest: UpdateManifest): Promise<{ ok: boolean; destPath?: string }>;
|
||||
onUpdateProgress(callback: (progress: UpdateProgress) => void): () => void;
|
||||
/** Inicia una descarga del sistema (Electron) desde una URL http(s). */
|
||||
downloadUrl(url: string): Promise<{ ok: boolean }>;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
@@ -25,6 +26,7 @@ from domain.batch_processor import BatchProcessor
|
||||
from domain.config import extractor_llm_config, normalizer_llm_config, settings
|
||||
from domain.config import get_allowed_document_types
|
||||
from domain.estimator import Estimator
|
||||
from domain.openai_balance import fetch_openai_balance
|
||||
from domain.service import InvoiceProcessingService
|
||||
from domain.strategies import OpenAICompatibleExtractor
|
||||
from infrastructure.database import (
|
||||
@@ -177,6 +179,7 @@ class ExtractionBatchRead(BaseModel):
|
||||
|
||||
class TaskRead(BaseModel):
|
||||
id: str
|
||||
job_id: str | None = None
|
||||
type: str
|
||||
status: str
|
||||
title: str | None
|
||||
@@ -205,6 +208,7 @@ def serialize_batch(batch: ExtractionBatch) -> dict[str, Any]:
|
||||
def serialize_task(task: Task) -> dict[str, Any]:
|
||||
return {
|
||||
"id": task.id,
|
||||
"job_id": task.job_id,
|
||||
"type": task.type,
|
||||
"status": task.status,
|
||||
"title": task.title,
|
||||
@@ -378,6 +382,12 @@ def list_models() -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
@app.get("/openai/balance")
|
||||
def openai_balance() -> dict[str, Any]:
|
||||
"""Crédito disponible en la cuenta OpenAI (o proveedor compatible)."""
|
||||
return fetch_openai_balance()
|
||||
|
||||
|
||||
@app.post("/schemas")
|
||||
def build_schema(request: SchemaRequest) -> dict[str, Any]:
|
||||
"""Descubre campos disponibles para que el cliente API elija `fields`."""
|
||||
@@ -1117,6 +1127,7 @@ class CreateCountJobRequest(BaseModel):
|
||||
results: list[CountJobFileCreate]
|
||||
download_job_id: str | None = None
|
||||
source: str | None = None
|
||||
started_at: str | None = None # ISO del cliente (inicio real del conteo)
|
||||
|
||||
|
||||
class CountJobFileRead(BaseModel):
|
||||
@@ -1179,6 +1190,17 @@ def create_count_job(request: CreateCountJobRequest) -> dict[str, Any]:
|
||||
"""Persiste los resultados del servicio de conteo de facturas en la BD compartida."""
|
||||
total = len(request.results)
|
||||
default_title = request.title or f"Conteo ({total} archivos)"
|
||||
finished = utcnow()
|
||||
started = finished
|
||||
if request.started_at:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(request.started_at.replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
started = parsed
|
||||
except ValueError:
|
||||
started = finished
|
||||
|
||||
with session_scope() as session:
|
||||
if request.job_id:
|
||||
job = session.get(Job, request.job_id)
|
||||
@@ -1212,9 +1234,9 @@ def create_count_job(request: CreateCountJobRequest) -> dict[str, Any]:
|
||||
"failed": sum(1 for r in request.results if r.status != "ok"),
|
||||
"nits": sorted({r.nit for r in request.results if r.nit}),
|
||||
},
|
||||
started_at=utcnow(),
|
||||
finished_at=utcnow(),
|
||||
updated_at=utcnow(),
|
||||
started_at=started,
|
||||
finished_at=finished,
|
||||
updated_at=finished,
|
||||
)
|
||||
session.add(task)
|
||||
session.flush()
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
"""Consulta de crédito / saldo disponible vía la API de OpenAI (y compatibles).
|
||||
|
||||
OpenAI no documenta un endpoint público de saldo con secret key; el endpoint
|
||||
histórico de credit_grants sigue siendo el más usado por herramientas de
|
||||
monitorización y por APIs compatibles. Se consulta con la key del servidor
|
||||
y se normaliza la respuesta para la UI.
|
||||
|
||||
Si EXTRACTOR_BASE_URL apunta a un mock/local, también se prueba la API oficial
|
||||
de OpenAI con la misma key.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from .config import extractor_llm_config
|
||||
|
||||
_log = logging.getLogger("domain.openai_balance")
|
||||
|
||||
_TIMEOUT = httpx.Timeout(connect=5.0, read=15.0, write=10.0, pool=5.0)
|
||||
_OPENAI_OFFICIAL = "https://api.openai.com/v1"
|
||||
|
||||
|
||||
def _is_openai_host(base_url: str) -> bool:
|
||||
host = (urlparse(base_url).hostname or "").lower()
|
||||
return host in {"api.openai.com", "openai.com"}
|
||||
|
||||
|
||||
def _is_local_or_mock(base_url: str) -> bool:
|
||||
host = (urlparse(base_url).hostname or "").lower()
|
||||
return host in {
|
||||
"localhost",
|
||||
"127.0.0.1",
|
||||
"host.docker.internal",
|
||||
"0.0.0.0",
|
||||
} or host.endswith(".local")
|
||||
|
||||
|
||||
def _candidate_roots(base_url: str) -> list[str]:
|
||||
cleaned = (base_url or _OPENAI_OFFICIAL).rstrip("/") + "/"
|
||||
roots = [cleaned]
|
||||
if cleaned.rstrip("/").endswith("/v1"):
|
||||
roots.append(cleaned.rstrip("/").removesuffix("/v1").rstrip("/") + "/")
|
||||
else:
|
||||
roots.append(urljoin(cleaned, "v1/"))
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for root in roots:
|
||||
if root not in seen:
|
||||
seen.add(root)
|
||||
out.append(root)
|
||||
return out
|
||||
|
||||
|
||||
def _candidate_urls(base_url: str) -> list[str]:
|
||||
"""URLs a probar: primero el proveedor configurado, luego OpenAI oficial si aplica."""
|
||||
paths = (
|
||||
"dashboard/billing/credit_grants",
|
||||
"v1/dashboard/billing/credit_grants",
|
||||
"credits", # OpenRouter-style
|
||||
"user/balance", # DeepSeek-style
|
||||
)
|
||||
urls: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
bases = [base_url or _OPENAI_OFFICIAL]
|
||||
# Con mock/local u otro proveedor, el saldo de cuenta OpenAI sigue en api.openai.com
|
||||
if not _is_openai_host(bases[0]):
|
||||
bases.append(_OPENAI_OFFICIAL)
|
||||
|
||||
for base in bases:
|
||||
for root in _candidate_roots(base):
|
||||
for path in paths:
|
||||
url = urljoin(root, path)
|
||||
if url not in seen:
|
||||
seen.add(url)
|
||||
urls.append(url)
|
||||
return urls
|
||||
|
||||
|
||||
def _as_float(value: Any) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_payload(payload: Any) -> dict[str, float] | None:
|
||||
"""Extrae total_available / granted / used de respuestas conocidas."""
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
|
||||
# OpenAI credit_grants
|
||||
total_available = _as_float(payload.get("total_available"))
|
||||
total_granted = _as_float(payload.get("total_granted"))
|
||||
total_used = _as_float(payload.get("total_used"))
|
||||
if total_available is not None:
|
||||
return {
|
||||
"total_available": total_available,
|
||||
"total_granted": total_granted if total_granted is not None else total_available,
|
||||
"total_used": total_used if total_used is not None else 0.0,
|
||||
}
|
||||
|
||||
# OpenRouter: { data: { total_credits, total_usage } }
|
||||
data = payload.get("data")
|
||||
if isinstance(data, dict):
|
||||
credits = _as_float(data.get("total_credits"))
|
||||
usage = _as_float(data.get("total_usage"))
|
||||
if credits is not None:
|
||||
used = usage or 0.0
|
||||
return {
|
||||
"total_available": credits - used,
|
||||
"total_granted": credits,
|
||||
"total_used": used,
|
||||
}
|
||||
|
||||
# DeepSeek: { balance_infos: [{ total_balance }] }
|
||||
infos = payload.get("balance_infos")
|
||||
if isinstance(infos, list) and infos:
|
||||
totals = [_as_float(item.get("total_balance")) for item in infos if isinstance(item, dict)]
|
||||
totals = [t for t in totals if t is not None]
|
||||
if totals:
|
||||
available = sum(totals)
|
||||
return {"total_available": available, "total_granted": available, "total_used": 0.0}
|
||||
|
||||
for key in ("balance", "available", "remaining", "credit_balance", "total_balance"):
|
||||
value = _as_float(payload.get(key))
|
||||
if value is not None:
|
||||
return {"total_available": value, "total_granted": value, "total_used": 0.0}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _friendly_error(detail: str, status_code: int, *, candidate: str = "") -> str:
|
||||
lower = detail.lower()
|
||||
if "session key" in lower or "browser" in lower or (
|
||||
status_code in {401, 403} and _is_openai_host(candidate)
|
||||
):
|
||||
return (
|
||||
"OpenAI no permite consultar el crédito con una API key "
|
||||
"(sk-proj / sk-...). Solo se ve en el panel web: "
|
||||
"https://platform.openai.com/settings/organization/billing/overview"
|
||||
)
|
||||
if status_code in {401, 403}:
|
||||
return detail or f"Acceso denegado al consultar el saldo (HTTP {status_code})."
|
||||
return detail or f"HTTP {status_code}"
|
||||
|
||||
|
||||
def fetch_openai_balance(
|
||||
*,
|
||||
api_key: str | None = None,
|
||||
base_url: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Devuelve el crédito disponible consultando la API del proveedor LLM."""
|
||||
config = extractor_llm_config()
|
||||
key = (api_key if api_key is not None else str(config["api_key"])).strip()
|
||||
url = (base_url if base_url is not None else str(config["base_url"])).strip()
|
||||
|
||||
if not key:
|
||||
return {
|
||||
"ok": False,
|
||||
"available": None,
|
||||
"currency": "USD",
|
||||
"source": None,
|
||||
"error": "API key de OpenAI no configurada (EXTRACTOR_API_KEY).",
|
||||
}
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
last_error = "No se pudo obtener el saldo de crédito."
|
||||
dead_hosts: set[str] = set()
|
||||
|
||||
with httpx.Client(timeout=_TIMEOUT, follow_redirects=True) as client:
|
||||
for candidate in _candidate_urls(url):
|
||||
host = (urlparse(candidate).hostname or "").lower()
|
||||
if host in dead_hosts:
|
||||
continue
|
||||
|
||||
try:
|
||||
response = client.get(candidate, headers=headers)
|
||||
except httpx.HTTPError as exc:
|
||||
last_error = f"Error de red al consultar {candidate}: {exc}"
|
||||
_log.warning(last_error)
|
||||
# Si el mock/local está caído, no insistir en ese host
|
||||
if _is_local_or_mock(candidate) or "Connection refused" in str(exc):
|
||||
dead_hosts.add(host)
|
||||
continue
|
||||
|
||||
if response.status_code == 404:
|
||||
continue
|
||||
|
||||
if response.status_code >= 400:
|
||||
detail = ""
|
||||
try:
|
||||
body = response.json()
|
||||
if isinstance(body, dict):
|
||||
err = body.get("error")
|
||||
if isinstance(err, dict):
|
||||
detail = str(err.get("message") or err)
|
||||
else:
|
||||
detail = str(body.get("detail") or body.get("message") or "")
|
||||
except Exception:
|
||||
detail = (response.text or "")[:300]
|
||||
last_error = _friendly_error(detail, response.status_code, candidate=candidate)
|
||||
# 401/403 en OpenAI billing suelen aplicar a todos los endpoints
|
||||
if response.status_code in {401, 403} and _is_openai_host(candidate):
|
||||
break
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = response.json()
|
||||
except Exception:
|
||||
last_error = f"Respuesta no JSON desde {candidate}"
|
||||
continue
|
||||
|
||||
normalized = _normalize_payload(payload)
|
||||
if normalized is None:
|
||||
last_error = f"Formato de saldo no reconocido desde {candidate}"
|
||||
continue
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"available": round(normalized["total_available"], 4),
|
||||
"total_granted": round(normalized["total_granted"], 4),
|
||||
"total_used": round(normalized["total_used"], 4),
|
||||
"currency": "USD",
|
||||
"source": candidate,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
return {
|
||||
"ok": False,
|
||||
"available": None,
|
||||
"currency": "USD",
|
||||
"source": None,
|
||||
"error": last_error,
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
coloredlogs==15.0.1
|
||||
fastapi==0.115.6
|
||||
python-multipart==0.0.20
|
||||
httpx==0.28.1
|
||||
openai==2.1.0
|
||||
pdf2image==1.17.0
|
||||
pillow==11.3.0
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONPATH=/app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY domain /app/domain
|
||||
COPY infrastructure /app/infrastructure
|
||||
COPY api /app/api
|
||||
|
||||
EXPOSE 8003
|
||||
CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8003"]
|
||||
@@ -0,0 +1,45 @@
|
||||
# poblacion_apta
|
||||
|
||||
Microservicio Python (FastAPI) que consolida varias limpiezas y aplica las validaciones
|
||||
de `Verificador_ST` para producir la **Población apta** (registros `PASO_BOT`).
|
||||
|
||||
## Puerto
|
||||
|
||||
**8003**
|
||||
|
||||
## Endpoints
|
||||
|
||||
| Método | Ruta | Descripción |
|
||||
|--------|------|-------------|
|
||||
| `GET` | `/health` | Liveness |
|
||||
| `POST` | `/jobs` | Crear job desde `cleaning_job_ids` (+ `job_id` Desktop) |
|
||||
| `POST` | `/jobs/from-csv` | Crear job desde uno o varios CSV (multipart) |
|
||||
| `GET` | `/jobs?job_id=` | Listar jobs |
|
||||
| `GET` | `/jobs/{id}` | Detalle + summary |
|
||||
| `GET` | `/jobs/{id}/rows` | Filas (`observacion` opcional) |
|
||||
| `GET` | `/jobs/{id}/export.csv?subset=aptos\|no_aptos` | CSV `\|` aptos o no aptos |
|
||||
|
||||
## Crear un job desde limpiezas
|
||||
|
||||
```bash
|
||||
POST /jobs
|
||||
{
|
||||
"job_id": "...",
|
||||
"cleaning_job_ids": ["uuid-limpieza-1", "uuid-limpieza-2"]
|
||||
}
|
||||
```
|
||||
|
||||
## Crear un job desde CSV
|
||||
|
||||
```bash
|
||||
POST /jobs/from-csv
|
||||
# multipart/form-data
|
||||
# files: uno o más .csv
|
||||
# job_id: (opcional) UUID del trabajo Desktop
|
||||
```
|
||||
|
||||
## Variables de entorno
|
||||
|
||||
| Variable | Descripción |
|
||||
|----------|-------------|
|
||||
| `DATABASE_URL` | PostgreSQL (`postgresql+psycopg://...`) |
|
||||
@@ -0,0 +1,517 @@
|
||||
"""API REST del microservicio Población apta."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, File, Form, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from domain.csv_io import load_csv_bytes
|
||||
from domain.verificador_st import (
|
||||
consolidado_to_records,
|
||||
export_no_aptos_csv,
|
||||
export_paso_bot_csv,
|
||||
process_cleaning_sources,
|
||||
)
|
||||
from domain.verificador_st.core import cargar_origen_servicio_desde_bytes
|
||||
from infrastructure.database import (
|
||||
CleanedRow,
|
||||
CleaningJob,
|
||||
Job,
|
||||
PoblacionAptaJob,
|
||||
PoblacionAptaRow,
|
||||
Task,
|
||||
init_db,
|
||||
session_scope,
|
||||
utcnow,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
init_db()
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="IPSExtractor Población Apta Service",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
|
||||
class PoblacionAptaRequest(BaseModel):
|
||||
job_id: str | None = None
|
||||
cleaning_job_ids: list[str] = Field(default_factory=list, min_length=1)
|
||||
|
||||
|
||||
def serialize_job(job: PoblacionAptaJob) -> dict[str, Any]:
|
||||
finished = (
|
||||
job.updated_at.isoformat()
|
||||
if job.status in {"completed", "failed"}
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"id": job.id,
|
||||
"job_id": job.job_id,
|
||||
"status": job.status,
|
||||
"request": job.request,
|
||||
"log": job.log,
|
||||
"summary": job.summary,
|
||||
"error": job.error,
|
||||
"created_at": job.created_at.isoformat(),
|
||||
"updated_at": job.updated_at.isoformat(),
|
||||
"started_at": job.created_at.isoformat(),
|
||||
"finished_at": finished,
|
||||
}
|
||||
|
||||
|
||||
def _validate_job_id(session: Any, job_id: str | None) -> None:
|
||||
if not job_id:
|
||||
return
|
||||
if session.get(Job, job_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Trabajo (job) no encontrado")
|
||||
|
||||
|
||||
def _task_title(request: dict[str, Any]) -> str:
|
||||
source = str(request.get("source") or "cleaning")
|
||||
if source == "csv":
|
||||
files = request.get("filenames") or []
|
||||
n = len(files) if isinstance(files, list) else 0
|
||||
return f"Población apta ({n} CSV)" if n else "Población apta (CSV)"
|
||||
n = len(request.get("cleaning_job_ids") or [])
|
||||
return f"Población apta ({n} limpiezas)"
|
||||
|
||||
|
||||
def _create_task(
|
||||
session: Any,
|
||||
*,
|
||||
job_id: str | None,
|
||||
poblacion_apta_job_id: str,
|
||||
status: str,
|
||||
request: dict[str, Any],
|
||||
error: str | None = None,
|
||||
title: str | None = None,
|
||||
started_at: Any | None = None,
|
||||
) -> str | None:
|
||||
if not job_id:
|
||||
return None
|
||||
now = utcnow()
|
||||
task = Task(
|
||||
job_id=job_id,
|
||||
type="poblacion_apta",
|
||||
status=status,
|
||||
title=title or _task_title(request),
|
||||
request=request,
|
||||
result={"poblacion_apta_job_id": poblacion_apta_job_id},
|
||||
error=error,
|
||||
started_at=started_at or now,
|
||||
finished_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(task)
|
||||
session.flush()
|
||||
return task.id
|
||||
|
||||
|
||||
def _persist_rows(
|
||||
session: Any,
|
||||
*,
|
||||
job_id: str,
|
||||
records: list[dict[str, Any]],
|
||||
) -> None:
|
||||
session.execute(
|
||||
delete(PoblacionAptaRow).where(PoblacionAptaRow.poblacion_apta_job_id == job_id)
|
||||
)
|
||||
for idx, row in enumerate(records, start=1):
|
||||
session.add(
|
||||
PoblacionAptaRow(
|
||||
poblacion_apta_job_id=job_id,
|
||||
cleaning_job_id=str(row.get("cleaning_job_id") or "") or None,
|
||||
row_index=idx,
|
||||
row_json=row,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _run_and_persist(
|
||||
*,
|
||||
pa_job_id: str,
|
||||
desktop_job_id: str | None,
|
||||
request_dump: dict[str, Any],
|
||||
sources: list[dict[str, Any]],
|
||||
mapa_servicios: dict[str, str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
consolidado, summary, log = process_cleaning_sources(
|
||||
sources, mapa_servicios=mapa_servicios
|
||||
)
|
||||
records = consolidado_to_records(consolidado)
|
||||
|
||||
with session_scope() as session:
|
||||
_persist_rows(session, job_id=pa_job_id, records=records)
|
||||
job = session.get(PoblacionAptaJob, pa_job_id)
|
||||
if job is None:
|
||||
raise RuntimeError("No se encontró el job de población apta recién creado.")
|
||||
job.status = "completed"
|
||||
job.log = log
|
||||
job.summary = summary
|
||||
job.updated_at = utcnow()
|
||||
_create_task(
|
||||
session,
|
||||
job_id=desktop_job_id,
|
||||
poblacion_apta_job_id=pa_job_id,
|
||||
status="completed",
|
||||
request=request_dump,
|
||||
started_at=job.created_at,
|
||||
)
|
||||
session.flush()
|
||||
return serialize_job(job)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
with session_scope() as session:
|
||||
job = session.get(PoblacionAptaJob, pa_job_id)
|
||||
started = job.created_at if job is not None else None
|
||||
if job is not None:
|
||||
job.status = "failed"
|
||||
job.error = str(exc)
|
||||
job.updated_at = utcnow()
|
||||
_create_task(
|
||||
session,
|
||||
job_id=desktop_job_id,
|
||||
poblacion_apta_job_id=pa_job_id,
|
||||
status="failed",
|
||||
request=request_dump,
|
||||
error=str(exc),
|
||||
started_at=started,
|
||||
)
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
|
||||
|
||||
def _load_cleaning_sources(
|
||||
session: Any,
|
||||
cleaning_ids: list[str],
|
||||
desktop_job_id: str | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
sources: list[dict[str, Any]] = []
|
||||
for cid in cleaning_ids:
|
||||
cleaning = session.get(CleaningJob, cid)
|
||||
if cleaning is None:
|
||||
raise HTTPException(status_code=404, detail=f"Job de limpieza no encontrado: {cid}")
|
||||
if cleaning.status != "completed":
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job de limpieza {cid} no está completed (status={cleaning.status})",
|
||||
)
|
||||
if desktop_job_id and cleaning.job_id and cleaning.job_id != desktop_job_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Job de limpieza {cid} no pertenece al trabajo {desktop_job_id}",
|
||||
)
|
||||
rows = session.scalars(
|
||||
select(CleanedRow)
|
||||
.where(CleanedRow.cleaning_job_id == cid)
|
||||
.order_by(CleanedRow.row_index, CleanedRow.id)
|
||||
).all()
|
||||
sources.append(
|
||||
{
|
||||
"cleaning_job_id": cid,
|
||||
"label": f"limpieza_{cid[:8]}",
|
||||
"rows": [r.row_json for r in rows],
|
||||
}
|
||||
)
|
||||
return sources
|
||||
|
||||
|
||||
async def _parse_origen_servicio(
|
||||
upload: UploadFile | None,
|
||||
) -> tuple[dict[str, str], str | None]:
|
||||
"""Lee el archivo Origen Servicio opcional. Devuelve (mapa, filename)."""
|
||||
if upload is None:
|
||||
return {}, None
|
||||
filename = upload.filename or "origen_servicio.xlsx"
|
||||
content = await upload.read()
|
||||
if not content:
|
||||
raise HTTPException(status_code=400, detail="El archivo Origen Servicio está vacío")
|
||||
try:
|
||||
mapa = cargar_origen_servicio_desde_bytes(content, filename)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"No se pudo leer Origen Servicio: {exc}",
|
||||
) from exc
|
||||
if not mapa:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
"Origen Servicio sin columnas de radicado/servicio reconocibles "
|
||||
"o sin filas válidas"
|
||||
),
|
||||
)
|
||||
return mapa, filename
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/jobs")
|
||||
def run_poblacion_apta(request: PoblacionAptaRequest) -> dict[str, Any]:
|
||||
"""Crea job desde limpiezas (JSON). Sin Origen Servicio; usar /jobs/from-cleaning."""
|
||||
cleaning_ids = [cid.strip() for cid in request.cleaning_job_ids if cid and cid.strip()]
|
||||
if not cleaning_ids:
|
||||
raise HTTPException(status_code=400, detail="cleaning_job_ids es requerido (mínimo 1)")
|
||||
|
||||
request_dump = request.model_dump()
|
||||
request_dump["cleaning_job_ids"] = cleaning_ids
|
||||
request_dump["source"] = "cleaning"
|
||||
|
||||
with session_scope() as session:
|
||||
_validate_job_id(session, request.job_id)
|
||||
sources = _load_cleaning_sources(session, cleaning_ids, request.job_id)
|
||||
job = PoblacionAptaJob(
|
||||
job_id=request.job_id,
|
||||
status="running",
|
||||
request=request_dump,
|
||||
updated_at=utcnow(),
|
||||
)
|
||||
session.add(job)
|
||||
session.flush()
|
||||
pa_job_id = job.id
|
||||
desktop_job_id = request.job_id
|
||||
|
||||
return _run_and_persist(
|
||||
pa_job_id=pa_job_id,
|
||||
desktop_job_id=desktop_job_id,
|
||||
request_dump=request_dump,
|
||||
sources=sources,
|
||||
)
|
||||
|
||||
|
||||
# Ruta estática ANTES de /jobs/{job_id} para evitar 405 (job_id="from-cleaning").
|
||||
@app.post("/jobs/from-cleaning")
|
||||
async def run_poblacion_apta_from_cleaning(
|
||||
cleaning_job_ids: str = Form(...),
|
||||
job_id: str | None = Form(None),
|
||||
origen_servicio: UploadFile | None = File(None),
|
||||
) -> dict[str, Any]:
|
||||
"""Limpiezas + Origen Servicio opcional (multipart)."""
|
||||
try:
|
||||
parsed = json.loads(cleaning_job_ids)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="cleaning_job_ids debe ser un JSON array de ids",
|
||||
) from exc
|
||||
if not isinstance(parsed, list):
|
||||
raise HTTPException(status_code=400, detail="cleaning_job_ids debe ser un array")
|
||||
|
||||
cleaning_ids = [str(cid).strip() for cid in parsed if str(cid).strip()]
|
||||
if not cleaning_ids:
|
||||
raise HTTPException(status_code=400, detail="cleaning_job_ids es requerido (mínimo 1)")
|
||||
|
||||
mapa, origen_filename = await _parse_origen_servicio(origen_servicio)
|
||||
|
||||
request_dump: dict[str, Any] = {
|
||||
"source": "cleaning",
|
||||
"job_id": job_id,
|
||||
"cleaning_job_ids": cleaning_ids,
|
||||
"origen_servicio": origen_filename,
|
||||
}
|
||||
|
||||
with session_scope() as session:
|
||||
_validate_job_id(session, job_id)
|
||||
sources = _load_cleaning_sources(session, cleaning_ids, job_id)
|
||||
job = PoblacionAptaJob(
|
||||
job_id=job_id,
|
||||
status="running",
|
||||
request=request_dump,
|
||||
updated_at=utcnow(),
|
||||
)
|
||||
session.add(job)
|
||||
session.flush()
|
||||
pa_job_id = job.id
|
||||
desktop_job_id = job_id
|
||||
|
||||
return _run_and_persist(
|
||||
pa_job_id=pa_job_id,
|
||||
desktop_job_id=desktop_job_id,
|
||||
request_dump=request_dump,
|
||||
sources=sources,
|
||||
mapa_servicios=mapa or None,
|
||||
)
|
||||
|
||||
|
||||
# Ruta estática ANTES de /jobs/{job_id} para evitar 405 (job_id="from-csv").
|
||||
@app.post("/jobs/from-csv")
|
||||
async def run_poblacion_apta_from_csv(
|
||||
files: list[UploadFile] = File(...),
|
||||
job_id: str | None = Form(None),
|
||||
origen_servicio: UploadFile | None = File(None),
|
||||
) -> dict[str, Any]:
|
||||
if not files:
|
||||
raise HTTPException(status_code=400, detail="Se requiere al menos un archivo CSV")
|
||||
|
||||
sources: list[dict[str, Any]] = []
|
||||
filenames: list[str] = []
|
||||
delimiters: list[str] = []
|
||||
|
||||
for upload in files:
|
||||
filename = upload.filename or "upload.csv"
|
||||
content = await upload.read()
|
||||
if not content:
|
||||
raise HTTPException(status_code=400, detail=f"El archivo {filename} está vacío")
|
||||
try:
|
||||
df, delimiter = load_csv_bytes(content, filename)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"No se pudo leer {filename}: {exc}",
|
||||
) from exc
|
||||
if df.empty:
|
||||
raise HTTPException(status_code=400, detail=f"El CSV {filename} no tiene filas")
|
||||
records = df.fillna("").astype(str).to_dict(orient="records")
|
||||
sources.append(
|
||||
{
|
||||
"cleaning_job_id": "",
|
||||
"label": filename,
|
||||
"rows": records,
|
||||
}
|
||||
)
|
||||
filenames.append(filename)
|
||||
delimiters.append(delimiter)
|
||||
|
||||
mapa, origen_filename = await _parse_origen_servicio(origen_servicio)
|
||||
|
||||
request_dump: dict[str, Any] = {
|
||||
"source": "csv",
|
||||
"job_id": job_id,
|
||||
"filenames": filenames,
|
||||
"delimiters": delimiters,
|
||||
"cleaning_job_ids": [],
|
||||
"origen_servicio": origen_filename,
|
||||
}
|
||||
|
||||
with session_scope() as session:
|
||||
_validate_job_id(session, job_id)
|
||||
job = PoblacionAptaJob(
|
||||
job_id=job_id,
|
||||
status="running",
|
||||
request=request_dump,
|
||||
updated_at=utcnow(),
|
||||
)
|
||||
session.add(job)
|
||||
session.flush()
|
||||
pa_job_id = job.id
|
||||
desktop_job_id = job_id
|
||||
|
||||
return _run_and_persist(
|
||||
pa_job_id=pa_job_id,
|
||||
desktop_job_id=desktop_job_id,
|
||||
request_dump=request_dump,
|
||||
sources=sources,
|
||||
mapa_servicios=mapa or None,
|
||||
)
|
||||
|
||||
|
||||
@app.get("/jobs")
|
||||
def list_jobs(job_id: str | None = None) -> list[dict[str, Any]]:
|
||||
with session_scope() as session:
|
||||
stmt = select(PoblacionAptaJob).order_by(PoblacionAptaJob.created_at.desc())
|
||||
if job_id:
|
||||
stmt = stmt.where(PoblacionAptaJob.job_id == job_id)
|
||||
jobs = session.scalars(stmt).all()
|
||||
return [serialize_job(job) for job in jobs]
|
||||
|
||||
|
||||
@app.get("/jobs/{job_id}")
|
||||
def get_job(job_id: str) -> dict[str, Any]:
|
||||
if job_id in {"from-csv", "from-cleaning"}:
|
||||
raise HTTPException(status_code=405, detail=f"Use POST /jobs/{job_id}")
|
||||
with session_scope() as session:
|
||||
job = session.get(PoblacionAptaJob, job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job de población apta no encontrado")
|
||||
return serialize_job(job)
|
||||
|
||||
|
||||
@app.get("/jobs/{job_id}/rows")
|
||||
def get_rows(
|
||||
job_id: str,
|
||||
limit: int = 500,
|
||||
offset: int = 0,
|
||||
observacion: str | None = Query(None),
|
||||
) -> dict[str, Any]:
|
||||
with session_scope() as session:
|
||||
job = session.get(PoblacionAptaJob, job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job de población apta no encontrado")
|
||||
stmt = (
|
||||
select(PoblacionAptaRow)
|
||||
.where(PoblacionAptaRow.poblacion_apta_job_id == job_id)
|
||||
.order_by(PoblacionAptaRow.row_index, PoblacionAptaRow.id)
|
||||
)
|
||||
rows = session.scalars(stmt).all()
|
||||
payload = [row.row_json for row in rows]
|
||||
if observacion:
|
||||
obs = observacion.strip().upper()
|
||||
if obs == "PASO_BOT":
|
||||
payload = [r for r in payload if str(r.get("OBSERVACION", "")).strip() == "PASO_BOT"]
|
||||
else:
|
||||
payload = [
|
||||
r
|
||||
for r in payload
|
||||
if obs in str(r.get("OBSERVACION", "")).upper()
|
||||
]
|
||||
total = len(payload)
|
||||
page = payload[offset : offset + limit]
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"rows": page,
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/jobs/{job_id}/export.csv")
|
||||
def export_csv(
|
||||
job_id: str,
|
||||
subset: str = Query("aptos", description="aptos | no_aptos"),
|
||||
) -> Response:
|
||||
subset_norm = (subset or "aptos").strip().lower()
|
||||
if subset_norm not in {"aptos", "no_aptos", "errores"}:
|
||||
raise HTTPException(status_code=400, detail="subset debe ser aptos o no_aptos")
|
||||
if subset_norm == "errores":
|
||||
subset_norm = "no_aptos"
|
||||
|
||||
with session_scope() as session:
|
||||
job = session.get(PoblacionAptaJob, job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job de población apta no encontrado")
|
||||
rows = session.scalars(
|
||||
select(PoblacionAptaRow)
|
||||
.where(PoblacionAptaRow.poblacion_apta_job_id == job_id)
|
||||
.order_by(PoblacionAptaRow.row_index, PoblacionAptaRow.id)
|
||||
).all()
|
||||
payload = [r.row_json for r in rows]
|
||||
if subset_norm == "aptos":
|
||||
csv_text = export_paso_bot_csv(payload)
|
||||
filename = f"poblacion_apta_aptos_{job_id[:8]}.csv"
|
||||
else:
|
||||
csv_text = export_no_aptos_csv(payload)
|
||||
filename = f"poblacion_apta_no_aptos_{job_id[:8]}.csv"
|
||||
|
||||
return Response(
|
||||
content=csv_text.encode("utf-8-sig"),
|
||||
media_type="text/csv; charset=utf-8",
|
||||
headers={
|
||||
"Content-Disposition": f'attachment; filename="{filename}"',
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
"""Dominio del servicio Población apta."""
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Carga robusta de CSV desde bytes (uploads)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv as _csv
|
||||
import os
|
||||
import tempfile
|
||||
from collections import Counter
|
||||
from typing import Optional
|
||||
|
||||
import pandas as pd
|
||||
|
||||
_CANDIDATE_DELIMITERS = ["|", ";", "\t", ",", ":"]
|
||||
|
||||
|
||||
def _detect_delimiter_from_sample(sample: str) -> str:
|
||||
if not sample:
|
||||
return ","
|
||||
try:
|
||||
dialect = _csv.Sniffer().sniff(sample, delimiters="".join(_CANDIDATE_DELIMITERS))
|
||||
sniffed = dialect.delimiter
|
||||
except _csv.Error:
|
||||
sniffed = None
|
||||
|
||||
lines = [ln for ln in sample.splitlines() if ln.strip()]
|
||||
if not lines:
|
||||
return sniffed or ","
|
||||
|
||||
scores: dict[str, tuple[int, int]] = {}
|
||||
for delim in _CANDIDATE_DELIMITERS:
|
||||
counts = [ln.count(delim) for ln in lines[:50]]
|
||||
if not counts or max(counts) == 0:
|
||||
continue
|
||||
mode_count, mode_freq = Counter(counts).most_common(1)[0]
|
||||
if mode_count == 0:
|
||||
continue
|
||||
scores[delim] = (mode_count, mode_freq)
|
||||
|
||||
if scores:
|
||||
best = max(scores.items(), key=lambda kv: (kv[1][0], kv[1][1]))
|
||||
if sniffed and sniffed in scores:
|
||||
sniffed_score = scores[sniffed]
|
||||
if sniffed_score[0] >= best[1][0] - 1:
|
||||
return sniffed
|
||||
return best[0]
|
||||
return sniffed or ","
|
||||
|
||||
|
||||
def load_csv_bytes(content: bytes, filename: str = "upload.csv") -> tuple[pd.DataFrame, str]:
|
||||
"""Carga un CSV desde bytes. Devuelve `(df, delimiter)`."""
|
||||
if not content:
|
||||
raise RuntimeError(f"El archivo {filename} está vacío")
|
||||
|
||||
sample = ""
|
||||
for enc in ("utf-8-sig", "utf-8", "latin-1", "cp1252"):
|
||||
try:
|
||||
sample = content[:65536].decode(enc, errors="replace")
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
delimiter = _detect_delimiter_from_sample(sample)
|
||||
|
||||
suffix = os.path.splitext(filename)[1] or ".csv"
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
|
||||
tmp.write(content)
|
||||
tmp_path = tmp.name
|
||||
|
||||
last_error: Optional[Exception] = None
|
||||
try:
|
||||
for enc in ("utf-8-sig", "utf-8", "latin-1", "cp1252"):
|
||||
try:
|
||||
df = pd.read_csv(
|
||||
tmp_path,
|
||||
delimiter=delimiter,
|
||||
dtype=str,
|
||||
encoding=enc,
|
||||
engine="python",
|
||||
on_bad_lines="skip",
|
||||
keep_default_na=False,
|
||||
)
|
||||
df.columns = [str(c).strip() for c in df.columns]
|
||||
return df, delimiter
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_error = exc
|
||||
raise RuntimeError(f"No se pudo leer el CSV {filename}: {last_error}")
|
||||
finally:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
@@ -0,0 +1,14 @@
|
||||
"""Paquete de validación Población apta (Verificador ST)."""
|
||||
from domain.verificador_st.pipeline import (
|
||||
consolidado_to_records,
|
||||
export_no_aptos_csv,
|
||||
export_paso_bot_csv,
|
||||
process_cleaning_sources,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"consolidado_to_records",
|
||||
"export_no_aptos_csv",
|
||||
"export_paso_bot_csv",
|
||||
"process_cleaning_sources",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,225 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Pipeline: filas limpias → consolidado Población apta."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from domain.verificador_st.core import (
|
||||
COLUMNAS_FINAL_IT,
|
||||
CSV_FINAL_IT_HEADERS,
|
||||
CSV_FINAL_IT_RENAME,
|
||||
aplicar_origen_servicio,
|
||||
construir_base_especial,
|
||||
construir_base_normal,
|
||||
es_archivo_especial,
|
||||
normalizar_registros,
|
||||
preparar_consolidado,
|
||||
preparar_resumen,
|
||||
)
|
||||
|
||||
|
||||
def _records_to_dataframe(records: list[dict[str, Any]]) -> pd.DataFrame:
|
||||
if not records:
|
||||
return pd.DataFrame()
|
||||
return pd.DataFrame(records).fillna("")
|
||||
|
||||
|
||||
def process_cleaning_sources(
|
||||
sources: list[dict[str, Any]],
|
||||
mapa_servicios: dict[str, str] | None = None,
|
||||
) -> tuple[pd.DataFrame, dict[str, Any], list[str]]:
|
||||
"""
|
||||
Procesa varias fuentes de limpieza.
|
||||
|
||||
Cada source: {
|
||||
"cleaning_job_id": str,
|
||||
"label": str, # archivo_origen
|
||||
"rows": list[dict],
|
||||
}
|
||||
|
||||
mapa_servicios: opcional, radicado -> tipo_servicio (Origen Servicio).
|
||||
"""
|
||||
log: list[str] = []
|
||||
ok_parts: list[pd.DataFrame] = []
|
||||
err_parts: list[pd.DataFrame] = []
|
||||
resumen_archivos: list[dict[str, Any]] = []
|
||||
mapa = mapa_servicios or {}
|
||||
|
||||
if mapa:
|
||||
# len(mapa) cuenta variantes de clave; estimar radicados únicos aproximados
|
||||
log.append(f"Origen Servicio cargado: {len(mapa)} claves de cruce")
|
||||
|
||||
for source in sources:
|
||||
cleaning_job_id = str(source.get("cleaning_job_id") or "")
|
||||
label = str(source.get("label") or cleaning_job_id)
|
||||
rows = source.get("rows") or []
|
||||
df = _records_to_dataframe(list(rows))
|
||||
log.append(f"Fuente {label}: {len(df)} filas")
|
||||
|
||||
if df.empty:
|
||||
resumen_archivos.append(
|
||||
{
|
||||
"archivo": label,
|
||||
"cleaning_job_id": cleaning_job_id,
|
||||
"filas": 0,
|
||||
"paso_bot": 0,
|
||||
"duplicados": 0,
|
||||
"con_error": 0,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if es_archivo_especial(df):
|
||||
base = construir_base_especial(df, label)
|
||||
log.append(f" tipo ESPECIAL")
|
||||
else:
|
||||
base = construir_base_normal(df, label)
|
||||
log.append(f" tipo NORMAL")
|
||||
|
||||
if mapa:
|
||||
base, encontrados, sin_cruce = aplicar_origen_servicio(base, mapa)
|
||||
log.append(
|
||||
f" Origen Servicio: {encontrados} con servicio / {sin_cruce} sin cruce"
|
||||
)
|
||||
|
||||
# Propagar cleaning_job_id para persistencia
|
||||
base["cleaning_job_id"] = cleaning_job_id
|
||||
|
||||
df_ok, df_err = normalizar_registros(base)
|
||||
if not df_ok.empty:
|
||||
df_ok = df_ok.copy()
|
||||
df_ok["cleaning_job_id"] = cleaning_job_id
|
||||
ok_parts.append(df_ok)
|
||||
if not df_err.empty:
|
||||
df_err = df_err.copy()
|
||||
df_err["cleaning_job_id"] = cleaning_job_id
|
||||
err_parts.append(df_err)
|
||||
|
||||
resumen_archivos.append(
|
||||
{
|
||||
"archivo": label,
|
||||
"cleaning_job_id": cleaning_job_id,
|
||||
"filas": len(df),
|
||||
"paso_bot": 0,
|
||||
"duplicados": 0,
|
||||
"con_error": 0,
|
||||
}
|
||||
)
|
||||
|
||||
df_ok_total = pd.concat(ok_parts, ignore_index=True) if ok_parts else pd.DataFrame()
|
||||
df_err_total = pd.concat(err_parts, ignore_index=True) if err_parts else pd.DataFrame()
|
||||
|
||||
consolidado = preparar_consolidado(df_ok_total, df_err_total)
|
||||
|
||||
# Reinyectar cleaning_job_id si se perdió en preparar_consolidado
|
||||
if "cleaning_job_id" not in consolidado.columns:
|
||||
consolidado["cleaning_job_id"] = ""
|
||||
# mapear por archivo_origen
|
||||
label_to_id = {
|
||||
str(s.get("label") or s.get("cleaning_job_id")): str(s.get("cleaning_job_id") or "")
|
||||
for s in sources
|
||||
}
|
||||
if "archivo_origen" in consolidado.columns:
|
||||
consolidado["cleaning_job_id"] = consolidado["archivo_origen"].map(
|
||||
lambda x: label_to_id.get(str(x), "")
|
||||
)
|
||||
|
||||
# Actualizar resumen por fuente
|
||||
if not consolidado.empty and "archivo_origen" in consolidado.columns:
|
||||
paso = (
|
||||
consolidado[consolidado["OBSERVACION"].eq("PASO_BOT")]
|
||||
.groupby("archivo_origen")
|
||||
.size()
|
||||
.to_dict()
|
||||
)
|
||||
dup = (
|
||||
consolidado[consolidado["OBSERVACION"].str.contains("RADICADO_DUPLICADO", na=False)]
|
||||
.groupby("archivo_origen")
|
||||
.size()
|
||||
.to_dict()
|
||||
)
|
||||
err = (
|
||||
consolidado[~consolidado["OBSERVACION"].eq("PASO_BOT")]
|
||||
.groupby("archivo_origen")
|
||||
.size()
|
||||
.to_dict()
|
||||
)
|
||||
for r in resumen_archivos:
|
||||
a = r.get("archivo", "")
|
||||
r["paso_bot"] = int(paso.get(a, 0))
|
||||
r["duplicados"] = int(dup.get(a, 0))
|
||||
r["con_error"] = int(err.get(a, 0))
|
||||
|
||||
resumen_obs, _ = preparar_resumen(consolidado, resumen_archivos)
|
||||
summary: dict[str, Any] = {
|
||||
"total": int(len(consolidado)),
|
||||
"paso_bot": int((consolidado["OBSERVACION"] == "PASO_BOT").sum()) if not consolidado.empty else 0,
|
||||
"con_error": int((consolidado["OBSERVACION"] != "PASO_BOT").sum()) if not consolidado.empty else 0,
|
||||
"duplicados": int(
|
||||
consolidado["OBSERVACION"].astype(str).str.contains("RADICADO_DUPLICADO", na=False).sum()
|
||||
)
|
||||
if not consolidado.empty
|
||||
else 0,
|
||||
"por_fuente": resumen_archivos,
|
||||
"por_observacion": resumen_obs.to_dict(orient="records") if not resumen_obs.empty else [],
|
||||
}
|
||||
log.append(
|
||||
f"Consolidado: {summary['total']} filas, "
|
||||
f"{summary['paso_bot']} PASO_BOT, {summary['con_error']} con observación"
|
||||
)
|
||||
return consolidado, summary, log
|
||||
|
||||
|
||||
def consolidado_to_records(df: pd.DataFrame) -> list[dict[str, Any]]:
|
||||
if df is None or df.empty:
|
||||
return []
|
||||
cols = [
|
||||
*COLUMNAS_FINAL_IT,
|
||||
"OBSERVACION",
|
||||
"archivo_origen",
|
||||
"tipo_archivo",
|
||||
"cleaning_job_id",
|
||||
]
|
||||
out = df.copy()
|
||||
for c in cols:
|
||||
if c not in out.columns:
|
||||
out[c] = ""
|
||||
records = out[cols].fillna("").astype(str).to_dict(orient="records")
|
||||
return records
|
||||
|
||||
|
||||
def export_paso_bot_csv(rows: list[dict[str, Any]]) -> str:
|
||||
"""CSV pipe-separated solo PASO_BOT, headers IT."""
|
||||
aptos = [r for r in rows if str(r.get("OBSERVACION", "")).strip() == "PASO_BOT"]
|
||||
if not aptos:
|
||||
df = pd.DataFrame(columns=CSV_FINAL_IT_HEADERS)
|
||||
else:
|
||||
df = pd.DataFrame(aptos)
|
||||
for c in COLUMNAS_FINAL_IT:
|
||||
if c not in df.columns:
|
||||
df[c] = ""
|
||||
df = df.reindex(columns=COLUMNAS_FINAL_IT).fillna("")
|
||||
df = df.rename(columns=CSV_FINAL_IT_RENAME)
|
||||
df = df.reindex(columns=CSV_FINAL_IT_HEADERS)
|
||||
return df.to_csv(sep="|", index=False, encoding="utf-8-sig")
|
||||
|
||||
|
||||
def export_no_aptos_csv(rows: list[dict[str, Any]]) -> str:
|
||||
"""CSV pipe-separated de filas que NO son PASO_BOT (incluye OBSERVACION)."""
|
||||
headers = [*CSV_FINAL_IT_HEADERS, "OBSERVACION"]
|
||||
no_aptos = [r for r in rows if str(r.get("OBSERVACION", "")).strip() != "PASO_BOT"]
|
||||
if not no_aptos:
|
||||
df = pd.DataFrame(columns=headers)
|
||||
else:
|
||||
df = pd.DataFrame(no_aptos)
|
||||
for c in COLUMNAS_FINAL_IT:
|
||||
if c not in df.columns:
|
||||
df[c] = ""
|
||||
if "OBSERVACION" not in df.columns:
|
||||
df["OBSERVACION"] = ""
|
||||
df = df.reindex(columns=[*COLUMNAS_FINAL_IT, "OBSERVACION"]).fillna("")
|
||||
df = df.rename(columns=CSV_FINAL_IT_RENAME)
|
||||
df = df.reindex(columns=headers)
|
||||
return df.to_csv(sep="|", index=False, encoding="utf-8-sig")
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Persistencia del servicio Población apta."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Iterator
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
create_engine,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, sessionmaker
|
||||
|
||||
|
||||
DATABASE_URL = os.getenv(
|
||||
"DATABASE_URL",
|
||||
"postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor",
|
||||
)
|
||||
|
||||
engine = create_engine(DATABASE_URL, pool_pre_ping=True)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def new_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
class Job(Base):
|
||||
__tablename__ = "jobs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
job_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
|
||||
class Task(Base):
|
||||
__tablename__ = "tasks"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("jobs.id"), nullable=True, index=True
|
||||
)
|
||||
type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
owner_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class CleaningJob(Base):
|
||||
__tablename__ = "cleaning_jobs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
extraction_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("jobs.id"), nullable=True, index=True
|
||||
)
|
||||
source: Mapped[str] = mapped_column(String(32), default="extraction", index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
log: Mapped[list[str]] = mapped_column(JSON, default=list)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
|
||||
class CleanedRow(Base):
|
||||
__tablename__ = "cleaned_rows"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
cleaning_job_id: Mapped[str] = mapped_column(ForeignKey("cleaning_jobs.id"), index=True)
|
||||
extraction_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
row_index: Mapped[int] = mapped_column(Integer, default=0)
|
||||
row_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
|
||||
class PoblacionAptaJob(Base):
|
||||
__tablename__ = "poblacion_apta_jobs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
||||
job_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("jobs.id"), nullable=True, index=True
|
||||
)
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
log: Mapped[list[str]] = mapped_column(JSON, default=list)
|
||||
summary: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
|
||||
class PoblacionAptaRow(Base):
|
||||
__tablename__ = "poblacion_apta_rows"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
poblacion_apta_job_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("poblacion_apta_jobs.id"), index=True
|
||||
)
|
||||
cleaning_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
row_index: Mapped[int] = mapped_column(Integer, default=0)
|
||||
row_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
if DATABASE_URL.startswith("postgresql"):
|
||||
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:
|
||||
connection.execute(text("SELECT pg_advisory_lock(74512002)"))
|
||||
try:
|
||||
Base.metadata.create_all(bind=connection)
|
||||
finally:
|
||||
connection.execute(text("SELECT pg_advisory_unlock(74512002)"))
|
||||
return
|
||||
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def session_scope() -> Iterator[Session]:
|
||||
session = SessionLocal()
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
@@ -0,0 +1,9 @@
|
||||
fastapi==0.115.6
|
||||
numpy==2.2.6
|
||||
openpyxl==3.1.5
|
||||
pandas==2.2.3
|
||||
psycopg[binary]==3.2.3
|
||||
pydantic==2.11.9
|
||||
python-multipart==0.0.20
|
||||
SQLAlchemy==2.0.36
|
||||
uvicorn[standard]==0.32.1
|
||||
Reference in New Issue
Block a user