feat: Integrate CSV cleaning functionality into the application, adding a new GUI tab for user interaction, enabling CSV loading, analysis, and rule application for data cleaning. Implement backend tracking for batch processing and enhance the BatchProcessor with tracking capabilities.
This commit is contained in:
erik
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Capa de persistencia SQLite para el servidor de tracking.
|
||||
|
||||
Guarda el historial completo de lotes de extracción, incluyendo
|
||||
métricas de progreso, costos y la ruta del CSV generado.
|
||||
"""
|
||||
import sqlite3
|
||||
import json
|
||||
import os
|
||||
from typing import Dict, Any, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
DB_PATH = os.path.join(os.path.dirname(__file__), "..", "tracking.db")
|
||||
|
||||
|
||||
def get_connection() -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
def init_db():
|
||||
"""Crea las tablas si no existen."""
|
||||
with get_connection() as conn:
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS batch_logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
batch_id TEXT NOT NULL,
|
||||
timestamp TEXT NOT NULL,
|
||||
level TEXT NOT NULL DEFAULT 'info',
|
||||
radicado TEXT,
|
||||
message TEXT NOT NULL,
|
||||
FOREIGN KEY (batch_id) REFERENCES batches(id)
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_batch_logs_batch_id ON batch_logs(batch_id)")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS batches (
|
||||
id TEXT PRIMARY KEY,
|
||||
root_path TEXT NOT NULL,
|
||||
csv_output_path TEXT,
|
||||
hostname TEXT,
|
||||
username TEXT,
|
||||
total_radicados INTEGER NOT NULL DEFAULT 0,
|
||||
config TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'processing',
|
||||
current_radicado_index INTEGER NOT NULL DEFAULT 0,
|
||||
success_count INTEGER NOT NULL DEFAULT 0,
|
||||
error_count INTEGER NOT NULL DEFAULT 0,
|
||||
total_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
total_cost_usd REAL NOT NULL DEFAULT 0.0,
|
||||
last_error TEXT,
|
||||
current_phase TEXT,
|
||||
current_radicado TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
|
||||
def insert_batch(batch: Dict[str, Any]):
|
||||
"""Inserta un nuevo lote en la base de datos."""
|
||||
with get_connection() as conn:
|
||||
conn.execute("""
|
||||
INSERT INTO batches (
|
||||
id, root_path, csv_output_path, hostname, username, total_radicados, config,
|
||||
status, current_radicado_index, success_count, error_count,
|
||||
total_tokens, total_cost_usd, last_error, current_phase, current_radicado,
|
||||
created_at, updated_at
|
||||
) VALUES (
|
||||
:id, :root_path, :csv_output_path, :hostname, :username, :total_radicados, :config,
|
||||
:status, :current_radicado_index, :success_count, :error_count,
|
||||
:total_tokens, :total_cost_usd, :last_error, :current_phase, :current_radicado,
|
||||
:created_at, :updated_at
|
||||
)
|
||||
""", {
|
||||
**batch,
|
||||
"config": json.dumps(batch.get("config") or {}),
|
||||
"created_at": _to_iso(batch.get("created_at")),
|
||||
"updated_at": _to_iso(batch.get("updated_at")),
|
||||
"current_phase": batch.get("current_phase"),
|
||||
"current_radicado": batch.get("current_radicado"),
|
||||
})
|
||||
conn.commit()
|
||||
|
||||
|
||||
def update_batch(batch_id: str, fields: Dict[str, Any]):
|
||||
"""Actualiza campos de un lote existente."""
|
||||
if not fields:
|
||||
return
|
||||
fields["updated_at"] = _to_iso(datetime.now())
|
||||
set_clause = ", ".join(f"{k} = :{k}" for k in fields)
|
||||
fields["batch_id"] = batch_id
|
||||
with get_connection() as conn:
|
||||
conn.execute(
|
||||
f"UPDATE batches SET {set_clause} WHERE id = :batch_id",
|
||||
fields
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_all_batches() -> List[Dict[str, Any]]:
|
||||
"""Retorna todos los lotes ordenados por fecha de creación descendente."""
|
||||
with get_connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM batches ORDER BY created_at DESC"
|
||||
).fetchall()
|
||||
return [_row_to_dict(r) for r in rows]
|
||||
|
||||
|
||||
def insert_log(batch_id: str, message: str, level: str = "info", radicado: str = None):
|
||||
"""Inserta una entrada de log para un lote."""
|
||||
with get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO batch_logs (batch_id, timestamp, level, radicado, message) VALUES (?, ?, ?, ?, ?)",
|
||||
(batch_id, datetime.now().isoformat(), level, radicado, message)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_logs(batch_id: str, limit: int = 200) -> List[Dict[str, Any]]:
|
||||
"""Retorna los logs de un lote, del más reciente al más antiguo."""
|
||||
with get_connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM batch_logs WHERE batch_id = ? ORDER BY id DESC LIMIT ?",
|
||||
(batch_id, limit)
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def get_batch(batch_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Retorna un lote por su ID."""
|
||||
with get_connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM batches WHERE id = ?", (batch_id,)
|
||||
).fetchone()
|
||||
return _row_to_dict(row) if row else None
|
||||
|
||||
|
||||
def _row_to_dict(row: sqlite3.Row) -> Dict[str, Any]:
|
||||
d = dict(row)
|
||||
try:
|
||||
d["config"] = json.loads(d.get("config") or "{}")
|
||||
except Exception:
|
||||
d["config"] = {}
|
||||
|
||||
# Asegurar que hostname y username estén presentes
|
||||
if "hostname" not in d: d["hostname"] = None
|
||||
if "username" not in d: d["username"] = None
|
||||
|
||||
return d
|
||||
|
||||
|
||||
def _to_iso(value) -> str:
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return datetime.now().isoformat()
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Servidor de tracking de IPSExtractor.
|
||||
|
||||
Persiste el historial de lotes en SQLite y distribuye actualizaciones
|
||||
en tiempo real vía WebSockets al dashboard web.
|
||||
"""
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.responses import FileResponse
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Dict, List, Any, Optional
|
||||
from pydantic import BaseModel
|
||||
import uuid
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
from .schemas.batch import BatchCreate, BatchUpdate, BatchRead, BatchStatus
|
||||
from .database import init_db, insert_batch, update_batch, get_all_batches, get_batch, insert_log, get_logs
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
init_db()
|
||||
yield
|
||||
|
||||
app = FastAPI(title="IPSExtractor Tracking API", lifespan=lifespan)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Cache en memoria para lotes activos (para broadcast rápido)
|
||||
active_batches: Dict[str, Dict[str, Any]] = {}
|
||||
active_connections: List[WebSocket] = []
|
||||
|
||||
|
||||
async def broadcast(message: dict):
|
||||
json_compatible_message = jsonable_encoder(message)
|
||||
disconnected = []
|
||||
for connection in active_connections:
|
||||
try:
|
||||
await connection.send_json(json_compatible_message)
|
||||
except Exception:
|
||||
disconnected.append(connection)
|
||||
for conn in disconnected:
|
||||
active_connections.remove(conn)
|
||||
|
||||
|
||||
@app.post("/batches", response_model=BatchRead)
|
||||
async def create_batch(batch: BatchCreate):
|
||||
batch_id = str(uuid.uuid4())
|
||||
now = datetime.now()
|
||||
new_batch = {
|
||||
"id": batch_id,
|
||||
**batch.model_dump(),
|
||||
"status": BatchStatus.PROCESSING,
|
||||
"current_radicado_index": 0,
|
||||
"success_count": 0,
|
||||
"error_count": 0,
|
||||
"total_tokens": 0,
|
||||
"total_cost_usd": 0.0,
|
||||
"last_error": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
active_batches[batch_id] = new_batch
|
||||
insert_batch(new_batch)
|
||||
await broadcast({"event": "batch_created", "data": new_batch})
|
||||
return new_batch
|
||||
|
||||
|
||||
@app.get("/batches", response_model=List[BatchRead])
|
||||
async def list_batches():
|
||||
# Retorna historial completo desde SQLite
|
||||
db_batches = get_all_batches()
|
||||
# Sobreescribe con datos en memoria si el lote está activo (más fresco)
|
||||
result = []
|
||||
for b in db_batches:
|
||||
if b["id"] in active_batches:
|
||||
result.append(active_batches[b["id"]])
|
||||
else:
|
||||
result.append(b)
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/batches/{batch_id}", response_model=BatchRead)
|
||||
async def get_batch_endpoint(batch_id: str):
|
||||
if batch_id in active_batches:
|
||||
return active_batches[batch_id]
|
||||
batch = get_batch(batch_id)
|
||||
if not batch:
|
||||
raise HTTPException(status_code=404, detail="Lote no encontrado")
|
||||
return batch
|
||||
|
||||
|
||||
@app.patch("/batches/{batch_id}", response_model=BatchRead)
|
||||
async def update_batch_endpoint(batch_id: str, update: BatchUpdate):
|
||||
if batch_id in active_batches:
|
||||
batch = active_batches[batch_id]
|
||||
update_data = update.model_dump(exclude_unset=True)
|
||||
for key, value in update_data.items():
|
||||
batch[key] = value
|
||||
batch["updated_at"] = datetime.now()
|
||||
|
||||
# Si el lote se completó, sacarlo del cache activo
|
||||
if batch.get("status") in (BatchStatus.COMPLETED, BatchStatus.FAILED):
|
||||
active_batches.pop(batch_id, None)
|
||||
|
||||
update_batch(batch_id, update_data)
|
||||
await broadcast({"event": "batch_updated", "data": batch})
|
||||
return batch
|
||||
|
||||
# Lote no está en memoria (completado), actualizar solo en DB
|
||||
batch = get_batch(batch_id)
|
||||
if not batch:
|
||||
raise HTTPException(status_code=404, detail="Lote no encontrado")
|
||||
update_data = update.model_dump(exclude_unset=True)
|
||||
update_batch(batch_id, update_data)
|
||||
batch.update(update_data)
|
||||
return batch
|
||||
|
||||
|
||||
class LogEntry(BaseModel):
|
||||
message: str
|
||||
level: str = "info"
|
||||
radicado: Optional[str] = None
|
||||
|
||||
class LogRead(BaseModel):
|
||||
id: int
|
||||
batch_id: str
|
||||
timestamp: str
|
||||
level: str
|
||||
radicado: Optional[str]
|
||||
message: str
|
||||
|
||||
@app.post("/batches/{batch_id}/logs", status_code=201)
|
||||
async def add_log(batch_id: str, entry: LogEntry):
|
||||
"""Añade una entrada de log para un lote."""
|
||||
insert_log(batch_id, entry.message, entry.level, entry.radicado)
|
||||
# Broadcast log también por WebSocket para actualizaciones en vivo
|
||||
await broadcast({"event": "log_added", "batch_id": batch_id, "data": {
|
||||
"level": entry.level, "radicado": entry.radicado, "message": entry.message
|
||||
}})
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@app.get("/batches/{batch_id}/logs", response_model=List[LogRead])
|
||||
async def list_logs(batch_id: str, limit: int = 200):
|
||||
"""Retorna el historial de logs de un lote."""
|
||||
return get_logs(batch_id, limit)
|
||||
|
||||
|
||||
@app.get("/download/{batch_id}")
|
||||
async def download_csv(batch_id: str):
|
||||
"""Sirve el archivo CSV generado por el lote de extracción."""
|
||||
batch = get_batch(batch_id)
|
||||
if not batch:
|
||||
raise HTTPException(status_code=404, detail="Lote no encontrado")
|
||||
|
||||
csv_path = batch.get("csv_output_path")
|
||||
if not csv_path:
|
||||
raise HTTPException(status_code=404, detail="Este lote no tiene un archivo CSV asociado")
|
||||
if not os.path.isfile(csv_path):
|
||||
raise HTTPException(status_code=404, detail=f"Archivo CSV no encontrado en el servidor: {csv_path}")
|
||||
|
||||
filename = os.path.basename(csv_path)
|
||||
return FileResponse(
|
||||
path=csv_path,
|
||||
media_type="text/csv",
|
||||
filename=filename,
|
||||
)
|
||||
|
||||
|
||||
@app.websocket("/ws/tracking")
|
||||
async def websocket_endpoint(websocket: WebSocket):
|
||||
await websocket.accept()
|
||||
active_connections.append(websocket)
|
||||
try:
|
||||
while True:
|
||||
await websocket.receive_text()
|
||||
except WebSocketDisconnect:
|
||||
if websocket in active_connections:
|
||||
active_connections.remove(websocket)
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, List, Dict
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class BatchStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
PROCESSING = "processing"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class BatchBase(BaseModel):
|
||||
root_path: str
|
||||
total_radicados: int
|
||||
csv_output_path: Optional[str] = None
|
||||
hostname: Optional[str] = None
|
||||
username: Optional[str] = None
|
||||
config: Optional[Dict] = None
|
||||
|
||||
|
||||
class BatchCreate(BatchBase):
|
||||
pass
|
||||
|
||||
|
||||
class BatchUpdate(BaseModel):
|
||||
status: Optional[BatchStatus] = None
|
||||
current_radicado_index: Optional[int] = None
|
||||
success_count: Optional[int] = None
|
||||
error_count: Optional[int] = None
|
||||
total_tokens: Optional[int] = None
|
||||
total_cost_usd: Optional[float] = None
|
||||
last_error: Optional[str] = None
|
||||
current_phase: Optional[str] = None
|
||||
current_radicado: Optional[str] = None
|
||||
|
||||
|
||||
class BatchRead(BatchBase):
|
||||
id: str
|
||||
status: BatchStatus
|
||||
current_radicado_index: int = 0
|
||||
success_count: int = 0
|
||||
error_count: int = 0
|
||||
total_tokens: int = 0
|
||||
total_cost_usd: float = 0.0
|
||||
last_error: Optional[str] = None
|
||||
current_phase: Optional[str] = None
|
||||
current_radicado: Optional[str] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
Reference in New Issue
Block a user