189 lines
7.4 KiB
Python
189 lines
7.4 KiB
Python
"""Persistencia del servicio sobre la base de datos de integracion."""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import uuid
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Iterator
|
|
|
|
from sqlalchemy import (
|
|
JSON,
|
|
Boolean,
|
|
DateTime,
|
|
ForeignKey,
|
|
Integer,
|
|
String,
|
|
Text,
|
|
create_engine,
|
|
text,
|
|
)
|
|
from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, sessionmaker
|
|
|
|
|
|
DATABASE_URL = os.getenv(
|
|
"DATABASE_URL",
|
|
"postgresql+psycopg://ipsextractor:ipsextractor@database:5432/ipsextractor",
|
|
)
|
|
|
|
engine = create_engine(
|
|
DATABASE_URL,
|
|
pool_pre_ping=True,
|
|
pool_size=10,
|
|
pool_timeout=30,
|
|
connect_args={"connect_timeout": 10},
|
|
)
|
|
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
|
|
|
|
|
class Base(DeclarativeBase):
|
|
pass
|
|
|
|
|
|
def utcnow() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def new_id() -> str:
|
|
return str(uuid.uuid4())
|
|
|
|
|
|
class Job(Base):
|
|
"""Contenedor de trabajo del usuario; puede abarcar múltiples módulos/servicios."""
|
|
__tablename__ = "jobs"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
|
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
|
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
# "metadata" es reservado por DeclarativeBase; el atributo Python usa job_metadata
|
|
# pero la columna en la BD sigue llamándose "metadata".
|
|
job_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict)
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
class ExtractionBatch(Base):
|
|
__tablename__ = "extraction_batches"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
|
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
|
root_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
job_module_id: Mapped[str | None] = mapped_column(
|
|
String(36), ForeignKey("tasks.id"), nullable=True, index=True
|
|
)
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
class Task(Base):
|
|
"""Módulo/resultado dentro de un Job; un job puede tener N tasks (una por servicio)."""
|
|
__tablename__ = "tasks"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
|
job_id: Mapped[str | None] = mapped_column(
|
|
String(36), ForeignKey("jobs.id"), nullable=True, index=True
|
|
)
|
|
type: Mapped[str] = mapped_column(String(64), index=True)
|
|
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
|
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
result: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
owner_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
|
|
|
|
class CountJobFile(Base):
|
|
"""Resultado del servicio de conteo para un PDF individual dentro de una tarea de conteo."""
|
|
__tablename__ = "count_job_files"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
|
job_module_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("tasks.id"), index=True
|
|
)
|
|
radicado: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
|
filename: Mapped[str] = mapped_column(String(512))
|
|
file_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
doc_type: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
|
nit: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
|
object_key: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
service_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
services: Mapped[list[Any]] = mapped_column(JSON, default=list)
|
|
status: Mapped[str] = mapped_column(String(32), default="ok", index=True)
|
|
is_valid: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
class ExtractionRow(Base):
|
|
__tablename__ = "extraction_rows"
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
|
batch_id: Mapped[str] = mapped_column(ForeignKey("extraction_batches.id"), index=True)
|
|
radicado: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
|
source_pdf: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
row_index: Mapped[int] = mapped_column(Integer, default=0)
|
|
row_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
raw_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
class CleaningJob(Base):
|
|
__tablename__ = "cleaning_jobs"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_id)
|
|
extraction_batch_id: Mapped[str | None] = mapped_column(
|
|
ForeignKey("extraction_batches.id"), nullable=True, index=True
|
|
)
|
|
job_id: Mapped[str | None] = mapped_column(
|
|
String(36), ForeignKey("jobs.id"), nullable=True, index=True
|
|
)
|
|
source: Mapped[str] = mapped_column(String(32), default="extraction", index=True)
|
|
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
|
request: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
log: Mapped[list[str]] = mapped_column(JSON, default=list)
|
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
class CleanedRow(Base):
|
|
__tablename__ = "cleaned_rows"
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
|
cleaning_job_id: Mapped[str] = mapped_column(ForeignKey("cleaning_jobs.id"), index=True)
|
|
extraction_batch_id: Mapped[str | None] = mapped_column(
|
|
ForeignKey("extraction_batches.id"), nullable=True, index=True
|
|
)
|
|
row_index: Mapped[int] = mapped_column(Integer, default=0)
|
|
row_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
def init_db() -> None:
|
|
"""Verifica conectividad con la BD.
|
|
|
|
Las migraciones son responsabilidad exclusiva de services/db (init container).
|
|
Este servicio asume que el esquema ya está actualizado al arrancar.
|
|
"""
|
|
with engine.connect() as conn:
|
|
conn.execute(text("SELECT 1"))
|
|
|
|
|
|
@contextmanager
|
|
def session_scope() -> Iterator[Session]:
|
|
session = SessionLocal()
|
|
try:
|
|
yield session
|
|
session.commit()
|
|
except Exception:
|
|
session.rollback()
|
|
raise
|
|
finally:
|
|
session.close()
|