314 lines
11 KiB
Plaintext
314 lines
11 KiB
Plaintext
// IPSExtractor — esquema de integración PostgreSQL
|
|
// Fuente autoritativa: services/extraction/infrastructure/database.py
|
|
// services/extraction/alembic/versions/
|
|
//
|
|
// Visualizar en https://dbdiagram.io (importar este archivo)
|
|
// o con la CLI de dbml: npx @dbml/cli dbml2sql docs/database.dbml
|
|
|
|
Project ipsextractor {
|
|
database_type: 'PostgreSQL'
|
|
Note: '''
|
|
Base de datos compartida entre todos los microservicios (extraction, cleaning,
|
|
count-facturas, bot-descarga). Cada servicio accede directamente a las tablas
|
|
que necesita. Las migraciones se gestionan con Alembic (services/db/alembic/).
|
|
Cada servicio es desacoplado; comparten solo la DB como fuente de verdad.
|
|
'''
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Grupos de tablas
|
|
// ---------------------------------------------------------------------------
|
|
|
|
TableGroup orchestration {
|
|
jobs
|
|
tasks
|
|
}
|
|
|
|
TableGroup pipeline {
|
|
extraction_batches
|
|
extraction_rows
|
|
cleaning_jobs
|
|
cleaned_rows
|
|
poblacion_apta_jobs
|
|
poblacion_apta_rows
|
|
}
|
|
|
|
TableGroup counting {
|
|
count_job_files
|
|
}
|
|
|
|
TableGroup bot_descarga {
|
|
download_workers
|
|
download_jobs
|
|
download_job_events
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Orchestration — trabajo multimodular del usuario
|
|
// ---------------------------------------------------------------------------
|
|
|
|
Table jobs {
|
|
id varchar(36) [pk, note: 'UUID v4']
|
|
status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed']
|
|
title text [note: 'Nombre descriptivo del trabajo']
|
|
metadata jsonb [not null, default: '{}', note: 'Datos adicionales específicos por tipo de trabajo']
|
|
created_at timestamptz [not null]
|
|
updated_at timestamptz [not null]
|
|
|
|
indexes {
|
|
status
|
|
}
|
|
|
|
Note: '''
|
|
Contenedor de trabajo del usuario. Un job puede tener múltiples tasks
|
|
(una por servicio/módulo utilizado). Permite agrupar conteo + extracción
|
|
+ limpieza de un mismo conjunto de documentos.
|
|
'''
|
|
}
|
|
|
|
Table tasks {
|
|
id varchar(36) [pk, note: 'UUID v4']
|
|
job_id varchar(36) [ref: > jobs.id, note: 'FK nullable — compat. con tasks anteriores']
|
|
type varchar(64) [not null, note: 'counting | extraction | cleaning | downloads']
|
|
status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed']
|
|
title text
|
|
request jsonb [not null, default: '{}', note: 'Payload de la petición que originó la tarea']
|
|
result jsonb [not null, default: '{}', note: 'Al completar incluye extraction_batch_id o conteos']
|
|
error text
|
|
owner_id varchar(36) [note: 'Reservado para multi-usuario']
|
|
created_at timestamptz [not null]
|
|
updated_at timestamptz [not null]
|
|
started_at timestamptz
|
|
finished_at timestamptz
|
|
|
|
indexes {
|
|
job_id
|
|
type
|
|
status
|
|
owner_id
|
|
}
|
|
|
|
Note: '''
|
|
Módulo/resultado dentro de un Job. Una task representa la ejecución
|
|
de un servicio específico (conteo, extracción, limpieza, bot de descarga).
|
|
job_id es nullable para compatibilidad con tasks creadas antes de jobs.
|
|
'''
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Bot de descarga — workers remotos + corridas PDF → MinIO
|
|
// ---------------------------------------------------------------------------
|
|
|
|
Table download_workers {
|
|
id varchar(36) [pk, note: 'ID del agent (ej. PC-01 / zer0)']
|
|
hostname text [not null, default: '']
|
|
max_concurrent integer [not null, default: 1]
|
|
status varchar(32) [not null, note: 'online | offline | busy']
|
|
last_heartbeat timestamptz [not null]
|
|
current_job_id varchar(36) [note: 'download_jobs.id en ejecución']
|
|
|
|
indexes {
|
|
status
|
|
}
|
|
|
|
Note: 'PC remota registrada por el agent BotDescarga (heartbeat).'
|
|
}
|
|
|
|
Table download_jobs {
|
|
id varchar(36) [pk, note: 'UUID v4 de la corrida']
|
|
job_id varchar(36) [null, ref: > jobs.id, note: 'Job Desktop contenedor']
|
|
task_id varchar(36) [null, ref: > tasks.id, note: 'Task type=downloads']
|
|
worker_id varchar(36) [not null, note: 'download_workers.id asignado']
|
|
status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed | cancelled']
|
|
title text
|
|
excel_key text [not null, default: '', note: 'Clave MinIO del Excel']
|
|
downloads_prefix text [not null, default: '', note: 'downloads/{id}/']
|
|
original_filename text [not null, default: '']
|
|
cancel_requested boolean [not null, default: false]
|
|
result jsonb [not null, default: '{}']
|
|
error text
|
|
created_at timestamptz [not null]
|
|
started_at timestamptz
|
|
finished_at timestamptz
|
|
|
|
indexes {
|
|
job_id
|
|
task_id
|
|
worker_id
|
|
status
|
|
(worker_id, status) [name: 'ix_download_jobs_worker_status']
|
|
}
|
|
|
|
Note: 'Corrida de descarga remota. PDFs en MinIO bajo downloads/{id}/.'
|
|
}
|
|
|
|
Table download_job_events {
|
|
id integer [pk, increment]
|
|
download_job_id varchar(36) [not null, ref: > download_jobs.id]
|
|
event_type varchar(64) [not null]
|
|
message text [not null]
|
|
timestamp timestamptz [not null]
|
|
radicado varchar(255)
|
|
nit varchar(64)
|
|
idx integer
|
|
total integer
|
|
metadata jsonb [not null, default: '{}']
|
|
|
|
indexes {
|
|
(download_job_id, id) [name: 'ix_download_job_events_job_id']
|
|
}
|
|
|
|
Note: 'Log incremental de progreso / errores de una corrida.'
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Counting — resultados del servicio de conteo de facturas médicas
|
|
// ---------------------------------------------------------------------------
|
|
|
|
Table count_job_files {
|
|
id varchar(36) [pk, note: 'UUID v4']
|
|
job_module_id varchar(36) [not null, ref: > tasks.id]
|
|
radicado varchar(255) [note: 'Identificador del radicado / carpeta de origen']
|
|
filename varchar(512) [not null, note: 'Nombre del archivo PDF']
|
|
file_path text [note: 'Ruta del archivo en el volumen montado (/data/...)']
|
|
doc_type varchar(128) [note: 'Tipo de documento (ej. Factura, Historia_Clinica)']
|
|
service_count integer [note: 'Número de servicios detectados por el modelo ML']
|
|
services jsonb [not null, default: '[]', note: 'Lista de ServiceRow detectados por el modelo']
|
|
status varchar(32) [not null, default: 'ok', note: 'ok | error']
|
|
is_valid boolean [note: 'null=sin revisar | true=válido | false=rechazado por el usuario']
|
|
error text [note: 'Mensaje de error si status=error']
|
|
created_at timestamptz [not null]
|
|
|
|
indexes {
|
|
job_module_id
|
|
radicado
|
|
status
|
|
is_valid
|
|
}
|
|
|
|
Note: '''
|
|
Resultado del servicio Go (services-count-facturas-medicas-serving) para
|
|
un PDF individual. El campo file_path permite que el servicio de extracción
|
|
procese el archivo directamente sin necesidad de re-subirlo.
|
|
is_valid=null hasta que el usuario revise; el umbral min_service_count
|
|
se aplica en el endpoint POST /tasks/extraction/from-count.
|
|
'''
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Extracción
|
|
// ---------------------------------------------------------------------------
|
|
|
|
Table extraction_batches {
|
|
id varchar(36) [pk, note: 'UUID v4']
|
|
job_module_id varchar(36) [ref: > tasks.id, note: 'FK nullable — tarea que origino este lote']
|
|
status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed']
|
|
root_path text [note: 'Ruta raíz de documentos (IPS_DOCUMENT_ROOT o "count://" si viene de conteo)']
|
|
request jsonb [not null, default: '{}', note: 'ExtractionRequest serializado']
|
|
error text
|
|
created_at timestamptz [not null]
|
|
updated_at timestamptz [not null]
|
|
|
|
indexes {
|
|
job_module_id
|
|
status
|
|
}
|
|
|
|
Note: 'Lote creado al iniciar una extracción; agrupa todas las filas extraídas.'
|
|
}
|
|
|
|
Table extraction_rows {
|
|
id integer [pk, increment]
|
|
batch_id varchar(36) [not null, ref: > extraction_batches.id]
|
|
radicado varchar(255) [note: 'Identificador del radicado IPS']
|
|
source_pdf text [note: 'Ruta del PDF origen dentro del volumen montado']
|
|
row_index integer [not null, default: 0, note: 'Orden estable dentro del lote']
|
|
row_json jsonb [not null, default: '{}', note: 'Fila aplanada con campos solicitados']
|
|
raw_json jsonb [not null, default: '{}', note: 'Resultado bruto del procesador por radicado']
|
|
created_at timestamptz [not null]
|
|
|
|
indexes {
|
|
batch_id
|
|
radicado
|
|
}
|
|
|
|
Note: 'Una o más filas por radicado según flatten_extraction_result().'
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Limpieza
|
|
// ---------------------------------------------------------------------------
|
|
|
|
Table cleaning_jobs {
|
|
id varchar(36) [pk, note: 'UUID v4']
|
|
extraction_batch_id varchar(36) [null, ref: > extraction_batches.id, note: 'Null si source=csv']
|
|
job_id varchar(36) [null, ref: > jobs.id, note: 'Trabajo Desktop al que pertenece']
|
|
source varchar(32) [not null, default: 'extraction', note: 'extraction | csv']
|
|
status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed']
|
|
request jsonb [not null, default: '{}', note: 'CleaningRequest (rules_per_column, etc.)']
|
|
log jsonb [not null, default: '[]', note: 'Lista de mensajes de auditoría del job']
|
|
error text
|
|
created_at timestamptz [not null]
|
|
updated_at timestamptz [not null]
|
|
|
|
indexes {
|
|
extraction_batch_id
|
|
job_id
|
|
source
|
|
status
|
|
}
|
|
|
|
Note: 'Ejecución de reglas de limpieza sobre filas de extracción o CSV subido.'
|
|
}
|
|
|
|
Table cleaned_rows {
|
|
id integer [pk, increment]
|
|
cleaning_job_id varchar(36) [not null, ref: > cleaning_jobs.id]
|
|
extraction_batch_id varchar(36) [null, ref: > extraction_batches.id]
|
|
row_index integer [not null, default: 0, note: 'Conserva el orden del lote original']
|
|
row_json jsonb [not null, default: '{}', note: 'Fila tras aplicar reglas de limpieza']
|
|
created_at timestamptz [not null]
|
|
|
|
indexes {
|
|
cleaning_job_id
|
|
extraction_batch_id
|
|
}
|
|
|
|
Note: 'Salida del servicio de limpieza.'
|
|
}
|
|
|
|
Table poblacion_apta_jobs {
|
|
id varchar(36) [pk, note: 'UUID v4']
|
|
job_id varchar(36) [null, ref: > jobs.id, note: 'Trabajo Desktop al que pertenece']
|
|
status varchar(32) [not null, default: 'pending', note: 'pending | running | completed | failed']
|
|
request jsonb [not null, default: '{}', note: 'cleaning_job_ids, job_id']
|
|
log jsonb [not null, default: '[]']
|
|
summary jsonb [not null, default: '{}', note: 'totales paso_bot / errores / por fuente']
|
|
error text
|
|
created_at timestamptz [not null]
|
|
updated_at timestamptz [not null]
|
|
|
|
indexes {
|
|
job_id
|
|
status
|
|
}
|
|
|
|
Note: 'Consolidación y validación Verificador ST sobre una o más limpiezas.'
|
|
}
|
|
|
|
Table poblacion_apta_rows {
|
|
id integer [pk, increment]
|
|
poblacion_apta_job_id varchar(36) [not null, ref: > poblacion_apta_jobs.id]
|
|
cleaning_job_id varchar(36) [null, note: 'Limpieza de origen de la fila']
|
|
row_index integer [not null, default: 0]
|
|
row_json jsonb [not null, default: '{}', note: 'Schema FINAL_IT + OBSERVACION']
|
|
created_at timestamptz [not null]
|
|
|
|
indexes {
|
|
poblacion_apta_job_id
|
|
cleaning_job_id
|
|
}
|
|
|
|
Note: 'Filas del consolidado Población apta (PASO_BOT y observaciones).'
|
|
}
|