fix: Update production API base URL in documentation and configuration files. Correct the URL in the desktop app README, migration documentation, and Vite configuration. Enhance ONNX model feature handling in the processing pipeline to align with the updated feature set, ensuring compatibility with the latest model version. Add tests for feature extraction to validate changes.

This commit is contained in:
erik
2026-07-09 03:13:14 -05:00
parent 328e5e0636
commit ff9abfc753
11 changed files with 449 additions and 169 deletions
+1 -1
View File
@@ -379,7 +379,7 @@ npm run dist
| Entorno | URL por defecto |
|---------|-----------------|
| Desarrollo (`npm run dev`) | `http://127.0.0.1:8189` |
| Instalador (build producción) | `https://pys1.builder.xyz` |
| Instalador (build producción) | `https://pys1.builker.xyz` |
La URL es configurable en **Configuración** de la app y se persiste en `localStorage`.
+1 -1
View File
@@ -46,7 +46,7 @@ Toda la comunicación pasa por el API Gateway.
| Entorno | URL por defecto |
|---------|-----------------|
| Desarrollo (`npm run dev`) | `http://127.0.0.1:8189` |
| Instalador (build producción) | `https://pys1.builder.xyz` |
| Instalador (build producción) | `https://pys1.builker.xyz` |
El cliente HTTP es [`src/renderer/lib/apiClient.ts`](src/renderer/lib/apiClient.ts).
### Indexación de carpetas
@@ -1,4 +1,4 @@
/** URL por defecto del API Gateway (dev: local Docker; instalador: pys1.builder.xyz). */
/** URL por defecto del API Gateway (dev: local Docker; instalador: pys1.builker.xyz). */
export const DEFAULT_API_BASE_URL: string = __DEFAULT_API_BASE_URL__;
/** Reemplaza localhost por 127.0.0.1 para evitar problemas de IPv6 en Windows/Docker. */
+1 -1
View File
@@ -7,7 +7,7 @@ const versionMeta = JSON.parse(
readFileSync("./src/shared/version.json", "utf8"),
) as { releaseDate: string };
const PRODUCTION_API_BASE_URL = "https://pys1.builder.xyz";
const PRODUCTION_API_BASE_URL = "https://pys1.builker.xyz";
const DEVELOPMENT_API_BASE_URL = "http://127.0.0.1:8189";
export default defineConfig(({ mode }) => ({
@@ -71,9 +71,10 @@ OCR palabra a palabra con coordenadas (X, Y, W, H)
▼ DBSCAN 1D sobre coordenada Y
Agrupación de palabras en filas candidatas
│
▼ 8 features por fila
first_token_is_code · currency_count · integer_count · x_span_norm
y_norm · token_count · special_char_ratio · digit_ratio
▼ 14 features por fila (pipeline v1.1.0)
first_token_is_code · first_token_is_service_code · has_service_table_pattern · has_header_keyword
currency_at_row_end · first_token_x_norm · is_likely_continuation · currency_count · integer_count
x_span_norm · y_norm · token_count · special_char_ratio · digit_ratio
│
▼ ONNX Runtime (Random Forest)
P(servicio) por fila → filtrar por threshold (inference.json)
@@ -103,6 +104,36 @@ pip install -r ../requirements-export.txt
python export_onnx.py --joblib /ruta/model.joblib --out ../models/latest
```
**Despliegue:** los archivos no van en la imagen Docker; se montan desde `models/latest/`. Tras copiarlos al servidor:
```bash
chmod 644 models/latest/model.onnx models/latest/inference.json
docker compose restart count-api
docker logs ipsextractor-count-api 2>&1 | grep -E "modelo ONNX|WARN: modelo"
# Debe aparecer: modelo ONNX cargado: /app/models/model.onnx
```
Si el log sigue mostrando `WARN: modelo ML no disponible`, verifica dentro del contenedor:
```bash
docker exec ipsextractor-count-api ls -la /app/models/
```
Sin modelo cargado, **todas las filas OCR salen con probability=1.0 (100%)** — modo heurístico.
### Error: `requests feature 13 but input tensor has 8 features`
El `model.onnx` desplegado **no coincide** con el pipeline Go. El modelo v1.1.0 usa **14 features** (ver `features.go` y `export_onnx.py`). Si el ONNX fue exportado con otro esquema verás errores como `requests feature 13 but input tensor has 8 features`.
**Exportar desde el joblib v1.1.0:**
```bash
python export_onnx.py \
--joblib /ruta/services-count-facturas-medicas-ml/models/v1.1.0/model.joblib \
--metadata /ruta/services-count-facturas-medicas-ml/models/v1.1.0/model_metadata.json \
--out ../models/latest
```
---
## Variables de entorno
@@ -1,125 +1,199 @@
services:␍
redis:␍
image: redis:7-alpine␍
command: redis-server --maxmemory 1gb --maxmemory-policy allkeys-lru␍
healthcheck:␍
test: ["CMD", "redis-cli", "ping"]␍
interval: 5s␍
timeout: 3s␍
retries: 5␍
␍
nats:␍
image: nats:2-alpine␍
command: ["-js", "-m", "8222"]␍
healthcheck:␍
test: ["CMD", "wget", "-qO-", "http://localhost:8222/healthz"]␍
interval: 5s␍
timeout: 3s␍
retries: 5␍
␍
minio:␍
image: minio/minio:latest␍
command: server /data --console-address ":9001"␍
environment:␍
MINIO_ROOT_USER: minioadmin␍
MINIO_ROOT_PASSWORD: minioadmin␍
volumes:␍
- minio_data:/data␍
ports:␍
- "9002:9001"␍
healthcheck:␍
test: ["CMD-SHELL", "curl -f http://localhost:9000/minio/health/live || exit 1"]␍
interval: 10s␍
timeout: 5s␍
retries: 5␍
start_period: 10s␍
␍
api:␍
build: .␍
command: ["/app/api"]␍
ports:␍
- "8080:8080"␍
depends_on:␍
redis:␍
condition: service_healthy␍
nats:␍
condition: service_healthy␍
minio:␍
condition: service_started␍
environment:␍
REDIS_URL: redis://redis:6379/0␍
NATS_URL: nats://nats:4222␍
MINIO_ENDPOINT: minio:9000␍
MINIO_ACCESS_KEY: minioadmin␍
MINIO_SECRET_KEY: minioadmin␍
MINIO_BUCKET: pdf-count␍
MODEL_PATH: /app/models/model.onnx␍
INFERENCE_CONFIG: /app/models/inference.json␍
SYNC_MAX_FILES: 50␍
JOB_MAX_FILES: 50000␍
QUEUE_MAX_PENDING: 100000␍
OPENAPI_PATH: /app/openapi.yaml␍
OCR_POOL_SIZE: 4␍
SYNC_PROCESS_CONCURRENCY: 4␍
JOB_INLINE_PROCESSING: "false"␍
LOG_PIPELINE_TIMING: "true"␍
volumes:␍
- ./models/latest:/app/models:ro␍
healthcheck:␍
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]␍
interval: 10s␍
timeout: 5s␍
retries: 5␍
␍
worker:␍
build: .␍
command: ["/app/worker"]␍
depends_on:␍
redis:␍
condition: service_healthy␍
nats:␍
condition: service_healthy␍
minio:␍
condition: service_started␍
environment:␍
REDIS_URL: redis://redis:6379/0␍
NATS_URL: nats://nats:4222␍
MINIO_ENDPOINT: minio:9000␍
MINIO_ACCESS_KEY: minioadmin␍
MINIO_SECRET_KEY: minioadmin␍
MINIO_BUCKET: pdf-count␍
MODEL_PATH: /app/models/model.onnx␍
INFERENCE_CONFIG: /app/models/inference.json␍
WORKER_CONCURRENCY: 4␍
PDF_PROCESS_TIMEOUT_SEC: 120␍
OCR_POOL_SIZE: 4␍
NATS_FETCH_MAX_WAIT_MS: 300␍
LOG_PIPELINE_TIMING: "true"␍
JOB_INLINE_PROCESSING: "false"␍
volumes:␍
- ./models/latest:/app/models:ro␍
␍
autoscaler:␍
build:␍
context: .␍
dockerfile: Dockerfile.autoscaler␍
depends_on:␍
redis:␍
condition: service_healthy␍
volumes:␍
- /var/run/docker.sock:/var/run/docker.sock␍
- .:/project:ro␍
working_dir: /project␍
environment:␍
REDIS_URL: redis://redis:6379/0␍
COMPOSE_FILE: /project/docker-compose.yml␍
COMPOSE_SERVICE: worker␍
MIN_WORKERS: 1␍
MAX_WORKERS: 8␍
WORKER_SLOTS: 4␍
POLL_INTERVAL_SEC: 5␍
SCALE_DOWN_IDLE_SEC: 60␍
␍
volumes:␍
minio_data:␍
␍
services:
redis:
image: redis:7-alpine
command: redis-server --maxmemory 1gb --maxmemory-policy allkeys-lru
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 5
nats:
image: nats:2-alpine
command: ["-js", "-m", "8222"]
healthcheck:
test: ["CMD", "wget", "-qO-", "http://localhost:8222/healthz"]
interval: 5s
timeout: 3s
retries: 5
minio:
image: minio/minio:latest
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER: minioadmin
MINIO_ROOT_PASSWORD: minioadmin
volumes:
- minio_data:/data
ports:
- "9002:9001"
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:9000/minio/health/live || exit 1"]
interval: 10s
timeout: 5s
retries: 5
start_period: 10s
api:
build: .
command: ["/app/api"]
ports:
- "8080:8080"
depends_on:
redis:
condition: service_healthy
nats:
condition: service_healthy
minio:
condition: service_started
environment:
REDIS_URL: redis://redis:6379/0
NATS_URL: nats://nats:4222
MINIO_ENDPOINT: minio:9000
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
MINIO_BUCKET: pdf-count
MODEL_PATH: /app/models/model.onnx
INFERENCE_CONFIG: /app/models/inference.json
SYNC_MAX_FILES: 50
JOB_MAX_FILES: 50000
QUEUE_MAX_PENDING: 100000
OPENAPI_PATH: /app/openapi.yaml
OCR_POOL_SIZE: 4
SYNC_PROCESS_CONCURRENCY: 4
JOB_INLINE_PROCESSING: "false"
LOG_PIPELINE_TIMING: "true"
volumes:
- ./models/latest:/app/models:ro
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
interval: 10s
timeout: 5s
retries: 5
worker:
build: .
command: ["/app/worker"]
depends_on:
redis:
condition: service_healthy
nats:
condition: service_healthy
minio:
condition: service_started
environment:
REDIS_URL: redis://redis:6379/0
NATS_URL: nats://nats:4222
MINIO_ENDPOINT: minio:9000
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
MINIO_BUCKET: pdf-count
MODEL_PATH: /app/models/model.onnx
INFERENCE_CONFIG: /app/models/inference.json
WORKER_CONCURRENCY: 4
PDF_PROCESS_TIMEOUT_SEC: 120
OCR_POOL_SIZE: 4
NATS_FETCH_MAX_WAIT_MS: 300
LOG_PIPELINE_TIMING: "true"
JOB_INLINE_PROCESSING: "false"
volumes:
- ./models/latest:/app/models:ro
autoscaler:
build:
context: .
dockerfile: Dockerfile.autoscaler
depends_on:
redis:
condition: service_healthy
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- .:/project:ro
working_dir: /project
environment:
REDIS_URL: redis://redis:6379/0
COMPOSE_FILE: /project/docker-compose.yml
COMPOSE_SERVICE: worker
MIN_WORKERS: 1
MAX_WORKERS: 8
WORKER_SLOTS: 4
POLL_INTERVAL_SEC: 5
SCALE_DOWN_IDLE_SEC: 60
volumes:
minio_data:
@@ -10,15 +10,18 @@ import (
ort "github.com/yalue/onnxruntime_go"
)
const defaultFeatureCount = 14
type InferenceMeta struct {
Threshold float64 `json:"threshold"`
FeatureColumns []string `json:"feature_columns"`
}
type Predictor struct {
session *ort.DynamicAdvancedSession
threshold float64
mu sync.Mutex
session *ort.DynamicAdvancedSession
threshold float64
featureCount int
mu sync.Mutex
}
func NewPredictor(modelPath, metaPath string) (*Predictor, error) {
@@ -41,6 +44,11 @@ func NewPredictor(modelPath, metaPath string) (*Predictor, error) {
meta.Threshold = 0.5
}
featureCount := len(meta.FeatureColumns)
if featureCount == 0 {
featureCount = defaultFeatureCount
}
session, err := ort.NewDynamicAdvancedSession(
modelPath,
[]string{"float_input"},
@@ -51,13 +59,21 @@ func NewPredictor(modelPath, metaPath string) (*Predictor, error) {
return nil, fmt.Errorf("onnx session: %w", err)
}
return &Predictor{session: session, threshold: meta.Threshold}, nil
return &Predictor{
session: session,
threshold: meta.Threshold,
featureCount: featureCount,
}, nil
}
func (p *Predictor) Threshold() float64 {
return p.threshold
}
func (p *Predictor) FeatureCount() int {
return p.featureCount
}
func (p *Predictor) Close() error {
if p.session != nil {
p.session.Destroy()
@@ -65,18 +81,24 @@ func (p *Predictor) Close() error {
return ort.DestroyEnvironment()
}
func (p *Predictor) PredictProba(features [][8]float32) ([]float64, error) {
func (p *Predictor) PredictProba(features [][]float32) ([]float64, error) {
if len(features) == 0 {
return nil, nil
}
n := len(features)
flat := make([]float32, 0, n*8)
for _, row := range features {
flat = append(flat, row[:]...)
flat := make([]float32, 0, n*p.featureCount)
for i, row := range features {
if len(row) != p.featureCount {
return nil, fmt.Errorf(
"fila %d: se esperaban %d features, se recibieron %d (revisa pipeline vs inference.json)",
i, p.featureCount, len(row),
)
}
flat = append(flat, row...)
}
inputShape := ort.NewShape(int64(n), 8)
inputShape := ort.NewShape(int64(n), int64(p.featureCount))
inputTensor, err := ort.NewTensor(inputShape, flat)
if err != nil {
return nil, err
@@ -7,15 +7,40 @@ import (
"unicode"
)
// FeatureCount debe coincidir con models v1.1.0 (services-count-facturas-medicas-ml).
const FeatureCount = 14
var (
codePattern = regexp.MustCompile(`^\d{4,6}$`)
currencyPattern = regexp.MustCompile(`^\d{1,3}(,\d{3})+(\.\d+)?$`)
codePattern = regexp.MustCompile(`^\d{4,6}$`)
serviceCodePattern = regexp.MustCompile(`(?i)^(\d{4,8}|\d+-\d+|[A-Z]{2,}\d{2,})$`)
currencyPattern = regexp.MustCompile(`^\d{1,3}(,\d{3})+(\.\d+)?$`)
headerKeywordPattern = regexp.MustCompile(`(?i)\b(FECHA|VALOR|SUBTOTAL|CUFE|RESOLUCION|CONTRATO|FACTURA|NIT|TELEFONO|TOTAL)\b`)
)
type rowFeatures struct {
Features [8]float32
TextoFila string
Cluster int
Features [FeatureCount]float32
TextoFila string
Cluster int
yNorm float32
// índices usados en el paso de continuación
hasServiceTablePattern int
currencyCount int
}
func isServiceCode(token string) bool {
return serviceCodePattern.MatchString(token)
}
func hasHeaderKeyword(text string) bool {
return headerKeywordPattern.MatchString(text)
}
func firstTokenOf(text string) string {
parts := strings.Fields(text)
if len(parts) == 0 {
return ""
}
return parts[0]
}
func extractRowFeatures(group []indexedBox, pageW, pageH int) rowFeatures {
@@ -54,13 +79,29 @@ func extractRowFeatures(group []indexedBox, pageW, pageH int) rowFeatures {
ySum += b.Y
}
xSpanNorm := 0.0
xSpanNorm := float32(0)
if pageW > 0 {
xSpanNorm = float64(xMax-xMin) / float64(pageW)
xSpanNorm = float32(xMax-xMin) / float32(pageW)
}
yNorm := 0.0
yNorm := float32(0)
if pageH > 0 {
yNorm = float64(ySum/len(group)) / float64(pageH)
yNorm = float32(ySum/len(group)) / float32(pageH)
}
firstTokenXNorm := float32(0)
if pageW > 0 {
firstTokenXNorm = float32(xMin) / float32(pageW)
}
tailStart := 0
if len(texts) >= 3 {
tailStart = len(texts) - 3
}
currencyAtRowEnd := 0
for _, t := range texts[tailStart:] {
if currencyPattern.MatchString(t) {
currencyAtRowEnd = 1
break
}
}
totalChars := len(textoFila)
@@ -83,22 +124,73 @@ func extractRowFeatures(group []indexedBox, pageW, pageH int) rowFeatures {
firstIsCode = 1
}
hasTablePattern := 0
if currencyCount >= 2 && integerCount >= 1 {
hasTablePattern = 1
}
headerKW := 0
if hasHeaderKeyword(textoFila) {
headerKW = 1
}
return rowFeatures{
Features: [8]float32{
Features: [FeatureCount]float32{
float32(firstIsCode),
boolToFloat(isServiceCode(firstToken)),
float32(hasTablePattern),
float32(headerKW),
float32(currencyAtRowEnd),
firstTokenXNorm,
0, // is_likely_continuation — segundo paso
float32(currencyCount),
float32(integerCount),
float32(xSpanNorm),
float32(yNorm),
xSpanNorm,
yNorm,
float32(len(texts)),
float32(special) / float32(totalChars),
float32(digits) / float32(totalChars),
},
TextoFila: textoFila,
Cluster: group[0].Cluster,
TextoFila: textoFila,
Cluster: group[0].Cluster,
yNorm: yNorm,
hasServiceTablePattern: hasTablePattern,
currencyCount: currencyCount,
}
}
func boolToFloat(v bool) float32 {
if v {
return 1
}
return 0
}
func buildFeatureRows(boxes []indexedBox, pageW, pageH int) []rowFeatures {
groups := groupByCluster(boxes)
rows := make([]rowFeatures, 0, len(groups))
for _, group := range groups {
rows = append(rows, extractRowFeatures(group, pageW, pageH))
}
if len(rows) == 0 {
return rows
}
sort.Slice(rows, func(i, j int) bool { return rows[i].yNorm < rows[j].yNorm })
for i := 1; i < len(rows); i++ {
prev := rows[i-1]
cur := &rows[i]
firstToken := firstTokenOf(cur.TextoFila)
if prev.hasServiceTablePattern == 1 &&
cur.currencyCount == 0 &&
!isServiceCode(firstToken) &&
!codePattern.MatchString(firstToken) {
cur.Features[6] = 1
}
}
return rows
}
func isDigits(s string) bool {
if s == "" {
return false
@@ -110,13 +202,3 @@ func isDigits(s string) bool {
}
return true
}
func buildFeatureRows(boxes []indexedBox, pageW, pageH int) []rowFeatures {
groups := groupByCluster(boxes)
order := sortClusters(groups)
rows := make([]rowFeatures, 0, len(groups))
for _, cid := range order {
rows = append(rows, extractRowFeatures(groups[cid], pageW, pageH))
}
return rows
}
@@ -0,0 +1,48 @@
package pipeline
import "testing"
func TestExtractRowFeaturesServiceTablePattern(t *testing.T) {
group := []indexedBox{
{Text: "890701", X: 10, Y: 200, Cluster: 0},
{Text: "CONSULTA", X: 100, Y: 201, Cluster: 0},
{Text: "16,277.00", X: 400, Y: 200, Cluster: 0},
{Text: "1", X: 500, Y: 201, Cluster: 0},
{Text: "16,277.00", X: 600, Y: 200, Cluster: 0},
}
row := extractRowFeatures(group, 1000, 2000)
if row.Features[2] != 1 {
t.Fatalf("has_service_table_pattern = %v, want 1", row.Features[2])
}
if row.Features[4] != 1 {
t.Fatalf("currency_at_row_end = %v, want 1", row.Features[4])
}
}
func TestBuildFeaturesContinuation(t *testing.T) {
boxes := []indexedBox{
{Text: "890701", X: 10, Y: 100, Cluster: 0},
{Text: "CONSULTA", X: 100, Y: 100, Cluster: 0},
{Text: "16,277.00", X: 400, Y: 100, Cluster: 0},
{Text: "1", X: 500, Y: 100, Cluster: 0},
{Text: "16,277.00", X: 600, Y: 100, Cluster: 0},
{Text: "DE", X: 100, Y: 120, Cluster: 1},
{Text: "SEGUIMIENTO", X: 200, Y: 120, Cluster: 1},
}
rows := buildFeatureRows(boxes, 1000, 2000)
if len(rows) != 2 {
t.Fatalf("rows = %d, want 2", len(rows))
}
if rows[0].Features[2] != 1 {
t.Fatal("first row should have service table pattern")
}
if rows[1].Features[6] != 1 {
t.Fatal("second row should be likely continuation")
}
}
func TestFeatureCount(t *testing.T) {
if FeatureCount != 14 {
t.Fatalf("FeatureCount = %d, want 14", FeatureCount)
}
}
@@ -88,9 +88,9 @@ func (p *Processor) ProcessPDF(filename string, pdfData []byte) domain.PDFResult
return result
}
features := make([][8]float32, len(rows))
features := make([][]float32, len(rows))
for i, r := range rows {
features[i] = r.Features
features[i] = r.Features[:]
}
tInfer := time.Now()
@@ -9,8 +9,15 @@ import joblib
from skl2onnx import convert_sklearn
from skl2onnx.common.data_types import FloatTensorType
# Alineado con services-count-facturas-medicas-ml v1.1.0 y pipeline Go.
FEATURE_COLUMNS = [
"first_token_is_code",
"first_token_is_service_code",
"has_service_table_pattern",
"has_header_keyword",
"currency_at_row_end",
"first_token_x_norm",
"is_likely_continuation",
"currency_count",
"integer_count",
"x_span_norm",
@@ -31,7 +38,7 @@ def parse_args() -> argparse.Namespace:
"--joblib",
type=Path,
required=True,
help="Ruta al model.joblib (ej. ../services-count/models/latest/model.joblib)",
help="Ruta al model.joblib (ej. models/v1.1.0/model.joblib)",
)
parser.add_argument(
"--out",
@@ -64,6 +71,21 @@ def main() -> None:
threshold = float(bundle.get("threshold", 0.5))
meta = load_metadata(args.metadata)
bundle_columns = list(bundle.get("feature_columns") or FEATURE_COLUMNS)
if bundle_columns != FEATURE_COLUMNS:
raise SystemExit(
f"feature_columns del joblib no coinciden con el pipeline Go.\n"
f" joblib: {bundle_columns}\n"
f" esperado: {FEATURE_COLUMNS}"
)
n_features = int(getattr(model, "n_features_in_", len(FEATURE_COLUMNS)))
if n_features != len(FEATURE_COLUMNS):
raise SystemExit(
f"El model.joblib tiene {n_features} features pero el pipeline Go produce "
f"{len(FEATURE_COLUMNS)}. Reentrena o usa un joblib compatible."
)
initial_type = [("float_input", FloatTensorType([None, len(FEATURE_COLUMNS)]))]
options = {id(model): {"zipmap": False}}
onnx_model = convert_sklearn(
@@ -84,7 +106,7 @@ def main() -> None:
inference = {
"threshold": threshold,
"feature_columns": FEATURE_COLUMNS,
"pipeline_version": meta.get("pipeline_version", "unknown"),
"pipeline_version": meta.get("pipeline_version", bundle.get("pipeline_version", "unknown")),
"ocr": meta.get("ocr", DEFAULT_OCR),
"dbscan": meta.get("dbscan", DEFAULT_DBSCAN),
}
@@ -93,6 +115,7 @@ def main() -> None:
print(f"ONNX: {onnx_path}")
print(f"Meta: {meta_path}")
print(f"Umbral: {threshold}")
print(f"Features: {len(FEATURE_COLUMNS)}")
if __name__ == "__main__":