fix: Update production API base URL in documentation and configuration files. Correct the URL in the desktop app README, migration documentation, and Vite configuration. Enhance ONNX model feature handling in the processing pipeline to align with the updated feature set, ensuring compatibility with the latest model version. Add tests for feature extraction to validate changes.
This commit is contained in:
@@ -46,7 +46,7 @@ Toda la comunicación pasa por el API Gateway.
|
||||
| Entorno | URL por defecto |
|
||||
|---------|-----------------|
|
||||
| Desarrollo (`npm run dev`) | `http://127.0.0.1:8189` |
|
||||
| Instalador (build producción) | `https://pys1.builder.xyz` |
|
||||
| Instalador (build producción) | `https://pys1.builker.xyz` |
|
||||
El cliente HTTP es [`src/renderer/lib/apiClient.ts`](src/renderer/lib/apiClient.ts).
|
||||
|
||||
### Indexación de carpetas
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/** URL por defecto del API Gateway (dev: local Docker; instalador: pys1.builder.xyz). */
|
||||
/** URL por defecto del API Gateway (dev: local Docker; instalador: pys1.builker.xyz). */
|
||||
export const DEFAULT_API_BASE_URL: string = __DEFAULT_API_BASE_URL__;
|
||||
|
||||
/** Reemplaza localhost por 127.0.0.1 para evitar problemas de IPv6 en Windows/Docker. */
|
||||
|
||||
@@ -7,7 +7,7 @@ const versionMeta = JSON.parse(
|
||||
readFileSync("./src/shared/version.json", "utf8"),
|
||||
) as { releaseDate: string };
|
||||
|
||||
const PRODUCTION_API_BASE_URL = "https://pys1.builder.xyz";
|
||||
const PRODUCTION_API_BASE_URL = "https://pys1.builker.xyz";
|
||||
const DEVELOPMENT_API_BASE_URL = "http://127.0.0.1:8189";
|
||||
|
||||
export default defineConfig(({ mode }) => ({
|
||||
|
||||
@@ -71,9 +71,10 @@ OCR palabra a palabra con coordenadas (X, Y, W, H)
|
||||
▼ DBSCAN 1D sobre coordenada Y
|
||||
Agrupación de palabras en filas candidatas
|
||||
│
|
||||
▼ 8 features por fila
|
||||
first_token_is_code · currency_count · integer_count · x_span_norm
|
||||
y_norm · token_count · special_char_ratio · digit_ratio
|
||||
▼ 14 features por fila (pipeline v1.1.0)
|
||||
first_token_is_code · first_token_is_service_code · has_service_table_pattern · has_header_keyword
|
||||
currency_at_row_end · first_token_x_norm · is_likely_continuation · currency_count · integer_count
|
||||
x_span_norm · y_norm · token_count · special_char_ratio · digit_ratio
|
||||
│
|
||||
▼ ONNX Runtime (Random Forest)
|
||||
P(servicio) por fila → filtrar por threshold (inference.json)
|
||||
@@ -103,6 +104,36 @@ pip install -r ../requirements-export.txt
|
||||
python export_onnx.py --joblib /ruta/model.joblib --out ../models/latest
|
||||
```
|
||||
|
||||
**Despliegue:** los archivos no van en la imagen Docker; se montan desde `models/latest/`. Tras copiarlos al servidor:
|
||||
|
||||
```bash
|
||||
chmod 644 models/latest/model.onnx models/latest/inference.json
|
||||
docker compose restart count-api
|
||||
docker logs ipsextractor-count-api 2>&1 | grep -E "modelo ONNX|WARN: modelo"
|
||||
# Debe aparecer: modelo ONNX cargado: /app/models/model.onnx
|
||||
```
|
||||
|
||||
Si el log sigue mostrando `WARN: modelo ML no disponible`, verifica dentro del contenedor:
|
||||
|
||||
```bash
|
||||
docker exec ipsextractor-count-api ls -la /app/models/
|
||||
```
|
||||
|
||||
Sin modelo cargado, **todas las filas OCR salen con probability=1.0 (100%)** — modo heurístico.
|
||||
|
||||
### Error: `requests feature 13 but input tensor has 8 features`
|
||||
|
||||
El `model.onnx` desplegado **no coincide** con el pipeline Go. El modelo v1.1.0 usa **14 features** (ver `features.go` y `export_onnx.py`). Si el ONNX fue exportado con otro esquema verás errores como `requests feature 13 but input tensor has 8 features`.
|
||||
|
||||
**Exportar desde el joblib v1.1.0:**
|
||||
|
||||
```bash
|
||||
python export_onnx.py \
|
||||
--joblib /ruta/services-count-facturas-medicas-ml/models/v1.1.0/model.joblib \
|
||||
--metadata /ruta/services-count-facturas-medicas-ml/models/v1.1.0/model_metadata.json \
|
||||
--out ../models/latest
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Variables de entorno
|
||||
|
||||
@@ -1,125 +1,199 @@
|
||||
services:␍
|
||||
redis:␍
|
||||
image: redis:7-alpine␍
|
||||
command: redis-server --maxmemory 1gb --maxmemory-policy allkeys-lru␍
|
||||
healthcheck:␍
|
||||
test: ["CMD", "redis-cli", "ping"]␍
|
||||
interval: 5s␍
|
||||
timeout: 3s␍
|
||||
retries: 5␍
|
||||
␍
|
||||
nats:␍
|
||||
image: nats:2-alpine␍
|
||||
command: ["-js", "-m", "8222"]␍
|
||||
healthcheck:␍
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:8222/healthz"]␍
|
||||
interval: 5s␍
|
||||
timeout: 3s␍
|
||||
retries: 5␍
|
||||
␍
|
||||
minio:␍
|
||||
image: minio/minio:latest␍
|
||||
command: server /data --console-address ":9001"␍
|
||||
environment:␍
|
||||
MINIO_ROOT_USER: minioadmin␍
|
||||
MINIO_ROOT_PASSWORD: minioadmin␍
|
||||
volumes:␍
|
||||
- minio_data:/data␍
|
||||
ports:␍
|
||||
- "9002:9001"␍
|
||||
healthcheck:␍
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:9000/minio/health/live || exit 1"]␍
|
||||
interval: 10s␍
|
||||
timeout: 5s␍
|
||||
retries: 5␍
|
||||
start_period: 10s␍
|
||||
␍
|
||||
api:␍
|
||||
build: .␍
|
||||
command: ["/app/api"]␍
|
||||
ports:␍
|
||||
- "8080:8080"␍
|
||||
depends_on:␍
|
||||
redis:␍
|
||||
condition: service_healthy␍
|
||||
nats:␍
|
||||
condition: service_healthy␍
|
||||
minio:␍
|
||||
condition: service_started␍
|
||||
environment:␍
|
||||
REDIS_URL: redis://redis:6379/0␍
|
||||
NATS_URL: nats://nats:4222␍
|
||||
MINIO_ENDPOINT: minio:9000␍
|
||||
MINIO_ACCESS_KEY: minioadmin␍
|
||||
MINIO_SECRET_KEY: minioadmin␍
|
||||
MINIO_BUCKET: pdf-count␍
|
||||
MODEL_PATH: /app/models/model.onnx␍
|
||||
INFERENCE_CONFIG: /app/models/inference.json␍
|
||||
SYNC_MAX_FILES: 50␍
|
||||
JOB_MAX_FILES: 50000␍
|
||||
QUEUE_MAX_PENDING: 100000␍
|
||||
OPENAPI_PATH: /app/openapi.yaml␍
|
||||
OCR_POOL_SIZE: 4␍
|
||||
SYNC_PROCESS_CONCURRENCY: 4␍
|
||||
JOB_INLINE_PROCESSING: "false"␍
|
||||
LOG_PIPELINE_TIMING: "true"␍
|
||||
volumes:␍
|
||||
- ./models/latest:/app/models:ro␍
|
||||
healthcheck:␍
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]␍
|
||||
interval: 10s␍
|
||||
timeout: 5s␍
|
||||
retries: 5␍
|
||||
␍
|
||||
worker:␍
|
||||
build: .␍
|
||||
command: ["/app/worker"]␍
|
||||
depends_on:␍
|
||||
redis:␍
|
||||
condition: service_healthy␍
|
||||
nats:␍
|
||||
condition: service_healthy␍
|
||||
minio:␍
|
||||
condition: service_started␍
|
||||
environment:␍
|
||||
REDIS_URL: redis://redis:6379/0␍
|
||||
NATS_URL: nats://nats:4222␍
|
||||
MINIO_ENDPOINT: minio:9000␍
|
||||
MINIO_ACCESS_KEY: minioadmin␍
|
||||
MINIO_SECRET_KEY: minioadmin␍
|
||||
MINIO_BUCKET: pdf-count␍
|
||||
MODEL_PATH: /app/models/model.onnx␍
|
||||
INFERENCE_CONFIG: /app/models/inference.json␍
|
||||
WORKER_CONCURRENCY: 4␍
|
||||
PDF_PROCESS_TIMEOUT_SEC: 120␍
|
||||
OCR_POOL_SIZE: 4␍
|
||||
NATS_FETCH_MAX_WAIT_MS: 300␍
|
||||
LOG_PIPELINE_TIMING: "true"␍
|
||||
JOB_INLINE_PROCESSING: "false"␍
|
||||
volumes:␍
|
||||
- ./models/latest:/app/models:ro␍
|
||||
␍
|
||||
autoscaler:␍
|
||||
build:␍
|
||||
context: .␍
|
||||
dockerfile: Dockerfile.autoscaler␍
|
||||
depends_on:␍
|
||||
redis:␍
|
||||
condition: service_healthy␍
|
||||
volumes:␍
|
||||
- /var/run/docker.sock:/var/run/docker.sock␍
|
||||
- .:/project:ro␍
|
||||
working_dir: /project␍
|
||||
environment:␍
|
||||
REDIS_URL: redis://redis:6379/0␍
|
||||
COMPOSE_FILE: /project/docker-compose.yml␍
|
||||
COMPOSE_SERVICE: worker␍
|
||||
MIN_WORKERS: 1␍
|
||||
MAX_WORKERS: 8␍
|
||||
WORKER_SLOTS: 4␍
|
||||
POLL_INTERVAL_SEC: 5␍
|
||||
SCALE_DOWN_IDLE_SEC: 60␍
|
||||
␍
|
||||
volumes:␍
|
||||
minio_data:␍
|
||||
␍
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
command: redis-server --maxmemory 1gb --maxmemory-policy allkeys-lru
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
nats:
|
||||
image: nats:2-alpine
|
||||
command: ["-js", "-m", "8222"]
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:8222/healthz"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
ports:
|
||||
- "9002:9001"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:9000/minio/health/live || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 10s
|
||||
api:
|
||||
build: .
|
||||
command: ["/app/api"]
|
||||
ports:
|
||||
- "8080:8080"
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
nats:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_started
|
||||
environment:
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
|
||||
NATS_URL: nats://nats:4222
|
||||
|
||||
MINIO_ENDPOINT: minio:9000
|
||||
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
|
||||
MINIO_BUCKET: pdf-count
|
||||
|
||||
MODEL_PATH: /app/models/model.onnx
|
||||
|
||||
INFERENCE_CONFIG: /app/models/inference.json
|
||||
|
||||
SYNC_MAX_FILES: 50
|
||||
|
||||
JOB_MAX_FILES: 50000
|
||||
|
||||
QUEUE_MAX_PENDING: 100000
|
||||
|
||||
OPENAPI_PATH: /app/openapi.yaml
|
||||
|
||||
OCR_POOL_SIZE: 4
|
||||
|
||||
SYNC_PROCESS_CONCURRENCY: 4
|
||||
|
||||
JOB_INLINE_PROCESSING: "false"
|
||||
|
||||
LOG_PIPELINE_TIMING: "true"
|
||||
|
||||
volumes:
|
||||
|
||||
- ./models/latest:/app/models:ro
|
||||
|
||||
healthcheck:
|
||||
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
|
||||
|
||||
interval: 10s
|
||||
|
||||
timeout: 5s
|
||||
|
||||
retries: 5
|
||||
|
||||
|
||||
|
||||
worker:
|
||||
|
||||
build: .
|
||||
|
||||
command: ["/app/worker"]
|
||||
|
||||
depends_on:
|
||||
|
||||
redis:
|
||||
|
||||
condition: service_healthy
|
||||
|
||||
nats:
|
||||
|
||||
condition: service_healthy
|
||||
|
||||
minio:
|
||||
|
||||
condition: service_started
|
||||
|
||||
environment:
|
||||
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
|
||||
NATS_URL: nats://nats:4222
|
||||
|
||||
MINIO_ENDPOINT: minio:9000
|
||||
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
|
||||
MINIO_BUCKET: pdf-count
|
||||
|
||||
MODEL_PATH: /app/models/model.onnx
|
||||
|
||||
INFERENCE_CONFIG: /app/models/inference.json
|
||||
|
||||
WORKER_CONCURRENCY: 4
|
||||
|
||||
PDF_PROCESS_TIMEOUT_SEC: 120
|
||||
|
||||
OCR_POOL_SIZE: 4
|
||||
|
||||
NATS_FETCH_MAX_WAIT_MS: 300
|
||||
|
||||
LOG_PIPELINE_TIMING: "true"
|
||||
|
||||
JOB_INLINE_PROCESSING: "false"
|
||||
|
||||
volumes:
|
||||
|
||||
- ./models/latest:/app/models:ro
|
||||
|
||||
|
||||
|
||||
autoscaler:
|
||||
|
||||
build:
|
||||
|
||||
context: .
|
||||
|
||||
dockerfile: Dockerfile.autoscaler
|
||||
|
||||
depends_on:
|
||||
|
||||
redis:
|
||||
|
||||
condition: service_healthy
|
||||
|
||||
volumes:
|
||||
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
|
||||
- .:/project:ro
|
||||
|
||||
working_dir: /project
|
||||
|
||||
environment:
|
||||
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
|
||||
COMPOSE_FILE: /project/docker-compose.yml
|
||||
|
||||
COMPOSE_SERVICE: worker
|
||||
|
||||
MIN_WORKERS: 1
|
||||
|
||||
MAX_WORKERS: 8
|
||||
|
||||
WORKER_SLOTS: 4
|
||||
|
||||
POLL_INTERVAL_SEC: 5
|
||||
|
||||
SCALE_DOWN_IDLE_SEC: 60
|
||||
|
||||
|
||||
|
||||
volumes:
|
||||
|
||||
minio_data:
|
||||
|
||||
|
||||
|
||||
@@ -10,15 +10,18 @@ import (
|
||||
ort "github.com/yalue/onnxruntime_go"
|
||||
)
|
||||
|
||||
const defaultFeatureCount = 14
|
||||
|
||||
type InferenceMeta struct {
|
||||
Threshold float64 `json:"threshold"`
|
||||
FeatureColumns []string `json:"feature_columns"`
|
||||
}
|
||||
|
||||
type Predictor struct {
|
||||
session *ort.DynamicAdvancedSession
|
||||
threshold float64
|
||||
mu sync.Mutex
|
||||
session *ort.DynamicAdvancedSession
|
||||
threshold float64
|
||||
featureCount int
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func NewPredictor(modelPath, metaPath string) (*Predictor, error) {
|
||||
@@ -41,6 +44,11 @@ func NewPredictor(modelPath, metaPath string) (*Predictor, error) {
|
||||
meta.Threshold = 0.5
|
||||
}
|
||||
|
||||
featureCount := len(meta.FeatureColumns)
|
||||
if featureCount == 0 {
|
||||
featureCount = defaultFeatureCount
|
||||
}
|
||||
|
||||
session, err := ort.NewDynamicAdvancedSession(
|
||||
modelPath,
|
||||
[]string{"float_input"},
|
||||
@@ -51,13 +59,21 @@ func NewPredictor(modelPath, metaPath string) (*Predictor, error) {
|
||||
return nil, fmt.Errorf("onnx session: %w", err)
|
||||
}
|
||||
|
||||
return &Predictor{session: session, threshold: meta.Threshold}, nil
|
||||
return &Predictor{
|
||||
session: session,
|
||||
threshold: meta.Threshold,
|
||||
featureCount: featureCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *Predictor) Threshold() float64 {
|
||||
return p.threshold
|
||||
}
|
||||
|
||||
func (p *Predictor) FeatureCount() int {
|
||||
return p.featureCount
|
||||
}
|
||||
|
||||
func (p *Predictor) Close() error {
|
||||
if p.session != nil {
|
||||
p.session.Destroy()
|
||||
@@ -65,18 +81,24 @@ func (p *Predictor) Close() error {
|
||||
return ort.DestroyEnvironment()
|
||||
}
|
||||
|
||||
func (p *Predictor) PredictProba(features [][8]float32) ([]float64, error) {
|
||||
func (p *Predictor) PredictProba(features [][]float32) ([]float64, error) {
|
||||
if len(features) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
n := len(features)
|
||||
flat := make([]float32, 0, n*8)
|
||||
for _, row := range features {
|
||||
flat = append(flat, row[:]...)
|
||||
flat := make([]float32, 0, n*p.featureCount)
|
||||
for i, row := range features {
|
||||
if len(row) != p.featureCount {
|
||||
return nil, fmt.Errorf(
|
||||
"fila %d: se esperaban %d features, se recibieron %d (revisa pipeline vs inference.json)",
|
||||
i, p.featureCount, len(row),
|
||||
)
|
||||
}
|
||||
flat = append(flat, row...)
|
||||
}
|
||||
|
||||
inputShape := ort.NewShape(int64(n), 8)
|
||||
inputShape := ort.NewShape(int64(n), int64(p.featureCount))
|
||||
inputTensor, err := ort.NewTensor(inputShape, flat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -7,15 +7,40 @@ import (
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// FeatureCount debe coincidir con models v1.1.0 (services-count-facturas-medicas-ml).
|
||||
const FeatureCount = 14
|
||||
|
||||
var (
|
||||
codePattern = regexp.MustCompile(`^\d{4,6}$`)
|
||||
currencyPattern = regexp.MustCompile(`^\d{1,3}(,\d{3})+(\.\d+)?$`)
|
||||
codePattern = regexp.MustCompile(`^\d{4,6}$`)
|
||||
serviceCodePattern = regexp.MustCompile(`(?i)^(\d{4,8}|\d+-\d+|[A-Z]{2,}\d{2,})$`)
|
||||
currencyPattern = regexp.MustCompile(`^\d{1,3}(,\d{3})+(\.\d+)?$`)
|
||||
headerKeywordPattern = regexp.MustCompile(`(?i)\b(FECHA|VALOR|SUBTOTAL|CUFE|RESOLUCION|CONTRATO|FACTURA|NIT|TELEFONO|TOTAL)\b`)
|
||||
)
|
||||
|
||||
type rowFeatures struct {
|
||||
Features [8]float32
|
||||
TextoFila string
|
||||
Cluster int
|
||||
Features [FeatureCount]float32
|
||||
TextoFila string
|
||||
Cluster int
|
||||
yNorm float32
|
||||
// índices usados en el paso de continuación
|
||||
hasServiceTablePattern int
|
||||
currencyCount int
|
||||
}
|
||||
|
||||
func isServiceCode(token string) bool {
|
||||
return serviceCodePattern.MatchString(token)
|
||||
}
|
||||
|
||||
func hasHeaderKeyword(text string) bool {
|
||||
return headerKeywordPattern.MatchString(text)
|
||||
}
|
||||
|
||||
func firstTokenOf(text string) string {
|
||||
parts := strings.Fields(text)
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
return parts[0]
|
||||
}
|
||||
|
||||
func extractRowFeatures(group []indexedBox, pageW, pageH int) rowFeatures {
|
||||
@@ -54,13 +79,29 @@ func extractRowFeatures(group []indexedBox, pageW, pageH int) rowFeatures {
|
||||
ySum += b.Y
|
||||
}
|
||||
|
||||
xSpanNorm := 0.0
|
||||
xSpanNorm := float32(0)
|
||||
if pageW > 0 {
|
||||
xSpanNorm = float64(xMax-xMin) / float64(pageW)
|
||||
xSpanNorm = float32(xMax-xMin) / float32(pageW)
|
||||
}
|
||||
yNorm := 0.0
|
||||
yNorm := float32(0)
|
||||
if pageH > 0 {
|
||||
yNorm = float64(ySum/len(group)) / float64(pageH)
|
||||
yNorm = float32(ySum/len(group)) / float32(pageH)
|
||||
}
|
||||
firstTokenXNorm := float32(0)
|
||||
if pageW > 0 {
|
||||
firstTokenXNorm = float32(xMin) / float32(pageW)
|
||||
}
|
||||
|
||||
tailStart := 0
|
||||
if len(texts) >= 3 {
|
||||
tailStart = len(texts) - 3
|
||||
}
|
||||
currencyAtRowEnd := 0
|
||||
for _, t := range texts[tailStart:] {
|
||||
if currencyPattern.MatchString(t) {
|
||||
currencyAtRowEnd = 1
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
totalChars := len(textoFila)
|
||||
@@ -83,22 +124,73 @@ func extractRowFeatures(group []indexedBox, pageW, pageH int) rowFeatures {
|
||||
firstIsCode = 1
|
||||
}
|
||||
|
||||
hasTablePattern := 0
|
||||
if currencyCount >= 2 && integerCount >= 1 {
|
||||
hasTablePattern = 1
|
||||
}
|
||||
headerKW := 0
|
||||
if hasHeaderKeyword(textoFila) {
|
||||
headerKW = 1
|
||||
}
|
||||
|
||||
return rowFeatures{
|
||||
Features: [8]float32{
|
||||
Features: [FeatureCount]float32{
|
||||
float32(firstIsCode),
|
||||
boolToFloat(isServiceCode(firstToken)),
|
||||
float32(hasTablePattern),
|
||||
float32(headerKW),
|
||||
float32(currencyAtRowEnd),
|
||||
firstTokenXNorm,
|
||||
0, // is_likely_continuation — segundo paso
|
||||
float32(currencyCount),
|
||||
float32(integerCount),
|
||||
float32(xSpanNorm),
|
||||
float32(yNorm),
|
||||
xSpanNorm,
|
||||
yNorm,
|
||||
float32(len(texts)),
|
||||
float32(special) / float32(totalChars),
|
||||
float32(digits) / float32(totalChars),
|
||||
},
|
||||
TextoFila: textoFila,
|
||||
Cluster: group[0].Cluster,
|
||||
TextoFila: textoFila,
|
||||
Cluster: group[0].Cluster,
|
||||
yNorm: yNorm,
|
||||
hasServiceTablePattern: hasTablePattern,
|
||||
currencyCount: currencyCount,
|
||||
}
|
||||
}
|
||||
|
||||
func boolToFloat(v bool) float32 {
|
||||
if v {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func buildFeatureRows(boxes []indexedBox, pageW, pageH int) []rowFeatures {
|
||||
groups := groupByCluster(boxes)
|
||||
rows := make([]rowFeatures, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
rows = append(rows, extractRowFeatures(group, pageW, pageH))
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return rows
|
||||
}
|
||||
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].yNorm < rows[j].yNorm })
|
||||
|
||||
for i := 1; i < len(rows); i++ {
|
||||
prev := rows[i-1]
|
||||
cur := &rows[i]
|
||||
firstToken := firstTokenOf(cur.TextoFila)
|
||||
if prev.hasServiceTablePattern == 1 &&
|
||||
cur.currencyCount == 0 &&
|
||||
!isServiceCode(firstToken) &&
|
||||
!codePattern.MatchString(firstToken) {
|
||||
cur.Features[6] = 1
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func isDigits(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
@@ -110,13 +202,3 @@ func isDigits(s string) bool {
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func buildFeatureRows(boxes []indexedBox, pageW, pageH int) []rowFeatures {
|
||||
groups := groupByCluster(boxes)
|
||||
order := sortClusters(groups)
|
||||
rows := make([]rowFeatures, 0, len(groups))
|
||||
for _, cid := range order {
|
||||
rows = append(rows, extractRowFeatures(groups[cid], pageW, pageH))
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package pipeline
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestExtractRowFeaturesServiceTablePattern(t *testing.T) {
|
||||
group := []indexedBox{
|
||||
{Text: "890701", X: 10, Y: 200, Cluster: 0},
|
||||
{Text: "CONSULTA", X: 100, Y: 201, Cluster: 0},
|
||||
{Text: "16,277.00", X: 400, Y: 200, Cluster: 0},
|
||||
{Text: "1", X: 500, Y: 201, Cluster: 0},
|
||||
{Text: "16,277.00", X: 600, Y: 200, Cluster: 0},
|
||||
}
|
||||
row := extractRowFeatures(group, 1000, 2000)
|
||||
if row.Features[2] != 1 {
|
||||
t.Fatalf("has_service_table_pattern = %v, want 1", row.Features[2])
|
||||
}
|
||||
if row.Features[4] != 1 {
|
||||
t.Fatalf("currency_at_row_end = %v, want 1", row.Features[4])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFeaturesContinuation(t *testing.T) {
|
||||
boxes := []indexedBox{
|
||||
{Text: "890701", X: 10, Y: 100, Cluster: 0},
|
||||
{Text: "CONSULTA", X: 100, Y: 100, Cluster: 0},
|
||||
{Text: "16,277.00", X: 400, Y: 100, Cluster: 0},
|
||||
{Text: "1", X: 500, Y: 100, Cluster: 0},
|
||||
{Text: "16,277.00", X: 600, Y: 100, Cluster: 0},
|
||||
{Text: "DE", X: 100, Y: 120, Cluster: 1},
|
||||
{Text: "SEGUIMIENTO", X: 200, Y: 120, Cluster: 1},
|
||||
}
|
||||
rows := buildFeatureRows(boxes, 1000, 2000)
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("rows = %d, want 2", len(rows))
|
||||
}
|
||||
if rows[0].Features[2] != 1 {
|
||||
t.Fatal("first row should have service table pattern")
|
||||
}
|
||||
if rows[1].Features[6] != 1 {
|
||||
t.Fatal("second row should be likely continuation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeatureCount(t *testing.T) {
|
||||
if FeatureCount != 14 {
|
||||
t.Fatalf("FeatureCount = %d, want 14", FeatureCount)
|
||||
}
|
||||
}
|
||||
@@ -88,9 +88,9 @@ func (p *Processor) ProcessPDF(filename string, pdfData []byte) domain.PDFResult
|
||||
return result
|
||||
}
|
||||
|
||||
features := make([][8]float32, len(rows))
|
||||
features := make([][]float32, len(rows))
|
||||
for i, r := range rows {
|
||||
features[i] = r.Features
|
||||
features[i] = r.Features[:]
|
||||
}
|
||||
|
||||
tInfer := time.Now()
|
||||
|
||||
@@ -9,8 +9,15 @@ import joblib
|
||||
from skl2onnx import convert_sklearn
|
||||
from skl2onnx.common.data_types import FloatTensorType
|
||||
|
||||
# Alineado con services-count-facturas-medicas-ml v1.1.0 y pipeline Go.
|
||||
FEATURE_COLUMNS = [
|
||||
"first_token_is_code",
|
||||
"first_token_is_service_code",
|
||||
"has_service_table_pattern",
|
||||
"has_header_keyword",
|
||||
"currency_at_row_end",
|
||||
"first_token_x_norm",
|
||||
"is_likely_continuation",
|
||||
"currency_count",
|
||||
"integer_count",
|
||||
"x_span_norm",
|
||||
@@ -31,7 +38,7 @@ def parse_args() -> argparse.Namespace:
|
||||
"--joblib",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Ruta al model.joblib (ej. ../services-count/models/latest/model.joblib)",
|
||||
help="Ruta al model.joblib (ej. models/v1.1.0/model.joblib)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--out",
|
||||
@@ -64,6 +71,21 @@ def main() -> None:
|
||||
threshold = float(bundle.get("threshold", 0.5))
|
||||
meta = load_metadata(args.metadata)
|
||||
|
||||
bundle_columns = list(bundle.get("feature_columns") or FEATURE_COLUMNS)
|
||||
if bundle_columns != FEATURE_COLUMNS:
|
||||
raise SystemExit(
|
||||
f"feature_columns del joblib no coinciden con el pipeline Go.\n"
|
||||
f" joblib: {bundle_columns}\n"
|
||||
f" esperado: {FEATURE_COLUMNS}"
|
||||
)
|
||||
|
||||
n_features = int(getattr(model, "n_features_in_", len(FEATURE_COLUMNS)))
|
||||
if n_features != len(FEATURE_COLUMNS):
|
||||
raise SystemExit(
|
||||
f"El model.joblib tiene {n_features} features pero el pipeline Go produce "
|
||||
f"{len(FEATURE_COLUMNS)}. Reentrena o usa un joblib compatible."
|
||||
)
|
||||
|
||||
initial_type = [("float_input", FloatTensorType([None, len(FEATURE_COLUMNS)]))]
|
||||
options = {id(model): {"zipmap": False}}
|
||||
onnx_model = convert_sklearn(
|
||||
@@ -84,7 +106,7 @@ def main() -> None:
|
||||
inference = {
|
||||
"threshold": threshold,
|
||||
"feature_columns": FEATURE_COLUMNS,
|
||||
"pipeline_version": meta.get("pipeline_version", "unknown"),
|
||||
"pipeline_version": meta.get("pipeline_version", bundle.get("pipeline_version", "unknown")),
|
||||
"ocr": meta.get("ocr", DEFAULT_OCR),
|
||||
"dbscan": meta.get("dbscan", DEFAULT_DBSCAN),
|
||||
}
|
||||
@@ -93,6 +115,7 @@ def main() -> None:
|
||||
print(f"ONNX: {onnx_path}")
|
||||
print(f"Meta: {meta_path}")
|
||||
print(f"Umbral: {threshold}")
|
||||
print(f"Features: {len(FEATURE_COLUMNS)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user