fix: Update production API base URL in documentation and configuration files. Correct the URL in the desktop app README, migration documentation, and Vite configuration. Enhance ONNX model feature handling in the processing pipeline to align with the updated feature set, ensuring compatibility with the latest model version. Add tests for feature extraction to validate changes.

2026-07-09 03:13:14 -05:00
parent 328e5e0636
commit ff9abfc753
11 changed files with 449 additions and 169 deletions
@@ -10,15 +10,18 @@ import (
 	ort "github.com/yalue/onnxruntime_go"
 )

+const defaultFeatureCount = 14
+
 type InferenceMeta struct {
 	Threshold      float64  `json:"threshold"`
 	FeatureColumns []string `json:"feature_columns"`
 }

 type Predictor struct {
-	session   *ort.DynamicAdvancedSession
-	threshold float64
-	mu        sync.Mutex
+	session      *ort.DynamicAdvancedSession
+	threshold    float64
+	featureCount int
+	mu           sync.Mutex
 }

 func NewPredictor(modelPath, metaPath string) (*Predictor, error) {
@@ -41,6 +44,11 @@ func NewPredictor(modelPath, metaPath string) (*Predictor, error) {
 		meta.Threshold = 0.5
 	}

+	featureCount := len(meta.FeatureColumns)
+	if featureCount == 0 {
+		featureCount = defaultFeatureCount
+	}
+
 	session, err := ort.NewDynamicAdvancedSession(
 		modelPath,
 		[]string{"float_input"},
@@ -51,13 +59,21 @@ func NewPredictor(modelPath, metaPath string) (*Predictor, error) {
 		return nil, fmt.Errorf("onnx session: %w", err)
 	}

-	return &Predictor{session: session, threshold: meta.Threshold}, nil
+	return &Predictor{
+		session:      session,
+		threshold:    meta.Threshold,
+		featureCount: featureCount,
+	}, nil
 }

 func (p *Predictor) Threshold() float64 {
 	return p.threshold
 }

+func (p *Predictor) FeatureCount() int {
+	return p.featureCount
+}
+
 func (p *Predictor) Close() error {
 	if p.session != nil {
 		p.session.Destroy()
@@ -65,18 +81,24 @@ func (p *Predictor) Close() error {
 	return ort.DestroyEnvironment()
 }

-func (p *Predictor) PredictProba(features [][8]float32) ([]float64, error) {
+func (p *Predictor) PredictProba(features [][]float32) ([]float64, error) {
 	if len(features) == 0 {
 		return nil, nil
 	}

 	n := len(features)
-	flat := make([]float32, 0, n*8)
-	for _, row := range features {
-		flat = append(flat, row[:]...)
+	flat := make([]float32, 0, n*p.featureCount)
+	for i, row := range features {
+		if len(row) != p.featureCount {
+			return nil, fmt.Errorf(
+				"fila %d: se esperaban %d features, se recibieron %d (revisa pipeline vs inference.json)",
+				i, p.featureCount, len(row),
+			)
+		}
+		flat = append(flat, row...)
 	}

-	inputShape := ort.NewShape(int64(n), 8)
+	inputShape := ort.NewShape(int64(n), int64(p.featureCount))
 	inputTensor, err := ort.NewTensor(inputShape, flat)
 	if err != nil {
 		return nil, err
@@ -7,15 +7,40 @@ import (
 	"unicode"
 )

+// FeatureCount debe coincidir con models v1.1.0 (services-count-facturas-medicas-ml).
+const FeatureCount = 14
+
 var (
-	codePattern     = regexp.MustCompile(`^\d{4,6}$`)
-	currencyPattern = regexp.MustCompile(`^\d{1,3}(,\d{3})+(\.\d+)?$`)
+	codePattern         = regexp.MustCompile(`^\d{4,6}$`)
+	serviceCodePattern  = regexp.MustCompile(`(?i)^(\d{4,8}|\d+-\d+|[A-Z]{2,}\d{2,})$`)
+	currencyPattern     = regexp.MustCompile(`^\d{1,3}(,\d{3})+(\.\d+)?$`)
+	headerKeywordPattern = regexp.MustCompile(`(?i)\b(FECHA|VALOR|SUBTOTAL|CUFE|RESOLUCION|CONTRATO|FACTURA|NIT|TELEFONO|TOTAL)\b`)
 )

 type rowFeatures struct {
-	Features   [8]float32
-	TextoFila  string
-	Cluster    int
+	Features  [FeatureCount]float32
+	TextoFila string
+	Cluster   int
+	yNorm     float32
+	// índices usados en el paso de continuación
+	hasServiceTablePattern int
+	currencyCount          int
+}
+
+func isServiceCode(token string) bool {
+	return serviceCodePattern.MatchString(token)
+}
+
+func hasHeaderKeyword(text string) bool {
+	return headerKeywordPattern.MatchString(text)
+}
+
+func firstTokenOf(text string) string {
+	parts := strings.Fields(text)
+	if len(parts) == 0 {
+		return ""
+	}
+	return parts[0]
 }

 func extractRowFeatures(group []indexedBox, pageW, pageH int) rowFeatures {
@@ -54,13 +79,29 @@ func extractRowFeatures(group []indexedBox, pageW, pageH int) rowFeatures {
 		ySum += b.Y
 	}

-	xSpanNorm := 0.0
+	xSpanNorm := float32(0)
 	if pageW > 0 {
-		xSpanNorm = float64(xMax-xMin) / float64(pageW)
+		xSpanNorm = float32(xMax-xMin) / float32(pageW)
 	}
-	yNorm := 0.0
+	yNorm := float32(0)
 	if pageH > 0 {
-		yNorm = float64(ySum/len(group)) / float64(pageH)
+		yNorm = float32(ySum/len(group)) / float32(pageH)
+	}
+	firstTokenXNorm := float32(0)
+	if pageW > 0 {
+		firstTokenXNorm = float32(xMin) / float32(pageW)
+	}
+
+	tailStart := 0
+	if len(texts) >= 3 {
+		tailStart = len(texts) - 3
+	}
+	currencyAtRowEnd := 0
+	for _, t := range texts[tailStart:] {
+		if currencyPattern.MatchString(t) {
+			currencyAtRowEnd = 1
+			break
+		}
 	}

 	totalChars := len(textoFila)
@@ -83,22 +124,73 @@ func extractRowFeatures(group []indexedBox, pageW, pageH int) rowFeatures {
 		firstIsCode = 1
 	}

+	hasTablePattern := 0
+	if currencyCount >= 2 && integerCount >= 1 {
+		hasTablePattern = 1
+	}
+	headerKW := 0
+	if hasHeaderKeyword(textoFila) {
+		headerKW = 1
+	}
+
 	return rowFeatures{
-		Features: [8]float32{
+		Features: [FeatureCount]float32{
 			float32(firstIsCode),
+			boolToFloat(isServiceCode(firstToken)),
+			float32(hasTablePattern),
+			float32(headerKW),
+			float32(currencyAtRowEnd),
+			firstTokenXNorm,
+			0, // is_likely_continuation — segundo paso
 			float32(currencyCount),
 			float32(integerCount),
-			float32(xSpanNorm),
-			float32(yNorm),
+			xSpanNorm,
+			yNorm,
 			float32(len(texts)),
 			float32(special) / float32(totalChars),
 			float32(digits) / float32(totalChars),
 		},
-		TextoFila: textoFila,
-		Cluster:   group[0].Cluster,
+		TextoFila:              textoFila,
+		Cluster:                group[0].Cluster,
+		yNorm:                  yNorm,
+		hasServiceTablePattern: hasTablePattern,
+		currencyCount:          currencyCount,
 	}
 }

+func boolToFloat(v bool) float32 {
+	if v {
+		return 1
+	}
+	return 0
+}
+
+func buildFeatureRows(boxes []indexedBox, pageW, pageH int) []rowFeatures {
+	groups := groupByCluster(boxes)
+	rows := make([]rowFeatures, 0, len(groups))
+	for _, group := range groups {
+		rows = append(rows, extractRowFeatures(group, pageW, pageH))
+	}
+	if len(rows) == 0 {
+		return rows
+	}
+
+	sort.Slice(rows, func(i, j int) bool { return rows[i].yNorm < rows[j].yNorm })
+
+	for i := 1; i < len(rows); i++ {
+		prev := rows[i-1]
+		cur := &rows[i]
+		firstToken := firstTokenOf(cur.TextoFila)
+		if prev.hasServiceTablePattern == 1 &&
+			cur.currencyCount == 0 &&
+			!isServiceCode(firstToken) &&
+			!codePattern.MatchString(firstToken) {
+			cur.Features[6] = 1
+		}
+	}
+	return rows
+}
+
 func isDigits(s string) bool {
 	if s == "" {
 		return false
@@ -110,13 +202,3 @@ func isDigits(s string) bool {
 	}
 	return true
 }
-
-func buildFeatureRows(boxes []indexedBox, pageW, pageH int) []rowFeatures {
-	groups := groupByCluster(boxes)
-	order := sortClusters(groups)
-	rows := make([]rowFeatures, 0, len(groups))
-	for _, cid := range order {
-		rows = append(rows, extractRowFeatures(groups[cid], pageW, pageH))
-	}
-	return rows
-}
@@ -0,0 +1,48 @@
+package pipeline
+
+import "testing"
+
+func TestExtractRowFeaturesServiceTablePattern(t *testing.T) {
+	group := []indexedBox{
+		{Text: "890701", X: 10, Y: 200, Cluster: 0},
+		{Text: "CONSULTA", X: 100, Y: 201, Cluster: 0},
+		{Text: "16,277.00", X: 400, Y: 200, Cluster: 0},
+		{Text: "1", X: 500, Y: 201, Cluster: 0},
+		{Text: "16,277.00", X: 600, Y: 200, Cluster: 0},
+	}
+	row := extractRowFeatures(group, 1000, 2000)
+	if row.Features[2] != 1 {
+		t.Fatalf("has_service_table_pattern = %v, want 1", row.Features[2])
+	}
+	if row.Features[4] != 1 {
+		t.Fatalf("currency_at_row_end = %v, want 1", row.Features[4])
+	}
+}
+
+func TestBuildFeaturesContinuation(t *testing.T) {
+	boxes := []indexedBox{
+		{Text: "890701", X: 10, Y: 100, Cluster: 0},
+		{Text: "CONSULTA", X: 100, Y: 100, Cluster: 0},
+		{Text: "16,277.00", X: 400, Y: 100, Cluster: 0},
+		{Text: "1", X: 500, Y: 100, Cluster: 0},
+		{Text: "16,277.00", X: 600, Y: 100, Cluster: 0},
+		{Text: "DE", X: 100, Y: 120, Cluster: 1},
+		{Text: "SEGUIMIENTO", X: 200, Y: 120, Cluster: 1},
+	}
+	rows := buildFeatureRows(boxes, 1000, 2000)
+	if len(rows) != 2 {
+		t.Fatalf("rows = %d, want 2", len(rows))
+	}
+	if rows[0].Features[2] != 1 {
+		t.Fatal("first row should have service table pattern")
+	}
+	if rows[1].Features[6] != 1 {
+		t.Fatal("second row should be likely continuation")
+	}
+}
+
+func TestFeatureCount(t *testing.T) {
+	if FeatureCount != 14 {
+		t.Fatalf("FeatureCount = %d, want 14", FeatureCount)
+	}
+}
@@ -88,9 +88,9 @@ func (p *Processor) ProcessPDF(filename string, pdfData []byte) domain.PDFResult
 		return result
 	}

-	features := make([][8]float32, len(rows))
+	features := make([][]float32, len(rows))
 	for i, r := range rows {
-		features[i] = r.Features
+		features[i] = r.Features[:]
 	}

 	tInfer := time.Now()