feat: Update testing documentation and introduce comprehensive report for IPSExtractor campaign. Add sections for executive summary, objectives, environment details, and methodology. Enhance README with compilation instructions and data sources for metrics analysis.

2026-07-13 21:10:43 -05:00
parent 6bf413c4ce
commit 311a4bd6d3
39 changed files with 1641 additions and 67 deletions
@@ -14,6 +14,12 @@ type PDFResult struct {
 	Status       string       `json:"status"`
 	Services     []ServiceRow `json:"services,omitempty"`
 	Error        string       `json:"error,omitempty"`
+	// Pipeline stage timings in milliseconds (populated by Processor).
+	RenderMs  int64 `json:"render_ms,omitempty"`
+	OCRMs     int64 `json:"ocr_ms,omitempty"`
+	ClusterMs int64 `json:"cluster_ms,omitempty"`
+	InferMs   int64 `json:"infer_ms,omitempty"`
+	TotalMs   int64 `json:"total_ms,omitempty"`
 }

 // CountResponse wraps the results of a synchronous batch request to POST /v1/count.
@@ -0,0 +1,61 @@
+package pipeline
+
+import "testing"
+
+func TestClusterRowsTwoBands(t *testing.T) {
+	boxes := []indexedBox{
+		{Text: "a", X: 0, Y: 100},
+		{Text: "b", X: 10, Y: 102},
+		{Text: "c", X: 0, Y: 300},
+		{Text: "d", X: 10, Y: 301},
+	}
+	ClusterRows(boxes, 10, 1)
+	if boxes[0].Cluster < 0 || boxes[1].Cluster < 0 {
+		t.Fatal("first band should be clustered")
+	}
+	if boxes[0].Cluster != boxes[1].Cluster {
+		t.Fatalf("same band: got %d and %d", boxes[0].Cluster, boxes[1].Cluster)
+	}
+	if boxes[2].Cluster != boxes[3].Cluster {
+		t.Fatalf("second band: got %d and %d", boxes[2].Cluster, boxes[3].Cluster)
+	}
+	if boxes[0].Cluster == boxes[2].Cluster {
+		t.Fatal("distant Y bands should differ")
+	}
+}
+
+func TestClusterRowsSameBand(t *testing.T) {
+	boxes := []indexedBox{
+		{Text: "a", Y: 50},
+		{Text: "b", Y: 52},
+		{Text: "c", Y: 54},
+	}
+	ClusterRows(boxes, 10, 1)
+	id := boxes[0].Cluster
+	for i, b := range boxes {
+		if b.Cluster != id {
+			t.Fatalf("box %d cluster=%d, want %d", i, b.Cluster, id)
+		}
+	}
+}
+
+func TestClusterRowsEmpty(t *testing.T) {
+	ClusterRows(nil, 10, 1) // no panic
+	ClusterRows([]indexedBox{}, 10, 1)
+}
+
+func TestClusterRowsNoiseMinSamples(t *testing.T) {
+	boxes := []indexedBox{
+		{Text: "lonely", Y: 10},
+		{Text: "pair-a", Y: 200},
+		{Text: "pair-b", Y: 201},
+	}
+	// minSamples=2: lonely point stays noise (-1); pair forms a cluster
+	ClusterRows(boxes, 5, 2)
+	if boxes[0].Cluster != -1 {
+		t.Fatalf("lonely cluster=%d, want -1 (noise)", boxes[0].Cluster)
+	}
+	if boxes[1].Cluster < 0 || boxes[1].Cluster != boxes[2].Cluster {
+		t.Fatalf("pair clusters=%d,%d", boxes[1].Cluster, boxes[2].Cluster)
+	}
+}
@@ -0,0 +1,44 @@
+package pipeline
+
+import "testing"
+
+func TestExtractRowFeaturesHeaderKeyword(t *testing.T) {
+	group := []indexedBox{
+		{Text: "FACTURA", X: 10, Y: 50, Cluster: 0},
+		{Text: "NIT", X: 100, Y: 50, Cluster: 0},
+		{Text: "123", X: 200, Y: 50, Cluster: 0},
+	}
+	row := extractRowFeatures(group, 1000, 2000)
+	if row.Features[3] != 1 {
+		t.Fatalf("has_header_keyword = %v, want 1", row.Features[3])
+	}
+}
+
+func TestExtractRowFeaturesServiceCode(t *testing.T) {
+	group := []indexedBox{
+		{Text: "AB12", X: 10, Y: 100, Cluster: 0},
+		{Text: "CONSULTA", X: 80, Y: 100, Cluster: 0},
+	}
+	row := extractRowFeatures(group, 1000, 2000)
+	if row.Features[1] != 1 {
+		t.Fatalf("first_token_is_service_code = %v, want 1", row.Features[1])
+	}
+}
+
+func TestBuildFeatureRowsEmpty(t *testing.T) {
+	rows := buildFeatureRows(nil, 1000, 2000)
+	if len(rows) != 0 {
+		t.Fatalf("rows = %d, want 0", len(rows))
+	}
+}
+
+func TestExtractRowFeaturesNoTablePattern(t *testing.T) {
+	group := []indexedBox{
+		{Text: "TEXTO", X: 10, Y: 100, Cluster: 0},
+		{Text: "LIBRE", X: 80, Y: 100, Cluster: 0},
+	}
+	row := extractRowFeatures(group, 1000, 2000)
+	if row.Features[2] != 0 {
+		t.Fatalf("has_service_table_pattern = %v, want 0", row.Features[2])
+	}
+}
@@ -32,28 +32,31 @@ func NewProcessor(predictor *model.Predictor, ocrEngine *ocr.Engine, opts Option
 func (p *Processor) ProcessPDF(filename string, pdfData []byte) domain.PDFResult {
 	start := time.Now()
 	result := domain.PDFResult{Filename: filename, Status: domain.StatusOK}
+	var renderDur, ocrDur, clusterDur, inferDur time.Duration

 	tRender := time.Now()
 	img, pageW, pageH, err := ocr.RenderPage(pdfData, p.opts.DPI)
-	renderDur := time.Since(tRender)
+	renderDur = time.Since(tRender)
 	if err != nil {
 		result.Status = domain.StatusError
 		result.Error = err.Error()
+		p.finishTiming(&result, renderDur, 0, 0, 0, time.Since(start))
 		return result
 	}

 	tOCR := time.Now()
 	boxes, err := p.ocrEngine.OCRImage(img)
-	ocrDur := time.Since(tOCR)
+	ocrDur = time.Since(tOCR)
 	if err != nil {
 		result.Status = domain.StatusError
 		result.Error = err.Error()
+		p.finishTiming(&result, renderDur, ocrDur, 0, 0, time.Since(start))
 		return result
 	}
 	if len(boxes) == 0 {
 		result.ServiceCount = 0
 		result.Services = []domain.ServiceRow{}
-		p.logTiming(filename, renderDur, ocrDur, 0, 0, time.Since(start))
+		p.finishTiming(&result, renderDur, ocrDur, 0, 0, time.Since(start))
 		return result
 	}

@@ -65,11 +68,11 @@ func (p *Processor) ProcessPDF(filename string, pdfData []byte) domain.PDFResult
 	ClusterRows(indexed, p.opts.DBSCANEps, p.opts.DBSCANMinSamples)

 	rows := buildFeatureRows(indexed, pageW, pageH)
-	clusterDur := time.Since(tCluster)
+	clusterDur = time.Since(tCluster)
 	if len(rows) == 0 {
 		result.ServiceCount = 0
 		result.Services = []domain.ServiceRow{}
-		p.logTiming(filename, renderDur, ocrDur, clusterDur, 0, time.Since(start))
+		p.finishTiming(&result, renderDur, ocrDur, clusterDur, 0, time.Since(start))
 		return result
 	}

@@ -84,7 +87,7 @@ func (p *Processor) ProcessPDF(filename string, pdfData []byte) domain.PDFResult
 		}
 		result.Services = services
 		result.ServiceCount = len(rows)
-		p.logTiming(filename, renderDur, ocrDur, clusterDur, 0, time.Since(start))
+		p.finishTiming(&result, renderDur, ocrDur, clusterDur, 0, time.Since(start))
 		return result
 	}

@@ -95,10 +98,11 @@ func (p *Processor) ProcessPDF(filename string, pdfData []byte) domain.PDFResult

 	tInfer := time.Now()
 	probs, err := p.predictor.PredictProba(features)
-	inferDur := time.Since(tInfer)
+	inferDur = time.Since(tInfer)
 	if err != nil {
 		result.Status = domain.StatusError
 		result.Error = fmt.Sprintf("inference: %v", err)
+		p.finishTiming(&result, renderDur, ocrDur, clusterDur, inferDur, time.Since(start))
 		return result
 	}

@@ -128,16 +132,21 @@ func (p *Processor) ProcessPDF(filename string, pdfData []byte) domain.PDFResult

 	result.Services = services
 	result.ServiceCount = serviceCount
-	p.logTiming(filename, renderDur, ocrDur, clusterDur, inferDur, time.Since(start))
+	p.finishTiming(&result, renderDur, ocrDur, clusterDur, inferDur, time.Since(start))
 	return result
 }

-func (p *Processor) logTiming(filename string, render, ocr, cluster, infer, total time.Duration) {
+func (p *Processor) finishTiming(result *domain.PDFResult, render, ocrD, cluster, infer, total time.Duration) {
+	result.RenderMs = render.Milliseconds()
+	result.OCRMs = ocrD.Milliseconds()
+	result.ClusterMs = cluster.Milliseconds()
+	result.InferMs = infer.Milliseconds()
+	result.TotalMs = total.Milliseconds()
 	if !p.opts.LogPipelineTiming {
 		return
 	}
 	log.Printf(
 		"pipeline filename=%s ocr_render=%s ocr_tesseract=%s cluster=%s infer=%s total=%s",
-		filename, render, ocr, cluster, infer, total,
+		result.Filename, render, ocrD, cluster, infer, total,
 	)
 }
@@ -0,0 +1,98 @@
+package pipeline
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/pys/pdf-count/internal/domain"
+	"github.com/pys/pdf-count/internal/ocr"
+)
+
+func testPDFDir(t *testing.T) string {
+	t.Helper()
+	if d := os.Getenv("COUNT_TEST_PDF_DIR"); d != "" {
+		return d
+	}
+	// services/services-count-facturas-medicas-serving → docs/plan_pruebas/data
+	candidates := []string{
+		filepath.Join("..", "..", "..", "..", "docs", "plan_pruebas", "data"),
+		filepath.Join("..", "..", "..", "docs", "plan_pruebas", "data"),
+	}
+	for _, c := range candidates {
+		abs, err := filepath.Abs(c)
+		if err != nil {
+			continue
+		}
+		if st, err := os.Stat(abs); err == nil && st.IsDir() {
+			return abs
+		}
+	}
+	return ""
+}
+
+func listFixturePDFs(t *testing.T, dir string) []string {
+	t.Helper()
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatalf("read pdf dir: %v", err)
+	}
+	var files []string
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		name := e.Name()
+		if strings.EqualFold(filepath.Ext(name), ".pdf") {
+			files = append(files, filepath.Join(dir, name))
+		}
+	}
+	return files
+}
+
+func TestProcessPDFTimingFixtures(t *testing.T) {
+	dir := testPDFDir(t)
+	if dir == "" {
+		t.Skip("PDF fixtures not found; set COUNT_TEST_PDF_DIR")
+	}
+	pdfs := listFixturePDFs(t, dir)
+	if len(pdfs) == 0 {
+		t.Skip("no PDF files in " + dir)
+	}
+
+	engine, err := ocr.NewEngine("spa", 1)
+	if err != nil {
+		t.Skipf("tesseract not available: %v", err)
+	}
+	defer engine.Close()
+
+	proc := NewProcessor(nil, engine, Options{
+		DPI:              200, // faster than 300 for timing smoke
+		DBSCANEps:        10,
+		DBSCANMinSamples: 1,
+		LogPipelineTiming: true,
+	})
+
+	t.Logf("pdf_dir=%s files=%d", dir, len(pdfs))
+	t.Logf("%-28s %8s %8s %8s %8s %8s %6s %s",
+		"file", "render", "ocr", "cluster", "infer", "total", "count", "status")
+
+	for _, path := range pdfs {
+		data, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatalf("read %s: %v", path, err)
+		}
+		name := filepath.Base(path)
+		res := proc.ProcessPDF(name, data)
+		t.Logf("%-28s %6dms %6dms %6dms %6dms %6dms %6d %s",
+			name, res.RenderMs, res.OCRMs, res.ClusterMs, res.InferMs, res.TotalMs, res.ServiceCount, res.Status)
+		if res.Status == domain.StatusError {
+			t.Errorf("%s failed: %s", name, res.Error)
+			continue
+		}
+		if res.TotalMs <= 0 {
+			t.Errorf("%s: TotalMs should be > 0", name)
+		}
+	}
+}