feat: Add .env.example configuration file and integrate Redis service into Docker Compose for improved task management. Update desktop application to support document indexing and extraction workflows, enhancing user experience with a new extraction wizard and stepper components. Revise README and documentation for clarity on setup and usage.
This commit is contained in:
erik
@@ -0,0 +1,112 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// ClusterRows runs 1D DBSCAN on Y coordinates (parity with sklearn DBSCAN).
|
||||
func ClusterRows(boxes []indexedBox, eps float64, minSamples int) {
|
||||
if len(boxes) == 0 {
|
||||
return
|
||||
}
|
||||
ys := make([]float64, len(boxes))
|
||||
for i, b := range boxes {
|
||||
ys[i] = float64(b.Y)
|
||||
}
|
||||
labels := dbscan1D(ys, eps, minSamples)
|
||||
for i := range boxes {
|
||||
boxes[i].Cluster = labels[i]
|
||||
}
|
||||
}
|
||||
|
||||
func dbscan1D(points []float64, eps float64, minSamples int) []int {
|
||||
n := len(points)
|
||||
labels := make([]int, n)
|
||||
for i := range labels {
|
||||
labels[i] = -1
|
||||
}
|
||||
clusterID := 0
|
||||
visited := make([]bool, n)
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
if visited[i] {
|
||||
continue
|
||||
}
|
||||
visited[i] = true
|
||||
neighbors := regionQuery(points, i, eps)
|
||||
if len(neighbors) < minSamples {
|
||||
continue
|
||||
}
|
||||
labels[i] = clusterID
|
||||
seed := append([]int(nil), neighbors...)
|
||||
for j := 0; j < len(seed); j++ {
|
||||
q := seed[j]
|
||||
if !visited[q] {
|
||||
visited[q] = true
|
||||
neighborsQ := regionQuery(points, q, eps)
|
||||
if len(neighborsQ) >= minSamples {
|
||||
seed = appendUnique(seed, neighborsQ)
|
||||
}
|
||||
}
|
||||
if labels[q] == -1 {
|
||||
labels[q] = clusterID
|
||||
}
|
||||
}
|
||||
clusterID++
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
func regionQuery(points []float64, idx int, eps float64) []int {
|
||||
var neighbors []int
|
||||
for i, p := range points {
|
||||
if math.Abs(p-points[idx]) <= eps {
|
||||
neighbors = append(neighbors, i)
|
||||
}
|
||||
}
|
||||
return neighbors
|
||||
}
|
||||
|
||||
func appendUnique(seed, add []int) []int {
|
||||
inSeed := make(map[int]bool, len(seed))
|
||||
for _, v := range seed {
|
||||
inSeed[v] = true
|
||||
}
|
||||
for _, v := range add {
|
||||
if !inSeed[v] {
|
||||
seed = append(seed, v)
|
||||
inSeed[v] = true
|
||||
}
|
||||
}
|
||||
return seed
|
||||
}
|
||||
|
||||
type indexedBox struct {
|
||||
Text string
|
||||
X int
|
||||
Y int
|
||||
W int
|
||||
H int
|
||||
Cluster int
|
||||
}
|
||||
|
||||
func groupByCluster(boxes []indexedBox) map[int][]indexedBox {
|
||||
groups := make(map[int][]indexedBox)
|
||||
for _, b := range boxes {
|
||||
if b.Cluster < 0 {
|
||||
continue
|
||||
}
|
||||
groups[b.Cluster] = append(groups[b.Cluster], b)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func sortClusters(groups map[int][]indexedBox) []int {
|
||||
keys := make([]int, 0, len(groups))
|
||||
for k := range groups {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Ints(keys)
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
var (
|
||||
codePattern = regexp.MustCompile(`^\d{4,6}$`)
|
||||
currencyPattern = regexp.MustCompile(`^\d{1,3}(,\d{3})+(\.\d+)?$`)
|
||||
)
|
||||
|
||||
type rowFeatures struct {
|
||||
Features [8]float32
|
||||
TextoFila string
|
||||
Cluster int
|
||||
}
|
||||
|
||||
func extractRowFeatures(group []indexedBox, pageW, pageH int) rowFeatures {
|
||||
sort.Slice(group, func(i, j int) bool { return group[i].X < group[j].X })
|
||||
|
||||
texts := make([]string, len(group))
|
||||
for i, b := range group {
|
||||
texts[i] = b.Text
|
||||
}
|
||||
textoFila := strings.Join(texts, " ")
|
||||
firstToken := ""
|
||||
if len(texts) > 0 {
|
||||
firstToken = texts[0]
|
||||
}
|
||||
|
||||
currencyCount := 0
|
||||
integerCount := 0
|
||||
for _, t := range texts {
|
||||
if currencyPattern.MatchString(t) {
|
||||
currencyCount++
|
||||
}
|
||||
if isDigits(t) {
|
||||
integerCount++
|
||||
}
|
||||
}
|
||||
|
||||
xMin, xMax := group[0].X, group[0].X
|
||||
ySum := 0
|
||||
for _, b := range group {
|
||||
if b.X < xMin {
|
||||
xMin = b.X
|
||||
}
|
||||
if b.X > xMax {
|
||||
xMax = b.X
|
||||
}
|
||||
ySum += b.Y
|
||||
}
|
||||
|
||||
xSpanNorm := 0.0
|
||||
if pageW > 0 {
|
||||
xSpanNorm = float64(xMax-xMin) / float64(pageW)
|
||||
}
|
||||
yNorm := 0.0
|
||||
if pageH > 0 {
|
||||
yNorm = float64(ySum/len(group)) / float64(pageH)
|
||||
}
|
||||
|
||||
totalChars := len(textoFila)
|
||||
if totalChars == 0 {
|
||||
totalChars = 1
|
||||
}
|
||||
special := 0
|
||||
digits := 0
|
||||
for _, c := range textoFila {
|
||||
if !unicode.IsLetter(c) && !unicode.IsNumber(c) && !unicode.IsSpace(c) {
|
||||
special++
|
||||
}
|
||||
if unicode.IsDigit(c) {
|
||||
digits++
|
||||
}
|
||||
}
|
||||
|
||||
firstIsCode := 0
|
||||
if codePattern.MatchString(firstToken) {
|
||||
firstIsCode = 1
|
||||
}
|
||||
|
||||
return rowFeatures{
|
||||
Features: [8]float32{
|
||||
float32(firstIsCode),
|
||||
float32(currencyCount),
|
||||
float32(integerCount),
|
||||
float32(xSpanNorm),
|
||||
float32(yNorm),
|
||||
float32(len(texts)),
|
||||
float32(special) / float32(totalChars),
|
||||
float32(digits) / float32(totalChars),
|
||||
},
|
||||
TextoFila: textoFila,
|
||||
Cluster: group[0].Cluster,
|
||||
}
|
||||
}
|
||||
|
||||
func isDigits(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
for _, c := range s {
|
||||
if c < '0' || c > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func buildFeatureRows(boxes []indexedBox, pageW, pageH int) []rowFeatures {
|
||||
groups := groupByCluster(boxes)
|
||||
order := sortClusters(groups)
|
||||
rows := make([]rowFeatures, 0, len(groups))
|
||||
for _, cid := range order {
|
||||
rows = append(rows, extractRowFeatures(groups[cid], pageW, pageH))
|
||||
}
|
||||
return rows
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/pys/pdf-count/internal/domain"
|
||||
"github.com/pys/pdf-count/internal/model"
|
||||
"github.com/pys/pdf-count/internal/ocr"
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
DPI int
|
||||
Lang string
|
||||
DBSCANEps float64
|
||||
DBSCANMinSamples int
|
||||
IncludeRejected bool
|
||||
LogPipelineTiming bool
|
||||
}
|
||||
|
||||
type Processor struct {
|
||||
predictor *model.Predictor
|
||||
ocrEngine *ocr.Engine
|
||||
opts Options
|
||||
}
|
||||
|
||||
func NewProcessor(predictor *model.Predictor, ocrEngine *ocr.Engine, opts Options) *Processor {
|
||||
return &Processor{predictor: predictor, ocrEngine: ocrEngine, opts: opts}
|
||||
}
|
||||
|
||||
func (p *Processor) ProcessPDF(filename string, pdfData []byte) domain.PDFResult {
|
||||
start := time.Now()
|
||||
result := domain.PDFResult{Filename: filename, Status: domain.StatusOK}
|
||||
|
||||
tRender := time.Now()
|
||||
img, pageW, pageH, err := ocr.RenderPage(pdfData, p.opts.DPI)
|
||||
renderDur := time.Since(tRender)
|
||||
if err != nil {
|
||||
result.Status = domain.StatusError
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
|
||||
tOCR := time.Now()
|
||||
boxes, err := p.ocrEngine.OCRImage(img)
|
||||
ocrDur := time.Since(tOCR)
|
||||
if err != nil {
|
||||
result.Status = domain.StatusError
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
if len(boxes) == 0 {
|
||||
result.ServiceCount = 0
|
||||
result.Services = []domain.ServiceRow{}
|
||||
p.logTiming(filename, renderDur, ocrDur, 0, 0, time.Since(start))
|
||||
return result
|
||||
}
|
||||
|
||||
tCluster := time.Now()
|
||||
indexed := make([]indexedBox, len(boxes))
|
||||
for i, b := range boxes {
|
||||
indexed[i] = indexedBox{Text: b.Text, X: b.X, Y: b.Y, W: b.W, H: b.H}
|
||||
}
|
||||
ClusterRows(indexed, p.opts.DBSCANEps, p.opts.DBSCANMinSamples)
|
||||
|
||||
rows := buildFeatureRows(indexed, pageW, pageH)
|
||||
clusterDur := time.Since(tCluster)
|
||||
if len(rows) == 0 {
|
||||
result.ServiceCount = 0
|
||||
result.Services = []domain.ServiceRow{}
|
||||
p.logTiming(filename, renderDur, ocrDur, clusterDur, 0, time.Since(start))
|
||||
return result
|
||||
}
|
||||
|
||||
features := make([][8]float32, len(rows))
|
||||
for i, r := range rows {
|
||||
features[i] = r.Features
|
||||
}
|
||||
|
||||
tInfer := time.Now()
|
||||
probs, err := p.predictor.PredictProba(features)
|
||||
inferDur := time.Since(tInfer)
|
||||
if err != nil {
|
||||
result.Status = domain.StatusError
|
||||
result.Error = fmt.Sprintf("inference: %v", err)
|
||||
return result
|
||||
}
|
||||
|
||||
threshold := p.predictor.Threshold()
|
||||
services := make([]domain.ServiceRow, 0)
|
||||
serviceCount := 0
|
||||
for i, row := range rows {
|
||||
prob := model.RoundProb(probs[i])
|
||||
isService := prob >= threshold
|
||||
if isService {
|
||||
serviceCount++
|
||||
}
|
||||
if p.opts.IncludeRejected {
|
||||
v := isService
|
||||
services = append(services, domain.ServiceRow{
|
||||
Text: row.TextoFila,
|
||||
Probability: prob,
|
||||
IsService: &v,
|
||||
})
|
||||
} else if isService {
|
||||
services = append(services, domain.ServiceRow{
|
||||
Text: row.TextoFila,
|
||||
Probability: prob,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
result.Services = services
|
||||
result.ServiceCount = serviceCount
|
||||
p.logTiming(filename, renderDur, ocrDur, clusterDur, inferDur, time.Since(start))
|
||||
return result
|
||||
}
|
||||
|
||||
func (p *Processor) logTiming(filename string, render, ocr, cluster, infer, total time.Duration) {
|
||||
if !p.opts.LogPipelineTiming {
|
||||
return
|
||||
}
|
||||
log.Printf(
|
||||
"pipeline filename=%s ocr_render=%s ocr_tesseract=%s cluster=%s infer=%s total=%s",
|
||||
filename, render, ocr, cluster, infer, total,
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user