123 lines
2.2 KiB
Go
123 lines
2.2 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"unicode"
|
|
)
|
|
|
|
var (
|
|
codePattern = regexp.MustCompile(`^\d{4,6}$`)
|
|
currencyPattern = regexp.MustCompile(`^\d{1,3}(,\d{3})+(\.\d+)?$`)
|
|
)
|
|
|
|
type rowFeatures struct {
|
|
Features [8]float32
|
|
TextoFila string
|
|
Cluster int
|
|
}
|
|
|
|
func extractRowFeatures(group []indexedBox, pageW, pageH int) rowFeatures {
|
|
sort.Slice(group, func(i, j int) bool { return group[i].X < group[j].X })
|
|
|
|
texts := make([]string, len(group))
|
|
for i, b := range group {
|
|
texts[i] = b.Text
|
|
}
|
|
textoFila := strings.Join(texts, " ")
|
|
firstToken := ""
|
|
if len(texts) > 0 {
|
|
firstToken = texts[0]
|
|
}
|
|
|
|
currencyCount := 0
|
|
integerCount := 0
|
|
for _, t := range texts {
|
|
if currencyPattern.MatchString(t) {
|
|
currencyCount++
|
|
}
|
|
if isDigits(t) {
|
|
integerCount++
|
|
}
|
|
}
|
|
|
|
xMin, xMax := group[0].X, group[0].X
|
|
ySum := 0
|
|
for _, b := range group {
|
|
if b.X < xMin {
|
|
xMin = b.X
|
|
}
|
|
if b.X > xMax {
|
|
xMax = b.X
|
|
}
|
|
ySum += b.Y
|
|
}
|
|
|
|
xSpanNorm := 0.0
|
|
if pageW > 0 {
|
|
xSpanNorm = float64(xMax-xMin) / float64(pageW)
|
|
}
|
|
yNorm := 0.0
|
|
if pageH > 0 {
|
|
yNorm = float64(ySum/len(group)) / float64(pageH)
|
|
}
|
|
|
|
totalChars := len(textoFila)
|
|
if totalChars == 0 {
|
|
totalChars = 1
|
|
}
|
|
special := 0
|
|
digits := 0
|
|
for _, c := range textoFila {
|
|
if !unicode.IsLetter(c) && !unicode.IsNumber(c) && !unicode.IsSpace(c) {
|
|
special++
|
|
}
|
|
if unicode.IsDigit(c) {
|
|
digits++
|
|
}
|
|
}
|
|
|
|
firstIsCode := 0
|
|
if codePattern.MatchString(firstToken) {
|
|
firstIsCode = 1
|
|
}
|
|
|
|
return rowFeatures{
|
|
Features: [8]float32{
|
|
float32(firstIsCode),
|
|
float32(currencyCount),
|
|
float32(integerCount),
|
|
float32(xSpanNorm),
|
|
float32(yNorm),
|
|
float32(len(texts)),
|
|
float32(special) / float32(totalChars),
|
|
float32(digits) / float32(totalChars),
|
|
},
|
|
TextoFila: textoFila,
|
|
Cluster: group[0].Cluster,
|
|
}
|
|
}
|
|
|
|
func isDigits(s string) bool {
|
|
if s == "" {
|
|
return false
|
|
}
|
|
for _, c := range s {
|
|
if c < '0' || c > '9' {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func buildFeatureRows(boxes []indexedBox, pageW, pageH int) []rowFeatures {
|
|
groups := groupByCluster(boxes)
|
|
order := sortClusters(groups)
|
|
rows := make([]rowFeatures, 0, len(groups))
|
|
for _, cid := range order {
|
|
rows = append(rows, extractRowFeatures(groups[cid], pageW, pageH))
|
|
}
|
|
return rows
|
|
}
|