query: precompute metricScore and update deps (#2893)

* query: precompute metricScore once per result

Normalize the search term and name/provider/desc at the metric boundary, drop the per-comparison distance cache, and score each result via prepareMetrics before sorting. Adds a search-flow benchmark to track regressions.

* update deps

go.mod

@@ -1,8 +1,8 @@
 module github.com/Jguer/yay/v13
 
 require (
-	github.com/Jguer/aur v1.3.0
-	github.com/Jguer/dyalpm v0.1.3
+	github.com/Jguer/aur v1.3.1
+	github.com/Jguer/dyalpm v0.1.4
 	github.com/Jguer/votar v1.0.0
 	github.com/Morganamilo/go-pacmanconf v0.0.0-20210502114700-cff030e927a5
 	github.com/Morganamilo/go-srcinfo v1.0.0
@@ -22,11 +22,11 @@ require (
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/ebitengine/purego v0.10.1 // indirect
+	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/itchyny/gojq v0.12.19 // indirect
 	github.com/itchyny/timefmt-go v0.1.8 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/ohler55/ojg v1.28.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	go.mongodb.org/mongo-driver v1.17.9 // indirect
@@ -34,4 +34,5 @@ require (
 )
 
 go 1.26
+
 toolchain go1.26.0

go.sum

@@ -1,7 +1,7 @@
-github.com/Jguer/aur v1.3.0 h1:skdjp/P9kB75TBaJmn9PKK/kCeA9QsgjdUrORZ3gldU=
-github.com/Jguer/aur v1.3.0/go.mod h1:F8Awo+WKzTxlXtNOO4pDQjMkePLZ+oMSbu+1fKLTTLo=
-github.com/Jguer/dyalpm v0.1.3 h1:hXwzc9LInpg4F1walco+1eS+AziK4ybqBWwUPUbfREg=
-github.com/Jguer/dyalpm v0.1.3/go.mod h1:QfKO3rYndbmh5ptIGYa1Uw5DOeIDPbCG1gJGbyqWaiI=
+github.com/Jguer/aur v1.3.1 h1:KZIsDlzolXRo8gSxsYXLLOF25Wk5vt6FJTeSTpKpDfs=
+github.com/Jguer/aur v1.3.1/go.mod h1:vg8FixA+sl+FEbGGsiM2y2vqtAiQb/XdHIJi94rL03I=
+github.com/Jguer/dyalpm v0.1.4 h1:oTS36Xgj/rESqlz71/hGeW9R9cmb77vDndQ0p3iDmMk=
+github.com/Jguer/dyalpm v0.1.4/go.mod h1:f78SsAmF2fdicKMcKB2j7sW7tHrJdrT+S3jRrIfuaWc=
 github.com/Jguer/votar v1.0.0 h1:drPYpV5Py5BeAQS8xezmT6uCEfLzotNjLf5yfmlHKTg=
 github.com/Jguer/votar v1.0.0/go.mod h1:rc6vgVlTqNjI4nAnPbDTbdxw/N7kXkbB8BcUDjeFbYQ=
 github.com/Morganamilo/go-pacmanconf v0.0.0-20210502114700-cff030e927a5 h1:TMscPjkb1ThXN32LuFY5bEYIcXZx3YlwzhS1GxNpn/c=
@@ -21,6 +21,8 @@ github.com/deckarep/golang-set/v2 v2.9.0 h1:prva4eP9UysWagLyKrtn074ughi0NnkIf0A4
 github.com/deckarep/golang-set/v2 v2.9.0/go.mod h1:EWknQXbs0mcFpat2QOoXV0Ee57cD+w6ZEN76BR2JVrM=
 github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
 github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
+github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
@@ -36,8 +38,6 @@ github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo79
 github.com/itchyny/timefmt-go v0.1.8/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI=
 github.com/leonelquinteros/gotext v1.7.2 h1:bDPndU8nt+/kRo1m4l/1OXiiy2v7Z7dfPQ9+YP7G1Mc=
 github.com/leonelquinteros/gotext v1.7.2/go.mod h1:9/haCkm5P7Jay1sxKDGJ5WIg4zkz8oZKw4ekNpALob8=
-github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
-github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/ohler55/ojg v1.28.1 h1:Xy93DelhLSZNeWv8GPKtP6qMqkUlZlAxBP/AQcC5RfY=

pkg/query/metric.go

@@ -25,26 +25,23 @@ func (a *abstractResults) aurSortByMetric(pkg *abstractResult) float64 {
 }
 
 func (a *abstractResults) GetMetric(pkg *abstractResult) float64 {
-	if v, ok := a.distanceCache[pkg.name]; ok {
-		return v
-	}
-
-	if strings.EqualFold(pkg.name, a.search) {
+	name := strings.ToLower(pkg.name)
+	if name == a.search {
 		return 1.0
 	}
 
-	sim := strutil.Similarity(pkg.name, a.search, a.metric)
+	sim := strutil.Similarity(name, a.search, a.metric)
 
 	for _, prov := range pkg.provides {
 		// If the package provides search, it's a perfect match
 		// AUR packages don't populate provides
-		candidate := strutil.Similarity(prov, a.search, a.metric) * 0.80
+		candidate := strutil.Similarity(strings.ToLower(prov), a.search, a.metric) * 0.80
 		if candidate > sim {
 			sim = candidate
 		}
 	}
 
-	simDesc := strutil.Similarity(pkg.description, a.search, a.metric)
+	simDesc := strutil.Similarity(strings.ToLower(pkg.description), a.search, a.metric)
 
 	// slightly overweight sync sources by always giving them max popularity
 	popularity := 1.0
@@ -52,11 +49,7 @@ func (a *abstractResults) GetMetric(pkg *abstractResult) float64 {
 		popularity = a.aurSortByMetric(pkg)
 	}
 
-	sim = sim*0.35 + simDesc*0.15 + popularity*0.50
-
-	a.distanceCache[pkg.name] = sim
-
-	return sim
+	return sim*0.35 + simDesc*0.15 + popularity*0.50
 }
 
 func (a *abstractResults) separateSourceScore(source string, score float64) float64 {

pkg/query/query_builder.go

@@ -96,6 +96,7 @@ type abstractResult struct {
 	firstSubmitted int
 	lastModified   int
 	provides       []string
+	metricScore    float64
 }
 
 type abstractResults struct {
@@ -106,7 +107,6 @@ type abstractResults struct {
 	sortByFunc      SortFunc
 	repoOrder       []string
 
-	distanceCache       map[string]float64
 	separateSourceCache map[string]float64
 }
 
@@ -158,9 +158,7 @@ func (a *abstractResults) GetSortFunc(sortBy string, bottomUp bool) SortFunc {
 				return cmpResult
 			}
 
-			metricA := a.calculateMetric(&pkgA)
-			metricB := a.calculateMetric(&pkgB)
-			return cmp.Compare(metricA, metricB)
+			return cmp.Compare(pkgA.metricScore, pkgB.metricScore)
 		}
 	}
 
@@ -175,23 +173,32 @@ func (a *abstractResults) GetSortFunc(sortBy string, bottomUp bool) SortFunc {
 	return sortFunc
 }
 
+// prepareMetrics computes the expensive Jaro-Winkler-based rank once per result
+// before sorting so the comparator stays cheap.
+func (a *abstractResults) prepareMetrics() {
+	a.separateSourceCache = make(map[string]float64, len(a.repoOrder))
+
+	for i := range a.results {
+		a.results[i].metricScore = a.calculateMetric(&a.results[i])
+	}
+}
+
 func (s *SourceQueryBuilder) Execute(ctx context.Context, dbExecutor db.Executor, pkgS []string) {
 	var aurErr error
 
 	pkgS = RemoveInvalidTargets(s.logger, pkgS, s.targetMode)
 
 	metric := &metrics.JaroWinkler{
-		CaseSensitive: false,
+		// Case-sensitive: we lower-case the corpus once in GetMetric, search string normalized in Execute.
+		CaseSensitive: true,
 	}
 
 	sortableResults := &abstractResults{
-		results:             []abstractResult{},
-		search:              strings.Join(pkgS, ""),
-		metric:              metric,
-		separateSources:     s.separateSources,
-		repoOrder:           dbExecutor.Repos(),
-		distanceCache:       map[string]float64{},
-		separateSourceCache: map[string]float64{},
+		results:         []abstractResult{},
+		search:          strings.ToLower(strings.Join(pkgS, "")),
+		metric:          metric,
+		separateSources: s.separateSources,
+		repoOrder:       dbExecutor.Repos(),
 	}
 	sortableResults.sortByFunc = sortableResults.GetSortFunc(s.sortBy, s.bottomUp)
 
@@ -251,6 +258,7 @@ func (s *SourceQueryBuilder) Execute(ctx context.Context, dbExecutor db.Executor
 			})
 		}
 	}
+	sortableResults.prepareMetrics()
 
 	slices.SortFunc(sortableResults.results, func(a, b abstractResult) int {
 		return sortableResults.sortByFunc(b, a)

pkg/query/search_flow_bench_test.go

@@ -0,0 +1,104 @@
+//go:build !integration
+
+package query
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/Jguer/aur"
+
+	"github.com/Jguer/yay/v13/pkg/db/mock"
+	mockaur "github.com/Jguer/yay/v13/pkg/dep/mock"
+	"github.com/Jguer/yay/v13/pkg/settings/parser"
+	"github.com/Jguer/yay/v13/pkg/text"
+)
+
+func buildSearchFlowBenchmarkData(repoCount, aurCount int) (*mock.DBExecutor, *mockaur.MockAUR, *text.Logger, []string) {
+	logger := text.NewLogger(io.Discard, io.Discard, strings.NewReader(""), false, "bench")
+	extraDB := mock.NewDB("extra")
+	communityDB := mock.NewDB("community")
+
+	repoPkgs := make([]mock.IPackage, 0, repoCount)
+	for i := range repoCount {
+		db := extraDB
+		if i%2 == 1 {
+			db = communityDB
+		}
+
+		repoPkgs = append(repoPkgs, &mock.Package{
+			PBase:         fmt.Sprintf("yay-tool-%04d", i),
+			PName:         fmt.Sprintf("yay-tool-%04d", i),
+			PVersion:      fmt.Sprintf("1.%d.0", i%17),
+			PDescription:  fmt.Sprintf("Yet another yay tool package %04d for search benchmarking", i),
+			PSize:         int64(1024 + i),
+			PISize:        int64(2048 + i),
+			PDB:           db,
+			PArchitecture: "x86_64",
+			PProvides:     mock.DependList{Depends: []mock.Depend{{Name: fmt.Sprintf("yay-provider-%04d", i)}, {Name: "yay"}}},
+		})
+	}
+
+	aurPkgs := make([]aur.Pkg, 0, aurCount)
+	for i := range aurCount {
+		aurPkgs = append(aurPkgs, aur.Pkg{
+			Description:    fmt.Sprintf("Yet another yay helper package %04d with search benchmark metadata", i),
+			FirstSubmitted: 1_500_000_000 + i,
+			ID:             2_000_000 + i,
+			LastModified:   1_760_000_000 + i,
+			Maintainer:     "bench",
+			Name:           fmt.Sprintf("yay-aur-%04d", i),
+			NumVotes:       100 + (i % 250),
+			OutOfDate:      0,
+			PackageBase:    fmt.Sprintf("yay-aur-%04d", i),
+			PackageBaseID:  300_000 + i,
+			Popularity:     1.0 + float64(i%100)/10,
+			URL:            "https://example.invalid/yay",
+			URLPath:        "/snapshot.tar.gz",
+			Version:        fmt.Sprintf("2.%d.0", i%23),
+			Provides:       []string{"yay"},
+		})
+	}
+
+	mockDB := &mock.DBExecutor{
+		ReposFn: func() []string {
+			return []string{"extra", "community"}
+		},
+		SyncPackagesFn: func(pkgs ...string) []mock.IPackage {
+			return repoPkgs
+		},
+		LocalPackageFn: func(string) mock.IPackage {
+			return nil
+		},
+	}
+
+	mockAUR := &mockaur.MockAUR{
+		GetFn: func(ctx context.Context, query *aur.Query) ([]aur.Pkg, error) {
+			return aurPkgs, nil
+		},
+	}
+
+	return mockDB, mockAUR, logger, []string{"yay"}
+}
+
+func BenchmarkExecuteSearchFlowLarge(b *testing.B) {
+	mockDB, mockAUR, logger, searchTerms := buildSearchFlowBenchmarkData(1500, 1500)
+
+	b.ReportAllocs()
+	for b.Loop() {
+		qb := NewSourceQueryBuilder(
+			mockAUR,
+			logger,
+			"",
+			parser.ModeAny,
+			"",
+			false,
+			false,
+			true,
+		)
+		qb.Execute(b.Context(), mockDB, searchTerms)
+	}
+}