From 19cd422953ec440015b880fc771de0bbdad38c82 Mon Sep 17 00:00:00 2001 From: Jo Date: Sat, 27 Jun 2026 16:47:41 +0200 Subject: [PATCH] query: precompute metricScore and update deps (#2893) * query: precompute metricScore once per result Normalize the search term and name/provider/desc at the metric boundary, drop the per-comparison distance cache, and score each result via prepareMetrics before sorting. Adds a search-flow benchmark to track regressions. * update deps --- go.mod | 7 +- go.sum | 12 ++-- pkg/query/metric.go | 19 ++--- pkg/query/query_builder.go | 32 +++++---- pkg/query/search_flow_bench_test.go | 104 ++++++++++++++++++++++++++++ 5 files changed, 140 insertions(+), 34 deletions(-) create mode 100644 pkg/query/search_flow_bench_test.go diff --git a/go.mod b/go.mod index eb55772f..c44474da 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/Jguer/yay/v13 require ( - github.com/Jguer/aur v1.3.0 - github.com/Jguer/dyalpm v0.1.3 + github.com/Jguer/aur v1.3.1 + github.com/Jguer/dyalpm v0.1.4 github.com/Jguer/votar v1.0.0 github.com/Morganamilo/go-pacmanconf v0.0.0-20210502114700-cff030e927a5 github.com/Morganamilo/go-srcinfo v1.0.0 @@ -22,11 +22,11 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/ebitengine/purego v0.10.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/itchyny/gojq v0.12.19 // indirect github.com/itchyny/timefmt-go v0.1.8 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/ohler55/ojg v1.28.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.mongodb.org/mongo-driver v1.17.9 // indirect @@ -34,4 +34,5 @@ require ( ) go 1.26 + toolchain go1.26.0 diff --git a/go.sum b/go.sum index f288bdf2..6db88fcd 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ -github.com/Jguer/aur v1.3.0 h1:skdjp/P9kB75TBaJmn9PKK/kCeA9QsgjdUrORZ3gldU= -github.com/Jguer/aur v1.3.0/go.mod h1:F8Awo+WKzTxlXtNOO4pDQjMkePLZ+oMSbu+1fKLTTLo= -github.com/Jguer/dyalpm v0.1.3 h1:hXwzc9LInpg4F1walco+1eS+AziK4ybqBWwUPUbfREg= -github.com/Jguer/dyalpm v0.1.3/go.mod h1:QfKO3rYndbmh5ptIGYa1Uw5DOeIDPbCG1gJGbyqWaiI= +github.com/Jguer/aur v1.3.1 h1:KZIsDlzolXRo8gSxsYXLLOF25Wk5vt6FJTeSTpKpDfs= +github.com/Jguer/aur v1.3.1/go.mod h1:vg8FixA+sl+FEbGGsiM2y2vqtAiQb/XdHIJi94rL03I= +github.com/Jguer/dyalpm v0.1.4 h1:oTS36Xgj/rESqlz71/hGeW9R9cmb77vDndQ0p3iDmMk= +github.com/Jguer/dyalpm v0.1.4/go.mod h1:f78SsAmF2fdicKMcKB2j7sW7tHrJdrT+S3jRrIfuaWc= github.com/Jguer/votar v1.0.0 h1:drPYpV5Py5BeAQS8xezmT6uCEfLzotNjLf5yfmlHKTg= github.com/Jguer/votar v1.0.0/go.mod h1:rc6vgVlTqNjI4nAnPbDTbdxw/N7kXkbB8BcUDjeFbYQ= github.com/Morganamilo/go-pacmanconf v0.0.0-20210502114700-cff030e927a5 h1:TMscPjkb1ThXN32LuFY5bEYIcXZx3YlwzhS1GxNpn/c= @@ -21,6 +21,8 @@ github.com/deckarep/golang-set/v2 v2.9.0 h1:prva4eP9UysWagLyKrtn074ughi0NnkIf0A4 github.com/deckarep/golang-set/v2 v2.9.0/go.mod h1:EWknQXbs0mcFpat2QOoXV0Ee57cD+w6ZEN76BR2JVrM= github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= @@ -36,8 +38,6 @@ github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo79 github.com/itchyny/timefmt-go v0.1.8/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI= github.com/leonelquinteros/gotext v1.7.2 h1:bDPndU8nt+/kRo1m4l/1OXiiy2v7Z7dfPQ9+YP7G1Mc= github.com/leonelquinteros/gotext v1.7.2/go.mod h1:9/haCkm5P7Jay1sxKDGJ5WIg4zkz8oZKw4ekNpALob8= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/ohler55/ojg v1.28.1 h1:Xy93DelhLSZNeWv8GPKtP6qMqkUlZlAxBP/AQcC5RfY= diff --git a/pkg/query/metric.go b/pkg/query/metric.go index acb65bad..2fcc78db 100644 --- a/pkg/query/metric.go +++ b/pkg/query/metric.go @@ -25,26 +25,23 @@ func (a *abstractResults) aurSortByMetric(pkg *abstractResult) float64 { } func (a *abstractResults) GetMetric(pkg *abstractResult) float64 { - if v, ok := a.distanceCache[pkg.name]; ok { - return v - } - - if strings.EqualFold(pkg.name, a.search) { + name := strings.ToLower(pkg.name) + if name == a.search { return 1.0 } - sim := strutil.Similarity(pkg.name, a.search, a.metric) + sim := strutil.Similarity(name, a.search, a.metric) for _, prov := range pkg.provides { // If the package provides search, it's a perfect match // AUR packages don't populate provides - candidate := strutil.Similarity(prov, a.search, a.metric) * 0.80 + candidate := strutil.Similarity(strings.ToLower(prov), a.search, a.metric) * 0.80 if candidate > sim { sim = candidate } } - simDesc := strutil.Similarity(pkg.description, a.search, a.metric) + simDesc := strutil.Similarity(strings.ToLower(pkg.description), a.search, a.metric) // slightly overweight sync sources by always giving them max popularity popularity := 1.0 @@ -52,11 +49,7 @@ func (a *abstractResults) GetMetric(pkg *abstractResult) float64 { popularity = a.aurSortByMetric(pkg) } - sim = sim*0.35 + simDesc*0.15 + popularity*0.50 - - a.distanceCache[pkg.name] = sim - - return sim + return sim*0.35 + simDesc*0.15 + popularity*0.50 } func (a *abstractResults) separateSourceScore(source string, score float64) float64 { diff --git a/pkg/query/query_builder.go b/pkg/query/query_builder.go index 0b6eef70..88c43355 100644 --- a/pkg/query/query_builder.go +++ b/pkg/query/query_builder.go @@ -96,6 +96,7 @@ type abstractResult struct { firstSubmitted int lastModified int provides []string + metricScore float64 } type abstractResults struct { @@ -106,7 +107,6 @@ type abstractResults struct { sortByFunc SortFunc repoOrder []string - distanceCache map[string]float64 separateSourceCache map[string]float64 } @@ -158,9 +158,7 @@ func (a *abstractResults) GetSortFunc(sortBy string, bottomUp bool) SortFunc { return cmpResult } - metricA := a.calculateMetric(&pkgA) - metricB := a.calculateMetric(&pkgB) - return cmp.Compare(metricA, metricB) + return cmp.Compare(pkgA.metricScore, pkgB.metricScore) } } @@ -175,23 +173,32 @@ func (a *abstractResults) GetSortFunc(sortBy string, bottomUp bool) SortFunc { return sortFunc } +// prepareMetrics computes the expensive Jaro-Winkler-based rank once per result +// before sorting so the comparator stays cheap. +func (a *abstractResults) prepareMetrics() { + a.separateSourceCache = make(map[string]float64, len(a.repoOrder)) + + for i := range a.results { + a.results[i].metricScore = a.calculateMetric(&a.results[i]) + } +} + func (s *SourceQueryBuilder) Execute(ctx context.Context, dbExecutor db.Executor, pkgS []string) { var aurErr error pkgS = RemoveInvalidTargets(s.logger, pkgS, s.targetMode) metric := &metrics.JaroWinkler{ - CaseSensitive: false, + // Case-sensitive: we lower-case the corpus once in GetMetric, search string normalized in Execute. + CaseSensitive: true, } sortableResults := &abstractResults{ - results: []abstractResult{}, - search: strings.Join(pkgS, ""), - metric: metric, - separateSources: s.separateSources, - repoOrder: dbExecutor.Repos(), - distanceCache: map[string]float64{}, - separateSourceCache: map[string]float64{}, + results: []abstractResult{}, + search: strings.ToLower(strings.Join(pkgS, "")), + metric: metric, + separateSources: s.separateSources, + repoOrder: dbExecutor.Repos(), } sortableResults.sortByFunc = sortableResults.GetSortFunc(s.sortBy, s.bottomUp) @@ -251,6 +258,7 @@ func (s *SourceQueryBuilder) Execute(ctx context.Context, dbExecutor db.Executor }) } } + sortableResults.prepareMetrics() slices.SortFunc(sortableResults.results, func(a, b abstractResult) int { return sortableResults.sortByFunc(b, a) diff --git a/pkg/query/search_flow_bench_test.go b/pkg/query/search_flow_bench_test.go new file mode 100644 index 00000000..a7598e6e --- /dev/null +++ b/pkg/query/search_flow_bench_test.go @@ -0,0 +1,104 @@ +//go:build !integration + +package query + +import ( + "context" + "fmt" + "io" + "strings" + "testing" + + "github.com/Jguer/aur" + + "github.com/Jguer/yay/v13/pkg/db/mock" + mockaur "github.com/Jguer/yay/v13/pkg/dep/mock" + "github.com/Jguer/yay/v13/pkg/settings/parser" + "github.com/Jguer/yay/v13/pkg/text" +) + +func buildSearchFlowBenchmarkData(repoCount, aurCount int) (*mock.DBExecutor, *mockaur.MockAUR, *text.Logger, []string) { + logger := text.NewLogger(io.Discard, io.Discard, strings.NewReader(""), false, "bench") + extraDB := mock.NewDB("extra") + communityDB := mock.NewDB("community") + + repoPkgs := make([]mock.IPackage, 0, repoCount) + for i := range repoCount { + db := extraDB + if i%2 == 1 { + db = communityDB + } + + repoPkgs = append(repoPkgs, &mock.Package{ + PBase: fmt.Sprintf("yay-tool-%04d", i), + PName: fmt.Sprintf("yay-tool-%04d", i), + PVersion: fmt.Sprintf("1.%d.0", i%17), + PDescription: fmt.Sprintf("Yet another yay tool package %04d for search benchmarking", i), + PSize: int64(1024 + i), + PISize: int64(2048 + i), + PDB: db, + PArchitecture: "x86_64", + PProvides: mock.DependList{Depends: []mock.Depend{{Name: fmt.Sprintf("yay-provider-%04d", i)}, {Name: "yay"}}}, + }) + } + + aurPkgs := make([]aur.Pkg, 0, aurCount) + for i := range aurCount { + aurPkgs = append(aurPkgs, aur.Pkg{ + Description: fmt.Sprintf("Yet another yay helper package %04d with search benchmark metadata", i), + FirstSubmitted: 1_500_000_000 + i, + ID: 2_000_000 + i, + LastModified: 1_760_000_000 + i, + Maintainer: "bench", + Name: fmt.Sprintf("yay-aur-%04d", i), + NumVotes: 100 + (i % 250), + OutOfDate: 0, + PackageBase: fmt.Sprintf("yay-aur-%04d", i), + PackageBaseID: 300_000 + i, + Popularity: 1.0 + float64(i%100)/10, + URL: "https://example.invalid/yay", + URLPath: "/snapshot.tar.gz", + Version: fmt.Sprintf("2.%d.0", i%23), + Provides: []string{"yay"}, + }) + } + + mockDB := &mock.DBExecutor{ + ReposFn: func() []string { + return []string{"extra", "community"} + }, + SyncPackagesFn: func(pkgs ...string) []mock.IPackage { + return repoPkgs + }, + LocalPackageFn: func(string) mock.IPackage { + return nil + }, + } + + mockAUR := &mockaur.MockAUR{ + GetFn: func(ctx context.Context, query *aur.Query) ([]aur.Pkg, error) { + return aurPkgs, nil + }, + } + + return mockDB, mockAUR, logger, []string{"yay"} +} + +func BenchmarkExecuteSearchFlowLarge(b *testing.B) { + mockDB, mockAUR, logger, searchTerms := buildSearchFlowBenchmarkData(1500, 1500) + + b.ReportAllocs() + for b.Loop() { + qb := NewSourceQueryBuilder( + mockAUR, + logger, + "", + parser.ModeAny, + "", + false, + false, + true, + ) + qb.Execute(b.Context(), mockDB, searchTerms) + } +}