diff --git a/cmd.go b/cmd.go index 6c8e6883..5b784bbb 100644 --- a/cmd.go +++ b/cmd.go @@ -330,7 +330,7 @@ func handleUpgrade(ctx context.Context, config *settings.Configuration, dbExecutor db.Executor, cmdArgs *parser.Arguments, ) error { if cmdArgs.ExistsArg("i", "install") { - return installLocalPKGBUILD(ctx, cmdArgs, dbExecutor, config.Runtime.AURClient, false) + return installLocalPKGBUILD(ctx, cmdArgs, dbExecutor, false) } return config.Runtime.CmdBuilder.Show(config.Runtime.CmdBuilder.BuildPacmanCmd(ctx, diff --git a/go.mod b/go.mod index 26ef933d..53976ada 100644 --- a/go.mod +++ b/go.mod @@ -24,4 +24,21 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) +require github.com/tidwall/gjson v1.14.3 + +require ( + github.com/josharian/intern v1.0.0 // indirect + github.com/pkg/profile v1.6.0 // indirect +) + +require ( + github.com/goccy/go-json v0.9.11 // indirect + github.com/itchyny/gojq v0.12.8 // indirect + github.com/itchyny/timefmt-go v0.1.3 // indirect + github.com/mailru/easyjson v0.7.7 + github.com/ohler55/ojg v1.14.4 + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect +) + go 1.19 diff --git a/go.sum b/go.sum index 25346a34..29037e69 100644 --- a/go.sum +++ b/go.sum @@ -17,16 +17,34 @@ github.com/bradleyjkemp/cupaloy v2.3.0+incompatible/go.mod h1:Au1Xw1sgaJ5iSFktEh github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/itchyny/gojq v0.12.8 h1:Zxcwq8w4IeR8JJYEtoG2MWJZUv0RGY6QqJcO1cqV8+A= +github.com/itchyny/gojq v0.12.8/go.mod h1:gE2kZ9fVRU0+JAksaTzjIlgnCa2akU+a1V0WXgJQN5c= +github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/leonelquinteros/gotext v1.5.0 h1:ODY7LzLpZWWSJdAHnzhreOr6cwLXTAmc914FOauSkBM= github.com/leonelquinteros/gotext v1.5.0/go.mod h1:OCiUVHuhP9LGFBQ1oAmdtNCHJCiHiQA8lf4nAifHkr0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/ohler55/ojg v1.14.4 h1:L2ds8AlB5t/QbqSfhRwvagJzQ7pgmdrefMIypQs0Xik= +github.com/ohler55/ojg v1.14.4/go.mod h1:7Ghirupn8NC8hSSDpI0gcjorPxj+vSVIONDWfliHR1k= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.6.0 h1:hUDfIISABYI59DyeB3OTay/HxSRwTQ8rB/H83k6r5dM= +github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -36,6 +54,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= +github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -44,6 +68,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc= @@ -53,6 +79,7 @@ golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20200221224223-e1da425f72fd/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= diff --git a/local_install.go b/local_install.go index e8bbcfe7..c96c7c0d 100644 --- a/local_install.go +++ b/local_install.go @@ -2,7 +2,6 @@ package main import ( "context" - "fmt" "os" "path/filepath" "strings" @@ -10,7 +9,7 @@ import ( "github.com/Jguer/aur" "github.com/Jguer/yay/v11/pkg/db" "github.com/Jguer/yay/v11/pkg/dep" - "github.com/Jguer/yay/v11/pkg/query" + "github.com/Jguer/yay/v11/pkg/metadata" "github.com/Jguer/yay/v11/pkg/settings/parser" "github.com/Jguer/yay/v11/pkg/topo" gosrc "github.com/Morganamilo/go-srcinfo" @@ -91,9 +90,13 @@ func installLocalPKGBUILD( ctx context.Context, cmdArgs *parser.Arguments, dbExecutor db.Executor, - aurClient aur.ClientInterface, ignoreProviders bool, ) error { + aurCache, err := metadata.NewAURCache("aur.json") + if err != nil { + return errors.Wrap(err, gotext.Get("failed to retrieve aur Cache")) + } + wd, err := os.Getwd() if err != nil { return errors.Wrap(err, gotext.Get("failed to retrieve working directory")) @@ -121,15 +124,19 @@ func installLocalPKGBUILD( for _, pkg := range aurPkgs { depSlice := dep.ComputeCombinedDepList(&pkg, false, false) - addNodes(dbExecutor, aurClient, pkg.Name, pkg.PackageBase, depSlice, graph) + addNodes(dbExecutor, aurCache, pkg.Name, pkg.PackageBase, depSlice, graph) } - fmt.Println(graph) + // fmt.Println(graph) + // aurCache.DebugInfo() + + // topoSorted := graph.TopoSortedLayers() + // fmt.Println(topoSorted, len(topoSorted)) return nil } -func addNodes(dbExecutor db.Executor, aurClient aur.ClientInterface, pkgName string, pkgBase string, deps []string, graph *topo.Graph[string]) { +func addNodes(dbExecutor db.Executor, aurCache *metadata.AURCache, pkgName string, pkgBase string, deps []string, graph *topo.Graph[string]) { graph.AddNode(pkgBase) graph.Alias(pkgBase, pkgName) graph.SetNodeInfo(pkgBase, &topo.NodeInfo{Color: "blue"}) @@ -137,17 +144,15 @@ func addNodes(dbExecutor db.Executor, aurClient aur.ClientInterface, pkgName str for _, depString := range deps { depName, _, _ := splitDep(depString) - graph.DependOn(depName, pkgBase) - - warnings := query.AURWarnings{} - if dbExecutor.LocalSatisfierExists(depString) { - graph.SetNodeInfo(depName, &topo.NodeInfo{Color: "green"}) continue - } else { - graph.SetNodeInfo(depName, &topo.NodeInfo{Color: "red"}) + } else if graph.Exists(depName) { + graph.DependOn(depName, pkgBase) + continue } + graph.DependOn(depName, pkgBase) + // Check ALPM if alpmPkg := dbExecutor.SyncSatisfier(depString); alpmPkg != nil { newDeps := alpmPkg.Depends().Slice() @@ -157,14 +162,21 @@ func addNodes(dbExecutor db.Executor, aurClient aur.ClientInterface, pkgName str newDepsSlice = append(newDepsSlice, newDep.Name) } - addNodes(dbExecutor, aurClient, alpmPkg.Name(), alpmPkg.Base(), newDepsSlice, graph) + if len(newDeps) == 0 { + continue + } + + addNodes(dbExecutor, aurCache, alpmPkg.Name(), alpmPkg.Base(), newDepsSlice, graph) // Check AUR - } else if aurPkgs, _ := query.AURInfo(context.TODO(), aurClient, []string{depName}, &warnings, 1); len(aurPkgs) != 0 { + } else if aurPkgs, _ := aurCache.FindDep(depName); len(aurPkgs) != 0 { pkg := aurPkgs[0] newDeps := dep.ComputeCombinedDepList(pkg, false, false) - newDepsSlice := make([]string, 0, len(newDeps)) - addNodes(dbExecutor, aurClient, pkg.PackageBase, pkg.Name, newDepsSlice, graph) + if len(newDeps) == 0 { + continue + } + + addNodes(dbExecutor, aurCache, pkg.PackageBase, pkg.Name, newDeps, graph) } } } diff --git a/pkg/metadata/metadata_aur.go b/pkg/metadata/metadata_aur.go new file mode 100644 index 00000000..95f76463 --- /dev/null +++ b/pkg/metadata/metadata_aur.go @@ -0,0 +1,124 @@ +package metadata + +import ( + "encoding/json" + "fmt" + "log" + + "github.com/Jguer/aur" + "github.com/itchyny/gojq" + "github.com/ohler55/ojg/oj" + "github.com/tidwall/gjson" +) + +type AURCache struct { + cache []byte + provideCache map[string][]*aur.Pkg + unmarshalledCache []interface{} + cacheHits int + gojqCode *gojq.Code +} + +func NewAURCache(cachePath string) (*AURCache, error) { + aurCache, err := MakeOrReadCache(cachePath) + if err != nil { + return nil, err + } + inputStruct, err := oj.Parse(aurCache) + + return &AURCache{ + cache: aurCache, + provideCache: make(map[string][]*aur.Pkg, 300), + unmarshalledCache: inputStruct.([]interface{}), + gojqCode: makeGoJQ(), + }, nil +} + +func (a *AURCache) DebugInfo() { + fmt.Println("Byte Cache", len(a.cache)) + fmt.Println("Entries Cached", len(a.provideCache)) + fmt.Println("Cache Hits", a.cacheHits) +} + +func (a *AURCache) FindDep(depName string) ([]*aur.Pkg, error) { + if pkgs, ok := a.provideCache[depName]; ok { + a.cacheHits++ + return pkgs, nil + } + + final, error := a.gojqGet(depName) + if error != nil { + return nil, error + } + + a.provideCache[depName] = final + + return final, nil +} + +func (a *AURCache) gjsonGet(depName string) ([]*aur.Pkg, error) { + dedupMap := make(map[string]bool) + queryProvides := fmt.Sprintf("#(Provides.#(==\"%s\"))#", depName) + queryNames := fmt.Sprintf("#(Name==\"%s\")#", depName) + queryBases := fmt.Sprintf("#(PackageBase==\"%s\")#", depName) + + results := gjson.GetManyBytes(a.cache, queryProvides, queryNames, queryBases) + + aggregated := append(append(results[0].Array(), results[1].Array()...), results[2].Array()...) + + final := make([]*aur.Pkg, 0, len(aggregated)) + + for i := range aggregated { + jsonString := aggregated[i].Raw + key := jsonString[:15] + + if _, ok := dedupMap[key]; !ok { + pkg := &aur.Pkg{} + json.Unmarshal([]byte(jsonString), pkg) + final = append(final, pkg) + dedupMap[key] = true + } + } + return final, nil +} + +func (a *AURCache) gojqGet(searchTerm string) ([]*aur.Pkg, error) { + final := make([]*aur.Pkg, 0, 1) + + iter := a.gojqCode.Run(a.unmarshalledCache, searchTerm) // or query.RunWithContext + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + return nil, err + } + + pkg := &aur.Pkg{} + bValue, err := gojq.Marshal(v) + if err != nil { + log.Fatalln(err) + } + + json.Unmarshal(bValue, pkg) + final = append(final, pkg) + } + + return final, nil +} + +func makeGoJQ() *gojq.Code { + pattern := ".[] | select((.PackageBase == $x) or (.Name == $x) or (.Provides[]? == ($x)))" + query, err := gojq.Parse(pattern) + if err != nil { + log.Fatalln(err) + } + + compiled, err := gojq.Compile(query, gojq.WithVariables([]string{"$x"})) + if err != nil { + log.Fatalln(err) + } + + return compiled +} diff --git a/pkg/metadata/metadata_downloader.go b/pkg/metadata/metadata_downloader.go new file mode 100644 index 00000000..02f94a25 --- /dev/null +++ b/pkg/metadata/metadata_downloader.go @@ -0,0 +1,103 @@ +package metadata + +import ( + "fmt" + "io" + "net/http" + "os" +) + +func main() { + // check if cache exists + cachePath := "aur.json" + cacheBytes, err := ReadCache(cachePath) + if err != nil { + fmt.Println(err) + return + } + + if len(cacheBytes) == 0 { + cacheBytes, err = MakeCache(cachePath) + if err != nil { + fmt.Println(err) + return + } + } +} + +func MakeOrReadCache(cachePath string) ([]byte, error) { + cacheBytes, err := ReadCache(cachePath) + if err != nil { + return nil, err + } + + if len(cacheBytes) == 0 { + cacheBytes, err = MakeCache(cachePath) + if err != nil { + return nil, err + } + } + + return cacheBytes, nil +} + +func ReadCache(cachePath string) ([]byte, error) { + fp, err := os.Open(cachePath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + + return nil, err + } + + defer fp.Close() + + s, err := io.ReadAll(fp) + if err != nil { + return nil, err + } + + return s, nil +} + +// Download the metadata for aur packages. +// create cache file +// write to cache file +func MakeCache(cachePath string) ([]byte, error) { + body, err := downloadAURMetadata() + if err != nil { + return nil, err + } + defer body.Close() + + s, err := io.ReadAll(body) + if err != nil { + return nil, err + } + + f, err := os.Create(cachePath) + if err != nil { + return nil, err + } + defer f.Close() + + if _, err = f.Write(s); err != nil { + return nil, err + } + + return s, err +} + +func downloadAURMetadata() (io.ReadCloser, error) { + resp, err := http.Get("https://aur.archlinux.org/packages-meta-ext-v1.json.gz") + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to download metadata: %s", resp.Status) + } + + return resp.Body, nil +} diff --git a/pkg/topo/dep.go b/pkg/topo/dep.go index eca2998e..4502cb02 100644 --- a/pkg/topo/dep.go +++ b/pkg/topo/dep.go @@ -43,6 +43,20 @@ func New[T comparable]() *Graph[T] { } } +func (g *Graph[T]) Len() int { + return len(g.nodes) +} + +func (g *Graph[T]) Exists(node T) bool { + // check aliases + if aliasNode, ok := g.alias[node]; ok { + node = aliasNode + } + + _, ok := g.nodes[node] + return ok +} + func (g *Graph[T]) Alias(node, alias T) error { if alias == node { return ErrSelfReferential