From cba56c3f9e60f8091e43516932bd72e1d393285e Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sat, 18 Aug 2018 00:16:04 +0100 Subject: [PATCH 1/6] Use downloadPkgbuilds for -G --- download.go | 53 +++++++++-------------------------------------------- install.go | 8 ++++---- vcs.go | 2 +- 3 files changed, 14 insertions(+), 49 deletions(-) diff --git a/download.go b/download.go index ae8c38d6..35283380 100644 --- a/download.go +++ b/download.go @@ -135,8 +135,11 @@ func getPkgbuilds(pkgs []string) error { } pkgs = removeInvalidTargets(pkgs) - aur, repo, err := packageSlices(pkgs) + info, err := aurInfoPrint(aur) + if err != nil { + return err + } if len(repo) > 0 { missing, err = getPkgbuildsfromABS(repo, wd) @@ -146,11 +149,13 @@ func getPkgbuilds(pkgs []string) error { } if len(aur) > 0 { - _missing, err := getPkgbuildsfromAUR(aur, wd) - if err != nil { + bases := getBases(info) + toSkip := pkgbuildsToSkip(bases, nil) + if _, err = downloadPkgbuilds(bases, toSkip, wd); err != nil { return err } - missing = missing || _missing + + missing = missing || len(aur) != len(info) } if missing { @@ -225,43 +230,3 @@ nextPkg: return } - -// GetPkgbuild downloads pkgbuild from the AUR. -func getPkgbuildsfromAUR(pkgs []string, dir string) (bool, error) { - missing := false - strippedPkgs := make([]string, 0) - for _, pkg := range pkgs { - _, name := splitDbFromName(pkg) - strippedPkgs = append(strippedPkgs, name) - } - - aq, err := aurInfoPrint(strippedPkgs) - if err != nil { - return missing, err - } - - for _, pkg := range aq { - if _, err := os.Stat(filepath.Join(dir, pkg.PackageBase)); err == nil { - fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name)), "directory already exists") - continue - } - - if shouldUseGit(filepath.Join(dir, pkg.PackageBase)) { - _, err = gitDownload(baseURL+"/"+pkg.PackageBase+".git", dir, pkg.PackageBase) - } else { - err = downloadAndUnpack(baseURL+aq[0].URLPath, dir) - } - - if err != nil { - fmt.Println(err) - } else { - fmt.Println(bold(yellow(arrow)), "Downloaded", cyan(pkg.PackageBase), "from AUR") - } - } - - if len(aq) != len(pkgs) { - missing = true - } - - return missing, err -} diff --git a/install.go b/install.go index 3275e379..42063d06 100644 --- a/install.go +++ b/install.go @@ -185,7 +185,7 @@ func install(parser *arguments) error { } toSkip := pkgbuildsToSkip(do.Aur, targets) - cloned, err := downloadPkgbuilds(do.Aur, toSkip) + cloned, err := downloadPkgbuilds(do.Aur, toSkip, config.BuildDir) if err != nil { return err } @@ -809,7 +809,7 @@ func mergePkgbuilds(bases []Base) error { return nil } -func downloadPkgbuilds(bases []Base, toSkip stringSet) (stringSet, error) { +func downloadPkgbuilds(bases []Base, toSkip stringSet, buildDir string) (stringSet, error) { cloned := make(stringSet) downloaded := 0 var wg sync.WaitGroup @@ -830,7 +830,7 @@ func downloadPkgbuilds(bases []Base, toSkip stringSet) (stringSet, error) { } if shouldUseGit(filepath.Join(config.BuildDir, pkg)) { - clone, err := gitDownload(baseURL+"/"+pkg+".git", config.BuildDir, pkg) + clone, err := gitDownload(baseURL+"/"+pkg+".git", buildDir, pkg) if err != nil { errs.Add(err) return @@ -841,7 +841,7 @@ func downloadPkgbuilds(bases []Base, toSkip stringSet) (stringSet, error) { mux.Unlock() } } else { - err := downloadAndUnpack(baseURL+base.URLPath(), config.BuildDir) + err := downloadAndUnpack(baseURL+base.URLPath(), buildDir) if err != nil { errs.Add(err) return diff --git a/vcs.go b/vcs.go index 53d949e2..584ed899 100644 --- a/vcs.go +++ b/vcs.go @@ -38,7 +38,7 @@ func createDevelDB() error { bases := getBases(info) toSkip := pkgbuildsToSkip(bases, sliceToStringSet(remoteNames)) - downloadPkgbuilds(bases, toSkip) + downloadPkgbuilds(bases, toSkip, config.BuildDir) srcinfos, _ := parseSrcinfoFiles(bases, false) for _, pkgbuild := range srcinfos { From a1edd09a52c68ba032b6d236c08e2351dd8efc78 Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sat, 18 Aug 2018 04:06:27 +0100 Subject: [PATCH 2/6] Fixup getPkgbuildsFromABS Make it output more similary to downloadPkgbuilds. Refactor to remove all the indention. --- download.go | 113 ++++++++++++++++++++++++++++------------------------ 1 file changed, 62 insertions(+), 51 deletions(-) diff --git a/download.go b/download.go index 35283380..d0af1861 100644 --- a/download.go +++ b/download.go @@ -8,6 +8,8 @@ import ( "os/exec" "path/filepath" "strings" + + alpm "github.com/jguer/go-alpm" ) // Decide what download method to use: @@ -136,6 +138,12 @@ func getPkgbuilds(pkgs []string) error { pkgs = removeInvalidTargets(pkgs) aur, repo, err := packageSlices(pkgs) + + for n := range aur { + _, pkg := splitDbFromName(aur[n]) + aur[n] = pkg + } + info, err := aurInfoPrint(aur) if err != nil { return err @@ -166,67 +174,70 @@ func getPkgbuilds(pkgs []string) error { } // GetPkgbuild downloads pkgbuild from the ABS. -func getPkgbuildsfromABS(pkgs []string, path string) (missing bool, err error) { +func getPkgbuildsfromABS(pkgs []string, path string) (bool, error) { + missing := false dbList, err := alpmHandle.SyncDbs() if err != nil { - return + return missing, err } -nextPkg: - for _, pkgN := range pkgs { + for n, pkgN := range pkgs { pkgDb, name := splitDbFromName(pkgN) + var pkg *alpm.Package + var err error + var url string - for _, db := range dbList.Slice() { - if pkgDb != "" && db.Name() != pkgDb { - continue + if pkgDb != "" { + if db, err := alpmHandle.SyncDbByName(pkgDb); err == nil { + pkg, err = db.PkgByName(name) } - - pkg, err := db.PkgByName(name) - if err == nil { - var url string - name := pkg.Base() - if name == "" { - name = pkg.Name() + } else { + dbList.ForEach(func(db alpm.Db) error { + if pkg, err = db.PkgByName(name); err == nil { + return fmt.Errorf("") } - - if _, err := os.Stat(filepath.Join(path, name)); err == nil { - fmt.Println(bold(red(arrow)), bold(cyan(name)), "directory already exists") - continue nextPkg - } - - switch db.Name() { - case "core", "extra": - url = "https://git.archlinux.org/svntogit/packages.git/snapshot/packages/" + name + ".tar.gz" - case "community", "multilib": - url = "https://git.archlinux.org/svntogit/community.git/snapshot/packages/" + name + ".tar.gz" - default: - fmt.Println(pkgN, "not in standard repositories") - continue nextPkg - } - - errD := downloadAndUnpack(url, cacheHome) - if errD != nil { - fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name())), bold(red(errD.Error()))) - } - - errD = exec.Command("mv", filepath.Join(cacheHome, "packages", name, "trunk"), filepath.Join(path, name)).Run() - if errD != nil { - fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name())), bold(red(errD.Error()))) - } else { - fmt.Println(bold(yellow(arrow)), "Downloaded", cyan(pkg.Name()), "from ABS") - } - - continue nextPkg - } + return nil + }) } - fmt.Println(pkgN, "could not find package in database") - missing = true + if pkg == nil { + fmt.Println(name, "could not find package in database") + missing = true + continue + } + + name = pkg.Base() + if name == "" { + name = pkg.Name() + } + + if err = os.RemoveAll(filepath.Join(path, name)); err != nil { + fmt.Println(err) + continue + } + + switch pkg.DB().Name() { + case "core", "extra", "testing": + url = "https://git.archlinux.org/svntogit/packages.git/snapshot/packages/" + name + ".tar.gz" + case "community", "multilib", "community-testing", "multilib-testing": + url = "https://git.archlinux.org/svntogit/community.git/snapshot/packages/" + name + ".tar.gz" + default: + fmt.Println(name, "not in standard repositories") + continue + } + + if err = downloadAndUnpack(url, cacheHome); err != nil { + fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name())), bold(red(err.Error()))) + } + + err = exec.Command("mv", filepath.Join(cacheHome, "packages", name, "trunk"), filepath.Join(path, name)).Run() + if err != nil { + fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name())), bold(red(err.Error()))) + } else { + fmt.Printf(bold(cyan("::"))+" Downloaded PKGBUILD from ABS (%d/%d): %s\n", n+1, len(pkgs), cyan(pkg.Name())) + } } - if _, err := os.Stat(filepath.Join(cacheHome, "packages")); err == nil { - os.RemoveAll(filepath.Join(cacheHome, "packages")) - } - - return + err = os.RemoveAll(filepath.Join(cacheHome, "packages")) + return missing, err } From a8e4f18e6bbe90d403e0113ef83fbf185bba064c Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sat, 18 Aug 2018 06:33:44 +0100 Subject: [PATCH 3/6] MultiError: add new lines between errors --- utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils.go b/utils.go index 3762dd56..28a48c09 100644 --- a/utils.go +++ b/utils.go @@ -139,10 +139,10 @@ func (err *MultiError) Error() string { str := "" for _, e := range err.Errors { - str += e.Error() + str += e.Error() + "\n" } - return str + return str[:len(str)-1] } func (err *MultiError) Add(e error) { From 8711562923408a97a4cbbc4de7f05f9245baa81b Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sat, 18 Aug 2018 06:38:32 +0100 Subject: [PATCH 4/6] capture: strip whitespace on outputs --- exec.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exec.go b/exec.go index facecf71..0049af48 100644 --- a/exec.go +++ b/exec.go @@ -25,8 +25,8 @@ func capture(cmd *exec.Cmd) (string, string, error) { cmd.Stdout = &outbuf cmd.Stderr = &errbuf err := cmd.Run() - stdout := outbuf.String() - stderr := errbuf.String() + stdout := strings.TrimSpace(outbuf.String()) + stderr := strings.TrimSpace(errbuf.String()) return stdout, stderr, err } From e547173d42c90ee1ad7d1a253fd3c2916927c0cf Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sat, 18 Aug 2018 06:41:16 +0100 Subject: [PATCH 5/6] Use goroutines for fetching abs packages --- download.go | 83 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 29 deletions(-) diff --git a/download.go b/download.go index d0af1861..e1bcefad 100644 --- a/download.go +++ b/download.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync" alpm "github.com/jguer/go-alpm" ) @@ -105,10 +106,10 @@ func gitDiff(path string, name string) error { } // DownloadAndUnpack downloads url tgz and extracts to path. -func downloadAndUnpack(url string, path string) (err error) { - err = os.MkdirAll(path, 0755) +func downloadAndUnpack(url string, path string) error { + err := os.MkdirAll(path, 0755) if err != nil { - return + return err } fileName := filepath.Base(url) @@ -118,15 +119,15 @@ func downloadAndUnpack(url string, path string) (err error) { err = downloadFile(tarLocation, url) if err != nil { - return + return err } - err = exec.Command(config.TarBin, "-xf", tarLocation, "-C", path).Run() + _, stderr, err := capture(exec.Command(config.TarBin, "-xf", tarLocation, "-C", path)) if err != nil { - return + return fmt.Errorf("%s", stderr) } - return + return nil } func getPkgbuilds(pkgs []string) error { @@ -175,17 +176,23 @@ func getPkgbuilds(pkgs []string) error { // GetPkgbuild downloads pkgbuild from the ABS. func getPkgbuildsfromABS(pkgs []string, path string) (bool, error) { - missing := false + var wg sync.WaitGroup + var mux sync.Mutex + var errs MultiError + names := make(map[string]string) + missing := make([]string, 0) + downloaded := 0 + dbList, err := alpmHandle.SyncDbs() if err != nil { - return missing, err + return false, err } - for n, pkgN := range pkgs { - pkgDb, name := splitDbFromName(pkgN) + for _, pkgN := range pkgs { var pkg *alpm.Package var err error var url string + pkgDb, name := splitDbFromName(pkgN) if pkgDb != "" { if db, err := alpmHandle.SyncDbByName(pkgDb); err == nil { @@ -201,8 +208,7 @@ func getPkgbuildsfromABS(pkgs []string, path string) (bool, error) { } if pkg == nil { - fmt.Println(name, "could not find package in database") - missing = true + missing = append(missing, name) continue } @@ -211,33 +217,52 @@ func getPkgbuildsfromABS(pkgs []string, path string) (bool, error) { name = pkg.Name() } - if err = os.RemoveAll(filepath.Join(path, name)); err != nil { - fmt.Println(err) - continue - } - switch pkg.DB().Name() { case "core", "extra", "testing": url = "https://git.archlinux.org/svntogit/packages.git/snapshot/packages/" + name + ".tar.gz" case "community", "multilib", "community-testing", "multilib-testing": url = "https://git.archlinux.org/svntogit/community.git/snapshot/packages/" + name + ".tar.gz" default: - fmt.Println(name, "not in standard repositories") + missing = append(missing, name) continue } - if err = downloadAndUnpack(url, cacheHome); err != nil { - fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name())), bold(red(err.Error()))) + if err = os.RemoveAll(filepath.Join(path, name)); err != nil { + fmt.Println(bold(red(smallArrow)), err) + continue } - err = exec.Command("mv", filepath.Join(cacheHome, "packages", name, "trunk"), filepath.Join(path, name)).Run() - if err != nil { - fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name())), bold(red(err.Error()))) - } else { - fmt.Printf(bold(cyan("::"))+" Downloaded PKGBUILD from ABS (%d/%d): %s\n", n+1, len(pkgs), cyan(pkg.Name())) - } + names[name] = url } - err = os.RemoveAll(filepath.Join(cacheHome, "packages")) - return missing, err + if len(missing) != 0 { + fmt.Println(yellow(bold(smallArrow)), "Missing ABS packages: ", cyan(strings.Join(missing, " "))) + } + + download := func(pkg string, url string) { + defer wg.Done() + if err := downloadAndUnpack(url, cacheHome); err != nil { + errs.Add(fmt.Errorf("%s Failed to get pkgbuild: %s: %s", bold(red(arrow)), bold(cyan(pkg)), bold(red(err.Error())))) + return + } + + _, stderr, err := capture(exec.Command("mv", filepath.Join(cacheHome, "packages", pkg, "trunk"), filepath.Join(path, pkg))) + mux.Lock() + downloaded++ + if err != nil { + errs.Add(fmt.Errorf("%s Failed to move %s: %s", bold(red(arrow)), bold(cyan(pkg)), bold(red(string(stderr))))) + } else { + fmt.Printf(bold(cyan("::"))+" Downloaded PKGBUILD from ABS (%d/%d): %s\n", downloaded, len(names), cyan(pkg)) + } + mux.Unlock() + } + + for name, url := range names { + wg.Add(1) + go download(name, url) + } + + wg.Wait() + errs.Add(os.RemoveAll(filepath.Join(cacheHome, "packages"))) + return len(missing) != 0, errs.Return() } From 03fee7b7d5b7949655f378043ca770f001c7240d Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sat, 18 Aug 2018 07:02:50 +0100 Subject: [PATCH 6/6] Expand -G documentation --- doc/yay.8 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/yay.8 b/doc/yay.8 index b0aa6584..7d558c76 100644 --- a/doc/yay.8 +++ b/doc/yay.8 @@ -31,7 +31,8 @@ Perform yay specific print operations. .TP .B \-G, \-\-getpkgbuild -Downloads PKGBUILD from ABS or AUR. +Downloads PKGBUILD from ABS or AUR. ABS pkgbuilds are always downloaded using +tarballs and taken from trunk. The ABS can only be used for Arch Linux repositories .RE If no arguments are provided 'yay \-Syu' will be performed. @@ -276,7 +277,7 @@ Show a detailed list of updates in a similar format to VerbosePkgLists. Upgrades can also be skipped using numbers, number ranges or repo names. Adidionally ^ can be used to invert the selection. -\fBWarning\fR: It is not recommended to skip updates from the repositores as +\fBWarning\fR: It is not recommended to skip updates from the repositories as this can lead to partial upgrades. This feature is intended to easily skip AUR updates on the fly that may be broken or have a long compile time. Ultimately it is up to the user what upgrades they skip.