Merge pull request #641 from Morganamilo/parallel-G

Use goroutines for fetching packages via -G

doc/yay.8

@@ -31,7 +31,8 @@ Perform yay specific print operations.
 
 .TP
 .B \-G, \-\-getpkgbuild
-Downloads PKGBUILD from ABS or AUR.
+Downloads PKGBUILD from ABS or AUR. ABS pkgbuilds are always downloaded using
+tarballs and taken from trunk. The ABS can only be used for Arch Linux repositories
 
 .RE
 If no arguments are provided 'yay \-Syu' will be performed.
@@ -276,7 +277,7 @@ Show a detailed list of updates in a similar format to VerbosePkgLists.
 Upgrades can also be skipped using numbers, number ranges or repo names.
 Adidionally ^ can be used to invert the selection.
 
-\fBWarning\fR: It is not recommended to skip updates from the repositores as
+\fBWarning\fR: It is not recommended to skip updates from the repositories as
 this can lead to partial upgrades. This feature is intended to easily skip AUR
 updates on the fly that may be broken or have a long compile time. Ultimately
 it is up to the user what upgrades they skip.

download.go

@@ -8,6 +8,9 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
+
+	alpm "github.com/jguer/go-alpm"
 )
 
 // Decide what download method to use:
@@ -103,10 +106,10 @@ func gitDiff(path string, name string) error {
 }
 
 // DownloadAndUnpack downloads url tgz and extracts to path.
-func downloadAndUnpack(url string, path string) (err error) {
-	err = os.MkdirAll(path, 0755)
+func downloadAndUnpack(url string, path string) error {
+	err := os.MkdirAll(path, 0755)
 	if err != nil {
-		return
+		return err
 	}
 
 	fileName := filepath.Base(url)
@@ -116,15 +119,15 @@ func downloadAndUnpack(url string, path string) (err error) {
 
 	err = downloadFile(tarLocation, url)
 	if err != nil {
-		return
+		return err
 	}
 
-	err = exec.Command(config.TarBin, "-xf", tarLocation, "-C", path).Run()
+	_, stderr, err := capture(exec.Command(config.TarBin, "-xf", tarLocation, "-C", path))
 	if err != nil {
-		return
+		return fmt.Errorf("%s", stderr)
 	}
 
-	return
+	return nil
 }
 
 func getPkgbuilds(pkgs []string) error {
@@ -135,9 +138,18 @@ func getPkgbuilds(pkgs []string) error {
 	}
 
 	pkgs = removeInvalidTargets(pkgs)
-
 	aur, repo, err := packageSlices(pkgs)
 
+	for n := range aur {
+		_, pkg := splitDbFromName(aur[n])
+		aur[n] = pkg
+	}
+
+	info, err := aurInfoPrint(aur)
+	if err != nil {
+		return err
+	}
+
 	if len(repo) > 0 {
 		missing, err = getPkgbuildsfromABS(repo, wd)
 		if err != nil {
@@ -146,11 +158,13 @@ func getPkgbuilds(pkgs []string) error {
 	}
 
 	if len(aur) > 0 {
-		_missing, err := getPkgbuildsfromAUR(aur, wd)
-		if err != nil {
+		bases := getBases(info)
+		toSkip := pkgbuildsToSkip(bases, nil)
+		if _, err = downloadPkgbuilds(bases, toSkip, wd); err != nil {
 			return err
 		}
-		missing = missing || _missing
+
+		missing = missing || len(aur) != len(info)
 	}
 
 	if missing {
@@ -161,107 +175,94 @@ func getPkgbuilds(pkgs []string) error {
 }
 
 // GetPkgbuild downloads pkgbuild from the ABS.
-func getPkgbuildsfromABS(pkgs []string, path string) (missing bool, err error) {
+func getPkgbuildsfromABS(pkgs []string, path string) (bool, error) {
+	var wg sync.WaitGroup
+	var mux sync.Mutex
+	var errs MultiError
+	names := make(map[string]string)
+	missing := make([]string, 0)
+	downloaded := 0
+
 	dbList, err := alpmHandle.SyncDbs()
 	if err != nil {
-		return
+		return false, err
 	}
 
-nextPkg:
 	for _, pkgN := range pkgs {
+		var pkg *alpm.Package
+		var err error
+		var url string
 		pkgDb, name := splitDbFromName(pkgN)
 
-		for _, db := range dbList.Slice() {
-			if pkgDb != "" && db.Name() != pkgDb {
-				continue
+		if pkgDb != "" {
+			if db, err := alpmHandle.SyncDbByName(pkgDb); err == nil {
+				pkg, err = db.PkgByName(name)
 			}
-
-			pkg, err := db.PkgByName(name)
-			if err == nil {
-				var url string
-				name := pkg.Base()
-				if name == "" {
-					name = pkg.Name()
+		} else {
+			dbList.ForEach(func(db alpm.Db) error {
+				if pkg, err = db.PkgByName(name); err == nil {
+					return fmt.Errorf("")
 				}
-
-				if _, err := os.Stat(filepath.Join(path, name)); err == nil {
-					fmt.Println(bold(red(arrow)), bold(cyan(name)), "directory already exists")
-					continue nextPkg
-				}
-
-				switch db.Name() {
-				case "core", "extra":
-					url = "https://git.archlinux.org/svntogit/packages.git/snapshot/packages/" + name + ".tar.gz"
-				case "community", "multilib":
-					url = "https://git.archlinux.org/svntogit/community.git/snapshot/packages/" + name + ".tar.gz"
-				default:
-					fmt.Println(pkgN, "not in standard repositories")
-					continue nextPkg
-				}
-
-				errD := downloadAndUnpack(url, cacheHome)
-				if errD != nil {
-					fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name())), bold(red(errD.Error())))
-				}
-
-				errD = exec.Command("mv", filepath.Join(cacheHome, "packages", name, "trunk"), filepath.Join(path, name)).Run()
-				if errD != nil {
-					fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name())), bold(red(errD.Error())))
-				} else {
-					fmt.Println(bold(yellow(arrow)), "Downloaded", cyan(pkg.Name()), "from ABS")
-				}
-
-				continue nextPkg
-			}
+				return nil
+			})
 		}
 
-		fmt.Println(pkgN, "could not find package in database")
-		missing = true
-	}
-
-	if _, err := os.Stat(filepath.Join(cacheHome, "packages")); err == nil {
-		os.RemoveAll(filepath.Join(cacheHome, "packages"))
-	}
-
-	return
-}
-
-// GetPkgbuild downloads pkgbuild from the AUR.
-func getPkgbuildsfromAUR(pkgs []string, dir string) (bool, error) {
-	missing := false
-	strippedPkgs := make([]string, 0)
-	for _, pkg := range pkgs {
-		_, name := splitDbFromName(pkg)
-		strippedPkgs = append(strippedPkgs, name)
-	}
-
-	aq, err := aurInfoPrint(strippedPkgs)
-	if err != nil {
-		return missing, err
-	}
-
-	for _, pkg := range aq {
-		if _, err := os.Stat(filepath.Join(dir, pkg.PackageBase)); err == nil {
-			fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name)), "directory already exists")
+		if pkg == nil {
+			missing = append(missing, name)
 			continue
 		}
 
-		if shouldUseGit(filepath.Join(dir, pkg.PackageBase)) {
-			_, err = gitDownload(baseURL+"/"+pkg.PackageBase+".git", dir, pkg.PackageBase)
-		} else {
-			err = downloadAndUnpack(baseURL+aq[0].URLPath, dir)
+		name = pkg.Base()
+		if name == "" {
+			name = pkg.Name()
 		}
 
+		switch pkg.DB().Name() {
+		case "core", "extra", "testing":
+			url = "https://git.archlinux.org/svntogit/packages.git/snapshot/packages/" + name + ".tar.gz"
+		case "community", "multilib", "community-testing", "multilib-testing":
+			url = "https://git.archlinux.org/svntogit/community.git/snapshot/packages/" + name + ".tar.gz"
+		default:
+			missing = append(missing, name)
+			continue
+		}
+
+		if err = os.RemoveAll(filepath.Join(path, name)); err != nil {
+			fmt.Println(bold(red(smallArrow)), err)
+			continue
+		}
+
+		names[name] = url
+	}
+
+	if len(missing) != 0 {
+		fmt.Println(yellow(bold(smallArrow)), "Missing ABS packages: ", cyan(strings.Join(missing, "  ")))
+	}
+
+	download := func(pkg string, url string) {
+		defer wg.Done()
+		if err := downloadAndUnpack(url, cacheHome); err != nil {
+			errs.Add(fmt.Errorf("%s Failed to get pkgbuild: %s: %s", bold(red(arrow)), bold(cyan(pkg)), bold(red(err.Error()))))
+			return
+		}
+
+		_, stderr, err := capture(exec.Command("mv", filepath.Join(cacheHome, "packages", pkg, "trunk"), filepath.Join(path, pkg)))
+		mux.Lock()
+		downloaded++
 		if err != nil {
-			fmt.Println(err)
+			errs.Add(fmt.Errorf("%s Failed to move %s: %s", bold(red(arrow)), bold(cyan(pkg)), bold(red(string(stderr)))))
 		} else {
-			fmt.Println(bold(yellow(arrow)), "Downloaded", cyan(pkg.PackageBase), "from AUR")
+			fmt.Printf(bold(cyan("::"))+" Downloaded PKGBUILD from ABS (%d/%d): %s\n", downloaded, len(names), cyan(pkg))
 		}
+		mux.Unlock()
 	}
 
-	if len(aq) != len(pkgs) {
-		missing = true
+	for name, url := range names {
+		wg.Add(1)
+		go download(name, url)
 	}
 
-	return missing, err
+	wg.Wait()
+	errs.Add(os.RemoveAll(filepath.Join(cacheHome, "packages")))
+	return len(missing) != 0, errs.Return()
 }

exec.go

@@ -25,8 +25,8 @@ func capture(cmd *exec.Cmd) (string, string, error) {
 	cmd.Stdout = &outbuf
 	cmd.Stderr = &errbuf
 	err := cmd.Run()
-	stdout := outbuf.String()
-	stderr := errbuf.String()
+	stdout := strings.TrimSpace(outbuf.String())
+	stderr := strings.TrimSpace(errbuf.String())
 
 	return stdout, stderr, err
 }

install.go

@@ -185,7 +185,7 @@ func install(parser *arguments) error {
 	}
 
 	toSkip := pkgbuildsToSkip(do.Aur, targets)
-	cloned, err := downloadPkgbuilds(do.Aur, toSkip)
+	cloned, err := downloadPkgbuilds(do.Aur, toSkip, config.BuildDir)
 	if err != nil {
 		return err
 	}
@@ -809,7 +809,7 @@ func mergePkgbuilds(bases []Base) error {
 	return nil
 }
 
-func downloadPkgbuilds(bases []Base, toSkip stringSet) (stringSet, error) {
+func downloadPkgbuilds(bases []Base, toSkip stringSet, buildDir string) (stringSet, error) {
 	cloned := make(stringSet)
 	downloaded := 0
 	var wg sync.WaitGroup
@@ -830,7 +830,7 @@ func downloadPkgbuilds(bases []Base, toSkip stringSet) (stringSet, error) {
 		}
 
 		if shouldUseGit(filepath.Join(config.BuildDir, pkg)) {
-			clone, err := gitDownload(baseURL+"/"+pkg+".git", config.BuildDir, pkg)
+			clone, err := gitDownload(baseURL+"/"+pkg+".git", buildDir, pkg)
 			if err != nil {
 				errs.Add(err)
 				return
@@ -841,7 +841,7 @@ func downloadPkgbuilds(bases []Base, toSkip stringSet) (stringSet, error) {
 				mux.Unlock()
 			}
 		} else {
-			err := downloadAndUnpack(baseURL+base.URLPath(), config.BuildDir)
+			err := downloadAndUnpack(baseURL+base.URLPath(), buildDir)
 			if err != nil {
 				errs.Add(err)
 				return

utils.go

@@ -139,10 +139,10 @@ func (err *MultiError) Error() string {
 	str := ""
 
 	for _, e := range err.Errors {
-		str += e.Error()
+		str += e.Error() + "\n"
 	}
 
-	return str
+	return str[:len(str)-1]
 }
 
 func (err *MultiError) Add(e error) {

vcs.go

@@ -38,7 +38,7 @@ func createDevelDB() error {
 
 	bases := getBases(info)
 	toSkip := pkgbuildsToSkip(bases, sliceToStringSet(remoteNames))
-	downloadPkgbuilds(bases, toSkip)
+	downloadPkgbuilds(bases, toSkip, config.BuildDir)
 	srcinfos, _ := parseSrcinfoFiles(bases, false)
 
 	for _, pkgbuild := range srcinfos {