From 5775e3c0b79b5598fd4016ca6ba6afd3cf4a24cc Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sun, 10 Jun 2018 00:42:58 +0100 Subject: [PATCH 1/7] Split git clone/pull and merge function --- download.go | 6 +++++- install.go | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/download.go b/download.go index 6bba35b9..7d761cf9 100644 --- a/download.go +++ b/download.go @@ -71,7 +71,11 @@ func gitDownload(url string, path string, name string) error { return fmt.Errorf("error fetching %s", name) } - err = passToGit(filepath.Join(path, name), "reset", "--hard", "HEAD") + return nil +} + +func gitMerge(url string, path string, name string) error { + err := passToGit(filepath.Join(path, name), "reset", "--hard", "HEAD") if err != nil { return fmt.Errorf("error resetting %s", name) } diff --git a/install.go b/install.go index 4ab48e31..8b45885d 100644 --- a/install.go +++ b/install.go @@ -636,10 +636,16 @@ func downloadPkgBuilds(pkgs []*rpc.Pkg, targets stringSet, bases map[string][]*r var err error if shouldUseGit(filepath.Join(config.BuildDir, pkg.PackageBase)) { - err = gitDownload(baseURL+"/"+pkg.PackageBase+".git", config.BuildDir, pkg.PackageBase) + err = gitDownload(baseURL + "/" + pkg.PackageBase + ".git", config.BuildDir, pkg.PackageBase) if err != nil { return hashes, err } + + err = gitMerge(baseURL + "/" + pkg.PackageBase + ".git", config.BuildDir, pkg.PackageBase) + if err != nil { + return hashes, err + } + } else { err = downloadAndUnpack(baseURL+pkg.URLPath, config.BuildDir) } From 1b8d9317e4aa6b69efcb3cc76c5a11ef1f55aa92 Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sun, 10 Jun 2018 01:04:04 +0100 Subject: [PATCH 2/7] Split pkgbuild downloading and up to date checks --- install.go | 12 +++++++++--- vcs.go | 3 ++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/install.go b/install.go index 8b45885d..5d7e371e 100644 --- a/install.go +++ b/install.go @@ -155,7 +155,8 @@ func install(parser *arguments) error { cleanBuilds(toClean) - oldHashes, err := downloadPkgBuilds(do.Aur, targets, do.Bases) + toSkip := pkgBuildsToSkip(do.Aur, targets) + oldHashes, err := downloadPkgBuilds(do.Aur, do.Bases, toSkip) if err != nil { return err } @@ -593,9 +594,8 @@ func tryParsesrcinfosFile(pkgs []*rpc.Pkg, srcinfos map[string]*gopkg.PKGBUILD, } } -func downloadPkgBuilds(pkgs []*rpc.Pkg, targets stringSet, bases map[string][]*rpc.Pkg) (map[string]string, error) { +func pkgBuildsToSkip(pkgs []*rpc.Pkg, targets stringSet) stringSet { toSkip := make(stringSet) - hashes := make(map[string]string) for _, pkg := range pkgs { if config.ReDownload == "no" || (config.ReDownload == "yes" && !targets.get(pkg.Name)) { @@ -614,6 +614,12 @@ func downloadPkgBuilds(pkgs []*rpc.Pkg, targets stringSet, bases map[string][]*r } } + return toSkip +} + +func downloadPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, toSkip stringSet) (map[string]string, error) { + hashes := make(map[string]string) + for k, pkg := range pkgs { if shouldUseGit(filepath.Join(config.BuildDir, pkg.PackageBase)) { hash, err := gitGetHash(config.BuildDir, pkg.PackageBase) diff --git a/vcs.go b/vcs.go index b6554a05..857185d3 100644 --- a/vcs.go +++ b/vcs.go @@ -43,7 +43,8 @@ func createDevelDB() error { bases := getBases(infoMap) - downloadPkgBuilds(info, sliceToStringSet(remoteNames), bases) + toSkip := pkgBuildsToSkip(info, sliceToStringSet(remoteNames)) + downloadPkgBuilds(info, bases, toSkip) tryParsesrcinfosFile(info, srcinfosStale, bases) for _, pkg := range info { From 73bd713217b253b94646c429b003ad51ec3b1692 Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sun, 10 Jun 2018 01:21:20 +0100 Subject: [PATCH 3/7] Split hash fetching and pkgbuild downloading --- install.go | 44 +++++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/install.go b/install.go index 5d7e371e..94df54e7 100644 --- a/install.go +++ b/install.go @@ -155,8 +155,13 @@ func install(parser *arguments) error { cleanBuilds(toClean) + oldHashes, err := getHashes(do.Aur) + if err != nil { + return err + } + toSkip := pkgBuildsToSkip(do.Aur, targets) - oldHashes, err := downloadPkgBuilds(do.Aur, do.Bases, toSkip) + err = downloadPkgBuilds(do.Aur, do.Bases, toSkip) if err != nil { return err } @@ -617,19 +622,25 @@ func pkgBuildsToSkip(pkgs []*rpc.Pkg, targets stringSet) stringSet { return toSkip } -func downloadPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, toSkip stringSet) (map[string]string, error) { +func getHashes(pkgs []*rpc.Pkg) (map[string]string, error) { hashes := make(map[string]string) - for k, pkg := range pkgs { + for _, pkg := range pkgs { if shouldUseGit(filepath.Join(config.BuildDir, pkg.PackageBase)) { hash, err := gitGetHash(config.BuildDir, pkg.PackageBase) - if err == nil { - hashes[pkg.PackageBase] = hash - } else { - hashes[pkg.PackageBase] = "" + if err != nil { + return hashes, err } - } + hashes[pkg.PackageBase] = hash + } + } + + return hashes, nil +} + +func downloadPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, toSkip stringSet) error { + for k, pkg := range pkgs { if toSkip.get(pkg.PackageBase) { str := bold(cyan("::") + " PKGBUILD up to date, Skipping (%d/%d): %s\n") fmt.Printf(str, k+1, len(pkgs), cyan(formatPkgbase(pkg, bases))) @@ -640,27 +651,26 @@ func downloadPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, toSkip stri fmt.Printf(str, k+1, len(pkgs), cyan(formatPkgbase(pkg, bases))) - var err error if shouldUseGit(filepath.Join(config.BuildDir, pkg.PackageBase)) { - err = gitDownload(baseURL + "/" + pkg.PackageBase + ".git", config.BuildDir, pkg.PackageBase) + err := gitDownload(baseURL + "/" + pkg.PackageBase + ".git", config.BuildDir, pkg.PackageBase) if err != nil { - return hashes, err + return err } err = gitMerge(baseURL + "/" + pkg.PackageBase + ".git", config.BuildDir, pkg.PackageBase) if err != nil { - return hashes, err + return err } } else { - err = downloadAndUnpack(baseURL+pkg.URLPath, config.BuildDir) - } - if err != nil { - return hashes, err + err := downloadAndUnpack(baseURL+pkg.URLPath, config.BuildDir) + if err != nil { + return err + } } } - return hashes, nil + return nil } func downloadPkgBuildsSources(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, incompatible stringSet) (err error) { From 43d2a6601ab7d710ca11fce04e936c5f17a28530 Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sun, 10 Jun 2018 01:29:54 +0100 Subject: [PATCH 4/7] Split pkgbuild downloading and merging --- install.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/install.go b/install.go index 94df54e7..af9a525c 100644 --- a/install.go +++ b/install.go @@ -166,6 +166,11 @@ func install(parser *arguments) error { return err } + err = mergePkgBuilds(do.Aur) + if err != nil { + return err + } + if len(toEdit) > 0 { if config.ShowDiffs { err = showPkgBuildDiffs(toEdit, do.Bases, oldHashes) @@ -639,6 +644,20 @@ func getHashes(pkgs []*rpc.Pkg) (map[string]string, error) { return hashes, nil } +func mergePkgBuilds(pkgs []*rpc.Pkg) error { + for _, pkg := range pkgs { + if shouldUseGit(filepath.Join(config.BuildDir, pkg.PackageBase)) { + err := gitMerge(baseURL + "/" + pkg.PackageBase + ".git", config.BuildDir, pkg.PackageBase) + if err != nil { + return err + } + } + } + + return nil +} + + func downloadPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, toSkip stringSet) error { for k, pkg := range pkgs { if toSkip.get(pkg.PackageBase) { @@ -656,12 +675,6 @@ func downloadPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, toSkip stri if err != nil { return err } - - err = gitMerge(baseURL + "/" + pkg.PackageBase + ".git", config.BuildDir, pkg.PackageBase) - if err != nil { - return err - } - } else { err := downloadAndUnpack(baseURL+pkg.URLPath, config.BuildDir) if err != nil { From 4f8b43cd604c63bbadbe199913802eae31c1971e Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sun, 10 Jun 2018 01:57:17 +0100 Subject: [PATCH 5/7] Show diffs before merging This is what 5775e3..43d2a6 has been leading up to. Git fetch will be called on all pkgbuilds, then the user is offered a chance to view the diffs. If they choose to continue, merging happens. This allows users to abort the install after viewing diffs and still be able to see thoes diffs again if they try to install later on. This also makes the git stuff a little more modular which should help in organzing diff showing + pkgbuild editing. --- download.go | 15 ++++++++++++--- install.go | 54 ++++++++++++++--------------------------------------- 2 files changed, 26 insertions(+), 43 deletions(-) diff --git a/download.go b/download.go index 7d761cf9..831c3832 100644 --- a/download.go +++ b/download.go @@ -44,13 +44,22 @@ func downloadFile(path string, url string) (err error) { return err } -func gitGetHash(path string, name string) (string, error) { +func gitHasDiff(path string, name string) (bool, error) { stdout, stderr, err := passToGitCapture(filepath.Join(path, name), "rev-parse", "HEAD") if err != nil { - return "", fmt.Errorf("%s%s", stderr, err) + return false, fmt.Errorf("%s%s", stderr, err) } - return strings.TrimSpace(stdout), nil + head := strings.TrimSpace(stdout) + + stdout, stderr, err = passToGitCapture(filepath.Join(path, name), "rev-parse", "HEAD@{upstream}") + if err != nil { + return false, fmt.Errorf("%s%s", stderr, err) + } + + upstream := strings.TrimSpace(stdout) + + return head != upstream, nil } func gitDownload(url string, path string, name string) error { diff --git a/install.go b/install.go index af9a525c..7f1ed71b 100644 --- a/install.go +++ b/install.go @@ -155,27 +155,17 @@ func install(parser *arguments) error { cleanBuilds(toClean) - oldHashes, err := getHashes(do.Aur) - if err != nil { - return err - } - toSkip := pkgBuildsToSkip(do.Aur, targets) err = downloadPkgBuilds(do.Aur, do.Bases, toSkip) if err != nil { return err } - - err = mergePkgBuilds(do.Aur) - if err != nil { - return err - } - + if len(toEdit) > 0 { if config.ShowDiffs { - err = showPkgBuildDiffs(toEdit, do.Bases, oldHashes) + err = showPkgBuildDiffs(toEdit, do.Bases) } else { - err = editPkgBuilds(toEdit, do.Bases, oldHashes) + err = editPkgBuilds(toEdit, do.Bases) } if err != nil { return err @@ -189,6 +179,12 @@ func install(parser *arguments) error { config.NoConfirm = oldValue } + err = mergePkgBuilds(do.Aur) + if err != nil { + return err + } + + //initial srcinfo parse before pkgver() bump err = parseSRCINFOFiles(do.Aur, srcinfosStale, do.Bases) if err != nil { @@ -504,26 +500,21 @@ func cleanBuilds(pkgs []*rpc.Pkg) { } } -func showPkgBuildDiffs(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, hashes map[string]string) error { +func showPkgBuildDiffs(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg) error { for _, pkg := range pkgs { dir := filepath.Join(config.BuildDir, pkg.PackageBase) if shouldUseGit(dir) { - hash, _ := hashes[pkg.PackageBase] - if hash == "" { - hash = gitEmptyTree - } - - head, err := gitGetHash(config.BuildDir, pkg.PackageBase) + hasDiff, err := gitHasDiff(config.BuildDir, pkg.PackageBase) if err != nil { return err } - if head == hash { + if !hasDiff { fmt.Printf("%s %s: %s\n", bold(yellow(arrow)), cyan(formatPkgbase(pkg, bases)), bold("No changes -- skipping")) continue } - args := []string{"diff", hash + "..HEAD", "--src-prefix", dir + "/", "--dst-prefix", dir + "/"} + args := []string{"diff", "HEAD..HEAD@{upstream}", "--src-prefix", dir + "/", "--dst-prefix", dir + "/"} if useColor { args = append(args, "--color=always") } else { @@ -548,7 +539,7 @@ func showPkgBuildDiffs(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, hashes map[ return nil } -func editPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, hashes map[string]string) error { +func editPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg) error { pkgbuilds := make([]string, 0, len(pkgs)) for _, pkg := range pkgs { dir := filepath.Join(config.BuildDir, pkg.PackageBase) @@ -627,23 +618,6 @@ func pkgBuildsToSkip(pkgs []*rpc.Pkg, targets stringSet) stringSet { return toSkip } -func getHashes(pkgs []*rpc.Pkg) (map[string]string, error) { - hashes := make(map[string]string) - - for _, pkg := range pkgs { - if shouldUseGit(filepath.Join(config.BuildDir, pkg.PackageBase)) { - hash, err := gitGetHash(config.BuildDir, pkg.PackageBase) - if err != nil { - return hashes, err - } - - hashes[pkg.PackageBase] = hash - } - } - - return hashes, nil -} - func mergePkgBuilds(pkgs []*rpc.Pkg) error { for _, pkg := range pkgs { if shouldUseGit(filepath.Join(config.BuildDir, pkg.PackageBase)) { From 2b925aefa3abcd058cf587ff61248514fbf8908d Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sun, 10 Jun 2018 02:02:24 +0100 Subject: [PATCH 6/7] Lint --- install.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/install.go b/install.go index 7f1ed71b..47b20d06 100644 --- a/install.go +++ b/install.go @@ -160,7 +160,7 @@ func install(parser *arguments) error { if err != nil { return err } - + if len(toEdit) > 0 { if config.ShowDiffs { err = showPkgBuildDiffs(toEdit, do.Bases) @@ -184,7 +184,6 @@ func install(parser *arguments) error { return err } - //initial srcinfo parse before pkgver() bump err = parseSRCINFOFiles(do.Aur, srcinfosStale, do.Bases) if err != nil { @@ -621,7 +620,7 @@ func pkgBuildsToSkip(pkgs []*rpc.Pkg, targets stringSet) stringSet { func mergePkgBuilds(pkgs []*rpc.Pkg) error { for _, pkg := range pkgs { if shouldUseGit(filepath.Join(config.BuildDir, pkg.PackageBase)) { - err := gitMerge(baseURL + "/" + pkg.PackageBase + ".git", config.BuildDir, pkg.PackageBase) + err := gitMerge(baseURL+"/"+pkg.PackageBase+".git", config.BuildDir, pkg.PackageBase) if err != nil { return err } @@ -631,7 +630,6 @@ func mergePkgBuilds(pkgs []*rpc.Pkg) error { return nil } - func downloadPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, toSkip stringSet) error { for k, pkg := range pkgs { if toSkip.get(pkg.PackageBase) { @@ -645,7 +643,7 @@ func downloadPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, toSkip stri fmt.Printf(str, k+1, len(pkgs), cyan(formatPkgbase(pkg, bases))) if shouldUseGit(filepath.Join(config.BuildDir, pkg.PackageBase)) { - err := gitDownload(baseURL + "/" + pkg.PackageBase + ".git", config.BuildDir, pkg.PackageBase) + err := gitDownload(baseURL+"/"+pkg.PackageBase+".git", config.BuildDir, pkg.PackageBase) if err != nil { return err } From 00da26a3ce1d60627df7d0f8fd4e0df85ba0c73e Mon Sep 17 00:00:00 2001 From: morganamilo Date: Sun, 10 Jun 2018 03:41:25 +0100 Subject: [PATCH 7/7] Always show diffs for newly cloned packages Newly cloned packages already start out at origin/master, so there is no diff to show. Track if we cloned a package and if so make sure to show the full diff --- download.go | 14 +++++++------- install.go | 47 +++++++++++++++++++++++++++++------------------ 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/download.go b/download.go index 831c3832..4b25a02f 100644 --- a/download.go +++ b/download.go @@ -62,25 +62,25 @@ func gitHasDiff(path string, name string) (bool, error) { return head != upstream, nil } -func gitDownload(url string, path string, name string) error { +func gitDownload(url string, path string, name string) (bool, error) { _, err := os.Stat(filepath.Join(path, name, ".git")) if os.IsNotExist(err) { err = passToGit(path, "clone", url, name) if err != nil { - return fmt.Errorf("error cloning %s", name) + return false, fmt.Errorf("error cloning %s", name) } - return nil + return true, nil } else if err != nil { - return fmt.Errorf("error reading %s", filepath.Join(path, name, ".git")) + return false, fmt.Errorf("error reading %s", filepath.Join(path, name, ".git")) } err = passToGit(filepath.Join(path, name), "fetch") if err != nil { - return fmt.Errorf("error fetching %s", name) + return false, fmt.Errorf("error fetching %s", name) } - return nil + return false, nil } func gitMerge(url string, path string, name string) error { @@ -248,7 +248,7 @@ func getPkgbuildsfromAUR(pkgs []string, dir string) (bool, error) { } if shouldUseGit(filepath.Join(dir, pkg.PackageBase)) { - err = gitDownload(baseURL+"/"+pkg.PackageBase+".git", dir, pkg.PackageBase) + _, err = gitDownload(baseURL+"/"+pkg.PackageBase+".git", dir, pkg.PackageBase) } else { err = downloadAndUnpack(baseURL+aq[0].URLPath, dir) } diff --git a/install.go b/install.go index 47b20d06..4ea59039 100644 --- a/install.go +++ b/install.go @@ -156,14 +156,14 @@ func install(parser *arguments) error { cleanBuilds(toClean) toSkip := pkgBuildsToSkip(do.Aur, targets) - err = downloadPkgBuilds(do.Aur, do.Bases, toSkip) + cloned, err := downloadPkgBuilds(do.Aur, do.Bases, toSkip) if err != nil { return err } if len(toEdit) > 0 { if config.ShowDiffs { - err = showPkgBuildDiffs(toEdit, do.Bases) + err = showPkgBuildDiffs(toEdit, do.Bases, cloned) } else { err = editPkgBuilds(toEdit, do.Bases) } @@ -499,27 +499,33 @@ func cleanBuilds(pkgs []*rpc.Pkg) { } } -func showPkgBuildDiffs(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg) error { +func showPkgBuildDiffs(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, cloned stringSet) error { for _, pkg := range pkgs { dir := filepath.Join(config.BuildDir, pkg.PackageBase) if shouldUseGit(dir) { - hasDiff, err := gitHasDiff(config.BuildDir, pkg.PackageBase) - if err != nil { - return err + start := "HEAD" + + if cloned.get(pkg.PackageBase) { + start = gitEmptyTree + } else { + hasDiff, err := gitHasDiff(config.BuildDir, pkg.PackageBase) + if err != nil { + return err + } + + if !hasDiff { + fmt.Printf("%s %s: %s\n", bold(yellow(arrow)), cyan(formatPkgbase(pkg, bases)), bold("No changes -- skipping")) + continue + } } - if !hasDiff { - fmt.Printf("%s %s: %s\n", bold(yellow(arrow)), cyan(formatPkgbase(pkg, bases)), bold("No changes -- skipping")) - continue - } - - args := []string{"diff", "HEAD..HEAD@{upstream}", "--src-prefix", dir + "/", "--dst-prefix", dir + "/"} + args := []string{"diff", start + "..HEAD@{upstream}", "--src-prefix", dir + "/", "--dst-prefix", dir + "/"} if useColor { args = append(args, "--color=always") } else { args = append(args, "--color=never") } - err = passToGit(dir, args...) + err := passToGit(dir, args...) if err != nil { return err } @@ -630,7 +636,9 @@ func mergePkgBuilds(pkgs []*rpc.Pkg) error { return nil } -func downloadPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, toSkip stringSet) error { +func downloadPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, toSkip stringSet) (stringSet, error) { + cloned := make(stringSet) + for k, pkg := range pkgs { if toSkip.get(pkg.PackageBase) { str := bold(cyan("::") + " PKGBUILD up to date, Skipping (%d/%d): %s\n") @@ -643,19 +651,22 @@ func downloadPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, toSkip stri fmt.Printf(str, k+1, len(pkgs), cyan(formatPkgbase(pkg, bases))) if shouldUseGit(filepath.Join(config.BuildDir, pkg.PackageBase)) { - err := gitDownload(baseURL+"/"+pkg.PackageBase+".git", config.BuildDir, pkg.PackageBase) + clone, err := gitDownload(baseURL+"/"+pkg.PackageBase+".git", config.BuildDir, pkg.PackageBase) if err != nil { - return err + return nil, err + } + if clone { + cloned.set(pkg.PackageBase) } } else { err := downloadAndUnpack(baseURL+pkg.URLPath, config.BuildDir) if err != nil { - return err + return nil, err } } } - return nil + return cloned, nil } func downloadPkgBuildsSources(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, incompatible stringSet) (err error) {