refactor(yay): move cfg inside of runtime (#2259)

* rework relationship between runtime and cfg

* separate runtime from cfg

* simplify instantiation logic

* move installer to appropriate package

* move operator to sync package

* add tests for srcinfo service

* consolidate srcinfo service in sync

* add logger to srcinfo

* add logger to preparer

* remove unused text functions

* remove remaining text.* from srcinfo

* remove global logger parts

* remove global org method exports

* remove global logger

* move text->input

* add rule to prevent fmt.Print

* update golangci go version

* remove outdated FAQs

* remove outdated FAQs
This commit is contained in:
Jo authored and GitHub committed 2023-08-06 21:39:41 +02:00
1 parent 7483393377
commit 8916cd174b
74 files changed
+1475 -1367

No files matched your search

+136
View File
@@ -0,0 +1,136 @@
package workdir
import (
"context"
"fmt"
"runtime"
"sync"
mapset "github.com/deckarep/golang-set/v2"
"github.com/leonelquinteros/gotext"
"github.com/Jguer/yay/v12/pkg/multierror"
"github.com/Jguer/yay/v12/pkg/settings/exe"
"github.com/Jguer/yay/v12/pkg/text"
)
type ErrDownloadSource struct {
inner error
pkgName string
errOut string
}
func (e ErrDownloadSource) Error() string {
return fmt.Sprintln(gotext.Get("error downloading sources: %s", text.Cyan(e.pkgName)),
"\n\t context:", e.inner.Error(), "\n\t", e.errOut)
}
func (e *ErrDownloadSource) Unwrap() error {
return e.inner
}
func downloadPKGBUILDSource(ctx context.Context,
cmdBuilder exe.ICmdBuilder, pkgBuildDir string, installIncompatible bool,
) error {
args := []string{"--verifysource", "--skippgpcheck", "-Ccf"}
if installIncompatible {
args = append(args, "--ignorearch")
}
err := cmdBuilder.Show(
cmdBuilder.BuildMakepkgCmd(ctx, pkgBuildDir, args...))
if err != nil {
return ErrDownloadSource{inner: err, pkgName: pkgBuildDir}
}
return nil
}
func downloadPKGBUILDSourceWorker(ctx context.Context, wg *sync.WaitGroup,
dirChannel <-chan string, valOut chan<- string, errOut chan<- error,
cmdBuilder exe.ICmdBuilder, incompatible bool,
) {
for pkgBuildDir := range dirChannel {
err := downloadPKGBUILDSource(ctx, cmdBuilder, pkgBuildDir, incompatible)
if err != nil {
errOut <- ErrDownloadSource{inner: err, pkgName: pkgBuildDir, errOut: ""}
} else {
valOut <- pkgBuildDir
}
}
wg.Done()
}
func downloadPKGBUILDSourceFanout(ctx context.Context, cmdBuilder exe.ICmdBuilder, pkgBuildDirs map[string]string,
incompatible bool, maxConcurrentDownloads int,
) error {
if len(pkgBuildDirs) == 0 {
return nil // no work to do
}
if len(pkgBuildDirs) == 1 {
for _, pkgBuildDir := range pkgBuildDirs {
return downloadPKGBUILDSource(ctx, cmdBuilder, pkgBuildDir, incompatible)
}
}
var (
numOfWorkers = runtime.NumCPU()
wg = &sync.WaitGroup{}
c = make(chan string)
fanInChanValues = make(chan string)
fanInChanErrors = make(chan error)
)
if maxConcurrentDownloads != 0 {
numOfWorkers = maxConcurrentDownloads
}
dedupSet := mapset.NewThreadUnsafeSet[string]()
go func() {
for _, pkgbuildDir := range pkgBuildDirs {
if !dedupSet.Contains(pkgbuildDir) {
c <- pkgbuildDir
dedupSet.Add(pkgbuildDir)
}
}
close(c)
}()
// Launch Workers
wg.Add(numOfWorkers)
for s := 0; s < numOfWorkers; s++ {
go downloadPKGBUILDSourceWorker(ctx, wg, c,
fanInChanValues, fanInChanErrors, cmdBuilder, incompatible)
}
go func() {
wg.Wait()
close(fanInChanValues)
close(fanInChanErrors)
}()
returnErr := multierror.MultiError{}
receiver:
for {
select {
case _, ok := <-fanInChanValues:
if !ok {
break receiver
}
case err, ok := <-fanInChanErrors:
if !ok {
break receiver
}
returnErr.Add(err)
}
}
return returnErr.Return()
}
+158
View File
@@ -0,0 +1,158 @@
//go:build !integration
// +build !integration
package workdir
import (
"context"
"fmt"
"os/exec"
"path/filepath"
"sync/atomic"
"testing"
"github.com/stretchr/testify/assert"
"github.com/Jguer/yay/v12/pkg/multierror"
"github.com/Jguer/yay/v12/pkg/settings/exe"
)
type TestMakepkgBuilder struct {
exe.ICmdBuilder
parentBuilder *exe.CmdBuilder
test *testing.T
passes uint32
want string
wantDir string
showError error
}
func (z *TestMakepkgBuilder) BuildMakepkgCmd(ctx context.Context, dir string, extraArgs ...string) *exec.Cmd {
cmd := z.parentBuilder.BuildMakepkgCmd(ctx, dir, extraArgs...)
if z.want != "" {
assert.Contains(z.test, cmd.String(), z.want)
}
if z.wantDir != "" {
assert.Equal(z.test, z.wantDir, cmd.Dir)
}
atomic.AddUint32(&z.passes, 1)
return cmd
}
func (z *TestMakepkgBuilder) Show(cmd *exec.Cmd) error {
return z.showError
}
// GIVEN 1 package
// WHEN downloadPKGBUILDSource is called
// THEN 1 call should be made to makepkg with the specified parameters and dir
func Test_downloadPKGBUILDSource(t *testing.T) {
t.Parallel()
cmdBuilder := &TestMakepkgBuilder{
parentBuilder: &exe.CmdBuilder{MakepkgConfPath: "/etc/not.conf", MakepkgFlags: []string{"--nocheck"}, MakepkgBin: "makepkg"},
test: t,
want: "makepkg --nocheck --config /etc/not.conf --verifysource --skippgpcheck -Ccf",
wantDir: "/tmp/yay-bin",
}
err := downloadPKGBUILDSource(context.Background(), cmdBuilder, filepath.Join("/tmp", "yay-bin"), false)
assert.NoError(t, err)
assert.Equal(t, 1, int(cmdBuilder.passes))
}
// GIVEN 1 package
// WHEN downloadPKGBUILDSource is called
// THEN 1 call should be made to makepkg which should return error
func Test_downloadPKGBUILDSourceError(t *testing.T) {
t.Parallel()
cmdBuilder := &TestMakepkgBuilder{
parentBuilder: &exe.CmdBuilder{MakepkgConfPath: "/etc/not.conf", MakepkgFlags: []string{"--nocheck"}, MakepkgBin: "makepkg"},
test: t,
want: "makepkg --nocheck --config /etc/not.conf --verifysource --skippgpcheck -Ccf",
wantDir: "/tmp/yay-bin",
showError: &exec.ExitError{},
}
err := downloadPKGBUILDSource(context.Background(), cmdBuilder, filepath.Join("/tmp", "yay-bin"), false)
assert.Error(t, err)
assert.EqualError(t, err, "error downloading sources: \x1b[36m/tmp/yay-bin\x1b[0m \n\t context: <nil> \n\t \n")
}
// GIVEN 5 packages
// WHEN downloadPKGBUILDSourceFanout is called
// THEN 5 calls should be made to makepkg
func Test_downloadPKGBUILDSourceFanout(t *testing.T) {
t.Parallel()
pkgBuildDirs := map[string]string{
"yay": "/tmp/yay",
"yay-bin": "/tmp/yay-bin",
"yay-git": "/tmp/yay-git",
"yay-v11": "/tmp/yay-v11",
"yay-v12": "/tmp/yay-v12",
}
for _, maxConcurrentDownloads := range []int{0, 3} {
t.Run(fmt.Sprintf("maxconcurrentdownloads set to %d", maxConcurrentDownloads), func(t *testing.T) {
cmdBuilder := &TestMakepkgBuilder{
parentBuilder: &exe.CmdBuilder{
MakepkgConfPath: "/etc/not.conf",
MakepkgFlags: []string{"--nocheck"}, MakepkgBin: "makepkg",
},
test: t,
}
err := downloadPKGBUILDSourceFanout(context.Background(), cmdBuilder, pkgBuildDirs, true, maxConcurrentDownloads)
assert.NoError(t, err)
assert.Equal(t, 5, int(cmdBuilder.passes))
})
}
}
// GIVEN 1 package
// WHEN downloadPKGBUILDSourceFanout is called
// THEN 1 calls should be made to makepkg without concurrency
func Test_downloadPKGBUILDSourceFanoutNoCC(t *testing.T) {
t.Parallel()
cmdBuilder := &TestMakepkgBuilder{
parentBuilder: &exe.CmdBuilder{
MakepkgConfPath: "/etc/not.conf",
MakepkgFlags: []string{"--nocheck"}, MakepkgBin: "makepkg",
},
test: t,
}
pkgBuildDirs := map[string]string{"yay": "/tmp/yay"}
err := downloadPKGBUILDSourceFanout(context.Background(), cmdBuilder, pkgBuildDirs, false, 0)
assert.NoError(t, err)
assert.Equal(t, 1, int(cmdBuilder.passes))
}
// GIVEN 5 packages
// WHEN downloadPKGBUILDSourceFanout is called
// THEN 5 calls should be made to makepkg
func Test_downloadPKGBUILDSourceFanoutError(t *testing.T) {
t.Parallel()
cmdBuilder := &TestMakepkgBuilder{
parentBuilder: &exe.CmdBuilder{
MakepkgConfPath: "/etc/not.conf",
MakepkgFlags: []string{"--nocheck"}, MakepkgBin: "makepkg",
},
test: t,
showError: &exec.ExitError{},
}
pkgBuildDirs := map[string]string{
"yay": "/tmp/yay",
"yay-bin": "/tmp/yay-bin",
"yay-git": "/tmp/yay-git",
"yay-v11": "/tmp/yay-v11",
"yay-v12": "/tmp/yay-v12",
}
err := downloadPKGBUILDSourceFanout(context.Background(), cmdBuilder, pkgBuildDirs, false, 0)
assert.Error(t, err)
assert.Equal(t, 5, int(cmdBuilder.passes))
assert.Len(t, err.(*multierror.MultiError).Errors, 5)
}
+62
View File
@@ -0,0 +1,62 @@
package workdir
import (
"context"
"github.com/leonelquinteros/gotext"
"github.com/Jguer/yay/v12/pkg/runtime"
"github.com/Jguer/yay/v12/pkg/settings"
"github.com/Jguer/yay/v12/pkg/settings/exe"
"github.com/Jguer/yay/v12/pkg/settings/parser"
"github.com/Jguer/yay/v12/pkg/text"
)
func removeMake(ctx context.Context, config *settings.Configuration,
cmdBuilder exe.ICmdBuilder, makeDeps []string, cmdArgs *parser.Arguments,
) error {
removeArguments := cmdArgs.CopyGlobal()
err := removeArguments.AddArg("R", "s", "u")
if err != nil {
return err
}
for _, pkg := range makeDeps {
removeArguments.AddTarget(pkg)
}
oldValue := settings.NoConfirm
settings.NoConfirm = true
err = cmdBuilder.Show(cmdBuilder.BuildPacmanCmd(ctx,
removeArguments, config.Mode, settings.NoConfirm))
settings.NoConfirm = oldValue
return err
}
func cleanAfter(ctx context.Context, run *runtime.Runtime,
cmdBuilder exe.ICmdBuilder, pkgbuildDirs map[string]string,
) {
run.Logger.Println(gotext.Get("removing untracked AUR files from cache..."))
i := 0
for _, dir := range pkgbuildDirs {
run.Logger.OperationInfoln(gotext.Get("Cleaning (%d/%d): %s", i+1, len(pkgbuildDirs), text.Cyan(dir)))
_, stderr, err := cmdBuilder.Capture(
cmdBuilder.BuildGitCmd(
ctx, dir, "reset", "--hard", "HEAD"))
if err != nil {
run.Logger.Errorln(gotext.Get("error resetting %s: %s", dir, stderr))
}
if err := run.CmdBuilder.Show(
run.CmdBuilder.BuildGitCmd(
ctx, dir, "clean", "-fx", "--exclude", "*.pkg.*")); err != nil {
run.Logger.Errorln(err)
}
i++
}
}
+39
View File
@@ -0,0 +1,39 @@
package workdir
import (
"context"
"errors"
"github.com/leonelquinteros/gotext"
"github.com/Jguer/yay/v12/pkg/settings/exe"
)
func gitMerge(ctx context.Context, cmdBuilder exe.ICmdBuilder, dir string) error {
_, stderr, err := cmdBuilder.Capture(
cmdBuilder.BuildGitCmd(ctx,
dir, "reset", "--hard", "HEAD"))
if err != nil {
return errors.New(gotext.Get("error resetting %s: %s", dir, stderr))
}
_, stderr, err = cmdBuilder.Capture(
cmdBuilder.BuildGitCmd(ctx,
dir, "merge", "--no-edit", "--ff"))
if err != nil {
return errors.New(gotext.Get("error merging %s: %s", dir, stderr))
}
return nil
}
func mergePkgbuilds(ctx context.Context, cmdBuilder exe.ICmdBuilder, pkgbuildDirs map[string]string) error {
for _, dir := range pkgbuildDirs {
err := gitMerge(ctx, cmdBuilder, dir)
if err != nil {
return err
}
}
return nil
}
+259
View File
@@ -0,0 +1,259 @@
package workdir
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/Jguer/yay/v12/pkg/db"
"github.com/Jguer/yay/v12/pkg/dep"
"github.com/Jguer/yay/v12/pkg/download"
"github.com/Jguer/yay/v12/pkg/menus"
"github.com/Jguer/yay/v12/pkg/runtime"
"github.com/Jguer/yay/v12/pkg/settings"
"github.com/Jguer/yay/v12/pkg/settings/exe"
"github.com/Jguer/yay/v12/pkg/settings/parser"
"github.com/Jguer/yay/v12/pkg/sync/build"
"github.com/Jguer/yay/v12/pkg/text"
gosrc "github.com/Morganamilo/go-srcinfo"
mapset "github.com/deckarep/golang-set/v2"
"github.com/leonelquinteros/gotext"
)
type HookType string
const (
// PreDownloadSourcesHook is called before sourcing a package
PreDownloadSourcesHook HookType = "pre-download-sources"
)
type HookFn func(ctx context.Context, run *runtime.Runtime, w io.Writer,
pkgbuildDirsByBase map[string]string, installed mapset.Set[string],
) error
type Hook struct {
Name string
Hookfn HookFn
Type HookType
}
type Preparer struct {
dbExecutor db.Executor
cmdBuilder exe.ICmdBuilder
cfg *settings.Configuration
hooks []Hook
downloadSources bool
log *text.Logger
makeDeps []string
}
func NewPreparerWithoutHooks(dbExecutor db.Executor, cmdBuilder exe.ICmdBuilder,
cfg *settings.Configuration, logger *text.Logger, downloadSources bool,
) *Preparer {
return &Preparer{
dbExecutor: dbExecutor,
cmdBuilder: cmdBuilder,
cfg: cfg,
hooks: []Hook{},
downloadSources: downloadSources,
log: logger,
}
}
func NewPreparer(dbExecutor db.Executor, cmdBuilder exe.ICmdBuilder,
cfg *settings.Configuration, logger *text.Logger,
) *Preparer {
preper := NewPreparerWithoutHooks(dbExecutor, cmdBuilder, cfg, logger, true)
if cfg.CleanMenu {
preper.hooks = append(preper.hooks, Hook{
Name: "clean",
Hookfn: menus.CleanFn,
Type: PreDownloadSourcesHook,
})
}
if cfg.DiffMenu {
preper.hooks = append(preper.hooks, Hook{
Name: "diff",
Hookfn: menus.DiffFn,
Type: PreDownloadSourcesHook,
})
}
if cfg.EditMenu {
preper.hooks = append(preper.hooks, Hook{
Name: "edit",
Hookfn: menus.EditFn,
Type: PreDownloadSourcesHook,
})
}
return preper
}
func (preper *Preparer) ShouldCleanAURDirs(run *runtime.Runtime, pkgBuildDirs map[string]string) build.PostInstallHookFunc {
if !preper.cfg.CleanAfter || len(pkgBuildDirs) == 0 {
return nil
}
preper.log.Debugln("added post install hook to clean up AUR dirs", pkgBuildDirs)
return func(ctx context.Context) error {
cleanAfter(ctx, run, run.CmdBuilder, pkgBuildDirs)
return nil
}
}
func (preper *Preparer) ShouldCleanMakeDeps(run *runtime.Runtime, cmdArgs *parser.Arguments) build.PostInstallHookFunc {
if len(preper.makeDeps) == 0 {
return nil
}
switch preper.cfg.RemoveMake {
case "yes":
break
case "no":
return nil
default:
isYesDefault := preper.cfg.RemoveMake == "askyes"
if !preper.log.ContinueTask(gotext.Get("Remove make dependencies after install?"),
isYesDefault, settings.NoConfirm) {
return nil
}
}
preper.log.Debugln("added post install hook to clean up AUR makedeps", preper.makeDeps)
return func(ctx context.Context) error {
return removeMake(ctx, preper.cfg, run.CmdBuilder, preper.makeDeps, cmdArgs)
}
}
func (preper *Preparer) Run(ctx context.Context, run *runtime.Runtime,
targets []map[string]*dep.InstallInfo,
) (pkgbuildDirsByBase map[string]string, err error) {
preper.Present(targets)
pkgBuildDirs, err := preper.PrepareWorkspace(ctx, run, targets)
if err != nil {
return nil, err
}
return pkgBuildDirs, nil
}
func (preper *Preparer) Present(targets []map[string]*dep.InstallInfo) {
pkgsBySourceAndReason := map[string]map[string][]string{}
for _, layer := range targets {
for pkgName, info := range layer {
source := dep.SourceNames[info.Source]
reason := dep.ReasonNames[info.Reason]
var pkgStr string
if info.Version != "" {
pkgStr = text.Cyan(fmt.Sprintf("%s-%s", pkgName, info.Version))
} else {
pkgStr = text.Cyan(pkgName)
}
if _, ok := pkgsBySourceAndReason[source]; !ok {
pkgsBySourceAndReason[source] = map[string][]string{}
}
pkgsBySourceAndReason[source][reason] = append(pkgsBySourceAndReason[source][reason], pkgStr)
if info.Reason == dep.MakeDep {
preper.makeDeps = append(preper.makeDeps, pkgName)
}
}
}
for source, pkgsByReason := range pkgsBySourceAndReason {
for reason, pkgs := range pkgsByReason {
preper.log.Printf(text.Bold("%s %s (%d):")+" %s\n",
source,
reason,
len(pkgs),
strings.Join(pkgs, ", "))
}
}
}
func (preper *Preparer) PrepareWorkspace(ctx context.Context,
run *runtime.Runtime, targets []map[string]*dep.InstallInfo,
) (map[string]string, error) {
aurBasesToClone := mapset.NewThreadUnsafeSet[string]()
pkgBuildDirsByBase := make(map[string]string, len(targets))
for _, layer := range targets {
for _, info := range layer {
if info.Source == dep.AUR {
pkgBase := *info.AURBase
pkgBuildDir := filepath.Join(preper.cfg.BuildDir, pkgBase)
if preper.needToCloneAURBase(info, pkgBuildDir) {
aurBasesToClone.Add(pkgBase)
}
pkgBuildDirsByBase[pkgBase] = pkgBuildDir
} else if info.Source == dep.SrcInfo {
pkgBase := *info.AURBase
pkgBuildDirsByBase[pkgBase] = *info.SrcinfoPath
}
}
}
if _, errA := download.AURPKGBUILDRepos(ctx,
preper.cmdBuilder, preper.log.Child("download"), aurBasesToClone.ToSlice(),
preper.cfg.AURURL, preper.cfg.BuildDir, false); errA != nil {
return nil, errA
}
if !preper.downloadSources {
return pkgBuildDirsByBase, nil
}
if err := mergePkgbuilds(ctx, preper.cmdBuilder, pkgBuildDirsByBase); err != nil {
return nil, err
}
remoteNames := preper.dbExecutor.InstalledRemotePackageNames()
remoteNamesCache := mapset.NewThreadUnsafeSet(remoteNames...)
for _, hookFn := range preper.hooks {
if hookFn.Type == PreDownloadSourcesHook {
if err := hookFn.Hookfn(ctx, run, os.Stdout, pkgBuildDirsByBase, remoteNamesCache); err != nil {
return nil, err
}
}
}
if errP := downloadPKGBUILDSourceFanout(ctx, preper.cmdBuilder,
pkgBuildDirsByBase, false, preper.cfg.MaxConcurrentDownloads); errP != nil {
preper.log.Errorln(errP)
}
return pkgBuildDirsByBase, nil
}
func (preper *Preparer) needToCloneAURBase(installInfo *dep.InstallInfo, pkgbuildDir string) bool {
if preper.cfg.ReDownload == "all" {
return true
}
srcinfoFile := filepath.Join(pkgbuildDir, ".SRCINFO")
if pkgbuild, err := gosrc.ParseFile(srcinfoFile); err == nil {
if db.VerCmp(pkgbuild.Version(), installInfo.Version) >= 0 {
preper.log.OperationInfoln(
gotext.Get("PKGBUILD up to date, skipping download: %s",
text.Cyan(*installInfo.AURBase)))
return false
}
}
return true
}
+72
View File
@@ -0,0 +1,72 @@
//go:build !integration
// +build !integration
package workdir
import (
"io"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/Jguer/yay/v12/pkg/settings"
"github.com/Jguer/yay/v12/pkg/text"
)
func newTestLogger() *text.Logger {
return text.NewLogger(io.Discard, io.Discard, strings.NewReader(""), true, "test")
}
// Test order of pre-download-sources hooks
func TestPreDownloadSourcesHooks(t *testing.T) {
testCases := []struct {
name string
cfg *settings.Configuration
wantHook []string
}{
{
name: "clean, diff, edit",
cfg: &settings.Configuration{
CleanMenu: true,
DiffMenu: true,
EditMenu: true,
},
wantHook: []string{"clean", "diff", "edit"},
},
{
name: "clean, edit",
cfg: &settings.Configuration{
CleanMenu: true,
DiffMenu: false,
EditMenu: true,
},
wantHook: []string{"clean", "edit"},
},
{
name: "clean, diff",
cfg: &settings.Configuration{
CleanMenu: true,
DiffMenu: true,
EditMenu: false,
},
wantHook: []string{"clean", "diff"},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
preper := NewPreparer(nil, nil, tc.cfg, newTestLogger())
assert.Len(t, preper.hooks, len(tc.wantHook))
got := make([]string, 0, len(preper.hooks))
for _, hook := range preper.hooks {
got = append(got, hook.Name)
}
assert.Equal(t, tc.wantHook, got)
})
}
}