deploy: walkLocal worker pool for performance

[ci skip]

common/hugo/version_current.go

@@ -17,7 +17,7 @@ package hugo
 // This should be the only one.
 var CurrentVersion = Version{
 	Major:      0,
-	Minor:      147,
+	Minor:      148,
-	PatchLevel: 9,
+	PatchLevel: 0,
-	Suffix:     "",
+	Suffix:     "-DEV",
 }

deploy/deploy.go

@@ -36,6 +36,7 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/gobwas/glob"
 	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/common/para"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/deploy/deployconfig"
 	"github.com/gohugoio/hugo/media"
@@ -487,7 +488,12 @@ func knownHiddenDirectory(name string) bool {
 // walkLocal walks the source directory and returns a flat list of files,
 // using localFile.SlashPath as the map keys.
 func (d *Deployer) walkLocal(fs afero.Fs, matchers []*deployconfig.Matcher, include, exclude glob.Glob, mediaTypes media.Types, mappath func(string) string) (map[string]*localFile, error) {
-	retval := map[string]*localFile{}
+	retval := make(map[string]*localFile)
+	var mu sync.Mutex
+
+	workers := para.New(d.cfg.Workers)
+	g, _ := workers.Start(context.Background())
+
 	err := afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
@@ -508,6 +514,8 @@ func (d *Deployer) walkLocal(fs afero.Fs, matchers []*deployconfig.Matcher, incl
 			return nil
 		}
 
+		// Process each file in a worker
+		g.Run(func() error {
 			// When a file system is HFS+, its filepath is in NFD form.
 			if runtime.GOOS == "darwin" {
 				path = norm.NFC.String(path)
@@ -541,12 +549,19 @@ func (d *Deployer) walkLocal(fs afero.Fs, matchers []*deployconfig.Matcher, incl
 			if err != nil {
 				return err
 			}
+			mu.Lock()
 			retval[lf.SlashPath] = lf
+			mu.Unlock()
+			return nil
+		})
 		return nil
 	})
 	if err != nil {
 		return nil, err
 	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
 	return retval, nil
 }
 

deploy/deploy_test.go

@@ -623,7 +623,7 @@ func TestEndToEndSync(t *testing.T) {
 				localFs:    test.fs,
 				bucket:     test.bucket,
 				mediaTypes: media.DefaultTypes,
-				cfg:        deployconfig.DeployConfig{MaxDeletes: -1},
+				cfg:        deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1},
 			}
 
 			// Initial deployment should sync remote with local.
@@ -706,7 +706,7 @@ func TestMaxDeletes(t *testing.T) {
 				localFs:    test.fs,
 				bucket:     test.bucket,
 				mediaTypes: media.DefaultTypes,
-				cfg:        deployconfig.DeployConfig{MaxDeletes: -1},
+				cfg:        deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1},
 			}
 
 			// Sync remote with local.
@@ -836,7 +836,7 @@ func TestIncludeExclude(t *testing.T) {
 			}
 			deployer := &Deployer{
 				localFs: fsTest.fs,
-				cfg:     deployconfig.DeployConfig{MaxDeletes: -1}, bucket: fsTest.bucket,
+				cfg:     deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1}, bucket: fsTest.bucket,
 				target:     tgt,
 				mediaTypes: media.DefaultTypes,
 			}
@@ -893,7 +893,7 @@ func TestIncludeExcludeRemoteDelete(t *testing.T) {
 			}
 			deployer := &Deployer{
 				localFs: fsTest.fs,
-				cfg:     deployconfig.DeployConfig{MaxDeletes: -1}, bucket: fsTest.bucket,
+				cfg:     deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1}, bucket: fsTest.bucket,
 				mediaTypes: media.DefaultTypes,
 			}
 
@@ -945,7 +945,7 @@ func TestCompression(t *testing.T) {
 			deployer := &Deployer{
 				localFs:    test.fs,
 				bucket:     test.bucket,
-				cfg:        deployconfig.DeployConfig{MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: ".*", Gzip: true, Re: regexp.MustCompile(".*")}}},
+				cfg:        deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: ".*", Gzip: true, Re: regexp.MustCompile(".*")}}},
 				mediaTypes: media.DefaultTypes,
 			}
 
@@ -1000,7 +1000,7 @@ func TestMatching(t *testing.T) {
 			deployer := &Deployer{
 				localFs:    test.fs,
 				bucket:     test.bucket,
-				cfg:        deployconfig.DeployConfig{MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: "^subdir/aaa$", Force: true, Re: regexp.MustCompile("^subdir/aaa$")}}},
+				cfg:        deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: "^subdir/aaa$", Force: true, Re: regexp.MustCompile("^subdir/aaa$")}}},
 				mediaTypes: media.DefaultTypes,
 			}
 
@@ -1097,5 +1097,6 @@ func verifyRemote(ctx context.Context, bucket *blob.Bucket, local []*fileData) (
 func newDeployer() *Deployer {
 	return &Deployer{
 		logger: loggers.NewDefault(),
+		cfg:    deployconfig.DeployConfig{Workers: 2},
 	}
 }

hugoreleaser.env

@@ -1,7 +1,8 @@
 # Release env. 
 # These will be replaced by script before release.
-HUGORELEASER_TAG=v0.147.8
+HUGORELEASER_TAG=v0.147.9
-HUGORELEASER_COMMITISH=10da2bd765d227761641f94d713d094e88b920ae
+HUGORELEASER_COMMITISH=29bdbde19c288d190e889294a862103c6efb70bf
+