Remove local clones & make hooks run on merge/edit/upload (#6672)

* Add options to git.Clone to make it more capable

* Begin the process of removing the local copy and tidy up

* Remove Wiki LocalCopy Checkouts

* Remove the last LocalRepo helpers

* Remove WithTemporaryFile

* Enable push-hooks for these routes

* Ensure tests cope with hooks

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Remove Repository.LocalCopyPath()

* Move temporary repo to use the standard temporary path

* Fix the tests

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Remove LocalWikiPath

* Fix missing remove

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Use AppURL for Oauth user link (#6894)

* Use AppURL for Oauth user link

Fix #6843

* Update oauth.go

* Update oauth.go

* internal/ssh: ignore env command totally (#6825)

* ssh: ignore env command totally

* Remove commented code 

Needed fix described in issue #6889

* Escape the commit message on issues update and title in telegram hook (#6901)

* update sdk to latest (#6903)

* improve description of branch protection (fix #6886) (#6906)

The branch protection description text were not quite accurate.

* Fix logging documentation (#6904)

* ENABLE_MACARON_REDIRECT should be REDIRECT_MACARON_LOG

* Allow DISABLE_ROUTER_LOG to be set in the [log] section

* [skip ci] Updated translations via Crowdin

* Move sdk structs to modules/structs (#6905)

* move sdk structs to moduels/structs

* fix tests

* fix fmt

* fix swagger

* fix vendor

modules/git/command.go

@@ -52,9 +52,15 @@ func (c *Command) AddArguments(args ...string) *Command {
 	return c
 }
 
-// RunInDirTimeoutPipeline executes the command in given directory with given timeout,
+// RunInDirTimeoutEnvPipeline executes the command in given directory with given timeout,
 // it pipes stdout and stderr to given io.Writer.
-func (c *Command) RunInDirTimeoutPipeline(timeout time.Duration, dir string, stdout, stderr io.Writer) error {
+func (c *Command) RunInDirTimeoutEnvPipeline(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer) error {
+	return c.RunInDirTimeoutEnvFullPipeline(env, timeout, dir, stdout, stderr, nil)
+}
+
+// RunInDirTimeoutEnvFullPipeline executes the command in given directory with given timeout,
+// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin.
+func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader) error {
 	if timeout == -1 {
 		timeout = DefaultCommandExecutionTimeout
 	}
@@ -69,9 +75,11 @@ func (c *Command) RunInDirTimeoutPipeline(timeout time.Duration, dir string, std
 	defer cancel()
 
 	cmd := exec.CommandContext(ctx, c.name, c.args...)
+	cmd.Env = env
 	cmd.Dir = dir
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
+	cmd.Stdin = stdin
 	if err := cmd.Start(); err != nil {
 		return err
 	}
@@ -83,12 +91,30 @@ func (c *Command) RunInDirTimeoutPipeline(timeout time.Duration, dir string, std
 	return ctx.Err()
 }
 
+// RunInDirTimeoutPipeline executes the command in given directory with given timeout,
+// it pipes stdout and stderr to given io.Writer.
+func (c *Command) RunInDirTimeoutPipeline(timeout time.Duration, dir string, stdout, stderr io.Writer) error {
+	return c.RunInDirTimeoutEnvPipeline(nil, timeout, dir, stdout, stderr)
+}
+
+// RunInDirTimeoutFullPipeline executes the command in given directory with given timeout,
+// it pipes stdout and stderr to given io.Writer, and stdin from the given io.Reader
+func (c *Command) RunInDirTimeoutFullPipeline(timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader) error {
+	return c.RunInDirTimeoutEnvFullPipeline(nil, timeout, dir, stdout, stderr, stdin)
+}
+
 // RunInDirTimeout executes the command in given directory with given timeout,
 // and returns stdout in []byte and error (combined with stderr).
 func (c *Command) RunInDirTimeout(timeout time.Duration, dir string) ([]byte, error) {
+	return c.RunInDirTimeoutEnv(nil, timeout, dir)
+}
+
+// RunInDirTimeoutEnv executes the command in given directory with given timeout,
+// and returns stdout in []byte and error (combined with stderr).
+func (c *Command) RunInDirTimeoutEnv(env []string, timeout time.Duration, dir string) ([]byte, error) {
 	stdout := new(bytes.Buffer)
 	stderr := new(bytes.Buffer)
-	if err := c.RunInDirTimeoutPipeline(timeout, dir, stdout, stderr); err != nil {
+	if err := c.RunInDirTimeoutEnvPipeline(env, timeout, dir, stdout, stderr); err != nil {
 		return nil, concatenateError(err, stderr.String())
 	}
 
@@ -101,7 +127,13 @@ func (c *Command) RunInDirTimeout(timeout time.Duration, dir string) ([]byte, er
 // RunInDirPipeline executes the command in given directory,
 // it pipes stdout and stderr to given io.Writer.
 func (c *Command) RunInDirPipeline(dir string, stdout, stderr io.Writer) error {
-	return c.RunInDirTimeoutPipeline(-1, dir, stdout, stderr)
+	return c.RunInDirFullPipeline(dir, stdout, stderr, nil)
+}
+
+// RunInDirFullPipeline executes the command in given directory,
+// it pipes stdout and stderr to given io.Writer.
+func (c *Command) RunInDirFullPipeline(dir string, stdout, stderr io.Writer, stdin io.Reader) error {
+	return c.RunInDirTimeoutFullPipeline(-1, dir, stdout, stderr, stdin)
 }
 
 // RunInDirBytes executes the command in given directory
@@ -113,7 +145,13 @@ func (c *Command) RunInDirBytes(dir string) ([]byte, error) {
 // RunInDir executes the command in given directory
 // and returns stdout in string and error (combined with stderr).
 func (c *Command) RunInDir(dir string) (string, error) {
-	stdout, err := c.RunInDirTimeout(-1, dir)
+	return c.RunInDirWithEnv(dir, nil)
+}
+
+// RunInDirWithEnv executes the command in given directory
+// and returns stdout in string and error (combined with stderr).
+func (c *Command) RunInDirWithEnv(dir string, env []string) (string, error) {
+	stdout, err := c.RunInDirTimeoutEnv(env, -1, dir)
 	if err != nil {
 		return "", err
 	}

modules/git/repo.go

@@ -109,11 +109,13 @@ func OpenRepository(repoPath string) (*Repository, error) {
 
 // CloneRepoOptions options when clone a repository
 type CloneRepoOptions struct {
-	Timeout time.Duration
-	Mirror  bool
-	Bare    bool
-	Quiet   bool
-	Branch  string
+	Timeout    time.Duration
+	Mirror     bool
+	Bare       bool
+	Quiet      bool
+	Branch     string
+	Shared     bool
+	NoCheckout bool
 }
 
 // Clone clones original repository to target path.
@@ -133,10 +135,17 @@ func Clone(from, to string, opts CloneRepoOptions) (err error) {
 	if opts.Quiet {
 		cmd.AddArguments("--quiet")
 	}
+	if opts.Shared {
+		cmd.AddArguments("-s")
+	}
+	if opts.NoCheckout {
+		cmd.AddArguments("--no-checkout")
+	}
+
 	if len(opts.Branch) > 0 {
 		cmd.AddArguments("-b", opts.Branch)
 	}
-	cmd.AddArguments(from, to)
+	cmd.AddArguments("--", from, to)
 
 	if opts.Timeout <= 0 {
 		opts.Timeout = -1
@@ -181,6 +190,7 @@ type PushOptions struct {
 	Remote string
 	Branch string
 	Force  bool
+	Env    []string
 }
 
 // Push pushs local commits to given remote branch.
@@ -190,7 +200,7 @@ func Push(repoPath string, opts PushOptions) error {
 		cmd.AddArguments("-f")
 	}
 	cmd.AddArguments(opts.Remote, opts.Branch)
-	_, err := cmd.RunInDir(repoPath)
+	_, err := cmd.RunInDirWithEnv(repoPath, opts.Env)
 	return err
 }
 

modules/git/repo_branch.go

@@ -17,7 +17,7 @@ const BranchPrefix = "refs/heads/"
 
 // IsReferenceExist returns true if given reference exists in the repository.
 func IsReferenceExist(repoPath, name string) bool {
-	_, err := NewCommand("show-ref", "--verify", name).RunInDir(repoPath)
+	_, err := NewCommand("show-ref", "--verify", "--", name).RunInDir(repoPath)
 	return err == nil
 }
 
@@ -145,9 +145,9 @@ func (repo *Repository) DeleteBranch(name string, opts DeleteBranchOptions) erro
 }
 
 // CreateBranch create a new branch
-func (repo *Repository) CreateBranch(branch, newBranch string) error {
+func (repo *Repository) CreateBranch(branch, oldbranchOrCommit string) error {
 	cmd := NewCommand("branch")
-	cmd.AddArguments(branch, newBranch)
+	cmd.AddArguments("--", branch, oldbranchOrCommit)
 
 	_, err := cmd.RunInDir(repo.Path)
 

modules/git/repo_index.go

@@ -0,0 +1,98 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package git
+
+import (
+	"bytes"
+	"strings"
+)
+
+// ReadTreeToIndex reads a treeish to the index
+func (repo *Repository) ReadTreeToIndex(treeish string) error {
+	if len(treeish) != 40 {
+		res, err := NewCommand("rev-parse", treeish).RunInDir(repo.Path)
+		if err != nil {
+			return err
+		}
+		if len(res) > 0 {
+			treeish = res[:len(res)-1]
+		}
+	}
+	id, err := NewIDFromString(treeish)
+	if err != nil {
+		return err
+	}
+	return repo.readTreeToIndex(id)
+}
+
+func (repo *Repository) readTreeToIndex(id SHA1) error {
+	_, err := NewCommand("read-tree", id.String()).RunInDir(repo.Path)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// EmptyIndex empties the index
+func (repo *Repository) EmptyIndex() error {
+	_, err := NewCommand("read-tree", "--empty").RunInDir(repo.Path)
+	return err
+}
+
+// LsFiles checks if the given filenames are in the index
+func (repo *Repository) LsFiles(filenames ...string) ([]string, error) {
+	cmd := NewCommand("ls-files", "-z", "--")
+	for _, arg := range filenames {
+		if arg != "" {
+			cmd.AddArguments(arg)
+		}
+	}
+	res, err := cmd.RunInDirBytes(repo.Path)
+	if err != nil {
+		return nil, err
+	}
+	filelist := make([]string, 0, len(filenames))
+	for _, line := range bytes.Split(res, []byte{'\000'}) {
+		filelist = append(filelist, string(line))
+	}
+
+	return filelist, err
+}
+
+// RemoveFilesFromIndex removes given filenames from the index - it does not check whether they are present.
+func (repo *Repository) RemoveFilesFromIndex(filenames ...string) error {
+	cmd := NewCommand("update-index", "--remove", "-z", "--index-info")
+	stdout := new(bytes.Buffer)
+	stderr := new(bytes.Buffer)
+	buffer := new(bytes.Buffer)
+	for _, file := range filenames {
+		if file != "" {
+			buffer.WriteString("0 0000000000000000000000000000000000000000\t")
+			buffer.WriteString(file)
+			buffer.WriteByte('\000')
+		}
+	}
+	return cmd.RunInDirFullPipeline(repo.Path, stdout, stderr, bytes.NewReader(buffer.Bytes()))
+}
+
+// AddObjectToIndex adds the provided object hash to the index at the provided filename
+func (repo *Repository) AddObjectToIndex(mode string, object SHA1, filename string) error {
+	cmd := NewCommand("update-index", "--add", "--replace", "--cacheinfo", mode, object.String(), filename)
+	_, err := cmd.RunInDir(repo.Path)
+	return err
+}
+
+// WriteTree writes the current index as a tree to the object db and returns its hash
+func (repo *Repository) WriteTree() (*Tree, error) {
+	res, err := NewCommand("write-tree").RunInDir(repo.Path)
+	if err != nil {
+		return nil, err
+	}
+	id, err := NewIDFromString(strings.TrimSpace(res))
+	if err != nil {
+		return nil, err
+	}
+	return NewTree(repo, id), nil
+}

modules/git/repo_object.go

@@ -4,6 +4,12 @@
 
 package git
 
+import (
+	"bytes"
+	"io"
+	"strings"
+)
+
 // ObjectType git object type
 type ObjectType string
 
@@ -17,3 +23,24 @@ const (
 	// ObjectTag tag object type
 	ObjectTag ObjectType = "tag"
 )
+
+// HashObject takes a reader and returns SHA1 hash for that reader
+func (repo *Repository) HashObject(reader io.Reader) (SHA1, error) {
+	idStr, err := repo.hashObject(reader)
+	if err != nil {
+		return SHA1{}, err
+	}
+	return NewIDFromString(idStr)
+}
+
+func (repo *Repository) hashObject(reader io.Reader) (string, error) {
+	cmd := NewCommand("hash-object", "-w", "--stdin")
+	stdout := new(bytes.Buffer)
+	stderr := new(bytes.Buffer)
+	err := cmd.RunInDirFullPipeline(repo.Path, stdout, stderr, reader)
+
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(stdout.String()), nil
+}

modules/git/repo_tree.go

@@ -6,6 +6,11 @@
 package git
 
 import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
 	"gopkg.in/src-d/go-git.v4/plumbing"
 )
 
@@ -47,3 +52,48 @@ func (repo *Repository) GetTree(idStr string) (*Tree, error) {
 	treeObject.ResolvedID = resolvedID
 	return treeObject, nil
 }
+
+// CommitTreeOpts represents the possible options to CommitTree
+type CommitTreeOpts struct {
+	Parents   []string
+	Message   string
+	KeyID     string
+	NoGPGSign bool
+}
+
+// CommitTree creates a commit from a given tree id for the user with provided message
+func (repo *Repository) CommitTree(sig *Signature, tree *Tree, opts CommitTreeOpts) (SHA1, error) {
+	commitTimeStr := time.Now().Format(time.UnixDate)
+
+	// Because this may call hooks we should pass in the environment
+	env := append(os.Environ(),
+		"GIT_AUTHOR_NAME="+sig.Name,
+		"GIT_AUTHOR_EMAIL="+sig.Email,
+		"GIT_AUTHOR_DATE="+commitTimeStr,
+		"GIT_COMMITTER_NAME="+sig.Name,
+		"GIT_COMMITTER_EMAIL="+sig.Email,
+		"GIT_COMMITTER_DATE="+commitTimeStr,
+	)
+	cmd := NewCommand("commit-tree", tree.ID.String())
+
+	for _, parent := range opts.Parents {
+		cmd.AddArguments("-p", parent)
+	}
+
+	cmd.AddArguments("-m", opts.Message)
+
+	if opts.KeyID != "" {
+		cmd.AddArguments(fmt.Sprintf("-S%s", opts.KeyID))
+	}
+
+	if opts.NoGPGSign {
+		cmd.AddArguments("--no-gpg-sign")
+	}
+
+	res, err := cmd.RunInDirWithEnv(repo.Path, env)
+
+	if err != nil {
+		return SHA1{}, err
+	}
+	return NewIDFromString(strings.TrimSpace(res))
+}