From 2af67f6044af1cad7136ce8c123e37ab090ca9bc Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Fri, 19 Apr 2019 14:17:27 +0200 Subject: [PATCH] Improve listing performance by using go-git (#6478) * Use go-git for tree reading and commit info lookup. Signed-off-by: Filip Navara * Use TreeEntry.IsRegular() instead of ObjectType that was removed. Signed-off-by: Filip Navara * Use the treePath to optimize commit info search. Signed-off-by: Filip Navara * Extract the latest commit at treePath along with the other commits. Signed-off-by: Filip Navara * Fix listing commit info for a directory that was created in one commit and never modified after. Signed-off-by: Filip Navara * Avoid nearly all external 'git' invocations when doing directory listing (.editorconfig code path is still hit). Signed-off-by: Filip Navara * Use go-git for reading blobs. Signed-off-by: Filip Navara * Make SHA1 type alias for plumbing.Hash in go-git. Signed-off-by: Filip Navara * Make Signature type alias for object.Signature in go-git. Signed-off-by: Filip Navara * Fix GetCommitsInfo for repository with only one commit. Signed-off-by: Filip Navara * Fix PGP signature verification. Signed-off-by: Filip Navara * Fix issues with walking commit graph across merges. Signed-off-by: Filip Navara * Fix typo in condition. Signed-off-by: Filip Navara * Speed up loading branch list by keeping the repository reference (and thus all the loaded packfile indexes). Signed-off-by: Filip Navara * Fix lising submodules. Signed-off-by: Filip Navara * Fix build Signed-off-by: Filip Navara * Add back commit cache because of name-rev Signed-off-by: Filip Navara * Fix tests Signed-off-by: Filip Navara * Fix code style * Fix spelling * Address PR feedback Signed-off-by: Filip Navara * Update vendor module list Signed-off-by: Filip Navara * Fix getting trees by commit id Signed-off-by: Filip Navara * Fix remaining unit test failures * Fix GetTreeBySHA * Avoid running `git name-rev` if not necessary Signed-off-by: Filip Navara * Move Branch code to git module * Clean up GPG signature verification and fix it for tagged commits * Address PR feedback (import formatting, copyright headers) * Make blob lookup by SHA working * Update tests to use public API * Allow getting content from any type of object through the blob interface * Change test to actually expect the object content that is in the GIT repository * Change one more test to actually expect the object content that is in the GIT repository * Add comments --- go.mod | 4 +- integrations/api_repo_git_blobs_test.go | 2 +- models/error.go | 15 - models/pull.go | 5 +- models/repo_branch.go | 57 +-- modules/context/repo.go | 3 +- modules/git/blob.go | 66 +-- modules/git/blob_test.go | 45 +- modules/git/commit.go | 80 +++- modules/git/commit_info.go | 522 ++++++++++------------- modules/git/commit_info_test.go | 4 +- modules/git/error.go | 15 + modules/git/parse.go | 25 +- modules/git/parse_test.go | 32 +- modules/git/repo.go | 31 +- modules/git/repo_blob.go | 16 +- modules/git/repo_blob_test.go | 3 +- modules/git/repo_branch.go | 66 ++- modules/git/repo_commit.go | 149 +++---- modules/git/repo_tag.go | 35 +- modules/git/repo_tree.go | 29 +- modules/git/sha1.go | 29 +- modules/git/signature.go | 9 +- modules/git/tree.go | 91 ++-- modules/git/tree_blob.go | 17 +- modules/git/tree_entry.go | 83 ++-- modules/git/tree_entry_test.go | 18 +- modules/repofiles/blob_test.go | 2 +- modules/repofiles/content.go | 2 +- modules/repofiles/temp_repo.go | 2 +- modules/repofiles/tree.go | 13 +- modules/repofiles/tree_test.go | 3 +- modules/repofiles/update.go | 2 +- routers/api/v1/convert/convert.go | 2 +- routers/api/v1/repo/branch.go | 5 +- routers/repo/commit.go | 1 + routers/repo/editor.go | 11 +- routers/repo/issue.go | 5 +- routers/repo/setting_protected_branch.go | 4 +- routers/repo/view.go | 11 +- routers/repo/wiki.go | 9 +- routers/repo/wiki_test.go | 3 +- templates/repo/diff/page.tmpl | 2 +- vendor/modules.txt | 14 +- 44 files changed, 759 insertions(+), 783 deletions(-) diff --git a/go.mod b/go.mod index 60a722ab97..419a50014b 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/dgrijalva/jwt-go v0.0.0-20161101193935-9ed569b5d1ac github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 // indirect github.com/elazarl/go-bindata-assetfs v0.0.0-20151224045452-57eb5e1fc594 // indirect - github.com/emirpasic/gods v1.12.0 // indirect + github.com/emirpasic/gods v1.12.0 github.com/etcd-io/bbolt v1.3.2 // indirect github.com/ethantkoenig/rupture v0.0.0-20180203182544-0a76f03a811a github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect @@ -127,7 +127,7 @@ require ( gopkg.in/ldap.v3 v3.0.2 gopkg.in/macaron.v1 v1.3.2 gopkg.in/redis.v2 v2.3.2 // indirect - gopkg.in/src-d/go-billy.v4 v4.3.0 // indirect + gopkg.in/src-d/go-billy.v4 v4.3.0 gopkg.in/src-d/go-git.v4 v4.10.0 gopkg.in/testfixtures.v2 v2.5.0 mvdan.cc/xurls/v2 v2.0.0 diff --git a/integrations/api_repo_git_blobs_test.go b/integrations/api_repo_git_blobs_test.go index 560f108fca..69bb6cde23 100644 --- a/integrations/api_repo_git_blobs_test.go +++ b/integrations/api_repo_git_blobs_test.go @@ -38,7 +38,7 @@ func TestAPIReposGitBlobs(t *testing.T) { var gitBlobResponse api.GitBlobResponse DecodeJSON(t, resp, &gitBlobResponse) assert.NotNil(t, gitBlobResponse) - expectedContent := "Y29tbWl0IDY1ZjFiZjI3YmMzYmY3MGY2NDY1NzY1ODYzNWU2NjA5NGVkYmNiNGQKQXV0aG9yOiB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+CkRhdGU6ICAgU3VuIE1hciAxOSAxNjo0Nzo1OSAyMDE3IC0wNDAwCgogICAgSW5pdGlhbCBjb21taXQKCmRpZmYgLS1naXQgYS9SRUFETUUubWQgYi9SRUFETUUubWQKbmV3IGZpbGUgbW9kZSAxMDA2NDQKaW5kZXggMDAwMDAwMC4uNGI0ODUxYQotLS0gL2Rldi9udWxsCisrKyBiL1JFQURNRS5tZApAQCAtMCwwICsxLDMgQEAKKyMgcmVwbzEKKworRGVzY3JpcHRpb24gZm9yIHJlcG8xClwgTm8gbmV3bGluZSBhdCBlbmQgb2YgZmlsZQo=" + expectedContent := "dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK" assert.Equal(t, expectedContent, gitBlobResponse.Content) // Tests a private repo with no token so will fail diff --git a/models/error.go b/models/error.go index 3dd2c79e85..6458594a03 100644 --- a/models/error.go +++ b/models/error.go @@ -874,21 +874,6 @@ func (err ErrUserDoesNotHaveAccessToRepo) Error() string { // |______ / |__| (____ /___| /\___ >___| / // \/ \/ \/ \/ \/ -// ErrBranchNotExist represents a "BranchNotExist" kind of error. -type ErrBranchNotExist struct { - Name string -} - -// IsErrBranchNotExist checks if an error is a ErrBranchNotExist. -func IsErrBranchNotExist(err error) bool { - _, ok := err.(ErrBranchNotExist) - return ok -} - -func (err ErrBranchNotExist) Error() string { - return fmt.Sprintf("branch does not exist [name: %s]", err.Name) -} - // ErrBranchAlreadyExists represents an error that branch with such name already exists. type ErrBranchAlreadyExists struct { BranchName string diff --git a/models/pull.go b/models/pull.go index 88f9b1f6e8..d059081a44 100644 --- a/models/pull.go +++ b/models/pull.go @@ -1,4 +1,5 @@ // Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -165,8 +166,8 @@ func (pr *PullRequest) APIFormat() *api.PullRequest { func (pr *PullRequest) apiFormat(e Engine) *api.PullRequest { var ( - baseBranch *Branch - headBranch *Branch + baseBranch *git.Branch + headBranch *git.Branch baseCommit *git.Commit headCommit *git.Commit err error diff --git a/models/repo_branch.go b/models/repo_branch.go index 1c62a3d67d..0958e23974 100644 --- a/models/repo_branch.go +++ b/models/repo_branch.go @@ -1,4 +1,5 @@ // Copyright 2016 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -86,53 +87,24 @@ func (repo *Repository) DeleteLocalBranch(branchName string) error { return deleteLocalBranch(repo.LocalCopyPath(), repo.DefaultBranch, branchName) } -// Branch holds the branch information -type Branch struct { - Path string - Name string -} - -// GetBranchesByPath returns a branch by it's path -func GetBranchesByPath(path string) ([]*Branch, error) { - gitRepo, err := git.OpenRepository(path) - if err != nil { - return nil, err - } - - brs, err := gitRepo.GetBranches() - if err != nil { - return nil, err - } - - branches := make([]*Branch, len(brs)) - for i := range brs { - branches[i] = &Branch{ - Path: path, - Name: brs[i], - } - } - return branches, nil -} - // CanCreateBranch returns true if repository meets the requirements for creating new branches. func (repo *Repository) CanCreateBranch() bool { return !repo.IsMirror } -// GetBranch returns a branch by it's name -func (repo *Repository) GetBranch(branch string) (*Branch, error) { - if !git.IsBranchExist(repo.RepoPath(), branch) { - return nil, ErrBranchNotExist{branch} +// GetBranch returns a branch by its name +func (repo *Repository) GetBranch(branch string) (*git.Branch, error) { + gitRepo, err := git.OpenRepository(repo.RepoPath()) + if err != nil { + return nil, err } - return &Branch{ - Path: repo.RepoPath(), - Name: branch, - }, nil + + return gitRepo.GetBranch(branch) } // GetBranches returns all the branches of a repository -func (repo *Repository) GetBranches() ([]*Branch, error) { - return GetBranchesByPath(repo.RepoPath()) +func (repo *Repository) GetBranches() ([]*git.Branch, error) { + return git.GetBranchesByPath(repo.RepoPath()) } // CheckBranchName validates branch name with existing repository branches @@ -257,12 +229,3 @@ func (repo *Repository) CreateNewBranchFromCommit(doer *User, commit, branchName return nil } - -// GetCommit returns all the commits of a branch -func (branch *Branch) GetCommit() (*git.Commit, error) { - gitRepo, err := git.OpenRepository(branch.Path) - if err != nil { - return nil, err - } - return gitRepo.GetBranchCommit(branch.Name) -} diff --git a/modules/context/repo.go b/modules/context/repo.go index 9d3fb7cfec..4670e01564 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -157,10 +157,11 @@ func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) { if treeEntry.Blob().Size() >= setting.UI.MaxDisplayFileSize { return nil, git.ErrNotExist{ID: "", RelPath: ".editorconfig"} } - reader, err := treeEntry.Blob().Data() + reader, err := treeEntry.Blob().DataAsync() if err != nil { return nil, err } + defer reader.Close() data, err := ioutil.ReadAll(reader) if err != nil { return nil, err diff --git a/modules/git/blob.go b/modules/git/blob.go index e194b973db..171b4a1010 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -1,76 +1,40 @@ // Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package git import ( - "bytes" "encoding/base64" - "fmt" "io" "io/ioutil" - "os" - "os/exec" + + "gopkg.in/src-d/go-git.v4/plumbing" ) // Blob represents a Git object. type Blob struct { - repo *Repository - *TreeEntry -} + ID SHA1 -// Data gets content of blob all at once and wrap it as io.Reader. -// This can be very slow and memory consuming for huge content. -func (b *Blob) Data() (io.Reader, error) { - stdout := new(bytes.Buffer) - stderr := new(bytes.Buffer) - - // Preallocate memory to save ~50% memory usage on big files. - stdout.Grow(int(b.Size() + 2048)) - - if err := b.DataPipeline(stdout, stderr); err != nil { - return nil, concatenateError(err, stderr.String()) - } - return stdout, nil -} - -// DataPipeline gets content of blob and write the result or error to stdout or stderr -func (b *Blob) DataPipeline(stdout, stderr io.Writer) error { - return NewCommand("show", b.ID.String()).RunInDirPipeline(b.repo.Path, stdout, stderr) -} - -type cmdReadCloser struct { - cmd *exec.Cmd - stdout io.Reader -} - -func (c cmdReadCloser) Read(p []byte) (int, error) { - return c.stdout.Read(p) -} - -func (c cmdReadCloser) Close() error { - io.Copy(ioutil.Discard, c.stdout) - return c.cmd.Wait() + gogitEncodedObj plumbing.EncodedObject + name string } // DataAsync gets a ReadCloser for the contents of a blob without reading it all. // Calling the Close function on the result will discard all unread output. func (b *Blob) DataAsync() (io.ReadCloser, error) { - cmd := exec.Command("git", "show", b.ID.String()) - cmd.Dir = b.repo.Path - cmd.Stderr = os.Stderr + return b.gogitEncodedObj.Reader() +} - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, fmt.Errorf("StdoutPipe: %v", err) - } +// Size returns the uncompressed size of the blob +func (b *Blob) Size() int64 { + return b.gogitEncodedObj.Size() +} - if err = cmd.Start(); err != nil { - return nil, fmt.Errorf("Start: %v", err) - } - - return cmdReadCloser{stdout: stdout, cmd: cmd}, nil +// Name returns name of the tree entry this blob object was created from (or empty string) +func (b *Blob) Name() string { + return b.name } // GetBlobContentBase64 Reads the content of the blob with a base64 encode and returns the encoded string diff --git a/modules/git/blob_test.go b/modules/git/blob_test.go index 39516c422c..66c046ecc8 100644 --- a/modules/git/blob_test.go +++ b/modules/git/blob_test.go @@ -1,11 +1,11 @@ // Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package git import ( - "bytes" "io/ioutil" "testing" @@ -13,20 +13,6 @@ import ( "github.com/stretchr/testify/require" ) -var repoSelf = &Repository{ - Path: "./", -} - -var testBlob = &Blob{ - repo: repoSelf, - TreeEntry: &TreeEntry{ - ID: MustIDFromString("a8d4b49dd073a4a38a7e58385eeff7cc52568697"), - ptree: &Tree{ - repo: repoSelf, - }, - }, -} - func TestBlob_Data(t *testing.T) { output := `Copyright (c) 2016 The Gitea Authors Copyright (c) 2015 The Gogs Authors @@ -49,10 +35,15 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ` + repo, err := OpenRepository("../../.git") + assert.NoError(t, err) + testBlob, err := repo.GetBlob("a8d4b49dd073a4a38a7e58385eeff7cc52568697") + assert.NoError(t, err) - r, err := testBlob.Data() + r, err := testBlob.DataAsync() assert.NoError(t, err) require.NotNil(t, r) + defer r.Close() data, err := ioutil.ReadAll(r) assert.NoError(t, err) @@ -60,21 +51,21 @@ THE SOFTWARE. } func Benchmark_Blob_Data(b *testing.B) { + repo, err := OpenRepository("../../.git") + if err != nil { + b.Fatal(err) + } + testBlob, err := repo.GetBlob("a8d4b49dd073a4a38a7e58385eeff7cc52568697") + if err != nil { + b.Fatal(err) + } + for i := 0; i < b.N; i++ { - r, err := testBlob.Data() + r, err := testBlob.DataAsync() if err != nil { b.Fatal(err) } + defer r.Close() ioutil.ReadAll(r) } } - -func Benchmark_Blob_DataPipeline(b *testing.B) { - stdout := new(bytes.Buffer) - for i := 0; i < b.N; i++ { - stdout.Reset() - if err := testBlob.DataPipeline(stdout, nil); err != nil { - b.Fatal(err) - } - } -} diff --git a/modules/git/commit.go b/modules/git/commit.go index dad67dada6..7b64a300ab 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -14,6 +14,8 @@ import ( "net/http" "strconv" "strings" + + "gopkg.in/src-d/go-git.v4/plumbing/object" ) // Commit represents a git commit. @@ -36,20 +38,59 @@ type CommitGPGSignature struct { Payload string //TODO check if can be reconstruct from the rest of commit information to not have duplicate data } -// similar to https://github.com/git/git/blob/3bc53220cb2dcf709f7a027a3f526befd021d858/commit.c#L1128 -func newGPGSignatureFromCommitline(data []byte, signatureStart int, tag bool) (*CommitGPGSignature, error) { - sig := new(CommitGPGSignature) - signatureEnd := bytes.LastIndex(data, []byte("-----END PGP SIGNATURE-----")) - if signatureEnd == -1 { - return nil, fmt.Errorf("end of commit signature not found") +func convertPGPSignature(c *object.Commit) *CommitGPGSignature { + if c.PGPSignature == "" { + return nil } - sig.Signature = strings.Replace(string(data[signatureStart:signatureEnd+27]), "\n ", "\n", -1) - if tag { - sig.Payload = string(data[:signatureStart-1]) - } else { - sig.Payload = string(data[:signatureStart-8]) + string(data[signatureEnd+27:]) + + var w strings.Builder + var err error + + if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil { + return nil + } + + for _, parent := range c.ParentHashes { + if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil { + return nil + } + } + + if _, err = fmt.Fprint(&w, "author "); err != nil { + return nil + } + + if err = c.Author.Encode(&w); err != nil { + return nil + } + + if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil { + return nil + } + + if err = c.Committer.Encode(&w); err != nil { + return nil + } + + if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil { + return nil + } + + return &CommitGPGSignature{ + Signature: c.PGPSignature, + Payload: w.String(), + } +} + +func convertCommit(c *object.Commit) *Commit { + return &Commit{ + ID: c.Hash, + CommitMessage: c.Message, + Committer: &c.Committer, + Author: &c.Author, + Signature: convertPGPSignature(c), + parents: c.ParentHashes, } - return sig, nil } // Message returns the commit message. Same as retrieving CommitMessage directly. @@ -281,11 +322,13 @@ func (c *Commit) GetSubModules() (*ObjectCache, error) { } return nil, err } - rd, err := entry.Blob().Data() + + rd, err := entry.Blob().DataAsync() if err != nil { return nil, err } + defer rd.Close() scanner := bufio.NewScanner(rd) c.submoduleCache = newObjectCache() var ismodule bool @@ -326,6 +369,17 @@ func (c *Commit) GetSubModule(entryname string) (*SubModule, error) { return nil, nil } +// GetBranchName gets the closes branch name (as returned by 'git name-rev') +func (c *Commit) GetBranchName() (string, error) { + data, err := NewCommand("name-rev", c.ID.String()).RunInDirBytes(c.repo.Path) + if err != nil { + return "", err + } + + // name-rev commitID output will be "COMMIT_ID master" or "COMMIT_ID master~12" + return strings.Split(strings.Split(string(data), " ")[1], "~")[0], nil +} + // CommitFileStatus represents status of files in a commit. type CommitFileStatus struct { Added []string diff --git a/modules/git/commit_info.go b/modules/git/commit_info.go index 971082be1f..02c6f710ad 100644 --- a/modules/git/commit_info.go +++ b/modules/git/commit_info.go @@ -5,325 +5,237 @@ package git import ( - "bufio" - "context" - "fmt" - "os/exec" - "path" - "runtime" - "strconv" - "strings" - "sync" - "time" + "github.com/emirpasic/gods/trees/binaryheap" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" ) -const ( - // parameters for searching for commit infos. If the untargeted search has - // not found any entries in the past 5 commits, and 12 or fewer entries - // remain, then we'll just let the targeted-searching threads finish off, - // and stop the untargeted search to not interfere. - deferToTargetedSearchColdStreak = 5 - deferToTargetedSearchNumRemainingEntries = 12 -) - -// getCommitsInfoState shared state while getting commit info for entries -type getCommitsInfoState struct { - lock sync.Mutex - /* read-only fields, can be read without the mutex */ - // entries and entryPaths are read-only after initialization, so they can - // safely be read without the mutex - entries []*TreeEntry - // set of filepaths to get info for - entryPaths map[string]struct{} - treePath string - headCommit *Commit - - /* mutable fields, must hold mutex to read or write */ - // map from filepath to commit - commits map[string]*Commit - // set of filepaths that have been or are being searched for in a target search - targetedPaths map[string]struct{} -} - -func (state *getCommitsInfoState) numRemainingEntries() int { - state.lock.Lock() - defer state.lock.Unlock() - return len(state.entries) - len(state.commits) -} - -// getTargetEntryPath Returns the next path for a targeted-searching thread to -// search for, or returns the empty string if nothing left to search for -func (state *getCommitsInfoState) getTargetedEntryPath() string { - var targetedEntryPath string - state.lock.Lock() - defer state.lock.Unlock() - for _, entry := range state.entries { - entryPath := path.Join(state.treePath, entry.Name()) - if _, ok := state.commits[entryPath]; ok { - continue - } else if _, ok = state.targetedPaths[entryPath]; ok { - continue - } - targetedEntryPath = entryPath - state.targetedPaths[entryPath] = struct{}{} - break - } - return targetedEntryPath -} - -// repeatedly perform targeted searches for unpopulated entries -func targetedSearch(state *getCommitsInfoState, done chan error, cache LastCommitCache) { - for { - entryPath := state.getTargetedEntryPath() - if len(entryPath) == 0 { - done <- nil - return - } - if cache != nil { - commit, err := cache.Get(state.headCommit.repo.Path, state.headCommit.ID.String(), entryPath) - if err == nil && commit != nil { - state.update(entryPath, commit) - continue - } - } - command := NewCommand("rev-list", "-1", state.headCommit.ID.String(), "--", entryPath) - output, err := command.RunInDir(state.headCommit.repo.Path) - if err != nil { - done <- err - return - } - id, err := NewIDFromString(strings.TrimSpace(output)) - if err != nil { - done <- err - return - } - commit, err := state.headCommit.repo.getCommit(id) - if err != nil { - done <- err - return - } - state.update(entryPath, commit) - if cache != nil { - cache.Put(state.headCommit.repo.Path, state.headCommit.ID.String(), entryPath, commit) - } - } -} - -func initGetCommitInfoState(entries Entries, headCommit *Commit, treePath string) *getCommitsInfoState { - entryPaths := make(map[string]struct{}, len(entries)) - for _, entry := range entries { - entryPaths[path.Join(treePath, entry.Name())] = struct{}{} - } - if treePath = path.Clean(treePath); treePath == "." { - treePath = "" - } - return &getCommitsInfoState{ - entries: entries, - entryPaths: entryPaths, - commits: make(map[string]*Commit, len(entries)), - targetedPaths: make(map[string]struct{}, len(entries)), - treePath: treePath, - headCommit: headCommit, - } -} - // GetCommitsInfo gets information of all commits that are corresponding to these entries -func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCommitCache) ([][]interface{}, error) { - state := initGetCommitInfoState(tes, commit, treePath) - if err := getCommitsInfo(state, cache); err != nil { - return nil, err +func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCommitCache) ([][]interface{}, *Commit, error) { + entryPaths := make([]string, len(tes)+1) + // Get the commit for the treePath itself + entryPaths[0] = "" + for i, entry := range tes { + entryPaths[i+1] = entry.Name() } - if len(state.commits) < len(state.entryPaths) { - return nil, fmt.Errorf("could not find commits for all entries") + + c, err := commit.repo.gogitRepo.CommitObject(plumbing.Hash(commit.ID)) + if err != nil { + return nil, nil, err } + revs, err := getLastCommitForPaths(c, treePath, entryPaths) + if err != nil { + return nil, nil, err + } + + commit.repo.gogitStorage.Close() + commitsInfo := make([][]interface{}, len(tes)) for i, entry := range tes { - commit, ok := state.commits[path.Join(treePath, entry.Name())] - if !ok { - return nil, fmt.Errorf("could not find commit for %s", entry.Name()) - } - switch entry.Type { - case ObjectCommit: - subModuleURL := "" - if subModule, err := state.headCommit.GetSubModule(entry.Name()); err != nil { - return nil, err - } else if subModule != nil { - subModuleURL = subModule.URL - } - subModuleFile := NewSubModuleFile(commit, subModuleURL, entry.ID.String()) - commitsInfo[i] = []interface{}{entry, subModuleFile} - default: - commitsInfo[i] = []interface{}{entry, commit} - } - } - return commitsInfo, nil -} - -func (state *getCommitsInfoState) cleanEntryPath(rawEntryPath string) (string, error) { - if rawEntryPath[0] == '"' { - var err error - rawEntryPath, err = strconv.Unquote(rawEntryPath) - if err != nil { - return rawEntryPath, err - } - } - var entryNameStartIndex int - if len(state.treePath) > 0 { - entryNameStartIndex = len(state.treePath) + 1 - } - - if index := strings.IndexByte(rawEntryPath[entryNameStartIndex:], '/'); index >= 0 { - return rawEntryPath[:entryNameStartIndex+index], nil - } - return rawEntryPath, nil -} - -// update report that the given path was last modified by the given commit. -// Returns whether state.commits was updated -func (state *getCommitsInfoState) update(entryPath string, commit *Commit) bool { - if _, ok := state.entryPaths[entryPath]; !ok { - return false - } - - var updated bool - state.lock.Lock() - defer state.lock.Unlock() - if _, ok := state.commits[entryPath]; !ok { - state.commits[entryPath] = commit - updated = true - } - return updated -} - -const getCommitsInfoPretty = "--pretty=format:%H %ct %s" - -func getCommitsInfo(state *getCommitsInfoState, cache LastCommitCache) error { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - - args := []string{"log", state.headCommit.ID.String(), getCommitsInfoPretty, "--name-status", "-c"} - if len(state.treePath) > 0 { - args = append(args, "--", state.treePath) - } - cmd := exec.CommandContext(ctx, "git", args...) - cmd.Dir = state.headCommit.repo.Path - - readCloser, err := cmd.StdoutPipe() - if err != nil { - return err - } - - if err := cmd.Start(); err != nil { - return err - } - // it's okay to ignore the error returned by cmd.Wait(); we expect the - // subprocess to sometimes have a non-zero exit status, since we may - // prematurely close stdout, resulting in a broken pipe. - defer cmd.Wait() - - numThreads := runtime.NumCPU() - done := make(chan error, numThreads) - for i := 0; i < numThreads; i++ { - go targetedSearch(state, done, cache) - } - - scanner := bufio.NewScanner(readCloser) - err = state.processGitLogOutput(scanner) - - // it is important that we close stdout here; if we do not close - // stdout, the subprocess will keep running, and the deffered call - // cmd.Wait() may block for a long time. - if closeErr := readCloser.Close(); closeErr != nil && err == nil { - err = closeErr - } - - for i := 0; i < numThreads; i++ { - doneErr := <-done - if doneErr != nil && err == nil { - err = doneErr - } - } - return err -} - -func (state *getCommitsInfoState) processGitLogOutput(scanner *bufio.Scanner) error { - // keep a local cache of seen paths to avoid acquiring a lock for paths - // we've already seen - seenPaths := make(map[string]struct{}, len(state.entryPaths)) - // number of consecutive commits without any finds - coldStreak := 0 - var commit *Commit - var err error - for scanner.Scan() { - line := scanner.Text() - if len(line) == 0 { // in-between commits - numRemainingEntries := state.numRemainingEntries() - if numRemainingEntries == 0 { - break - } - if coldStreak >= deferToTargetedSearchColdStreak && - numRemainingEntries <= deferToTargetedSearchNumRemainingEntries { - // stop this untargeted search, and let the targeted-search threads - // finish the work - break - } - continue - } - if line[0] >= 'A' && line[0] <= 'X' { // a file was changed by the current commit - // look for the last tab, since for copies (C) and renames (R) two - // filenames are printed: src, then dest - tabIndex := strings.LastIndexByte(line, '\t') - if tabIndex < 1 { - return fmt.Errorf("misformatted line: %s", line) - } - entryPath, err := state.cleanEntryPath(line[tabIndex+1:]) - if err != nil { - return err - } - if _, ok := seenPaths[entryPath]; !ok { - if state.update(entryPath, commit) { - coldStreak = 0 + if rev, ok := revs[entry.Name()]; ok { + entryCommit := convertCommit(rev) + if entry.IsSubModule() { + subModuleURL := "" + if subModule, err := commit.GetSubModule(entry.Name()); err != nil { + return nil, nil, err + } else if subModule != nil { + subModuleURL = subModule.URL } - seenPaths[entryPath] = struct{}{} + subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) + commitsInfo[i] = []interface{}{entry, subModuleFile} + } else { + commitsInfo[i] = []interface{}{entry, entryCommit} } + } else { + commitsInfo[i] = []interface{}{entry, nil} + } + } + + // Retrieve the commit for the treePath itself (see above). We basically + // get it for free during the tree traversal and it's used for listing + // pages to display information about newest commit for a given path. + var treeCommit *Commit + if rev, ok := revs[""]; ok { + treeCommit = convertCommit(rev) + } + return commitsInfo, treeCommit, nil +} + +type commitAndPaths struct { + commit *object.Commit + // Paths that are still on the branch represented by commit + paths []string + // Set of hashes for the paths + hashes map[string]plumbing.Hash +} + +func getCommitTree(c *object.Commit, treePath string) (*object.Tree, error) { + tree, err := c.Tree() + if err != nil { + return nil, err + } + + // Optimize deep traversals by focusing only on the specific tree + if treePath != "" { + tree, err = tree.Tree(treePath) + if err != nil { + return nil, err + } + } + + return tree, nil +} + +func getFullPath(treePath, path string) string { + if treePath != "" { + if path != "" { + return treePath + "/" + path + } + return treePath + } + return path +} + +func getFileHashes(c *object.Commit, treePath string, paths []string) (map[string]plumbing.Hash, error) { + tree, err := getCommitTree(c, treePath) + if err == object.ErrDirectoryNotFound { + // The whole tree didn't exist, so return empty map + return make(map[string]plumbing.Hash), nil + } + if err != nil { + return nil, err + } + + hashes := make(map[string]plumbing.Hash) + for _, path := range paths { + if path != "" { + entry, err := tree.FindEntry(path) + if err == nil { + hashes[path] = entry.Hash + } + } else { + hashes[path] = tree.Hash + } + } + + return hashes, nil +} + +func getLastCommitForPaths(c *object.Commit, treePath string, paths []string) (map[string]*object.Commit, error) { + // We do a tree traversal with nodes sorted by commit time + seen := make(map[plumbing.Hash]bool) + heap := binaryheap.NewWith(func(a, b interface{}) int { + if a.(*commitAndPaths).commit.Committer.When.Before(b.(*commitAndPaths).commit.Committer.When) { + return 1 + } + return -1 + }) + + result := make(map[string]*object.Commit) + initialHashes, err := getFileHashes(c, treePath, paths) + if err != nil { + return nil, err + } + + // Start search from the root commit and with full set of paths + heap.Push(&commitAndPaths{c, paths, initialHashes}) + + for { + cIn, ok := heap.Pop() + if !ok { + break + } + current := cIn.(*commitAndPaths) + currentID := current.commit.ID() + + if seen[currentID] { continue } + seen[currentID] = true - // a new commit - commit, err = parseCommitInfo(line) - if err != nil { - return err + // Load the parent commits for the one we are currently examining + numParents := current.commit.NumParents() + var parents []*object.Commit + for i := 0; i < numParents; i++ { + parent, err := current.commit.Parent(i) + if err != nil { + break + } + parents = append(parents, parent) } - coldStreak++ - } - return scanner.Err() -} -// parseCommitInfo parse a commit from a line of `git log` output. Expects the -// line to be formatted according to getCommitsInfoPretty. -func parseCommitInfo(line string) (*Commit, error) { - if len(line) < 43 { - return nil, fmt.Errorf("invalid git output: %s", line) + // Examine the current commit and set of interesting paths + numOfParentsWithPath := make([]int, len(current.paths)) + pathChanged := make([]bool, len(current.paths)) + parentHashes := make([]map[string]plumbing.Hash, len(parents)) + for j, parent := range parents { + parentHashes[j], err = getFileHashes(parent, treePath, current.paths) + if err != nil { + break + } + + for i, path := range current.paths { + if parentHashes[j][path] != plumbing.ZeroHash { + numOfParentsWithPath[i]++ + if parentHashes[j][path] != current.hashes[path] { + pathChanged[i] = true + } + } + } + } + + var remainingPaths []string + for i, path := range current.paths { + switch numOfParentsWithPath[i] { + case 0: + // The path didn't exist in any parent, so it must have been created by + // this commit. The results could already contain some newer change from + // different path, so don't override that. + if result[path] == nil { + result[path] = current.commit + } + case 1: + // The file is present on exactly one parent, so check if it was changed + // and save the revision if it did. + if pathChanged[i] { + if result[path] == nil { + result[path] = current.commit + } + } else { + remainingPaths = append(remainingPaths, path) + } + default: + // The file is present on more than one of the parent paths, so this is + // a merge. We have to examine all the parent trees to find out where + // the change occurred. pathChanged[i] would tell us that the file was + // changed during the merge, but it wouldn't tell us the relevant commit + // that introduced it. + remainingPaths = append(remainingPaths, path) + } + } + + if len(remainingPaths) > 0 { + // Add the parent nodes along with remaining paths to the heap for further + // processing. + for j, parent := range parents { + if seen[parent.ID()] { + continue + } + + // Combine remainingPath with paths available on the parent branch + // and make union of them + var remainingPathsForParent []string + for _, path := range remainingPaths { + if parentHashes[j][path] != plumbing.ZeroHash { + remainingPathsForParent = append(remainingPathsForParent, path) + } + } + + heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]}) + } + } } - ref, err := NewIDFromString(line[:40]) - if err != nil { - return nil, err - } - spaceIndex := strings.IndexByte(line[41:], ' ') - if spaceIndex < 0 { - return nil, fmt.Errorf("invalid git output: %s", line) - } - unixSeconds, err := strconv.Atoi(line[41 : 41+spaceIndex]) - if err != nil { - return nil, err - } - message := line[spaceIndex+42:] - return &Commit{ - ID: ref, - CommitMessage: message, - Committer: &Signature{ - When: time.Unix(int64(unixSeconds), 0), - }, - }, nil + + return result, nil } diff --git a/modules/git/commit_info_test.go b/modules/git/commit_info_test.go index 120a9a737c..d7d863b032 100644 --- a/modules/git/commit_info_test.go +++ b/modules/git/commit_info_test.go @@ -51,7 +51,7 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) { assert.NoError(t, err) entries, err := tree.ListEntries() assert.NoError(t, err) - commitsInfo, err := entries.GetCommitsInfo(commit, testCase.Path, nil) + commitsInfo, _, err := entries.GetCommitsInfo(commit, testCase.Path, nil) assert.NoError(t, err) assert.Len(t, commitsInfo, len(testCase.ExpectedIDs)) for _, commitInfo := range commitsInfo { @@ -107,7 +107,7 @@ func BenchmarkEntries_GetCommitsInfo(b *testing.B) { b.ResetTimer() b.Run(benchmark.name, func(b *testing.B) { for i := 0; i < b.N; i++ { - _, err := entries.GetCommitsInfo(commit, "", nil) + _, _, err := entries.GetCommitsInfo(commit, "", nil) if err != nil { b.Fatal(err) } diff --git a/modules/git/error.go b/modules/git/error.go index 1aae5a37a2..6e4f26de13 100644 --- a/modules/git/error.go +++ b/modules/git/error.go @@ -64,3 +64,18 @@ func IsErrUnsupportedVersion(err error) bool { func (err ErrUnsupportedVersion) Error() string { return fmt.Sprintf("Operation requires higher version [required: %s]", err.Required) } + +// ErrBranchNotExist represents a "BranchNotExist" kind of error. +type ErrBranchNotExist struct { + Name string +} + +// IsErrBranchNotExist checks if an error is a ErrBranchNotExist. +func IsErrBranchNotExist(err error) bool { + _, ok := err.(ErrBranchNotExist) + return ok +} + +func (err ErrBranchNotExist) Error() string { + return fmt.Sprintf("branch does not exist [name: %s]", err.Name) +} diff --git a/modules/git/parse.go b/modules/git/parse.go index 5c964f16ee..22861b1d2c 100644 --- a/modules/git/parse.go +++ b/modules/git/parse.go @@ -8,6 +8,10 @@ import ( "bytes" "fmt" "strconv" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/filemode" + "gopkg.in/src-d/go-git.v4/plumbing/object" ) // ParseTreeEntries parses the output of a `git ls-tree` command. @@ -20,30 +24,26 @@ func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { for pos := 0; pos < len(data); { // expect line to be of the form " \t" entry := new(TreeEntry) + entry.gogitTreeEntry = &object.TreeEntry{} entry.ptree = ptree if pos+6 > len(data) { return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data)) } switch string(data[pos : pos+6]) { case "100644": - entry.mode = EntryModeBlob - entry.Type = ObjectBlob + entry.gogitTreeEntry.Mode = filemode.Regular pos += 12 // skip over "100644 blob " case "100755": - entry.mode = EntryModeExec - entry.Type = ObjectBlob + entry.gogitTreeEntry.Mode = filemode.Executable pos += 12 // skip over "100755 blob " case "120000": - entry.mode = EntryModeSymlink - entry.Type = ObjectBlob + entry.gogitTreeEntry.Mode = filemode.Symlink pos += 12 // skip over "120000 blob " case "160000": - entry.mode = EntryModeCommit - entry.Type = ObjectCommit + entry.gogitTreeEntry.Mode = filemode.Submodule pos += 14 // skip over "160000 object " case "040000": - entry.mode = EntryModeTree - entry.Type = ObjectTree + entry.gogitTreeEntry.Mode = filemode.Dir pos += 12 // skip over "040000 tree " default: return nil, fmt.Errorf("unknown type: %v", string(data[pos:pos+6])) @@ -57,6 +57,7 @@ func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { return nil, fmt.Errorf("Invalid ls-tree output: %v", err) } entry.ID = id + entry.gogitTreeEntry.Hash = plumbing.Hash(id) pos += 41 // skip over sha and trailing space end := pos + bytes.IndexByte(data[pos:], '\n') @@ -66,12 +67,12 @@ func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { // In case entry name is surrounded by double quotes(it happens only in git-shell). if data[pos] == '"' { - entry.name, err = strconv.Unquote(string(data[pos:end])) + entry.gogitTreeEntry.Name, err = strconv.Unquote(string(data[pos:end])) if err != nil { return nil, fmt.Errorf("Invalid ls-tree output: %v", err) } } else { - entry.name = string(data[pos:end]) + entry.gogitTreeEntry.Name = string(data[pos:end]) } pos = end + 1 diff --git a/modules/git/parse_test.go b/modules/git/parse_test.go index 66936cbdf0..d249623949 100644 --- a/modules/git/parse_test.go +++ b/modules/git/parse_test.go @@ -8,6 +8,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + "gopkg.in/src-d/go-git.v4/plumbing/filemode" + "gopkg.in/src-d/go-git.v4/plumbing/object" ) func TestParseTreeEntries(t *testing.T) { @@ -23,10 +25,12 @@ func TestParseTreeEntries(t *testing.T) { Input: "100644 blob 61ab7345a1a3bbc590068ccae37b8515cfc5843c\texample/file2.txt\n", Expected: []*TreeEntry{ { - mode: EntryModeBlob, - Type: ObjectBlob, - ID: MustIDFromString("61ab7345a1a3bbc590068ccae37b8515cfc5843c"), - name: "example/file2.txt", + ID: MustIDFromString("61ab7345a1a3bbc590068ccae37b8515cfc5843c"), + gogitTreeEntry: &object.TreeEntry{ + Hash: MustIDFromString("61ab7345a1a3bbc590068ccae37b8515cfc5843c"), + Name: "example/file2.txt", + Mode: filemode.Regular, + }, }, }, }, @@ -35,16 +39,20 @@ func TestParseTreeEntries(t *testing.T) { "040000 tree 1d01fb729fb0db5881daaa6030f9f2d3cd3d5ae8\texample\n", Expected: []*TreeEntry{ { - ID: MustIDFromString("61ab7345a1a3bbc590068ccae37b8515cfc5843c"), - Type: ObjectBlob, - mode: EntryModeSymlink, - name: "example/\n.txt", + ID: MustIDFromString("61ab7345a1a3bbc590068ccae37b8515cfc5843c"), + gogitTreeEntry: &object.TreeEntry{ + Hash: MustIDFromString("61ab7345a1a3bbc590068ccae37b8515cfc5843c"), + Name: "example/\n.txt", + Mode: filemode.Symlink, + }, }, { - ID: MustIDFromString("1d01fb729fb0db5881daaa6030f9f2d3cd3d5ae8"), - Type: ObjectTree, - mode: EntryModeTree, - name: "example", + ID: MustIDFromString("1d01fb729fb0db5881daaa6030f9f2d3cd3d5ae8"), + gogitTreeEntry: &object.TreeEntry{ + Hash: MustIDFromString("1d01fb729fb0db5881daaa6030f9f2d3cd3d5ae8"), + Name: "example", + Mode: filemode.Dir, + }, }, }, }, diff --git a/modules/git/repo.go b/modules/git/repo.go index 4306730920..f86c4aae5c 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -16,14 +16,20 @@ import ( "time" "github.com/Unknwon/com" + "gopkg.in/src-d/go-billy.v4/osfs" + gogit "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-git.v4/storage/filesystem" ) // Repository represents a Git repository. type Repository struct { Path string - commitCache *ObjectCache - tagCache *ObjectCache + tagCache *ObjectCache + + gogitRepo *gogit.Repository + gogitStorage *filesystem.Storage } const prettyLogFormat = `--pretty=format:%H` @@ -77,10 +83,25 @@ func OpenRepository(repoPath string) (*Repository, error) { return nil, errors.New("no such file or directory") } + fs := osfs.New(repoPath) + _, err = fs.Stat(".git") + if err == nil { + fs, err = fs.Chroot(".git") + if err != nil { + return nil, err + } + } + storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true}) + gogitRepo, err := gogit.Open(storage, fs) + if err != nil { + return nil, err + } + return &Repository{ - Path: repoPath, - commitCache: newObjectCache(), - tagCache: newObjectCache(), + Path: repoPath, + gogitRepo: gogitRepo, + gogitStorage: storage, + tagCache: newObjectCache(), }, nil } diff --git a/modules/git/repo_blob.go b/modules/git/repo_blob.go index a9445a1f7a..db63491ce4 100644 --- a/modules/git/repo_blob.go +++ b/modules/git/repo_blob.go @@ -4,19 +4,19 @@ package git +import ( + "gopkg.in/src-d/go-git.v4/plumbing" +) + func (repo *Repository) getBlob(id SHA1) (*Blob, error) { - if _, err := NewCommand("cat-file", "-p", id.String()).RunInDir(repo.Path); err != nil { + encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, plumbing.Hash(id)) + if err != nil { return nil, ErrNotExist{id.String(), ""} } return &Blob{ - repo: repo, - TreeEntry: &TreeEntry{ - ID: id, - ptree: &Tree{ - repo: repo, - }, - }, + ID: id, + gogitEncodedObj: encodedObj, }, nil } diff --git a/modules/git/repo_blob_test.go b/modules/git/repo_blob_test.go index 074365f164..128a227829 100644 --- a/modules/git/repo_blob_test.go +++ b/modules/git/repo_blob_test.go @@ -30,8 +30,9 @@ func TestRepository_GetBlob_Found(t *testing.T) { blob, err := r.GetBlob(testCase.OID) assert.NoError(t, err) - dataReader, err := blob.Data() + dataReader, err := blob.DataAsync() assert.NoError(t, err) + defer dataReader.Close() data, err := ioutil.ReadAll(dataReader) assert.NoError(t, err) diff --git a/modules/git/repo_branch.go b/modules/git/repo_branch.go index 6414abbec5..83689ee9dc 100644 --- a/modules/git/repo_branch.go +++ b/modules/git/repo_branch.go @@ -9,7 +9,6 @@ import ( "fmt" "strings" - "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" ) @@ -29,13 +28,19 @@ func IsBranchExist(repoPath, name string) bool { // IsBranchExist returns true if given branch exists in current repository. func (repo *Repository) IsBranchExist(name string) bool { - return IsBranchExist(repo.Path, name) + _, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true) + if err != nil { + return false + } + return true } // Branch represents a Git branch. type Branch struct { Name string Path string + + gitRepo *Repository } // GetHEADBranch returns corresponding branch of HEAD. @@ -51,8 +56,9 @@ func (repo *Repository) GetHEADBranch() (*Branch, error) { } return &Branch{ - Name: stdout[len(BranchPrefix):], - Path: stdout, + Name: stdout[len(BranchPrefix):], + Path: stdout, + gitRepo: repo, }, nil } @@ -64,23 +70,56 @@ func (repo *Repository) SetDefaultBranch(name string) error { // GetBranches returns all branches of the repository. func (repo *Repository) GetBranches() ([]string, error) { - r, err := git.PlainOpen(repo.Path) + var branchNames []string + + branches, err := repo.gogitRepo.Branches() if err != nil { return nil, err } - branchIter, err := r.Branches() + branches.ForEach(func(branch *plumbing.Reference) error { + branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix)) + return nil + }) + + // TODO: Sort? + + return branchNames, nil +} + +// GetBranch returns a branch by it's name +func (repo *Repository) GetBranch(branch string) (*Branch, error) { + if !repo.IsBranchExist(branch) { + return nil, ErrBranchNotExist{branch} + } + return &Branch{ + Path: repo.Path, + Name: branch, + gitRepo: repo, + }, nil +} + +// GetBranchesByPath returns a branch by it's path +func GetBranchesByPath(path string) ([]*Branch, error) { + gitRepo, err := OpenRepository(path) if err != nil { return nil, err } - branches := make([]string, 0) - if err = branchIter.ForEach(func(branch *plumbing.Reference) error { - branches = append(branches, branch.Name().Short()) - return nil - }); err != nil { + + brs, err := gitRepo.GetBranches() + if err != nil { return nil, err } + branches := make([]*Branch, len(brs)) + for i := range brs { + branches[i] = &Branch{ + Path: path, + Name: brs[i], + gitRepo: gitRepo, + } + } + return branches, nil } @@ -132,3 +171,8 @@ func (repo *Repository) RemoveRemote(name string) error { _, err := NewCommand("remote", "remove", name).RunInDir(repo.Path) return err } + +// GetCommit returns the head commit of a branch +func (branch *Branch) GetCommit() (*Commit, error) { + return branch.gitRepo.GetBranchCommit(branch.Name) +} diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go index 7c65d6e921..b631f9341e 100644 --- a/modules/git/repo_commit.go +++ b/modules/git/repo_commit.go @@ -1,4 +1,5 @@ // Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -7,22 +8,23 @@ package git import ( "bytes" "container/list" + "fmt" "strconv" "strings" "github.com/mcuadros/go-version" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" ) // GetRefCommitID returns the last commit ID string of given reference (branch or tag). func (repo *Repository) GetRefCommitID(name string) (string, error) { - stdout, err := NewCommand("show-ref", "--verify", name).RunInDir(repo.Path) + ref, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true) if err != nil { - if strings.Contains(err.Error(), "not a valid ref") { - return "", ErrNotExist{name, ""} - } return "", err } - return strings.Split(stdout, " ")[0], nil + + return ref.Hash().String(), nil } // GetBranchCommitID returns last commit ID string of given branch. @@ -42,114 +44,69 @@ func (repo *Repository) GetTagCommitID(name string) (string, error) { return strings.TrimSpace(stdout), nil } -// parseCommitData parses commit information from the (uncompressed) raw -// data from the commit object. -// \n\n separate headers from message -func parseCommitData(data []byte) (*Commit, error) { - commit := new(Commit) - commit.parents = make([]SHA1, 0, 1) - // we now have the contents of the commit object. Let's investigate... - nextline := 0 -l: - for { - eol := bytes.IndexByte(data[nextline:], '\n') - switch { - case eol > 0: - line := data[nextline : nextline+eol] - spacepos := bytes.IndexByte(line, ' ') - reftype := line[:spacepos] - switch string(reftype) { - case "tree", "object": - id, err := NewIDFromString(string(line[spacepos+1:])) - if err != nil { - return nil, err - } - commit.Tree.ID = id - case "parent": - // A commit can have one or more parents - oid, err := NewIDFromString(string(line[spacepos+1:])) - if err != nil { - return nil, err - } - commit.parents = append(commit.parents, oid) - case "author", "tagger": - sig, err := newSignatureFromCommitline(line[spacepos+1:]) - if err != nil { - return nil, err - } - commit.Author = sig - case "committer": - sig, err := newSignatureFromCommitline(line[spacepos+1:]) - if err != nil { - return nil, err - } - commit.Committer = sig - case "gpgsig": - sig, err := newGPGSignatureFromCommitline(data, nextline+spacepos+1, false) - if err != nil { - return nil, err - } - commit.Signature = sig - } - nextline += eol + 1 - case eol == 0: - cm := string(data[nextline+1:]) - - // Tag GPG signatures are stored below the commit message - sigindex := strings.Index(cm, "-----BEGIN PGP SIGNATURE-----") - if sigindex != -1 { - sig, err := newGPGSignatureFromCommitline(data, (nextline+1)+sigindex, true) - if err == nil && sig != nil { - // remove signature from commit message - if sigindex == 0 { - cm = "" - } else { - cm = cm[:sigindex-1] - } - commit.Signature = sig - } - } - - commit.CommitMessage = cm - break l - default: - break l - } +func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature { + if t.PGPSignature == "" { + return nil + } + + var w strings.Builder + var err error + + if _, err = fmt.Fprintf(&w, + "object %s\ntype %s\ntag %s\ntagger ", + t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil { + return nil + } + + if err = t.Tagger.Encode(&w); err != nil { + return nil + } + + if _, err = fmt.Fprintf(&w, "\n\n"); err != nil { + return nil + } + + if _, err = fmt.Fprintf(&w, t.Message); err != nil { + return nil + } + + return &CommitGPGSignature{ + Signature: t.PGPSignature, + Payload: strings.TrimSpace(w.String()) + "\n", } - return commit, nil } func (repo *Repository) getCommit(id SHA1) (*Commit, error) { - c, ok := repo.commitCache.Get(id.String()) - if ok { - log("Hit cache: %s", id) - return c.(*Commit), nil - } + var tagObject *object.Tag - data, err := NewCommand("cat-file", "-p", id.String()).RunInDirBytes(repo.Path) - if err != nil { - if strings.Contains(err.Error(), "fatal: Not a valid object name") { - return nil, ErrNotExist{id.String(), ""} + gogitCommit, err := repo.gogitRepo.CommitObject(plumbing.Hash(id)) + if err == plumbing.ErrObjectNotFound { + tagObject, err = repo.gogitRepo.TagObject(plumbing.Hash(id)) + if err == nil { + gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target) } - return nil, err } - - commit, err := parseCommitData(data) if err != nil { return nil, err } + + commit := convertCommit(gogitCommit) commit.repo = repo - commit.ID = id - data, err = NewCommand("name-rev", id.String()).RunInDirBytes(repo.Path) + if tagObject != nil { + commit.CommitMessage = strings.TrimSpace(tagObject.Message) + commit.Author = &tagObject.Tagger + commit.Signature = convertPGPSignatureForTag(tagObject) + } + + tree, err := gogitCommit.Tree() if err != nil { return nil, err } - // name-rev commitID output will be "COMMIT_ID master" or "COMMIT_ID master~12" - commit.Branch = strings.Split(strings.Split(string(data), " ")[1], "~")[0] + commit.Tree.ID = tree.Hash + commit.Tree.gogitTree = tree - repo.commitCache.Set(id.String(), commit) return commit, nil } diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go index 84825d7dc3..8c72528933 100644 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -1,4 +1,5 @@ // Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -8,6 +9,7 @@ import ( "strings" "github.com/mcuadros/go-version" + "gopkg.in/src-d/go-git.v4/plumbing" ) // TagPrefix tags prefix path on the repository @@ -20,7 +22,11 @@ func IsTagExist(repoPath, name string) bool { // IsTagExist returns true if given tag exists in the repository. func (repo *Repository) IsTagExist(name string) bool { - return IsTagExist(repo.Path, name) + _, err := repo.gogitRepo.Reference(plumbing.ReferenceName(TagPrefix+name), true) + if err != nil { + return false + } + return true } // CreateTag create one tag in the repository @@ -122,28 +128,25 @@ func (repo *Repository) GetTagInfos() ([]*Tag, error) { // GetTags returns all tags of the repository. func (repo *Repository) GetTags() ([]string, error) { - cmd := NewCommand("tag", "-l") - if version.Compare(gitVersion, "2.0.0", ">=") { - cmd.AddArguments("--sort=-v:refname") - } + var tagNames []string - stdout, err := cmd.RunInDir(repo.Path) + tags, err := repo.gogitRepo.Tags() if err != nil { return nil, err } - tags := strings.Split(stdout, "\n") - tags = tags[:len(tags)-1] + tags.ForEach(func(tag *plumbing.Reference) error { + tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix)) + return nil + }) - if version.Compare(gitVersion, "2.0.0", "<") { - version.Sort(tags) + version.Sort(tagNames) - // Reverse order - for i := 0; i < len(tags)/2; i++ { - j := len(tags) - i - 1 - tags[i], tags[j] = tags[j], tags[i] - } + // Reverse order + for i := 0; i < len(tagNames)/2; i++ { + j := len(tagNames) - i - 1 + tagNames[i], tagNames[j] = tagNames[j], tagNames[i] } - return tags, nil + return tagNames, nil } diff --git a/modules/git/repo_tree.go b/modules/git/repo_tree.go index 3fa491d529..8a024fe6ac 100644 --- a/modules/git/repo_tree.go +++ b/modules/git/repo_tree.go @@ -1,19 +1,23 @@ // Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package git +import ( + "gopkg.in/src-d/go-git.v4/plumbing" +) + func (repo *Repository) getTree(id SHA1) (*Tree, error) { - treePath := filepathFromSHA1(repo.Path, id.String()) - if isFile(treePath) { - _, err := NewCommand("ls-tree", id.String()).RunInDir(repo.Path) - if err != nil { - return nil, ErrNotExist{id.String(), ""} - } + gogitTree, err := repo.gogitRepo.TreeObject(plumbing.Hash(id)) + if err != nil { + return nil, err } - return NewTree(repo, id), nil + tree := NewTree(repo, id) + tree.gogitTree = gogitTree + return tree, nil } // GetTree find the tree object in the repository. @@ -31,5 +35,14 @@ func (repo *Repository) GetTree(idStr string) (*Tree, error) { if err != nil { return nil, err } - return repo.getTree(id) + commitObject, err := repo.gogitRepo.CommitObject(plumbing.Hash(id)) + if err != nil { + return nil, err + } + treeObject, err := repo.getTree(SHA1(commitObject.TreeHash)) + if err != nil { + return nil, err + } + treeObject.CommitID = id + return treeObject, nil } diff --git a/modules/git/sha1.go b/modules/git/sha1.go index 6c9d53949d..57b06fe738 100644 --- a/modules/git/sha1.go +++ b/modules/git/sha1.go @@ -1,44 +1,23 @@ // Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package git import ( - "bytes" "encoding/hex" "fmt" "strings" + + "gopkg.in/src-d/go-git.v4/plumbing" ) // EmptySHA defines empty git SHA const EmptySHA = "0000000000000000000000000000000000000000" // SHA1 a git commit name -type SHA1 [20]byte - -// Equal returns true if s has the same SHA1 as caller. -// Support 40-length-string, []byte, SHA1. -func (id SHA1) Equal(s2 interface{}) bool { - switch v := s2.(type) { - case string: - if len(v) != 40 { - return false - } - return v == id.String() - case []byte: - return bytes.Equal(v, id[:]) - case SHA1: - return v == id - default: - return false - } -} - -// String returns string (hex) representation of the Oid. -func (id SHA1) String() string { - return hex.EncodeToString(id[:]) -} +type SHA1 = plumbing.Hash // MustID always creates a new SHA1 from a [20]byte array with no validation of input. func MustID(b []byte) SHA1 { diff --git a/modules/git/signature.go b/modules/git/signature.go index e6ab247fd7..3f67bceb09 100644 --- a/modules/git/signature.go +++ b/modules/git/signature.go @@ -1,4 +1,5 @@ // Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -8,14 +9,12 @@ import ( "bytes" "strconv" "time" + + "gopkg.in/src-d/go-git.v4/plumbing/object" ) // Signature represents the Author or Committer information. -type Signature struct { - Email string - Name string - When time.Time -} +type Signature = object.Signature const ( // GitTimeLayout is the (default) time layout used by git. diff --git a/modules/git/tree.go b/modules/git/tree.go index 5ec22a3a6f..8f55d7a8c5 100644 --- a/modules/git/tree.go +++ b/modules/git/tree.go @@ -1,29 +1,31 @@ // Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package git import ( + "io" "strings" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" ) // Tree represents a flat directory listing. type Tree struct { - ID SHA1 - repo *Repository + ID SHA1 + CommitID SHA1 + repo *Repository + + gogitTree *object.Tree // parent tree ptree *Tree - - entries Entries - entriesParsed bool - - entriesRecursive Entries - entriesRecursiveParsed bool } -// NewTree create a new tree according the repository and commit id +// NewTree create a new tree according the repository and tree id func NewTree(repo *Repository, id SHA1) *Tree { return &Tree{ ID: id, @@ -60,39 +62,68 @@ func (t *Tree) SubTree(rpath string) (*Tree, error) { return g, nil } +func (t *Tree) loadTreeObject() error { + gogitTree, err := t.repo.gogitRepo.TreeObject(plumbing.Hash(t.ID)) + if err != nil { + return err + } + + t.gogitTree = gogitTree + return nil +} + // ListEntries returns all entries of current tree. func (t *Tree) ListEntries() (Entries, error) { - if t.entriesParsed { - return t.entries, nil + if t.gogitTree == nil { + err := t.loadTreeObject() + if err != nil { + return nil, err + } } - stdout, err := NewCommand("ls-tree", t.ID.String()).RunInDirBytes(t.repo.Path) - if err != nil { - return nil, err + entries := make([]*TreeEntry, len(t.gogitTree.Entries)) + for i, entry := range t.gogitTree.Entries { + entries[i] = &TreeEntry{ + ID: entry.Hash, + gogitTreeEntry: &t.gogitTree.Entries[i], + ptree: t, + } } - t.entries, err = parseTreeEntries(stdout, t) - if err == nil { - t.entriesParsed = true - } - - return t.entries, err + return entries, nil } // ListEntriesRecursive returns all entries of current tree recursively including all subtrees func (t *Tree) ListEntriesRecursive() (Entries, error) { - if t.entriesRecursiveParsed { - return t.entriesRecursive, nil - } - stdout, err := NewCommand("ls-tree", "-t", "-r", t.ID.String()).RunInDirBytes(t.repo.Path) - if err != nil { - return nil, err + if t.gogitTree == nil { + err := t.loadTreeObject() + if err != nil { + return nil, err + } } - t.entriesRecursive, err = parseTreeEntries(stdout, t) - if err == nil { - t.entriesRecursiveParsed = true + var entries []*TreeEntry + seen := map[plumbing.Hash]bool{} + walker := object.NewTreeWalker(t.gogitTree, true, seen) + for { + _, entry, err := walker.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if seen[entry.Hash] { + continue + } + + convertedEntry := &TreeEntry{ + ID: entry.Hash, + gogitTreeEntry: &entry, + ptree: t, + } + entries = append(entries, convertedEntry) } - return t.entriesRecursive, err + return entries, nil } diff --git a/modules/git/tree_blob.go b/modules/git/tree_blob.go index a37f6b2279..14237df6e8 100644 --- a/modules/git/tree_blob.go +++ b/modules/git/tree_blob.go @@ -1,4 +1,5 @@ // Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -7,15 +8,23 @@ package git import ( "path" "strings" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/filemode" + "gopkg.in/src-d/go-git.v4/plumbing/object" ) // GetTreeEntryByPath get the tree entries according the sub dir func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { if len(relpath) == 0 { return &TreeEntry{ - ID: t.ID, - Type: ObjectTree, - mode: EntryModeTree, + ID: t.ID, + //Type: ObjectTree, + gogitTreeEntry: &object.TreeEntry{ + Name: "", + Mode: filemode.Dir, + Hash: plumbing.Hash(t.ID), + }, }, nil } @@ -30,7 +39,7 @@ func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { return nil, err } for _, v := range entries { - if v.name == name { + if v.Name() == name { return v, nil } } diff --git a/modules/git/tree_entry.go b/modules/git/tree_entry.go index 5b74e9a695..fe2fd14f97 100644 --- a/modules/git/tree_entry.go +++ b/modules/git/tree_entry.go @@ -1,4 +1,5 @@ // Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -7,8 +8,11 @@ package git import ( "io" "sort" - "strconv" "strings" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/filemode" + "gopkg.in/src-d/go-git.v4/plumbing/object" ) // EntryMode the type of the object in the git tree @@ -18,28 +22,23 @@ type EntryMode int // one of these. const ( // EntryModeBlob - EntryModeBlob EntryMode = 0x0100644 + EntryModeBlob EntryMode = 0100644 // EntryModeExec - EntryModeExec EntryMode = 0x0100755 + EntryModeExec EntryMode = 0100755 // EntryModeSymlink - EntryModeSymlink EntryMode = 0x0120000 + EntryModeSymlink EntryMode = 0120000 // EntryModeCommit - EntryModeCommit EntryMode = 0x0160000 + EntryModeCommit EntryMode = 0160000 // EntryModeTree - EntryModeTree EntryMode = 0x0040000 + EntryModeTree EntryMode = 0040000 ) // TreeEntry the leaf in the git tree type TreeEntry struct { - ID SHA1 - Type ObjectType + ID SHA1 - mode EntryMode - name string - - ptree *Tree - - committed bool + gogitTreeEntry *object.TreeEntry + ptree *Tree size int64 sized bool @@ -47,12 +46,24 @@ type TreeEntry struct { // Name returns the name of the entry func (te *TreeEntry) Name() string { - return te.name + return te.gogitTreeEntry.Name } // Mode returns the mode of the entry func (te *TreeEntry) Mode() EntryMode { - return te.mode + return EntryMode(te.gogitTreeEntry.Mode) +} + +// Type returns the type of the entry (commit, tree, blob) +func (te *TreeEntry) Type() string { + switch te.Mode() { + case EntryModeCommit: + return "commit" + case EntryModeTree: + return "tree" + default: + return "blob" + } } // Size returns the size of the entry @@ -63,36 +74,47 @@ func (te *TreeEntry) Size() int64 { return te.size } - stdout, err := NewCommand("cat-file", "-s", te.ID.String()).RunInDir(te.ptree.repo.Path) + file, err := te.ptree.gogitTree.TreeEntryFile(te.gogitTreeEntry) if err != nil { return 0 } te.sized = true - te.size, _ = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64) + te.size = file.Size return te.size } // IsSubModule if the entry is a sub module func (te *TreeEntry) IsSubModule() bool { - return te.mode == EntryModeCommit + return te.gogitTreeEntry.Mode == filemode.Submodule } // IsDir if the entry is a sub dir func (te *TreeEntry) IsDir() bool { - return te.mode == EntryModeTree + return te.gogitTreeEntry.Mode == filemode.Dir } // IsLink if the entry is a symlink func (te *TreeEntry) IsLink() bool { - return te.mode == EntryModeSymlink + return te.gogitTreeEntry.Mode == filemode.Symlink } -// Blob retrun the blob object the entry +// IsRegular if the entry is a regular file +func (te *TreeEntry) IsRegular() bool { + return te.gogitTreeEntry.Mode == filemode.Regular +} + +// Blob returns the blob object the entry func (te *TreeEntry) Blob() *Blob { + encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash) + if err != nil { + return nil + } + return &Blob{ - repo: te.ptree.repo, - TreeEntry: te, + ID: te.gogitTreeEntry.Hash, + gogitEncodedObj: encodedObj, + name: te.Name(), } } @@ -103,10 +125,11 @@ func (te *TreeEntry) FollowLink() (*TreeEntry, error) { } // read the link - r, err := te.Blob().Data() + r, err := te.Blob().DataAsync() if err != nil { return nil, err } + defer r.Close() buf := make([]byte, te.Size()) _, err = io.ReadFull(r, buf) if err != nil { @@ -140,18 +163,18 @@ func (te *TreeEntry) GetSubJumpablePathName() string { if te.IsSubModule() || !te.IsDir() { return "" } - tree, err := te.ptree.SubTree(te.name) + tree, err := te.ptree.SubTree(te.Name()) if err != nil { - return te.name + return te.Name() } entries, _ := tree.ListEntries() if len(entries) == 1 && entries[0].IsDir() { name := entries[0].GetSubJumpablePathName() if name != "" { - return te.name + "/" + name + return te.Name() + "/" + name } } - return te.name + return te.Name() } // Entries a list of entry @@ -167,7 +190,7 @@ var sorter = []func(t1, t2 *TreeEntry, cmp func(s1, s2 string) bool) bool{ return (t1.IsDir() || t1.IsSubModule()) && !t2.IsDir() && !t2.IsSubModule() }, func(t1, t2 *TreeEntry, cmp func(s1, s2 string) bool) bool { - return cmp(t1.name, t2.name) + return cmp(t1.Name(), t2.Name()) }, } diff --git a/modules/git/tree_entry_test.go b/modules/git/tree_entry_test.go index 52920bc00e..c65a691ecf 100644 --- a/modules/git/tree_entry_test.go +++ b/modules/git/tree_entry_test.go @@ -8,18 +8,20 @@ import ( "testing" "github.com/stretchr/testify/assert" + "gopkg.in/src-d/go-git.v4/plumbing/filemode" + "gopkg.in/src-d/go-git.v4/plumbing/object" ) func getTestEntries() Entries { return Entries{ - &TreeEntry{name: "v1.0", mode: EntryModeTree}, - &TreeEntry{name: "v2.0", mode: EntryModeTree}, - &TreeEntry{name: "v2.1", mode: EntryModeTree}, - &TreeEntry{name: "v2.12", mode: EntryModeTree}, - &TreeEntry{name: "v2.2", mode: EntryModeTree}, - &TreeEntry{name: "v12.0", mode: EntryModeTree}, - &TreeEntry{name: "abc", mode: EntryModeBlob}, - &TreeEntry{name: "bcd", mode: EntryModeBlob}, + &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "v1.0", Mode: filemode.Dir}}, + &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "v2.0", Mode: filemode.Dir}}, + &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "v2.1", Mode: filemode.Dir}}, + &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "v2.12", Mode: filemode.Dir}}, + &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "v2.2", Mode: filemode.Dir}}, + &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "v12.0", Mode: filemode.Dir}}, + &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "abc", Mode: filemode.Regular}}, + &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "bcd", Mode: filemode.Regular}}, } } diff --git a/modules/repofiles/blob_test.go b/modules/repofiles/blob_test.go index 260b775fc4..55320345f9 100644 --- a/modules/repofiles/blob_test.go +++ b/modules/repofiles/blob_test.go @@ -27,7 +27,7 @@ func TestGetBlobBySHA(t *testing.T) { gbr, err := GetBlobBySHA(ctx.Repo.Repository, ctx.Params(":sha")) expectedGBR := &api.GitBlobResponse{ - Content: "Y29tbWl0IDY1ZjFiZjI3YmMzYmY3MGY2NDY1NzY1ODYzNWU2NjA5NGVkYmNiNGQKQXV0aG9yOiB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+CkRhdGU6ICAgU3VuIE1hciAxOSAxNjo0Nzo1OSAyMDE3IC0wNDAwCgogICAgSW5pdGlhbCBjb21taXQKCmRpZmYgLS1naXQgYS9SRUFETUUubWQgYi9SRUFETUUubWQKbmV3IGZpbGUgbW9kZSAxMDA2NDQKaW5kZXggMDAwMDAwMC4uNGI0ODUxYQotLS0gL2Rldi9udWxsCisrKyBiL1JFQURNRS5tZApAQCAtMCwwICsxLDMgQEAKKyMgcmVwbzEKKworRGVzY3JpcHRpb24gZm9yIHJlcG8xClwgTm8gbmV3bGluZSBhdCBlbmQgb2YgZmlsZQo=", + Content: "dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK", Encoding: "base64", URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/65f1bf27bc3bf70f64657658635e66094edbcb4d", SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", diff --git a/modules/repofiles/content.go b/modules/repofiles/content.go index d55ca497cd..d35c5a6a67 100644 --- a/modules/repofiles/content.go +++ b/modules/repofiles/content.go @@ -61,7 +61,7 @@ func GetFileContents(repo *models.Repository, treePath, ref string) (*api.FileCo HTMLURL: htmlURL.String(), GitURL: gitURL.String(), DownloadURL: downloadURL.String(), - Type: string(entry.Type), + Type: entry.Type(), Links: &api.FileLinksResponse{ Self: selfURL.String(), GitURL: gitURL.String(), diff --git a/modules/repofiles/temp_repo.go b/modules/repofiles/temp_repo.go index 5bf64d52a8..46e03f5650 100644 --- a/modules/repofiles/temp_repo.go +++ b/modules/repofiles/temp_repo.go @@ -58,7 +58,7 @@ func (t *TemporaryUploadRepository) Clone(branch string) error { fmt.Sprintf("Clone (git clone -s --bare): %s", t.basePath), "git", "clone", "-s", "--bare", "-b", branch, t.repo.RepoPath(), t.basePath); err != nil { if matched, _ := regexp.MatchString(".*Remote branch .* not found in upstream origin.*", stderr); matched { - return models.ErrBranchNotExist{ + return git.ErrBranchNotExist{ Name: branch, } } else if matched, _ := regexp.MatchString(".* repository .* does not exist.*", stderr); matched { diff --git a/modules/repofiles/tree.go b/modules/repofiles/tree.go index 8766ed36d0..5b4e7aeb28 100644 --- a/modules/repofiles/tree.go +++ b/modules/repofiles/tree.go @@ -23,7 +23,7 @@ func GetTreeBySHA(repo *models.Repository, sha string, page, perPage int, recurs } } tree := new(api.GitTreeResponse) - tree.SHA = gitTree.ID.String() + tree.SHA = gitTree.CommitID.String() tree.URL = repo.APIURL() + "/git/trees/" + tree.SHA var entries git.Entries if recursive { @@ -74,11 +74,12 @@ func GetTreeBySHA(repo *models.Repository, sha string, page, perPage int, recurs tree.Entries = make([]api.GitEntry, rangeEnd-rangeStart) for e := rangeStart; e < rangeEnd; e++ { i := e - rangeStart - tree.Entries[i].Path = entries[e].Name() - tree.Entries[i].Mode = fmt.Sprintf("%06x", entries[e].Mode()) - tree.Entries[i].Type = string(entries[e].Type) - tree.Entries[i].Size = entries[e].Size() - tree.Entries[i].SHA = entries[e].ID.String() + + tree.Entries[e].Path = entries[e].Name() + tree.Entries[e].Mode = fmt.Sprintf("%06o", entries[e].Mode()) + tree.Entries[e].Type = entries[e].Type() + tree.Entries[e].Size = entries[e].Size() + tree.Entries[e].SHA = entries[e].ID.String() if entries[e].IsDir() { copy(treeURL[copyPos:], entries[e].ID.String()) diff --git a/modules/repofiles/tree_test.go b/modules/repofiles/tree_test.go index 266dc91670..c211cfcfd2 100644 --- a/modules/repofiles/tree_test.go +++ b/modules/repofiles/tree_test.go @@ -46,5 +46,6 @@ func TestGetTreeBySHA(t *testing.T) { Page: 1, TotalCount: 1, } - assert.EqualValues(t, tree, expectedTree) + + assert.EqualValues(t, expectedTree, tree) } diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 216df18cd3..e9b3077535 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -62,7 +62,7 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up BranchName: opts.NewBranch, } } - if err != nil && !models.IsErrBranchNotExist(err) { + if err != nil && !git.IsErrBranchNotExist(err) { return nil, err } } else { diff --git a/routers/api/v1/convert/convert.go b/routers/api/v1/convert/convert.go index aca21db4ed..f132dedb55 100644 --- a/routers/api/v1/convert/convert.go +++ b/routers/api/v1/convert/convert.go @@ -27,7 +27,7 @@ func ToEmail(email *models.EmailAddress) *api.Email { } // ToBranch convert a commit and branch to an api.Branch -func ToBranch(repo *models.Repository, b *models.Branch, c *git.Commit) *api.Branch { +func ToBranch(repo *models.Repository, b *git.Branch, c *git.Commit) *api.Branch { return &api.Branch{ Name: b.Name, Commit: ToCommit(repo, c), diff --git a/routers/api/v1/repo/branch.go b/routers/api/v1/repo/branch.go index e20eef6139..b9a23d3b51 100644 --- a/routers/api/v1/repo/branch.go +++ b/routers/api/v1/repo/branch.go @@ -1,12 +1,13 @@ // Copyright 2016 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package repo import ( - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/routers/api/v1/convert" api "code.gitea.io/sdk/gitea" @@ -47,7 +48,7 @@ func GetBranch(ctx *context.APIContext) { } branch, err := ctx.Repo.Repository.GetBranch(ctx.Repo.BranchName) if err != nil { - if models.IsErrBranchNotExist(err) { + if git.IsErrBranchNotExist(err) { ctx.NotFound(err) } else { ctx.Error(500, "GetBranch", err) diff --git a/routers/repo/commit.go b/routers/repo/commit.go index a1e9b8bc5d..9263bcac7f 100644 --- a/routers/repo/commit.go +++ b/routers/repo/commit.go @@ -243,6 +243,7 @@ func Diff(ctx *context.Context) { ctx.Data["BeforeSourcePath"] = setting.AppSubURL + "/" + path.Join(userName, repoName, "src", "commit", parents[0]) } ctx.Data["RawPath"] = setting.AppSubURL + "/" + path.Join(userName, repoName, "raw", "commit", commitID) + ctx.Data["BranchName"], err = commit.GetBranchName() ctx.HTML(200, tplDiff) } diff --git a/routers/repo/editor.go b/routers/repo/editor.go index b5bb2f0ce3..4314218bf3 100644 --- a/routers/repo/editor.go +++ b/routers/repo/editor.go @@ -95,11 +95,12 @@ func editFile(ctx *context.Context, isNewFile bool) { return } - dataRc, err := blob.Data() + dataRc, err := blob.DataAsync() if err != nil { ctx.NotFound("blob.Data", err) return } + defer dataRc.Close() ctx.Data["FileSize"] = blob.Size() ctx.Data["FileName"] = blob.Name() @@ -251,9 +252,9 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo } else if models.IsErrRepoFileAlreadyExists(err) { ctx.Data["Err_TreePath"] = true ctx.RenderWithErr(ctx.Tr("repo.editor.file_already_exists", form.TreePath), tplEditFile, &form) - } else if models.IsErrBranchNotExist(err) { + } else if git.IsErrBranchNotExist(err) { // For when a user adds/updates a file to a branch that no longer exists - if branchErr, ok := err.(models.ErrBranchNotExist); ok { + if branchErr, ok := err.(git.ErrBranchNotExist); ok { ctx.RenderWithErr(ctx.Tr("repo.editor.branch_does_not_exist", branchErr.Name), tplEditFile, &form) } else { ctx.Error(500, err.Error()) @@ -417,9 +418,9 @@ func DeleteFilePost(ctx *context.Context, form auth.DeleteRepoFileForm) { } else { ctx.ServerError("DeleteRepoFile", err) } - } else if models.IsErrBranchNotExist(err) { + } else if git.IsErrBranchNotExist(err) { // For when a user deletes a file to a branch that no longer exists - if branchErr, ok := err.(models.ErrBranchNotExist); ok { + if branchErr, ok := err.(git.ErrBranchNotExist); ok { ctx.RenderWithErr(ctx.Tr("repo.editor.branch_does_not_exist", branchErr.Name), tplEditFile, &form) } else { ctx.Error(500, err.Error()) diff --git a/routers/repo/issue.go b/routers/repo/issue.go index 42f3ddf4e8..7d235d84ef 100644 --- a/routers/repo/issue.go +++ b/routers/repo/issue.go @@ -9,7 +9,6 @@ import ( "bytes" "errors" "fmt" - "io" "io/ioutil" "net/http" "strconv" @@ -363,7 +362,6 @@ func RetrieveRepoMetas(ctx *context.Context, repo *models.Repository) []*models. } func getFileContentFromDefaultBranch(ctx *context.Context, filename string) (string, bool) { - var r io.Reader var bytes []byte if ctx.Repo.Commit == nil { @@ -381,10 +379,11 @@ func getFileContentFromDefaultBranch(ctx *context.Context, filename string) (str if entry.Blob().Size() >= setting.UI.MaxDisplayFileSize { return "", false } - r, err = entry.Blob().Data() + r, err := entry.Blob().DataAsync() if err != nil { return "", false } + defer r.Close() bytes, err = ioutil.ReadAll(r) if err != nil { return "", false diff --git a/routers/repo/setting_protected_branch.go b/routers/repo/setting_protected_branch.go index 05ad0ec0ec..b5a115b6a4 100644 --- a/routers/repo/setting_protected_branch.go +++ b/routers/repo/setting_protected_branch.go @@ -103,7 +103,7 @@ func SettingsProtectedBranch(c *context.Context) { protectBranch, err := models.GetProtectedBranchBy(c.Repo.Repository.ID, branch) if err != nil { - if !models.IsErrBranchNotExist(err) { + if !git.IsErrBranchNotExist(err) { c.ServerError("GetProtectBranchOfRepoByName", err) return } @@ -152,7 +152,7 @@ func SettingsProtectedBranchPost(ctx *context.Context, f auth.ProtectBranchForm) protectBranch, err := models.GetProtectedBranchBy(ctx.Repo.Repository.ID, branch) if err != nil { - if !models.IsErrBranchNotExist(err) { + if !git.IsErrBranchNotExist(err) { ctx.ServerError("GetProtectBranchOfRepoByName", err) return } diff --git a/routers/repo/view.go b/routers/repo/view.go index e883ebbf89..d20f94dfba 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -49,7 +49,8 @@ func renderDirectory(ctx *context.Context, treeLink string) { } entries.CustomSort(base.NaturalSortLess) - ctx.Data["Files"], err = entries.GetCommitsInfo(ctx.Repo.Commit, ctx.Repo.TreePath, nil) + var latestCommit *git.Commit + ctx.Data["Files"], latestCommit, err = entries.GetCommitsInfo(ctx.Repo.Commit, ctx.Repo.TreePath, nil) if err != nil { ctx.ServerError("GetCommitsInfo", err) return @@ -178,14 +179,6 @@ func renderDirectory(ctx *context.Context, treeLink string) { // Show latest commit info of repository in table header, // or of directory if not in root directory. - latestCommit := ctx.Repo.Commit - if len(ctx.Repo.TreePath) > 0 { - latestCommit, err = ctx.Repo.Commit.GetCommitByPath(ctx.Repo.TreePath) - if err != nil { - ctx.ServerError("GetCommitByPath", err) - return - } - } ctx.Data["LatestCommit"] = latestCommit ctx.Data["LatestCommitVerification"] = models.ParseCommitWithSignature(latestCommit) ctx.Data["LatestCommitUser"] = models.ValidateCommitWithEmail(latestCommit) diff --git a/routers/repo/wiki.go b/routers/repo/wiki.go index 790214e8e5..c0fb370dd9 100644 --- a/routers/repo/wiki.go +++ b/routers/repo/wiki.go @@ -57,7 +57,7 @@ func findEntryForFile(commit *git.Commit, target string) (*git.TreeEntry, error) return nil, err } for _, entry := range entries { - if entry.Type == git.ObjectBlob && entry.Name() == target { + if entry.IsRegular() && entry.Name() == target { return entry, nil } } @@ -81,11 +81,12 @@ func findWikiRepoCommit(ctx *context.Context) (*git.Repository, *git.Commit, err // wikiContentsByEntry returns the contents of the wiki page referenced by the // given tree entry. Writes to ctx if an error occurs. func wikiContentsByEntry(ctx *context.Context, entry *git.TreeEntry) []byte { - reader, err := entry.Blob().Data() + reader, err := entry.Blob().DataAsync() if err != nil { ctx.ServerError("Blob.Data", err) return nil } + defer reader.Close() content, err := ioutil.ReadAll(reader) if err != nil { ctx.ServerError("ReadAll", err) @@ -125,7 +126,7 @@ func renderWikiPage(ctx *context.Context, isViewPage bool) (*git.Repository, *gi } pages := make([]PageMeta, 0, len(entries)) for _, entry := range entries { - if entry.Type != git.ObjectBlob { + if !entry.IsRegular() { continue } wikiName, err := models.WikiFilenameToName(entry.Name()) @@ -259,7 +260,7 @@ func WikiPages(ctx *context.Context) { } pages := make([]PageMeta, 0, len(entries)) for _, entry := range entries { - if entry.Type != git.ObjectBlob { + if !entry.IsRegular() { continue } c, err := wikiRepo.GetCommitByPath(entry.Name()) diff --git a/routers/repo/wiki_test.go b/routers/repo/wiki_test.go index 7302f9e48b..4687d24f6b 100644 --- a/routers/repo/wiki_test.go +++ b/routers/repo/wiki_test.go @@ -40,8 +40,9 @@ func wikiContent(t *testing.T, repo *models.Repository, wikiName string) string if !assert.NotNil(t, entry) { return "" } - reader, err := entry.Blob().Data() + reader, err := entry.Blob().DataAsync() assert.NoError(t, err) + defer reader.Close() bytes, err := ioutil.ReadAll(reader) assert.NoError(t, err) return string(bytes) diff --git a/templates/repo/diff/page.tmpl b/templates/repo/diff/page.tmpl index 3f383add94..c8f5a3d9f0 100644 --- a/templates/repo/diff/page.tmpl +++ b/templates/repo/diff/page.tmpl @@ -13,7 +13,7 @@ {{if IsMultilineCommitMessage .Commit.Message}} 
{{RenderCommitBody .Commit.Message $.RepoLink $.Repository.ComposeMetas}}
{{end}} - {{.Commit.Branch}} + {{.BranchName}}
diff --git a/vendor/modules.txt b/vendor/modules.txt index 940b7fc875..08db5c7113 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -410,8 +410,8 @@ gopkg.in/macaron.v1 # gopkg.in/redis.v2 v2.3.2 gopkg.in/redis.v2 # gopkg.in/src-d/go-billy.v4 v4.3.0 -gopkg.in/src-d/go-billy.v4 gopkg.in/src-d/go-billy.v4/osfs +gopkg.in/src-d/go-billy.v4 gopkg.in/src-d/go-billy.v4/util gopkg.in/src-d/go-billy.v4/helper/chroot gopkg.in/src-d/go-billy.v4/helper/polyfill @@ -419,13 +419,14 @@ gopkg.in/src-d/go-billy.v4/helper/polyfill gopkg.in/src-d/go-git.v4 gopkg.in/src-d/go-git.v4/config gopkg.in/src-d/go-git.v4/plumbing -gopkg.in/src-d/go-git.v4/internal/revision gopkg.in/src-d/go-git.v4/plumbing/cache gopkg.in/src-d/go-git.v4/plumbing/filemode +gopkg.in/src-d/go-git.v4/plumbing/object +gopkg.in/src-d/go-git.v4/storage/filesystem +gopkg.in/src-d/go-git.v4/internal/revision gopkg.in/src-d/go-git.v4/plumbing/format/gitignore gopkg.in/src-d/go-git.v4/plumbing/format/index gopkg.in/src-d/go-git.v4/plumbing/format/packfile -gopkg.in/src-d/go-git.v4/plumbing/object gopkg.in/src-d/go-git.v4/plumbing/protocol/packp gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/capability gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/sideband @@ -434,7 +435,6 @@ gopkg.in/src-d/go-git.v4/plumbing/storer gopkg.in/src-d/go-git.v4/plumbing/transport gopkg.in/src-d/go-git.v4/plumbing/transport/client gopkg.in/src-d/go-git.v4/storage -gopkg.in/src-d/go-git.v4/storage/filesystem gopkg.in/src-d/go-git.v4/storage/memory gopkg.in/src-d/go-git.v4/utils/diff gopkg.in/src-d/go-git.v4/utils/ioutil @@ -444,16 +444,16 @@ gopkg.in/src-d/go-git.v4/utils/merkletrie/index gopkg.in/src-d/go-git.v4/utils/merkletrie/noder gopkg.in/src-d/go-git.v4/internal/url gopkg.in/src-d/go-git.v4/plumbing/format/config +gopkg.in/src-d/go-git.v4/plumbing/format/diff gopkg.in/src-d/go-git.v4/utils/binary gopkg.in/src-d/go-git.v4/plumbing/format/idxfile -gopkg.in/src-d/go-git.v4/plumbing/format/diff +gopkg.in/src-d/go-git.v4/plumbing/format/objfile +gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit gopkg.in/src-d/go-git.v4/plumbing/format/pktline gopkg.in/src-d/go-git.v4/plumbing/transport/file gopkg.in/src-d/go-git.v4/plumbing/transport/git gopkg.in/src-d/go-git.v4/plumbing/transport/http gopkg.in/src-d/go-git.v4/plumbing/transport/ssh -gopkg.in/src-d/go-git.v4/plumbing/format/objfile -gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit gopkg.in/src-d/go-git.v4/utils/merkletrie/internal/frame gopkg.in/src-d/go-git.v4/plumbing/transport/internal/common gopkg.in/src-d/go-git.v4/plumbing/transport/server