gitea/models/git/commit_status.go

555 lines
17 KiB
Go
Raw Normal View History

2017-04-21 19:32:31 +08:00
// Copyright 2017 Gitea. All rights reserved.
// SPDX-License-Identifier: MIT
2017-04-21 19:32:31 +08:00
package git
2017-04-21 19:32:31 +08:00
import (
"context"
"crypto/sha1"
"errors"
2017-04-21 19:32:31 +08:00
"fmt"
"net/url"
"strconv"
2017-04-21 19:32:31 +08:00
"strings"
"time"
2017-04-21 19:32:31 +08:00
asymkey_model "code.gitea.io/gitea/models/asymkey"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
2017-04-21 19:32:31 +08:00
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/translation"
"xorm.io/builder"
"xorm.io/xorm"
2017-04-21 19:32:31 +08:00
)
// CommitStatus holds a single Status of a single Commit
type CommitStatus struct {
ID int64 `xorm:"pk autoincr"`
Index int64 `xorm:"INDEX UNIQUE(repo_sha_index)"`
RepoID int64 `xorm:"INDEX UNIQUE(repo_sha_index)"`
Repo *repo_model.Repository `xorm:"-"`
State api.CommitStatusState `xorm:"VARCHAR(7) NOT NULL"`
SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_sha_index)"`
TargetURL string `xorm:"TEXT"`
Description string `xorm:"TEXT"`
ContextHash string `xorm:"VARCHAR(64) index"`
Context string `xorm:"TEXT"`
Creator *user_model.User `xorm:"-"`
2017-04-21 19:32:31 +08:00
CreatorID int64
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
2017-04-21 19:32:31 +08:00
}
func init() {
db.RegisterModel(new(CommitStatus))
db.RegisterModel(new(CommitStatusIndex))
}
func postgresGetCommitStatusIndex(ctx context.Context, repoID int64, sha string) (int64, error) {
res, err := db.GetEngine(ctx).Query("INSERT INTO `commit_status_index` (repo_id, sha, max_index) "+
"VALUES (?,?,1) ON CONFLICT (repo_id, sha) DO UPDATE SET max_index = `commit_status_index`.max_index+1 RETURNING max_index",
repoID, sha)
if err != nil {
return 0, err
}
if len(res) == 0 {
return 0, db.ErrGetResourceIndexFailed
}
return strconv.ParseInt(string(res[0]["max_index"]), 10, 64)
}
func mysqlGetCommitStatusIndex(ctx context.Context, repoID int64, sha string) (int64, error) {
if _, err := db.GetEngine(ctx).Exec("INSERT INTO `commit_status_index` (repo_id, sha, max_index) "+
"VALUES (?,?,1) ON DUPLICATE KEY UPDATE max_index = max_index+1",
repoID, sha); err != nil {
return 0, err
}
var idx int64
_, err := db.GetEngine(ctx).SQL("SELECT max_index FROM `commit_status_index` WHERE repo_id = ? AND sha = ?",
repoID, sha).Get(&idx)
if err != nil {
return 0, err
}
if idx == 0 {
return 0, errors.New("cannot get the correct index")
}
return idx, nil
}
func mssqlGetCommitStatusIndex(ctx context.Context, repoID int64, sha string) (int64, error) {
if _, err := db.GetEngine(ctx).Exec(`
MERGE INTO commit_status_index WITH (HOLDLOCK) AS target
USING (SELECT ? AS repo_id, ? AS sha) AS source
(repo_id, sha)
ON target.repo_id = source.repo_id AND target.sha = source.sha
WHEN MATCHED
THEN UPDATE
SET max_index = max_index + 1
WHEN NOT MATCHED
THEN INSERT (repo_id, sha, max_index)
VALUES (?, ?, 1);
`, repoID, sha, repoID, sha); err != nil {
return 0, err
}
var idx int64
_, err := db.GetEngine(ctx).SQL("SELECT max_index FROM `commit_status_index` WHERE repo_id = ? AND sha = ?",
repoID, sha).Get(&idx)
if err != nil {
return 0, err
}
if idx == 0 {
return 0, errors.New("cannot get the correct index")
}
return idx, nil
}
// GetNextCommitStatusIndex retried 3 times to generate a resource index
func GetNextCommitStatusIndex(ctx context.Context, repoID int64, sha string) (int64, error) {
_, err := git.NewIDFromString(sha)
if err != nil {
return 0, git.ErrInvalidSHA{SHA: sha}
}
switch {
case setting.Database.Type.IsPostgreSQL():
return postgresGetCommitStatusIndex(ctx, repoID, sha)
case setting.Database.Type.IsMySQL():
return mysqlGetCommitStatusIndex(ctx, repoID, sha)
case setting.Database.Type.IsMSSQL():
return mssqlGetCommitStatusIndex(ctx, repoID, sha)
}
e := db.GetEngine(ctx)
// try to update the max_index to next value, and acquire the write-lock for the record
res, err := e.Exec("UPDATE `commit_status_index` SET max_index=max_index+1 WHERE repo_id=? AND sha=?", repoID, sha)
if err != nil {
return 0, fmt.Errorf("update failed: %w", err)
}
affected, err := res.RowsAffected()
if err != nil {
return 0, err
}
if affected == 0 {
// this slow path is only for the first time of creating a resource index
_, errIns := e.Exec("INSERT INTO `commit_status_index` (repo_id, sha, max_index) VALUES (?, ?, 0)", repoID, sha)
res, err = e.Exec("UPDATE `commit_status_index` SET max_index=max_index+1 WHERE repo_id=? AND sha=?", repoID, sha)
if err != nil {
return 0, fmt.Errorf("update2 failed: %w", err)
}
affected, err = res.RowsAffected()
if err != nil {
return 0, fmt.Errorf("RowsAffected failed: %w", err)
}
// if the update still can not update any records, the record must not exist and there must be some errors (insert error)
if affected == 0 {
if errIns == nil {
return 0, errors.New("impossible error when GetNextCommitStatusIndex, insert and update both succeeded but no record is updated")
}
return 0, fmt.Errorf("insert failed: %w", errIns)
}
}
// now, the new index is in database (protected by the transaction and write-lock)
var newIdx int64
has, err := e.SQL("SELECT max_index FROM `commit_status_index` WHERE repo_id=? AND sha=?", repoID, sha).Get(&newIdx)
if err != nil {
return 0, fmt.Errorf("select failed: %w", err)
}
if !has {
return 0, errors.New("impossible error when GetNextCommitStatusIndex, upsert succeeded but no record can be selected")
}
return newIdx, nil
}
func (status *CommitStatus) loadRepository(ctx context.Context) (err error) {
2017-04-21 19:32:31 +08:00
if status.Repo == nil {
status.Repo, err = repo_model.GetRepositoryByID(ctx, status.RepoID)
2017-04-21 19:32:31 +08:00
if err != nil {
return fmt.Errorf("getRepositoryByID [%d]: %w", status.RepoID, err)
2017-04-21 19:32:31 +08:00
}
}
return nil
}
func (status *CommitStatus) loadCreator(ctx context.Context) (err error) {
2017-04-21 19:32:31 +08:00
if status.Creator == nil && status.CreatorID > 0 {
status.Creator, err = user_model.GetUserByID(ctx, status.CreatorID)
2017-04-21 19:32:31 +08:00
if err != nil {
return fmt.Errorf("getUserByID [%d]: %w", status.CreatorID, err)
2017-04-21 19:32:31 +08:00
}
}
return nil
}
func (status *CommitStatus) loadAttributes(ctx context.Context) (err error) {
if err := status.loadRepository(ctx); err != nil {
return err
}
return status.loadCreator(ctx)
}
2017-04-21 19:32:31 +08:00
// APIURL returns the absolute APIURL to this commit-status.
func (status *CommitStatus) APIURL(ctx context.Context) string {
_ = status.loadAttributes(ctx)
return status.Repo.APIURL() + "/statuses/" + url.PathEscape(status.SHA)
2017-04-21 19:32:31 +08:00
}
// LocaleString returns the locale string name of the Status
func (status *CommitStatus) LocaleString(lang translation.Locale) string {
return lang.TrString("repo.commitstatus." + status.State.String())
}
// HideActionsURL set `TargetURL` to an empty string if the status comes from Gitea Actions
func (status *CommitStatus) HideActionsURL(ctx context.Context) {
if status.RepoID == 0 {
return
}
if status.Repo == nil {
if err := status.loadRepository(ctx); err != nil {
log.Error("loadRepository: %v", err)
return
}
}
prefix := fmt.Sprintf("%s/actions", status.Repo.Link())
if strings.HasPrefix(status.TargetURL, prefix) {
status.TargetURL = ""
}
}
// CalcCommitStatus returns commit status state via some status, the commit statues should order by id desc
func CalcCommitStatus(statuses []*CommitStatus) *CommitStatus {
var lastStatus *CommitStatus
state := api.CommitStatusSuccess
for _, status := range statuses {
if status.State.NoBetterThan(state) {
state = status.State
lastStatus = status
}
}
if lastStatus == nil {
if len(statuses) > 0 {
lastStatus = statuses[0]
} else {
lastStatus = &CommitStatus{}
}
}
return lastStatus
}
// CommitStatusOptions holds the options for query commit statuses
type CommitStatusOptions struct {
db.ListOptions
RepoID int64
SHA string
State string
SortType string
}
func (opts *CommitStatusOptions) ToConds() builder.Cond {
var cond builder.Cond = builder.Eq{
"repo_id": opts.RepoID,
"sha": opts.SHA,
}
switch opts.State {
case "pending", "success", "error", "failure", "warning":
cond = cond.And(builder.Eq{
"state": opts.State,
})
}
return cond
}
func (opts *CommitStatusOptions) ToOrders() string {
switch opts.SortType {
case "oldest":
return "created_unix ASC"
case "recentupdate":
return "updated_unix DESC"
case "leastupdate":
return "updated_unix ASC"
case "leastindex":
return "`index` DESC"
case "highestindex":
return "`index` ASC"
default:
return "created_unix DESC"
}
2017-04-21 19:32:31 +08:00
}
// CommitStatusIndex represents a table for commit status index
type CommitStatusIndex struct {
ID int64
RepoID int64 `xorm:"unique(repo_sha)"`
SHA string `xorm:"unique(repo_sha)"`
MaxIndex int64 `xorm:"index"`
}
2017-04-21 19:32:31 +08:00
// GetLatestCommitStatus returns all statuses with a unique context for a given commit.
func GetLatestCommitStatus(ctx context.Context, repoID int64, sha string, listOptions db.ListOptions) ([]*CommitStatus, int64, error) {
getBase := func() *xorm.Session {
return db.GetEngine(ctx).Table(&CommitStatus{}).
Where("repo_id = ?", repoID).And("sha = ?", sha)
}
indices := make([]int64, 0, 10)
sess := getBase().Select("max( `index` ) as `index`").
GroupBy("context_hash").OrderBy("max( `index` ) desc")
if !listOptions.IsListAll() {
sess = db.SetSessionPagination(sess, &listOptions)
}
count, err := sess.FindAndCount(&indices)
if err != nil {
return nil, count, err
}
statuses := make([]*CommitStatus, 0, len(indices))
if len(indices) == 0 {
return statuses, count, nil
}
return statuses, count, getBase().And(builder.In("`index`", indices)).Find(&statuses)
2017-04-21 19:32:31 +08:00
}
// GetLatestCommitStatusForPairs returns all statuses with a unique context for a given list of repo-sha pairs
func GetLatestCommitStatusForPairs(ctx context.Context, repoSHAs []RepoSHA) (map[int64][]*CommitStatus, error) {
type result struct {
Index int64
RepoID int64
SHA string
}
results := make([]result, 0, len(repoSHAs))
getBase := func() *xorm.Session {
return db.GetEngine(ctx).Table(&CommitStatus{})
}
// Create a disjunction of conditions for each repoID and SHA pair
conds := make([]builder.Cond, 0, len(repoSHAs))
for _, repoSHA := range repoSHAs {
conds = append(conds, builder.Eq{"repo_id": repoSHA.RepoID, "sha": repoSHA.SHA})
}
sess := getBase().Where(builder.Or(conds...)).
Select("max( `index` ) as `index`, repo_id, sha").
GroupBy("context_hash, repo_id, sha").OrderBy("max( `index` ) desc")
err := sess.Find(&results)
if err != nil {
return nil, err
}
repoStatuses := make(map[int64][]*CommitStatus)
if len(results) > 0 {
statuses := make([]*CommitStatus, 0, len(results))
conds = make([]builder.Cond, 0, len(results))
for _, result := range results {
cond := builder.Eq{
"`index`": result.Index,
"repo_id": result.RepoID,
"sha": result.SHA,
}
conds = append(conds, cond)
}
err = getBase().Where(builder.Or(conds...)).Find(&statuses)
if err != nil {
return nil, err
}
// Group the statuses by repo ID
for _, status := range statuses {
repoStatuses[status.RepoID] = append(repoStatuses[status.RepoID], status)
}
}
return repoStatuses, nil
}
// GetLatestCommitStatusForRepoCommitIDs returns all statuses with a unique context for a given list of repo-sha pairs
func GetLatestCommitStatusForRepoCommitIDs(ctx context.Context, repoID int64, commitIDs []string) (map[string][]*CommitStatus, error) {
type result struct {
Index int64
SHA string
}
getBase := func() *xorm.Session {
return db.GetEngine(ctx).Table(&CommitStatus{}).Where("repo_id = ?", repoID)
}
results := make([]result, 0, len(commitIDs))
conds := make([]builder.Cond, 0, len(commitIDs))
for _, sha := range commitIDs {
conds = append(conds, builder.Eq{"sha": sha})
}
sess := getBase().And(builder.Or(conds...)).
Select("max( `index` ) as `index`, sha").
GroupBy("context_hash, sha").OrderBy("max( `index` ) desc")
err := sess.Find(&results)
if err != nil {
return nil, err
}
repoStatuses := make(map[string][]*CommitStatus)
if len(results) > 0 {
statuses := make([]*CommitStatus, 0, len(results))
conds = make([]builder.Cond, 0, len(results))
for _, result := range results {
conds = append(conds, builder.Eq{"`index`": result.Index, "sha": result.SHA})
}
err = getBase().And(builder.Or(conds...)).Find(&statuses)
if err != nil {
return nil, err
}
// Group the statuses by commit
for _, status := range statuses {
repoStatuses[status.SHA] = append(repoStatuses[status.SHA], status)
}
}
return repoStatuses, nil
}
// FindRepoRecentCommitStatusContexts returns repository's recent commit status contexts
func FindRepoRecentCommitStatusContexts(ctx context.Context, repoID int64, before time.Duration) ([]string, error) {
start := timeutil.TimeStampNow().AddDuration(-before)
var contexts []string
if err := db.GetEngine(ctx).Table("commit_status").
Where("repo_id = ?", repoID).And("updated_unix >= ?", start).
Cols("context").Distinct().Find(&contexts); err != nil {
return nil, err
}
return contexts, nil
}
2017-04-21 19:32:31 +08:00
// NewCommitStatusOptions holds options for creating a CommitStatus
type NewCommitStatusOptions struct {
Repo *repo_model.Repository
Creator *user_model.User
SHA git.ObjectID
2017-04-21 19:32:31 +08:00
CommitStatus *CommitStatus
}
// NewCommitStatus save commit statuses into database
func NewCommitStatus(ctx context.Context, opts NewCommitStatusOptions) error {
2017-04-21 19:32:31 +08:00
if opts.Repo == nil {
return fmt.Errorf("NewCommitStatus[nil, %s]: no repository specified", opts.SHA)
2017-04-21 19:32:31 +08:00
}
repoPath := opts.Repo.RepoPath()
2017-04-21 19:32:31 +08:00
if opts.Creator == nil {
return fmt.Errorf("NewCommitStatus[%s, %s]: no user specified", repoPath, opts.SHA)
2017-04-21 19:32:31 +08:00
}
ctx, committer, err := db.TxContext(ctx)
if err != nil {
return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", opts.Repo.ID, opts.Creator.ID, opts.SHA, err)
2017-04-21 19:32:31 +08:00
}
defer committer.Close()
2017-04-21 19:32:31 +08:00
// Get the next Status Index
idx, err := GetNextCommitStatusIndex(ctx, opts.Repo.ID, opts.SHA.String())
if err != nil {
return fmt.Errorf("generate commit status index failed: %w", err)
}
opts.CommitStatus.Description = strings.TrimSpace(opts.CommitStatus.Description)
opts.CommitStatus.Context = strings.TrimSpace(opts.CommitStatus.Context)
opts.CommitStatus.TargetURL = strings.TrimSpace(opts.CommitStatus.TargetURL)
opts.CommitStatus.SHA = opts.SHA.String()
opts.CommitStatus.CreatorID = opts.Creator.ID
opts.CommitStatus.RepoID = opts.Repo.ID
opts.CommitStatus.Index = idx
log.Debug("NewCommitStatus[%s, %s]: %d", repoPath, opts.SHA, opts.CommitStatus.Index)
opts.CommitStatus.ContextHash = hashCommitStatusContext(opts.CommitStatus.Context)
2017-04-21 19:32:31 +08:00
// Insert new CommitStatus
if _, err = db.GetEngine(ctx).Insert(opts.CommitStatus); err != nil {
return fmt.Errorf("insert CommitStatus[%s, %s]: %w", repoPath, opts.SHA, err)
2017-04-21 19:32:31 +08:00
}
return committer.Commit()
2017-04-21 19:32:31 +08:00
}
// SignCommitWithStatuses represents a commit with validation of signature and status state.
type SignCommitWithStatuses struct {
Status *CommitStatus
Statuses []*CommitStatus
*asymkey_model.SignCommit
}
// ParseCommitsWithStatus checks commits latest statuses and calculates its worst status state
func ParseCommitsWithStatus(ctx context.Context, oldCommits []*asymkey_model.SignCommit, repo *repo_model.Repository) []*SignCommitWithStatuses {
newCommits := make([]*SignCommitWithStatuses, 0, len(oldCommits))
for _, c := range oldCommits {
commit := &SignCommitWithStatuses{
SignCommit: c,
}
statuses, _, err := GetLatestCommitStatus(ctx, repo.ID, commit.ID.String(), db.ListOptions{})
if err != nil {
Better logging (#6038) (#6095) * Panic don't fatal on create new logger Fixes #5854 Signed-off-by: Andrew Thornton <art27@cantab.net> * partial broken * Update the logging infrastrcture Signed-off-by: Andrew Thornton <art27@cantab.net> * Reset the skip levels for Fatal and Error Signed-off-by: Andrew Thornton <art27@cantab.net> * broken ncsa * More log.Error fixes Signed-off-by: Andrew Thornton <art27@cantab.net> * Remove nal * set log-levels to lowercase * Make console_test test all levels * switch to lowercased levels * OK now working * Fix vetting issues * Fix lint * Fix tests * change default logging to match current gitea * Improve log testing Signed-off-by: Andrew Thornton <art27@cantab.net> * reset error skip levels to 0 * Update documentation and access logger configuration * Redirect the router log back to gitea if redirect macaron log but also allow setting the log level - i.e. TRACE * Fix broken level caching * Refactor the router log * Add Router logger * Add colorizing options * Adjust router colors * Only create logger if they will be used * update app.ini.sample * rename Attribute ColorAttribute * Change from white to green for function * Set fatal/error levels * Restore initial trace logger * Fix Trace arguments in modules/auth/auth.go * Properly handle XORMLogger * Improve admin/config page * fix fmt * Add auto-compression of old logs * Update error log levels * Remove the unnecessary skip argument from Error, Fatal and Critical * Add stacktrace support * Fix tests * Remove x/sync from vendors? * Add stderr option to console logger * Use filepath.ToSlash to protect against Windows in tests * Remove prefixed underscores from names in colors.go * Remove not implemented database logger This was removed from Gogs on 4 Mar 2016 but left in the configuration since then. * Ensure that log paths are relative to ROOT_PATH * use path.Join * rename jsonConfig to logConfig * Rename "config" to "jsonConfig" to make it clearer * Requested changes * Requested changes: XormLogger * Try to color the windows terminal If successful default to colorizing the console logs * fixup * Colorize initially too * update vendor * Colorize logs on default and remove if this is not a colorizing logger * Fix documentation * fix test * Use go-isatty to detect if on windows we are on msys or cygwin * Fix spelling mistake * Add missing vendors * More changes * Rationalise the ANSI writer protection * Adjust colors on advice from @0x5c * Make Flags a comma separated list * Move to use the windows constant for ENABLE_VIRTUAL_TERMINAL_PROCESSING * Ensure matching is done on the non-colored message - to simpify EXPRESSION
2019-04-02 15:48:31 +08:00
log.Error("GetLatestCommitStatus: %v", err)
} else {
commit.Statuses = statuses
commit.Status = CalcCommitStatus(statuses)
}
newCommits = append(newCommits, commit)
}
return newCommits
}
// hashCommitStatusContext hash context
func hashCommitStatusContext(context string) string {
return fmt.Sprintf("%x", sha1.Sum([]byte(context)))
}
// ConvertFromGitCommit converts git commits into SignCommitWithStatuses
func ConvertFromGitCommit(ctx context.Context, commits []*git.Commit, repo *repo_model.Repository) []*SignCommitWithStatuses {
return ParseCommitsWithStatus(ctx,
asymkey_model.ParseCommitsWithSignature(
Add context cache as a request level cache (#22294) To avoid duplicated load of the same data in an HTTP request, we can set a context cache to do that. i.e. Some pages may load a user from a database with the same id in different areas on the same page. But the code is hidden in two different deep logic. How should we share the user? As a result of this PR, now if both entry functions accept `context.Context` as the first parameter and we just need to refactor `GetUserByID` to reuse the user from the context cache. Then it will not be loaded twice on an HTTP request. But of course, sometimes we would like to reload an object from the database, that's why `RemoveContextData` is also exposed. The core context cache is here. It defines a new context ```go type cacheContext struct { ctx context.Context data map[any]map[any]any lock sync.RWMutex } var cacheContextKey = struct{}{} func WithCacheContext(ctx context.Context) context.Context { return context.WithValue(ctx, cacheContextKey, &cacheContext{ ctx: ctx, data: make(map[any]map[any]any), }) } ``` Then you can use the below 4 methods to read/write/del the data within the same context. ```go func GetContextData(ctx context.Context, tp, key any) any func SetContextData(ctx context.Context, tp, key, value any) func RemoveContextData(ctx context.Context, tp, key any) func GetWithContextCache[T any](ctx context.Context, cacheGroupKey string, cacheTargetID any, f func() (T, error)) (T, error) ``` Then let's take a look at how `system.GetString` implement it. ```go func GetSetting(ctx context.Context, key string) (string, error) { return cache.GetWithContextCache(ctx, contextCacheKey, key, func() (string, error) { return cache.GetString(genSettingCacheKey(key), func() (string, error) { res, err := GetSettingNoCache(ctx, key) if err != nil { return "", err } return res.SettingValue, nil }) }) } ``` First, it will check if context data include the setting object with the key. If not, it will query from the global cache which may be memory or a Redis cache. If not, it will get the object from the database. In the end, if the object gets from the global cache or database, it will be set into the context cache. An object stored in the context cache will only be destroyed after the context disappeared.
2023-02-15 21:37:34 +08:00
ctx,
user_model.ValidateCommitsWithEmails(ctx, commits),
repo.GetTrustModel(),
func(user *user_model.User) (bool, error) {
return repo_model.IsOwnerMemberCollaborator(ctx, repo, user.ID)
},
),
repo,
)
}
// CommitStatusesHideActionsURL hide Gitea Actions urls
func CommitStatusesHideActionsURL(ctx context.Context, statuses []*CommitStatus) {
idToRepos := make(map[int64]*repo_model.Repository)
for _, status := range statuses {
if status == nil {
continue
}
if status.Repo == nil {
status.Repo = idToRepos[status.RepoID]
}
status.HideActionsURL(ctx)
idToRepos[status.RepoID] = status.Repo
}
}