THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Browse Source

Add push webhook support for mirrored repositories (#4127)

pull/2917/merge
Lauris BH 9 months ago
parent
commit
fa4663e61e

+ 2
- 2
Gopkg.lock View File

@@ -3,11 +3,11 @@
[[projects]]
  branch = "master"
  digest = "1:42f77a668e3bd06812ef254f334d0d0a62346969fbcd3fa3a613e75067343751"
  digest = "1:835585f8450b4ec12252d032b0f13e6571ecf846e49076f69067f2503a7c1e07"
  name = "code.gitea.io/git"
  packages = ["."]
  pruneopts = "NUT"
  revision = "31f4b8e8c805438ac6d8914b38accb1d8aaf695e"
  revision = "6ef79e80b3b06ca13a1f3a7b940903ebc73b44cb"
[[projects]]
  branch = "master"

+ 68
- 0
models/action.go View File

@@ -47,6 +47,9 @@ const (
	ActionReopenPullRequest                       // 15
	ActionDeleteTag                               // 16
	ActionDeleteBranch                            // 17
	ActionMirrorSyncPush                          // 18
	ActionMirrorSyncCreate                        // 19
	ActionMirrorSyncDelete                        // 20
)
var (
@@ -736,6 +739,71 @@ func MergePullRequestAction(actUser *User, repo *Repository, pull *Issue) error
	return mergePullRequestAction(x, actUser, repo, pull)
}
func mirrorSyncAction(e Engine, opType ActionType, repo *Repository, refName string, data []byte) error {
	if err := notifyWatchers(e, &Action{
		ActUserID: repo.OwnerID,
		ActUser:   repo.MustOwner(),
		OpType:    opType,
		RepoID:    repo.ID,
		Repo:      repo,
		IsPrivate: repo.IsPrivate,
		RefName:   refName,
		Content:   string(data),
	}); err != nil {
		return fmt.Errorf("notifyWatchers: %v", err)
	}
	return nil
}
// MirrorSyncPushActionOptions mirror synchronization action options.
type MirrorSyncPushActionOptions struct {
	RefName     string
	OldCommitID string
	NewCommitID string
	Commits     *PushCommits
}
// MirrorSyncPushAction adds new action for mirror synchronization of pushed commits.
func MirrorSyncPushAction(repo *Repository, opts MirrorSyncPushActionOptions) error {
	if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum {
		opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum]
	}
	apiCommits := opts.Commits.ToAPIPayloadCommits(repo.HTMLURL())
	opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID)
	apiPusher := repo.MustOwner().APIFormat()
	if err := PrepareWebhooks(repo, HookEventPush, &api.PushPayload{
		Ref:        opts.RefName,
		Before:     opts.OldCommitID,
		After:      opts.NewCommitID,
		CompareURL: setting.AppURL + opts.Commits.CompareURL,
		Commits:    apiCommits,
		Repo:       repo.APIFormat(AccessModeOwner),
		Pusher:     apiPusher,
		Sender:     apiPusher,
	}); err != nil {
		return fmt.Errorf("PrepareWebhooks: %v", err)
	}
	data, err := json.Marshal(opts.Commits)
	if err != nil {
		return err
	}
	return mirrorSyncAction(x, ActionMirrorSyncPush, repo, opts.RefName, data)
}
// MirrorSyncCreateAction adds new action for mirror synchronization of new reference.
func MirrorSyncCreateAction(repo *Repository, refName string) error {
	return mirrorSyncAction(x, ActionMirrorSyncCreate, repo, refName, nil)
}
// MirrorSyncDeleteAction adds new action for mirror synchronization of delete reference.
func MirrorSyncDeleteAction(repo *Repository, refName string) error {
	return mirrorSyncAction(x, ActionMirrorSyncDelete, repo, refName, nil)
}
// GetFeedsOptions options for retrieving feeds
type GetFeedsOptions struct {
	RequestedUser    *User

+ 136
- 11
models/repo_mirror.go View File

@@ -1,4 +1,5 @@
// Copyright 2016 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
@@ -6,6 +7,7 @@ package models
import (
	"fmt"
	"strings"
	"time"
	"code.gitea.io/git"
@@ -119,8 +121,68 @@ func (m *Mirror) SaveAddress(addr string) error {
	return cfg.SaveToIndent(configPath, "\t")
}
// gitShortEmptySha Git short empty SHA
const gitShortEmptySha = "0000000"
// mirrorSyncResult contains information of a updated reference.
// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
type mirrorSyncResult struct {
	refName     string
	oldCommitID string
	newCommitID string
}
// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
func parseRemoteUpdateOutput(output string) []*mirrorSyncResult {
	results := make([]*mirrorSyncResult, 0, 3)
	lines := strings.Split(output, "\n")
	for i := range lines {
		// Make sure reference name is presented before continue
		idx := strings.Index(lines[i], "-> ")
		if idx == -1 {
			continue
		}
		refName := lines[i][idx+3:]
		switch {
		case strings.HasPrefix(lines[i], " * "): // New reference
			results = append(results, &mirrorSyncResult{
				refName:     refName,
				oldCommitID: gitShortEmptySha,
			})
		case strings.HasPrefix(lines[i], " - "): // Delete reference
			results = append(results, &mirrorSyncResult{
				refName:     refName,
				newCommitID: gitShortEmptySha,
			})
		case strings.HasPrefix(lines[i], "   "): // New commits of a reference
			delimIdx := strings.Index(lines[i][3:], " ")
			if delimIdx == -1 {
				log.Error(2, "SHA delimiter not found: %q", lines[i])
				continue
			}
			shas := strings.Split(lines[i][3:delimIdx+3], "..")
			if len(shas) != 2 {
				log.Error(2, "Expect two SHAs but not what found: %q", lines[i])
				continue
			}
			results = append(results, &mirrorSyncResult{
				refName:     refName,
				oldCommitID: shas[0],
				newCommitID: shas[1],
			})
		default:
			log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i])
		}
	}
	return results
}
// runSync returns true if sync finished without error.
func (m *Mirror) runSync() bool {
func (m *Mirror) runSync() ([]*mirrorSyncResult, bool) {
	repoPath := m.Repo.RepoPath()
	wikiPath := m.Repo.WikiPath()
	timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
@@ -130,28 +192,30 @@ func (m *Mirror) runSync() bool {
		gitArgs = append(gitArgs, "--prune")
	}
	if _, stderr, err := process.GetManager().ExecDir(
	_, stderr, err := process.GetManager().ExecDir(
		timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
		"git", gitArgs...); err != nil {
		"git", gitArgs...)
	if err != nil {
		// sanitize the output, since it may contain the remote address, which may
		// contain a password
		message, err := sanitizeOutput(stderr, repoPath)
		if err != nil {
			log.Error(4, "sanitizeOutput: %v", err)
			return false
			return nil, false
		}
		desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, message)
		log.Error(4, desc)
		if err = CreateRepositoryNotice(desc); err != nil {
			log.Error(4, "CreateRepositoryNotice: %v", err)
		}
		return false
		return nil, false
	}
	output := stderr
	gitRepo, err := git.OpenRepository(repoPath)
	if err != nil {
		log.Error(4, "OpenRepository: %v", err)
		return false
		return nil, false
	}
	if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
		log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
@@ -170,21 +234,21 @@ func (m *Mirror) runSync() bool {
			message, err := sanitizeOutput(stderr, wikiPath)
			if err != nil {
				log.Error(4, "sanitizeOutput: %v", err)
				return false
				return nil, false
			}
			desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, message)
			log.Error(4, desc)
			if err = CreateRepositoryNotice(desc); err != nil {
				log.Error(4, "CreateRepositoryNotice: %v", err)
			}
			return false
			return nil, false
		}
	}
	branches, err := m.Repo.GetBranches()
	if err != nil {
		log.Error(4, "GetBranches: %v", err)
		return false
		return nil, false
	}
	for i := range branches {
@@ -192,7 +256,7 @@ func (m *Mirror) runSync() bool {
	}
	m.UpdatedUnix = util.TimeStampNow()
	return true
	return parseRemoteUpdateOutput(output), true
}
func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) {
@@ -268,7 +332,8 @@ func SyncMirrors() {
			continue
		}
		if !m.runSync() {
		results, ok := m.runSync()
		if !ok {
			continue
		}
@@ -278,6 +343,66 @@ func SyncMirrors() {
			continue
		}
		var gitRepo *git.Repository
		if len(results) == 0 {
			log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID)
		} else {
			gitRepo, err = git.OpenRepository(m.Repo.RepoPath())
			if err != nil {
				log.Error(2, "OpenRepository [%d]: %v", m.RepoID, err)
				continue
			}
		}
		for _, result := range results {
			// Discard GitHub pull requests, i.e. refs/pull/*
			if strings.HasPrefix(result.refName, "refs/pull/") {
				continue
			}
			// Create reference
			if result.oldCommitID == gitShortEmptySha {
				if err = MirrorSyncCreateAction(m.Repo, result.refName); err != nil {
					log.Error(2, "MirrorSyncCreateAction [repo_id: %d]: %v", m.RepoID, err)
				}
				continue
			}
			// Delete reference
			if result.newCommitID == gitShortEmptySha {
				if err = MirrorSyncDeleteAction(m.Repo, result.refName); err != nil {
					log.Error(2, "MirrorSyncDeleteAction [repo_id: %d]: %v", m.RepoID, err)
				}
				continue
			}
			// Push commits
			oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID)
			if err != nil {
				log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err)
				continue
			}
			newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID)
			if err != nil {
				log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err)
				continue
			}
			commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID)
			if err != nil {
				log.Error(2, "CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err)
				continue
			}
			if err = MirrorSyncPushAction(m.Repo, MirrorSyncPushActionOptions{
				RefName:     result.refName,
				OldCommitID: oldCommitID,
				NewCommitID: newCommitID,
				Commits:     ListToPushCommits(commits),
			}); err != nil {
				log.Error(2, "MirrorSyncPushAction [repo_id: %d]: %v", m.RepoID, err)
				continue
			}
		}
		// Get latest commit date and update to current repository updated time
		commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath())
		if err != nil {

+ 2
- 0
modules/templates/helper.go View File

@@ -391,6 +391,8 @@ func ActionIcon(opType models.ActionType) string {
		return "issue-closed"
	case models.ActionReopenIssue, models.ActionReopenPullRequest:
		return "issue-reopened"
	case models.ActionMirrorSyncPush, models.ActionMirrorSyncCreate, models.ActionMirrorSyncDelete:
		return "repo-clone"
	default:
		return "invalid type"
	}

+ 3
- 0
options/locale/locale_en-US.ini View File

@@ -1663,6 +1663,9 @@ push_tag = pushed tag <a href="%s/src/tag/%s">%[2]s</a> to <a href="%[1]s">%[3]s
delete_tag = deleted tag %[2]s from <a href="%[1]s">%[3]s</a>
delete_branch = deleted branch %[2]s from <a href="%[1]s">%[3]s</a>
compare_commits = Compare %d commits
mirror_sync_push = synced commits to <a href="%[1]s/src/%[2]s">%[3]s</a> at <a href="%[1]s">%[4]s</a> from mirror
mirror_sync_create = synced new reference <a href="%s/src/%s">%[2]s</a> to <a href="%[1]s">%[3]s</a> from mirror
mirror_sync_delete = synced and deleted reference <code>%[2]s</code> at <a href="%[1]s">%[3]s</a> from mirror
[tool]
ago = %s ago

+ 9
- 2
templates/user/dashboard/feeds.tmpl View File

@@ -5,7 +5,7 @@
		</div>
		<div class="ui grid">
			<div class="ui thirteen wide column">
				<div class="{{if eq .GetOpType 5}}push news{{end}}">
				<div class="{{if or (eq .GetOpType 5) (eq .GetOpType 18)}}push news{{end}}">
					<p>
						<a href="{{AppSubUrl}}/{{.GetActUserName}}" title="{{.GetActFullName}}">{{.ShortActUserName}}</a>
						{{if eq .GetOpType 1}}
@@ -49,9 +49,16 @@
						{{else if eq .GetOpType 17}}
							{{ $index := index .GetIssueInfos 0}}
							{{$.i18n.Tr "action.delete_branch" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}}
						{{else if eq .GetOpType 18}}
							{{ $branchLink := .GetBranch | EscapePound}}
							{{$.i18n.Tr "action.mirror_sync_push" .GetRepoLink $branchLink .GetBranch .ShortRepoPath | Str2html}}
						{{else if eq .GetOpType 19}}
							{{$.i18n.Tr "action.mirror_sync_create" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}}
						{{else if eq .GetOpType 20}}
							{{$.i18n.Tr "action.mirror_sync_delete" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}}
						{{end}}
					</p>
					{{if eq .GetOpType 5}}
					{{if or (eq .GetOpType 5) (eq .GetOpType 18)}}
						<div class="content">
							<ul>
								{{ $push := ActionContent2Commits .}}

+ 22
- 2
vendor/code.gitea.io/git/commit.go View File

@@ -34,14 +34,18 @@ type CommitGPGSignature struct {
}
// similar to https://github.com/git/git/blob/3bc53220cb2dcf709f7a027a3f526befd021d858/commit.c#L1128
func newGPGSignatureFromCommitline(data []byte, signatureStart int) (*CommitGPGSignature, error) {
func newGPGSignatureFromCommitline(data []byte, signatureStart int, tag bool) (*CommitGPGSignature, error) {
	sig := new(CommitGPGSignature)
	signatureEnd := bytes.LastIndex(data, []byte("-----END PGP SIGNATURE-----"))
	if signatureEnd == -1 {
		return nil, fmt.Errorf("end of commit signature not found")
	}
	sig.Signature = strings.Replace(string(data[signatureStart:signatureEnd+27]), "\n ", "\n", -1)
	sig.Payload = string(data[:signatureStart-8]) + string(data[signatureEnd+27:])
	if tag {
		sig.Payload = string(data[:signatureStart-1])
	} else {
		sig.Payload = string(data[:signatureStart-8]) + string(data[signatureEnd+27:])
	}
	return sig, nil
}
@@ -274,3 +278,19 @@ func (c *Commit) GetSubModule(entryname string) (*SubModule, error) {
	}
	return nil, nil
}
// GetFullCommitID returns full length (40) of commit ID by given short SHA in a repository.
func GetFullCommitID(repoPath, shortID string) (string, error) {
	if len(shortID) >= 40 {
		return shortID, nil
	}
	commitID, err := NewCommand("rev-parse", shortID).RunInDir(repoPath)
	if err != nil {
		if strings.Contains(err.Error(), "exit status 128") {
			return "", ErrNotExist{shortID, ""}
		}
		return "", err
	}
	return strings.TrimSpace(commitID), nil
}

+ 15
- 2
vendor/code.gitea.io/git/repo_commit.go View File

@@ -78,7 +78,7 @@ l:
				}
				commit.Committer = sig
			case "gpgsig":
				sig, err := newGPGSignatureFromCommitline(data, nextline+spacepos+1)
				sig, err := newGPGSignatureFromCommitline(data, nextline+spacepos+1, false)
				if err != nil {
					return nil, err
				}
@@ -86,7 +86,20 @@ l:
			}
			nextline += eol + 1
		case eol == 0:
			commit.CommitMessage = string(data[nextline+1:])
			cm := string(data[nextline+1:])
			// Tag GPG signatures are stored below the commit message
			sigindex := strings.Index(cm, "-----BEGIN PGP SIGNATURE-----")
			if sigindex != -1 {
				sig, err := newGPGSignatureFromCommitline(data, (nextline+1)+sigindex, true)
				if err == nil && sig != nil {
					// remove signature from commit message
					cm = cm[:sigindex-1]
					commit.Signature = sig
				}
			}
			commit.CommitMessage = cm
			break l
		default:
			break l

Loading…
Cancel
Save