THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Browse Source

Add push webhook support for mirrored repositories (#4127)

pull/2917/merge
Lauris BH 7 months ago
parent
commit
fa4663e61e

+ 2
- 2
Gopkg.lock View File

@@ -3,11 +3,11 @@
3 3
 
4 4
 [[projects]]
5 5
   branch = "master"
6
-  digest = "1:42f77a668e3bd06812ef254f334d0d0a62346969fbcd3fa3a613e75067343751"
6
+  digest = "1:835585f8450b4ec12252d032b0f13e6571ecf846e49076f69067f2503a7c1e07"
7 7
   name = "code.gitea.io/git"
8 8
   packages = ["."]
9 9
   pruneopts = "NUT"
10
-  revision = "31f4b8e8c805438ac6d8914b38accb1d8aaf695e"
10
+  revision = "6ef79e80b3b06ca13a1f3a7b940903ebc73b44cb"
11 11
 
12 12
 [[projects]]
13 13
   branch = "master"

+ 68
- 0
models/action.go View File

@@ -47,6 +47,9 @@ const (
47 47
 	ActionReopenPullRequest                       // 15
48 48
 	ActionDeleteTag                               // 16
49 49
 	ActionDeleteBranch                            // 17
50
+	ActionMirrorSyncPush                          // 18
51
+	ActionMirrorSyncCreate                        // 19
52
+	ActionMirrorSyncDelete                        // 20
50 53
 )
51 54
 
52 55
 var (
@@ -736,6 +739,71 @@ func MergePullRequestAction(actUser *User, repo *Repository, pull *Issue) error
736 739
 	return mergePullRequestAction(x, actUser, repo, pull)
737 740
 }
738 741
 
742
+func mirrorSyncAction(e Engine, opType ActionType, repo *Repository, refName string, data []byte) error {
743
+	if err := notifyWatchers(e, &Action{
744
+		ActUserID: repo.OwnerID,
745
+		ActUser:   repo.MustOwner(),
746
+		OpType:    opType,
747
+		RepoID:    repo.ID,
748
+		Repo:      repo,
749
+		IsPrivate: repo.IsPrivate,
750
+		RefName:   refName,
751
+		Content:   string(data),
752
+	}); err != nil {
753
+		return fmt.Errorf("notifyWatchers: %v", err)
754
+	}
755
+	return nil
756
+}
757
+
758
+// MirrorSyncPushActionOptions mirror synchronization action options.
759
+type MirrorSyncPushActionOptions struct {
760
+	RefName     string
761
+	OldCommitID string
762
+	NewCommitID string
763
+	Commits     *PushCommits
764
+}
765
+
766
+// MirrorSyncPushAction adds new action for mirror synchronization of pushed commits.
767
+func MirrorSyncPushAction(repo *Repository, opts MirrorSyncPushActionOptions) error {
768
+	if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum {
769
+		opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum]
770
+	}
771
+
772
+	apiCommits := opts.Commits.ToAPIPayloadCommits(repo.HTMLURL())
773
+
774
+	opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID)
775
+	apiPusher := repo.MustOwner().APIFormat()
776
+	if err := PrepareWebhooks(repo, HookEventPush, &api.PushPayload{
777
+		Ref:        opts.RefName,
778
+		Before:     opts.OldCommitID,
779
+		After:      opts.NewCommitID,
780
+		CompareURL: setting.AppURL + opts.Commits.CompareURL,
781
+		Commits:    apiCommits,
782
+		Repo:       repo.APIFormat(AccessModeOwner),
783
+		Pusher:     apiPusher,
784
+		Sender:     apiPusher,
785
+	}); err != nil {
786
+		return fmt.Errorf("PrepareWebhooks: %v", err)
787
+	}
788
+
789
+	data, err := json.Marshal(opts.Commits)
790
+	if err != nil {
791
+		return err
792
+	}
793
+
794
+	return mirrorSyncAction(x, ActionMirrorSyncPush, repo, opts.RefName, data)
795
+}
796
+
797
+// MirrorSyncCreateAction adds new action for mirror synchronization of new reference.
798
+func MirrorSyncCreateAction(repo *Repository, refName string) error {
799
+	return mirrorSyncAction(x, ActionMirrorSyncCreate, repo, refName, nil)
800
+}
801
+
802
+// MirrorSyncDeleteAction adds new action for mirror synchronization of delete reference.
803
+func MirrorSyncDeleteAction(repo *Repository, refName string) error {
804
+	return mirrorSyncAction(x, ActionMirrorSyncDelete, repo, refName, nil)
805
+}
806
+
739 807
 // GetFeedsOptions options for retrieving feeds
740 808
 type GetFeedsOptions struct {
741 809
 	RequestedUser    *User

+ 136
- 11
models/repo_mirror.go View File

@@ -1,4 +1,5 @@
1 1
 // Copyright 2016 The Gogs Authors. All rights reserved.
2
+// Copyright 2018 The Gitea Authors. All rights reserved.
2 3
 // Use of this source code is governed by a MIT-style
3 4
 // license that can be found in the LICENSE file.
4 5
 
@@ -6,6 +7,7 @@ package models
6 7
 
7 8
 import (
8 9
 	"fmt"
10
+	"strings"
9 11
 	"time"
10 12
 
11 13
 	"code.gitea.io/git"
@@ -119,8 +121,68 @@ func (m *Mirror) SaveAddress(addr string) error {
119 121
 	return cfg.SaveToIndent(configPath, "\t")
120 122
 }
121 123
 
124
+// gitShortEmptySha Git short empty SHA
125
+const gitShortEmptySha = "0000000"
126
+
127
+// mirrorSyncResult contains information of a updated reference.
128
+// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
129
+// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
130
+type mirrorSyncResult struct {
131
+	refName     string
132
+	oldCommitID string
133
+	newCommitID string
134
+}
135
+
136
+// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
137
+func parseRemoteUpdateOutput(output string) []*mirrorSyncResult {
138
+	results := make([]*mirrorSyncResult, 0, 3)
139
+	lines := strings.Split(output, "\n")
140
+	for i := range lines {
141
+		// Make sure reference name is presented before continue
142
+		idx := strings.Index(lines[i], "-> ")
143
+		if idx == -1 {
144
+			continue
145
+		}
146
+
147
+		refName := lines[i][idx+3:]
148
+
149
+		switch {
150
+		case strings.HasPrefix(lines[i], " * "): // New reference
151
+			results = append(results, &mirrorSyncResult{
152
+				refName:     refName,
153
+				oldCommitID: gitShortEmptySha,
154
+			})
155
+		case strings.HasPrefix(lines[i], " - "): // Delete reference
156
+			results = append(results, &mirrorSyncResult{
157
+				refName:     refName,
158
+				newCommitID: gitShortEmptySha,
159
+			})
160
+		case strings.HasPrefix(lines[i], "   "): // New commits of a reference
161
+			delimIdx := strings.Index(lines[i][3:], " ")
162
+			if delimIdx == -1 {
163
+				log.Error(2, "SHA delimiter not found: %q", lines[i])
164
+				continue
165
+			}
166
+			shas := strings.Split(lines[i][3:delimIdx+3], "..")
167
+			if len(shas) != 2 {
168
+				log.Error(2, "Expect two SHAs but not what found: %q", lines[i])
169
+				continue
170
+			}
171
+			results = append(results, &mirrorSyncResult{
172
+				refName:     refName,
173
+				oldCommitID: shas[0],
174
+				newCommitID: shas[1],
175
+			})
176
+
177
+		default:
178
+			log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i])
179
+		}
180
+	}
181
+	return results
182
+}
183
+
122 184
 // runSync returns true if sync finished without error.
123
-func (m *Mirror) runSync() bool {
185
+func (m *Mirror) runSync() ([]*mirrorSyncResult, bool) {
124 186
 	repoPath := m.Repo.RepoPath()
125 187
 	wikiPath := m.Repo.WikiPath()
126 188
 	timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
@@ -130,28 +192,30 @@ func (m *Mirror) runSync() bool {
130 192
 		gitArgs = append(gitArgs, "--prune")
131 193
 	}
132 194
 
133
-	if _, stderr, err := process.GetManager().ExecDir(
195
+	_, stderr, err := process.GetManager().ExecDir(
134 196
 		timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
135
-		"git", gitArgs...); err != nil {
197
+		"git", gitArgs...)
198
+	if err != nil {
136 199
 		// sanitize the output, since it may contain the remote address, which may
137 200
 		// contain a password
138 201
 		message, err := sanitizeOutput(stderr, repoPath)
139 202
 		if err != nil {
140 203
 			log.Error(4, "sanitizeOutput: %v", err)
141
-			return false
204
+			return nil, false
142 205
 		}
143 206
 		desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, message)
144 207
 		log.Error(4, desc)
145 208
 		if err = CreateRepositoryNotice(desc); err != nil {
146 209
 			log.Error(4, "CreateRepositoryNotice: %v", err)
147 210
 		}
148
-		return false
211
+		return nil, false
149 212
 	}
213
+	output := stderr
150 214
 
151 215
 	gitRepo, err := git.OpenRepository(repoPath)
152 216
 	if err != nil {
153 217
 		log.Error(4, "OpenRepository: %v", err)
154
-		return false
218
+		return nil, false
155 219
 	}
156 220
 	if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
157 221
 		log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
@@ -170,21 +234,21 @@ func (m *Mirror) runSync() bool {
170 234
 			message, err := sanitizeOutput(stderr, wikiPath)
171 235
 			if err != nil {
172 236
 				log.Error(4, "sanitizeOutput: %v", err)
173
-				return false
237
+				return nil, false
174 238
 			}
175 239
 			desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, message)
176 240
 			log.Error(4, desc)
177 241
 			if err = CreateRepositoryNotice(desc); err != nil {
178 242
 				log.Error(4, "CreateRepositoryNotice: %v", err)
179 243
 			}
180
-			return false
244
+			return nil, false
181 245
 		}
182 246
 	}
183 247
 
184 248
 	branches, err := m.Repo.GetBranches()
185 249
 	if err != nil {
186 250
 		log.Error(4, "GetBranches: %v", err)
187
-		return false
251
+		return nil, false
188 252
 	}
189 253
 
190 254
 	for i := range branches {
@@ -192,7 +256,7 @@ func (m *Mirror) runSync() bool {
192 256
 	}
193 257
 
194 258
 	m.UpdatedUnix = util.TimeStampNow()
195
-	return true
259
+	return parseRemoteUpdateOutput(output), true
196 260
 }
197 261
 
198 262
 func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) {
@@ -268,7 +332,8 @@ func SyncMirrors() {
268 332
 			continue
269 333
 		}
270 334
 
271
-		if !m.runSync() {
335
+		results, ok := m.runSync()
336
+		if !ok {
272 337
 			continue
273 338
 		}
274 339
 
@@ -278,6 +343,66 @@ func SyncMirrors() {
278 343
 			continue
279 344
 		}
280 345
 
346
+		var gitRepo *git.Repository
347
+		if len(results) == 0 {
348
+			log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID)
349
+		} else {
350
+			gitRepo, err = git.OpenRepository(m.Repo.RepoPath())
351
+			if err != nil {
352
+				log.Error(2, "OpenRepository [%d]: %v", m.RepoID, err)
353
+				continue
354
+			}
355
+		}
356
+
357
+		for _, result := range results {
358
+			// Discard GitHub pull requests, i.e. refs/pull/*
359
+			if strings.HasPrefix(result.refName, "refs/pull/") {
360
+				continue
361
+			}
362
+
363
+			// Create reference
364
+			if result.oldCommitID == gitShortEmptySha {
365
+				if err = MirrorSyncCreateAction(m.Repo, result.refName); err != nil {
366
+					log.Error(2, "MirrorSyncCreateAction [repo_id: %d]: %v", m.RepoID, err)
367
+				}
368
+				continue
369
+			}
370
+
371
+			// Delete reference
372
+			if result.newCommitID == gitShortEmptySha {
373
+				if err = MirrorSyncDeleteAction(m.Repo, result.refName); err != nil {
374
+					log.Error(2, "MirrorSyncDeleteAction [repo_id: %d]: %v", m.RepoID, err)
375
+				}
376
+				continue
377
+			}
378
+
379
+			// Push commits
380
+			oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID)
381
+			if err != nil {
382
+				log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err)
383
+				continue
384
+			}
385
+			newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID)
386
+			if err != nil {
387
+				log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err)
388
+				continue
389
+			}
390
+			commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID)
391
+			if err != nil {
392
+				log.Error(2, "CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err)
393
+				continue
394
+			}
395
+			if err = MirrorSyncPushAction(m.Repo, MirrorSyncPushActionOptions{
396
+				RefName:     result.refName,
397
+				OldCommitID: oldCommitID,
398
+				NewCommitID: newCommitID,
399
+				Commits:     ListToPushCommits(commits),
400
+			}); err != nil {
401
+				log.Error(2, "MirrorSyncPushAction [repo_id: %d]: %v", m.RepoID, err)
402
+				continue
403
+			}
404
+		}
405
+
281 406
 		// Get latest commit date and update to current repository updated time
282 407
 		commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath())
283 408
 		if err != nil {

+ 2
- 0
modules/templates/helper.go View File

@@ -391,6 +391,8 @@ func ActionIcon(opType models.ActionType) string {
391 391
 		return "issue-closed"
392 392
 	case models.ActionReopenIssue, models.ActionReopenPullRequest:
393 393
 		return "issue-reopened"
394
+	case models.ActionMirrorSyncPush, models.ActionMirrorSyncCreate, models.ActionMirrorSyncDelete:
395
+		return "repo-clone"
394 396
 	default:
395 397
 		return "invalid type"
396 398
 	}

+ 3
- 0
options/locale/locale_en-US.ini View File

@@ -1663,6 +1663,9 @@ push_tag = pushed tag <a href="%s/src/tag/%s">%[2]s</a> to <a href="%[1]s">%[3]s
1663 1663
 delete_tag = deleted tag %[2]s from <a href="%[1]s">%[3]s</a>
1664 1664
 delete_branch = deleted branch %[2]s from <a href="%[1]s">%[3]s</a>
1665 1665
 compare_commits = Compare %d commits
1666
+mirror_sync_push = synced commits to <a href="%[1]s/src/%[2]s">%[3]s</a> at <a href="%[1]s">%[4]s</a> from mirror
1667
+mirror_sync_create = synced new reference <a href="%s/src/%s">%[2]s</a> to <a href="%[1]s">%[3]s</a> from mirror
1668
+mirror_sync_delete = synced and deleted reference <code>%[2]s</code> at <a href="%[1]s">%[3]s</a> from mirror
1666 1669
 
1667 1670
 [tool]
1668 1671
 ago = %s ago

+ 9
- 2
templates/user/dashboard/feeds.tmpl View File

@@ -5,7 +5,7 @@
5 5
 		</div>
6 6
 		<div class="ui grid">
7 7
 			<div class="ui thirteen wide column">
8
-				<div class="{{if eq .GetOpType 5}}push news{{end}}">
8
+				<div class="{{if or (eq .GetOpType 5) (eq .GetOpType 18)}}push news{{end}}">
9 9
 					<p>
10 10
 						<a href="{{AppSubUrl}}/{{.GetActUserName}}" title="{{.GetActFullName}}">{{.ShortActUserName}}</a>
11 11
 						{{if eq .GetOpType 1}}
@@ -49,9 +49,16 @@
49 49
 						{{else if eq .GetOpType 17}}
50 50
 							{{ $index := index .GetIssueInfos 0}}
51 51
 							{{$.i18n.Tr "action.delete_branch" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}}
52
+						{{else if eq .GetOpType 18}}
53
+							{{ $branchLink := .GetBranch | EscapePound}}
54
+							{{$.i18n.Tr "action.mirror_sync_push" .GetRepoLink $branchLink .GetBranch .ShortRepoPath | Str2html}}
55
+						{{else if eq .GetOpType 19}}
56
+							{{$.i18n.Tr "action.mirror_sync_create" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}}
57
+						{{else if eq .GetOpType 20}}
58
+							{{$.i18n.Tr "action.mirror_sync_delete" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}}
52 59
 						{{end}}
53 60
 					</p>
54
-					{{if eq .GetOpType 5}}
61
+					{{if or (eq .GetOpType 5) (eq .GetOpType 18)}}
55 62
 						<div class="content">
56 63
 							<ul>
57 64
 								{{ $push := ActionContent2Commits .}}

+ 22
- 2
vendor/code.gitea.io/git/commit.go View File

@@ -34,14 +34,18 @@ type CommitGPGSignature struct {
34 34
 }
35 35
 
36 36
 // similar to https://github.com/git/git/blob/3bc53220cb2dcf709f7a027a3f526befd021d858/commit.c#L1128
37
-func newGPGSignatureFromCommitline(data []byte, signatureStart int) (*CommitGPGSignature, error) {
37
+func newGPGSignatureFromCommitline(data []byte, signatureStart int, tag bool) (*CommitGPGSignature, error) {
38 38
 	sig := new(CommitGPGSignature)
39 39
 	signatureEnd := bytes.LastIndex(data, []byte("-----END PGP SIGNATURE-----"))
40 40
 	if signatureEnd == -1 {
41 41
 		return nil, fmt.Errorf("end of commit signature not found")
42 42
 	}
43 43
 	sig.Signature = strings.Replace(string(data[signatureStart:signatureEnd+27]), "\n ", "\n", -1)
44
-	sig.Payload = string(data[:signatureStart-8]) + string(data[signatureEnd+27:])
44
+	if tag {
45
+		sig.Payload = string(data[:signatureStart-1])
46
+	} else {
47
+		sig.Payload = string(data[:signatureStart-8]) + string(data[signatureEnd+27:])
48
+	}
45 49
 	return sig, nil
46 50
 }
47 51
 
@@ -274,3 +278,19 @@ func (c *Commit) GetSubModule(entryname string) (*SubModule, error) {
274 278
 	}
275 279
 	return nil, nil
276 280
 }
281
+
282
+// GetFullCommitID returns full length (40) of commit ID by given short SHA in a repository.
283
+func GetFullCommitID(repoPath, shortID string) (string, error) {
284
+	if len(shortID) >= 40 {
285
+		return shortID, nil
286
+	}
287
+
288
+	commitID, err := NewCommand("rev-parse", shortID).RunInDir(repoPath)
289
+	if err != nil {
290
+		if strings.Contains(err.Error(), "exit status 128") {
291
+			return "", ErrNotExist{shortID, ""}
292
+		}
293
+		return "", err
294
+	}
295
+	return strings.TrimSpace(commitID), nil
296
+}

+ 15
- 2
vendor/code.gitea.io/git/repo_commit.go View File

@@ -78,7 +78,7 @@ l:
78 78
 				}
79 79
 				commit.Committer = sig
80 80
 			case "gpgsig":
81
-				sig, err := newGPGSignatureFromCommitline(data, nextline+spacepos+1)
81
+				sig, err := newGPGSignatureFromCommitline(data, nextline+spacepos+1, false)
82 82
 				if err != nil {
83 83
 					return nil, err
84 84
 				}
@@ -86,7 +86,20 @@ l:
86 86
 			}
87 87
 			nextline += eol + 1
88 88
 		case eol == 0:
89
-			commit.CommitMessage = string(data[nextline+1:])
89
+			cm := string(data[nextline+1:])
90
+
91
+			// Tag GPG signatures are stored below the commit message
92
+			sigindex := strings.Index(cm, "-----BEGIN PGP SIGNATURE-----")
93
+			if sigindex != -1 {
94
+				sig, err := newGPGSignatureFromCommitline(data, (nextline+1)+sigindex, true)
95
+				if err == nil && sig != nil {
96
+					// remove signature from commit message
97
+					cm = cm[:sigindex-1]
98
+					commit.Signature = sig
99
+				}
100
+			}
101
+
102
+			commit.CommitMessage = cm
90 103
 			break l
91 104
 		default:
92 105
 			break l

Loading…
Cancel
Save