THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Browse Source

Restore Graceful Restarting & Socket Activation (#7274)

* Prevent deadlock in indexer initialisation during graceful restart

* Move from gracehttp to our own service to add graceful ssh

* Add timeout for start of indexers and make hammer time configurable

* Fix issue with re-initialization in indexer during tests

* move the code to detect use of closed to graceful

* Handle logs gracefully - add a pid suffix just before restart

* Move to using a cond and a holder for indexers

* use time.Since

* Add some comments and attribution

* update modules.txt

* Use zero to disable timeout

* Move RestartProcess to its own file

* Add cleanup routine
pull/8531/head
zeripath GitHub 1 month ago
parent
commit
167e8f18da
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
45 changed files with 1199 additions and 2006 deletions
  1. +26
    -9
      cmd/web.go
  2. +18
    -26
      cmd/web_graceful.go
  3. +18
    -0
      cmd/web_windows.go
  4. +9
    -0
      custom/conf/app.ini.sample
  5. +3
    -0
      docs/content/doc/advanced/config-cheat-sheet.en-us.md
  6. +0
    -5
      go.mod
  7. +0
    -10
      go.sum
  8. +25
    -2
      models/repo_indexer.go
  9. +38
    -0
      modules/graceful/cleanup.go
  10. +209
    -0
      modules/graceful/net.go
  11. +67
    -0
      modules/graceful/restart.go
  12. +267
    -0
      modules/graceful/server.go
  13. +119
    -0
      modules/graceful/server_hooks.go
  14. +45
    -0
      modules/graceful/server_http.go
  15. +93
    -0
      modules/graceful/server_signals.go
  16. +130
    -63
      modules/indexer/issues/indexer.go
  17. +2
    -12
      modules/indexer/issues/indexer_test.go
  18. +41
    -11
      modules/indexer/repo.go
  19. +3
    -0
      modules/setting/indexer.go
  20. +10
    -1
      modules/setting/log.go
  21. +4
    -0
      modules/setting/setting.go
  22. +1
    -6
      modules/ssh/ssh.go
  23. +30
    -0
      modules/ssh/ssh_graceful.go
  24. +24
    -0
      modules/ssh/ssh_windows.go
  25. +11
    -6
      routers/init.go
  26. +6
    -0
      routers/install.go
  27. +0
    -21
      vendor/github.com/facebookgo/clock/LICENSE
  28. +0
    -104
      vendor/github.com/facebookgo/clock/README.md
  29. +0
    -363
      vendor/github.com/facebookgo/clock/clock.go
  30. +0
    -186
      vendor/github.com/facebookgo/grace/gracehttp/http.go
  31. +0
    -252
      vendor/github.com/facebookgo/grace/gracenet/net.go
  32. +0
    -23
      vendor/github.com/facebookgo/httpdown/.travis.yml
  33. +0
    -376
      vendor/github.com/facebookgo/httpdown/httpdown.go
  34. +0
    -30
      vendor/github.com/facebookgo/httpdown/license
  35. +0
    -33
      vendor/github.com/facebookgo/httpdown/patents
  36. +0
    -41
      vendor/github.com/facebookgo/httpdown/readme.md
  37. +0
    -20
      vendor/github.com/facebookgo/stats/.travis.yml
  38. +0
    -35
      vendor/github.com/facebookgo/stats/aggregation.go
  39. +0
    -112
      vendor/github.com/facebookgo/stats/counter.go
  40. +0
    -30
      vendor/github.com/facebookgo/stats/license
  41. +0
    -33
      vendor/github.com/facebookgo/stats/patents
  42. +0
    -4
      vendor/github.com/facebookgo/stats/readme.md
  43. +0
    -166
      vendor/github.com/facebookgo/stats/stats.go
  44. +0
    -17
      vendor/github.com/facebookgo/stats/stopper.go
  45. +0
    -9
      vendor/modules.txt

+ 26
- 9
cmd/web.go View File

@@ -75,17 +75,13 @@ func runLetsEncrypt(listenAddr, domain, directory, email string, m http.Handler)
}
go func() {
log.Info("Running Let's Encrypt handler on %s", setting.HTTPAddr+":"+setting.PortToRedirect)
var err = http.ListenAndServe(setting.HTTPAddr+":"+setting.PortToRedirect, certManager.HTTPHandler(http.HandlerFunc(runLetsEncryptFallbackHandler))) // all traffic coming into HTTP will be redirect to HTTPS automatically (LE HTTP-01 validation happens here)
// all traffic coming into HTTP will be redirect to HTTPS automatically (LE HTTP-01 validation happens here)
var err = runHTTP(setting.HTTPAddr+":"+setting.PortToRedirect, certManager.HTTPHandler(http.HandlerFunc(runLetsEncryptFallbackHandler)))
if err != nil {
log.Fatal("Failed to start the Let's Encrypt handler on port %s: %v", setting.PortToRedirect, err)
}
}()
server := &http.Server{
Addr: listenAddr,
Handler: m,
TLSConfig: certManager.TLSConfig(),
}
return server.ListenAndServeTLS("", "")
return runHTTPSWithTLSConfig(listenAddr, certManager.TLSConfig(), context2.ClearHandler(m))
}

func runLetsEncryptFallbackHandler(w http.ResponseWriter, r *http.Request) {
@@ -101,12 +97,21 @@ func runLetsEncryptFallbackHandler(w http.ResponseWriter, r *http.Request) {
}

func runWeb(ctx *cli.Context) error {
if os.Getppid() > 1 && len(os.Getenv("LISTEN_FDS")) > 0 {
log.Info("Restarting Gitea on PID: %d from parent PID: %d", os.Getpid(), os.Getppid())
} else {
log.Info("Starting Gitea on PID: %d", os.Getpid())
}

// Set pid file setting
if ctx.IsSet("pid") {
setting.CustomPID = ctx.String("pid")
}

// Perform global initialization
routers.GlobalInit()

// Set up Macaron
m := routes.NewMacaron()
routes.RegisterRoutes(m)

@@ -164,6 +169,7 @@ func runWeb(ctx *cli.Context) error {
var err error
switch setting.Protocol {
case setting.HTTP:
NoHTTPRedirector()
err = runHTTP(listenAddr, context2.ClearHandler(m))
case setting.HTTPS:
if setting.EnableLetsEncrypt {
@@ -172,9 +178,15 @@ func runWeb(ctx *cli.Context) error {
}
if setting.RedirectOtherPort {
go runHTTPRedirector()
} else {
NoHTTPRedirector()
}
err = runHTTPS(listenAddr, setting.CertFile, setting.KeyFile, context2.ClearHandler(m))
case setting.FCGI:
NoHTTPRedirector()
// FCGI listeners are provided as stdin - this is orthogonal to the LISTEN_FDS approach
// in graceful and systemD
NoMainListener()
var listener net.Listener
listener, err = net.Listen("tcp", listenAddr)
if err != nil {
@@ -187,6 +199,10 @@ func runWeb(ctx *cli.Context) error {
}()
err = fcgi.Serve(listener, context2.ClearHandler(m))
case setting.UnixSocket:
// This could potentially be inherited using LISTEN_FDS but currently
// these cannot be inherited
NoHTTPRedirector()
NoMainListener()
if err := os.Remove(listenAddr); err != nil && !os.IsNotExist(err) {
log.Fatal("Failed to remove unix socket directory %s: %v", listenAddr, err)
}
@@ -207,8 +223,9 @@ func runWeb(ctx *cli.Context) error {
}

if err != nil {
log.Fatal("Failed to start server: %v", err)
log.Critical("Failed to start server: %v", err)
}

log.Info("HTTP Listener: %s Closed", listenAddr)
log.Close()
return nil
}

+ 18
- 26
cmd/web_graceful.go View File

@@ -10,36 +10,28 @@ import (
"crypto/tls"
"net/http"

"code.gitea.io/gitea/modules/log"

"github.com/facebookgo/grace/gracehttp"
"code.gitea.io/gitea/modules/graceful"
)

func runHTTP(listenAddr string, m http.Handler) error {
return gracehttp.Serve(&http.Server{
Addr: listenAddr,
Handler: m,
})
return graceful.HTTPListenAndServe("tcp", listenAddr, m)
}

func runHTTPS(listenAddr, certFile, keyFile string, m http.Handler) error {
config := &tls.Config{
MinVersion: tls.VersionTLS10,
}
if config.NextProtos == nil {
config.NextProtos = []string{"http/1.1"}
}

config.Certificates = make([]tls.Certificate, 1)
var err error
config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
log.Fatal("Failed to load https cert file %s: %v", listenAddr, err)
}

return gracehttp.Serve(&http.Server{
Addr: listenAddr,
Handler: m,
TLSConfig: config,
})
return graceful.HTTPListenAndServeTLS("tcp", listenAddr, certFile, keyFile, m)
}

func runHTTPSWithTLSConfig(listenAddr string, tlsConfig *tls.Config, m http.Handler) error {
return graceful.HTTPListenAndServeTLSConfig("tcp", listenAddr, tlsConfig, m)
}

// NoHTTPRedirector tells our cleanup routine that we will not be using a fallback http redirector
func NoHTTPRedirector() {
graceful.InformCleanup()
}

// NoMainListener tells our cleanup routine that we will not be using a possibly provided listener
// for our main HTTP/HTTPS service
func NoMainListener() {
graceful.InformCleanup()
}

+ 18
- 0
cmd/web_windows.go View File

@@ -7,6 +7,7 @@
package cmd

import (
"crypto/tls"
"net/http"
)

@@ -17,3 +18,20 @@ func runHTTP(listenAddr string, m http.Handler) error {
func runHTTPS(listenAddr, certFile, keyFile string, m http.Handler) error {
return http.ListenAndServeTLS(listenAddr, certFile, keyFile, m)
}

func runHTTPSWithTLSConfig(listenAddr string, tlsConfig *tls.Config, m http.Handler) error {
server := &http.Server{
Addr: listenAddr,
Handler: m,
TLSConfig: tlsConfig,
}
return server.ListenAndServeTLS("", "")
}

// NoHTTPRedirector is a no-op on Windows
func NoHTTPRedirector() {
}

// NoMainListener is a no-op on Windows
func NoMainListener() {
}

+ 9
- 0
custom/conf/app.ini.sample View File

@@ -244,6 +244,12 @@ LFS_CONTENT_PATH = data/lfs
LFS_JWT_SECRET =
; LFS authentication validity period (in time.Duration), pushes taking longer than this may fail.
LFS_HTTP_AUTH_EXPIRY = 20m
; Allow graceful restarts using SIGHUP to fork
ALLOW_GRACEFUL_RESTARTS = true
; After a restart the parent will finish ongoing requests before
; shutting down. Force shutdown if this process takes longer than this delay.
; set to a negative value to disable
GRACEFUL_HAMMER_TIME = 60s
; Static resources, includes resources on custom/, public/ and all uploaded avatars web browser cache time, default is 6h
STATIC_CACHE_TIME = 6h

@@ -299,6 +305,9 @@ ISSUE_INDEXER_QUEUE_DIR = indexers/issues.queue
ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
; Batch queue number, default is 20
ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20
; Timeout the indexer if it takes longer than this to start.
; Set to zero to disable timeout.
STARTUP_TIMEOUT=30s

; repo indexer by default disabled, since it uses a lot of disk space
REPO_INDEXER_ENABLED = false


+ 3
- 0
docs/content/doc/advanced/config-cheat-sheet.en-us.md View File

@@ -157,6 +157,8 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
- `LETSENCRYPT_ACCEPTTOS`: **false**: This is an explicit check that you accept the terms of service for Let's Encrypt.
- `LETSENCRYPT_DIRECTORY`: **https**: Directory that Letsencrypt will use to cache information such as certs and private keys.
- `LETSENCRYPT_EMAIL`: **email@example.com**: Email used by Letsencrypt to notify about problems with issued certificates. (No default)
- `ALLOW_GRACEFUL_RESTARTS`: **true**: Perform a graceful restart on SIGHUP
- `GRACEFUL_HAMMER_TIME`: **60s**: After a restart the parent process will stop accepting new connections and will allow requests to finish before stopping. Shutdown will be forced if it takes longer than this time.

## Database (`database`)

@@ -189,6 +191,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
- `REPO_INDEXER_EXCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **exclude** from the index. Files that match this list will not be indexed, even if they match in `REPO_INDEXER_INCLUDE`.
- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.
- `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
- `STARTUP_TIMEOUT`: **30s**: If the indexer takes longer than this timeout to start - fail. (This timeout will be added to the hammer time above for child processes - as bleve will not start until the previous parent is shutdown.) Set to zero to never timeout.

## Admin (`admin`)
- `DEFAULT_EMAIL_NOTIFICATIONS`: **enabled**: Default configuration for email notifications for users (user configurable). Options: enabled, onmention, disabled


+ 0
- 5
go.mod View File

@@ -32,13 +32,8 @@ require (
github.com/emirpasic/gods v1.12.0
github.com/etcd-io/bbolt v1.3.2 // indirect
github.com/ethantkoenig/rupture v0.0.0-20180203182544-0a76f03a811a
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
github.com/facebookgo/grace v0.0.0-20160926231715-5729e484473f
github.com/facebookgo/httpdown v0.0.0-20160323221027-a3b1354551a2 // indirect
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 // indirect
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
github.com/gliderlabs/ssh v0.2.2
github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd // indirect


+ 0
- 10
go.sum View File

@@ -142,20 +142,10 @@ github.com/etcd-io/bbolt v1.3.2 h1:RLRQ0TKLX7DlBRXAJHvbmXL17Q3KNnTBtZ9B6Qo+/Y0=
github.com/etcd-io/bbolt v1.3.2/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
github.com/ethantkoenig/rupture v0.0.0-20180203182544-0a76f03a811a h1:M1bRpaZAn4GSsqu3hdK2R8H0AH9O6vqCTCbm2oAFGfE=
github.com/ethantkoenig/rupture v0.0.0-20180203182544-0a76f03a811a/go.mod h1:MkKY/CB98aVE4VxO63X5vTQKUgcn+3XP15LMASe3lYs=
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw=
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA=
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7VRjQhwPlR/v0glZXNYzBHdNUzf/Am2Nmg=
github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg=
github.com/facebookgo/grace v0.0.0-20160926231715-5729e484473f h1:0mlfEUWnUDVZnqWEVHGerL5bKYDKMEmT/Qk/W/3nGuo=
github.com/facebookgo/grace v0.0.0-20160926231715-5729e484473f/go.mod h1:KigFdumBXUPSwzLDbeuzyt0elrL7+CP7TKuhrhT4bcU=
github.com/facebookgo/httpdown v0.0.0-20160323221027-a3b1354551a2 h1:3Zvf9wRhl1cOhckN1oRGWPOkIhOketmEcrQ4TeFAoR4=
github.com/facebookgo/httpdown v0.0.0-20160323221027-a3b1354551a2/go.mod h1:TUV/fX3XrTtBQb5+ttSUJzcFgLNpILONFTKmBuk5RSw=
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 h1:0YtRCqIZs2+Tz49QuH6cJVw/IFqzo39gEqZ0iYLxD2M=
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4/go.mod h1:vsJz7uE339KUCpBXx3JAJzSRH7Uk4iGGyJzR529qDIA=
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y=
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=


+ 25
- 2
models/repo_indexer.go View File

@@ -8,10 +8,12 @@ import (
"fmt"
"strconv"
"strings"
"time"

"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/indexer"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
@@ -70,9 +72,30 @@ func InitRepoIndexer() {
if !setting.Indexer.RepoIndexerEnabled {
return
}
waitChannel := make(chan time.Duration)
repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength)
indexer.InitRepoIndexer(populateRepoIndexerAsynchronously)
go processRepoIndexerOperationQueue()
go func() {
start := time.Now()
log.Info("Initializing Repository Indexer")
indexer.InitRepoIndexer(populateRepoIndexerAsynchronously)
go processRepoIndexerOperationQueue()
waitChannel <- time.Since(start)
}()
if setting.Indexer.StartupTimeout > 0 {
go func() {
timeout := setting.Indexer.StartupTimeout
if graceful.IsChild && setting.GracefulHammerTime > 0 {
timeout += setting.GracefulHammerTime
}
select {
case duration := <-waitChannel:
log.Info("Repository Indexer Initialization took %v", duration)
case <-time.After(timeout):
log.Fatal("Repository Indexer Initialization Timed-Out after: %v", timeout)
}
}()

}
}

// populateRepoIndexerAsynchronously asynchronously populates the repo indexer


+ 38
- 0
modules/graceful/cleanup.go View File

@@ -0,0 +1,38 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package graceful

import "sync"

var cleanupWaitGroup sync.WaitGroup

func init() {
cleanupWaitGroup = sync.WaitGroup{}

// There are three places that could inherit sockets:
//
// * HTTP or HTTPS main listener
// * HTTP redirection fallback
// * SSH
//
// If you add an additional place you must increment this number
// and add a function to call InformCleanup if it's not going to be used
cleanupWaitGroup.Add(3)

// Wait till we're done getting all of the listeners and then close
// the unused ones
go func() {
cleanupWaitGroup.Wait()
// Ignore the error here there's not much we can do with it
// They're logged in the CloseProvidedListeners function
_ = CloseProvidedListeners()
}()
}

// InformCleanup tells the cleanup wait group that we have either taken a listener
// or will not be taking a listener
func InformCleanup() {
cleanupWaitGroup.Done()
}

+ 209
- 0
modules/graceful/net.go View File

@@ -0,0 +1,209 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// This code is heavily inspired by the archived gofacebook/gracenet/net.go handler

package graceful

import (
"fmt"
"net"
"os"
"strconv"
"strings"
"sync"

"code.gitea.io/gitea/modules/log"
)

const (
listenFDs = "LISTEN_FDS"
startFD = 3
)

// In order to keep the working directory the same as when we started we record
// it at startup.
var originalWD, _ = os.Getwd()

var (
once = sync.Once{}
mutex = sync.Mutex{}

providedListeners = []net.Listener{}
activeListeners = []net.Listener{}
)

func getProvidedFDs() (savedErr error) {
// Only inherit the provided FDS once but we will save the error so that repeated calls to this function will return the same error
once.Do(func() {
mutex.Lock()
defer mutex.Unlock()

numFDs := os.Getenv(listenFDs)
if numFDs == "" {
return
}
n, err := strconv.Atoi(numFDs)
if err != nil {
savedErr = fmt.Errorf("%s is not a number: %s. Err: %v", listenFDs, numFDs, err)
return
}

for i := startFD; i < n+startFD; i++ {
file := os.NewFile(uintptr(i), fmt.Sprintf("listener_FD%d", i))

l, err := net.FileListener(file)
if err == nil {
// Close the inherited file if it's a listener
if err = file.Close(); err != nil {
savedErr = fmt.Errorf("error closing provided socket fd %d: %s", i, err)
return
}
providedListeners = append(providedListeners, l)
continue
}

// If needed we can handle packetconns here.
savedErr = fmt.Errorf("Error getting provided socket fd %d: %v", i, err)
return
}
})
return savedErr
}

// CloseProvidedListeners closes all unused provided listeners.
func CloseProvidedListeners() error {
mutex.Lock()
defer mutex.Unlock()
var returnableError error
for _, l := range providedListeners {
err := l.Close()
if err != nil {
log.Error("Error in closing unused provided listener: %v", err)
if returnableError != nil {
returnableError = fmt.Errorf("%v & %v", returnableError, err)
} else {
returnableError = err
}
}
}
providedListeners = []net.Listener{}

return returnableError
}

// GetListener obtains a listener for the local network address. The network must be
// a stream-oriented network: "tcp", "tcp4", "tcp6", "unix" or "unixpacket". It
// returns an provided net.Listener for the matching network and address, or
// creates a new one using net.Listen.
func GetListener(network, address string) (net.Listener, error) {
// Add a deferral to say that we've tried to grab a listener
defer InformCleanup()
switch network {
case "tcp", "tcp4", "tcp6":
tcpAddr, err := net.ResolveTCPAddr(network, address)
if err != nil {
return nil, err
}
return GetListenerTCP(network, tcpAddr)
case "unix", "unixpacket":
unixAddr, err := net.ResolveUnixAddr(network, address)
if err != nil {
return nil, err
}
return GetListenerUnix(network, unixAddr)
default:
return nil, net.UnknownNetworkError(network)
}
}

// GetListenerTCP announces on the local network address. The network must be:
// "tcp", "tcp4" or "tcp6". It returns a provided net.Listener for the
// matching network and address, or creates a new one using net.ListenTCP.
func GetListenerTCP(network string, address *net.TCPAddr) (*net.TCPListener, error) {
if err := getProvidedFDs(); err != nil {
return nil, err
}

mutex.Lock()
defer mutex.Unlock()

// look for a provided listener
for i, l := range providedListeners {
if isSameAddr(l.Addr(), address) {
providedListeners = append(providedListeners[:i], providedListeners[i+1:]...)

activeListeners = append(activeListeners, l)
return l.(*net.TCPListener), nil
}
}

// no provided listener for this address -> make a fresh listener
l, err := net.ListenTCP(network, address)
if err != nil {
return nil, err
}
activeListeners = append(activeListeners, l)
return l, nil
}

// GetListenerUnix announces on the local network address. The network must be:
// "unix" or "unixpacket". It returns a provided net.Listener for the
// matching network and address, or creates a new one using net.ListenUnix.
func GetListenerUnix(network string, address *net.UnixAddr) (*net.UnixListener, error) {
if err := getProvidedFDs(); err != nil {
return nil, err
}

mutex.Lock()
defer mutex.Unlock()

// look for a provided listener
for i, l := range providedListeners {
if isSameAddr(l.Addr(), address) {
providedListeners = append(providedListeners[:i], providedListeners[i+1:]...)
activeListeners = append(activeListeners, l)
return l.(*net.UnixListener), nil
}
}

// make a fresh listener
l, err := net.ListenUnix(network, address)
if err != nil {
return nil, err
}
activeListeners = append(activeListeners, l)
return l, nil
}

func isSameAddr(a1, a2 net.Addr) bool {
// If the addresses are not on the same network fail.
if a1.Network() != a2.Network() {
return false
}

// If the two addresses have the same string representation they're equal
a1s := a1.String()
a2s := a2.String()
if a1s == a2s {
return true
}

// This allows for ipv6 vs ipv4 local addresses to compare as equal. This
// scenario is common when listening on localhost.
const ipv6prefix = "[::]"
a1s = strings.TrimPrefix(a1s, ipv6prefix)
a2s = strings.TrimPrefix(a2s, ipv6prefix)
const ipv4prefix = "0.0.0.0"
a1s = strings.TrimPrefix(a1s, ipv4prefix)
a2s = strings.TrimPrefix(a2s, ipv4prefix)
return a1s == a2s
}

func getActiveListeners() []net.Listener {
mutex.Lock()
defer mutex.Unlock()
listeners := make([]net.Listener, len(activeListeners))
copy(listeners, activeListeners)
return listeners
}

+ 67
- 0
modules/graceful/restart.go View File

@@ -0,0 +1,67 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// This code is heavily inspired by the archived gofacebook/gracenet/net.go handler

package graceful

import (
"fmt"
"os"
"os/exec"
"strings"
)

// RestartProcess starts a new process passing it the active listeners. It
// doesn't fork, but starts a new process using the same environment and
// arguments as when it was originally started. This allows for a newly
// deployed binary to be started. It returns the pid of the newly started
// process when successful.
func RestartProcess() (int, error) {
listeners := getActiveListeners()

// Extract the fds from the listeners.
files := make([]*os.File, len(listeners))
for i, l := range listeners {
var err error
// Now, all our listeners actually have File() functions so instead of
// individually casting we just use a hacky interface
files[i], err = l.(filer).File()
if err != nil {
return 0, err
}
// Remember to close these at the end.
defer files[i].Close()
}

// Use the original binary location. This works with symlinks such that if
// the file it points to has been changed we will use the updated symlink.
argv0, err := exec.LookPath(os.Args[0])
if err != nil {
return 0, err
}

// Pass on the environment and replace the old count key with the new one.
var env []string
for _, v := range os.Environ() {
if !strings.HasPrefix(v, listenFDs+"=") {
env = append(env, v)
}
}
env = append(env, fmt.Sprintf("%s=%d", listenFDs, len(listeners)))

allFiles := append([]*os.File{os.Stdin, os.Stdout, os.Stderr}, files...)
process, err := os.StartProcess(argv0, os.Args, &os.ProcAttr{
Dir: originalWD,
Env: env,
Files: allFiles,
})
if err != nil {
return 0, err
}
return process.Pid, nil
}

type filer interface {
File() (*os.File, error)
}

+ 267
- 0
modules/graceful/server.go View File

@@ -0,0 +1,267 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// This code is highly inspired by endless go

package graceful

import (
"crypto/tls"
"net"
"os"
"strings"
"sync"
"syscall"
"time"

"code.gitea.io/gitea/modules/log"
)

type state uint8

const (
stateInit state = iota
stateRunning
stateShuttingDown
stateTerminate
)

var (
// RWMutex for when adding servers or shutting down
runningServerReg sync.RWMutex
// ensure we only fork once
runningServersForked bool

// DefaultReadTimeOut default read timeout
DefaultReadTimeOut time.Duration
// DefaultWriteTimeOut default write timeout
DefaultWriteTimeOut time.Duration
// DefaultMaxHeaderBytes default max header bytes
DefaultMaxHeaderBytes int

// IsChild reports if we are a fork iff LISTEN_FDS is set and our parent PID is not 1
IsChild = len(os.Getenv(listenFDs)) > 0 && os.Getppid() > 1
)

func init() {
runningServerReg = sync.RWMutex{}

DefaultMaxHeaderBytes = 0 // use http.DefaultMaxHeaderBytes - which currently is 1 << 20 (1MB)
}

// ServeFunction represents a listen.Accept loop
type ServeFunction = func(net.Listener) error

// Server represents our graceful server
type Server struct {
network string
address string
listener net.Listener
PreSignalHooks map[os.Signal][]func()
PostSignalHooks map[os.Signal][]func()
wg sync.WaitGroup
sigChan chan os.Signal
state state
lock *sync.RWMutex
BeforeBegin func(network, address string)
OnShutdown func()
}

// NewServer creates a server on network at provided address
func NewServer(network, address string) *Server {
runningServerReg.Lock()
defer runningServerReg.Unlock()

if IsChild {
log.Info("Restarting new server: %s:%s on PID: %d", network, address, os.Getpid())
} else {
log.Info("Starting new server: %s:%s on PID: %d", network, address, os.Getpid())
}
srv := &Server{
wg: sync.WaitGroup{},
sigChan: make(chan os.Signal),
PreSignalHooks: map[os.Signal][]func(){},
PostSignalHooks: map[os.Signal][]func(){},
state: stateInit,
lock: &sync.RWMutex{},
network: network,
address: address,
}

srv.BeforeBegin = func(network, addr string) {
log.Debug("Starting server on %s:%s (PID: %d)", network, addr, syscall.Getpid())
}

return srv
}

// ListenAndServe listens on the provided network address and then calls Serve
// to handle requests on incoming connections.
func (srv *Server) ListenAndServe(serve ServeFunction) error {
go srv.handleSignals()

l, err := GetListener(srv.network, srv.address)
if err != nil {
log.Error("Unable to GetListener: %v", err)
return err
}

srv.listener = newWrappedListener(l, srv)

if IsChild {
_ = syscall.Kill(syscall.Getppid(), syscall.SIGTERM)
}

srv.BeforeBegin(srv.network, srv.address)

return srv.Serve(serve)
}

// ListenAndServeTLS listens on the provided network address and then calls
// Serve to handle requests on incoming TLS connections.
//
// Filenames containing a certificate and matching private key for the server must
// be provided. If the certificate is signed by a certificate authority, the
// certFile should be the concatenation of the server's certificate followed by the
// CA's certificate.
func (srv *Server) ListenAndServeTLS(certFile, keyFile string, serve ServeFunction) error {
config := &tls.Config{}
if config.NextProtos == nil {
config.NextProtos = []string{"http/1.1"}
}

config.Certificates = make([]tls.Certificate, 1)
var err error
config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
log.Error("Failed to load https cert file %s for %s:%s: %v", certFile, srv.network, srv.address, err)
return err
}
return srv.ListenAndServeTLSConfig(config, serve)
}

// ListenAndServeTLSConfig listens on the provided network address and then calls
// Serve to handle requests on incoming TLS connections.
func (srv *Server) ListenAndServeTLSConfig(tlsConfig *tls.Config, serve ServeFunction) error {
go srv.handleSignals()

l, err := GetListener(srv.network, srv.address)
if err != nil {
log.Error("Unable to get Listener: %v", err)
return err
}

wl := newWrappedListener(l, srv)
srv.listener = tls.NewListener(wl, tlsConfig)

if IsChild {
_ = syscall.Kill(syscall.Getppid(), syscall.SIGTERM)
}
srv.BeforeBegin(srv.network, srv.address)

return srv.Serve(serve)
}

// Serve accepts incoming HTTP connections on the wrapped listener l, creating a new
// service goroutine for each. The service goroutines read requests and then call
// handler to reply to them. Handler is typically nil, in which case the
// DefaultServeMux is used.
//
// In addition to the standard Serve behaviour each connection is added to a
// sync.Waitgroup so that all outstanding connections can be served before shutting
// down the server.
func (srv *Server) Serve(serve ServeFunction) error {
defer log.Debug("Serve() returning... (PID: %d)", syscall.Getpid())
srv.setState(stateRunning)
err := serve(srv.listener)
log.Debug("Waiting for connections to finish... (PID: %d)", syscall.Getpid())
srv.wg.Wait()
srv.setState(stateTerminate)
// use of closed means that the listeners are closed - i.e. we should be shutting down - return nil
if err != nil && strings.Contains(err.Error(), "use of closed") {
return nil
}
return err
}

func (srv *Server) getState() state {
srv.lock.RLock()
defer srv.lock.RUnlock()

return srv.state
}

func (srv *Server) setState(st state) {
srv.lock.Lock()
defer srv.lock.Unlock()

srv.state = st
}

type wrappedListener struct {
net.Listener
stopped bool
server *Server
}

func newWrappedListener(l net.Listener, srv *Server) *wrappedListener {
return &wrappedListener{
Listener: l,
server: srv,
}
}

func (wl *wrappedListener) Accept() (net.Conn, error) {
var c net.Conn
// Set keepalive on TCPListeners connections.
if tcl, ok := wl.Listener.(*net.TCPListener); ok {
tc, err := tcl.AcceptTCP()
if err != nil {
return nil, err
}
_ = tc.SetKeepAlive(true) // see http.tcpKeepAliveListener
_ = tc.SetKeepAlivePeriod(3 * time.Minute) // see http.tcpKeepAliveListener
c = tc
} else {
var err error
c, err = wl.Listener.Accept()
if err != nil {
return nil, err
}
}

c = wrappedConn{
Conn: c,
server: wl.server,
}

wl.server.wg.Add(1)
return c, nil
}

func (wl *wrappedListener) Close() error {
if wl.stopped {
return syscall.EINVAL
}

wl.stopped = true
return wl.Listener.Close()
}

func (wl *wrappedListener) File() (*os.File, error) {
// returns a dup(2) - FD_CLOEXEC flag *not* set so the listening socket can be passed to child processes
return wl.Listener.(filer).File()
}

type wrappedConn struct {
net.Conn
server *Server
}

func (w wrappedConn) Close() error {
err := w.Conn.Close()
if err == nil {
w.server.wg.Done()
}
return err
}

+ 119
- 0
modules/graceful/server_hooks.go View File

@@ -0,0 +1,119 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package graceful

import (
"errors"
"fmt"
"os"
"runtime"
"time"

"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)

// shutdown closes the listener so that no new connections are accepted
// and starts a goroutine that will hammer (stop all running requests) the server
// after setting.GracefulHammerTime.
func (srv *Server) shutdown() {
// only shutdown if we're running.
if srv.getState() != stateRunning {
return
}

srv.setState(stateShuttingDown)
if setting.GracefulHammerTime >= 0 {
go srv.hammerTime(setting.GracefulHammerTime)
}

if srv.OnShutdown != nil {
srv.OnShutdown()
}
err := srv.listener.Close()
if err != nil {
log.Error("PID: %d Listener.Close() error: %v", os.Getpid(), err)
} else {
log.Info("PID: %d Listener (%s) closed.", os.Getpid(), srv.listener.Addr())
}
}

// hammerTime forces the server to shutdown in a given timeout - whether it
// finished outstanding requests or not. if Read/WriteTimeout are not set or the
// max header size is very big a connection could hang...
//
// srv.Serve() will not return until all connections are served. this will
// unblock the srv.wg.Wait() in Serve() thus causing ListenAndServe* functions to
// return.
func (srv *Server) hammerTime(d time.Duration) {
defer func() {
// We call srv.wg.Done() until it panics.
// This happens if we call Done() when the WaitGroup counter is already at 0
// So if it panics -> we're done, Serve() will return and the
// parent will goroutine will exit.
if r := recover(); r != nil {
log.Error("WaitGroup at 0: Error: %v", r)
}
}()
if srv.getState() != stateShuttingDown {
return
}
time.Sleep(d)
log.Warn("Forcefully shutting down parent")
for {
if srv.getState() == stateTerminate {
break
}
srv.wg.Done()

// Give other goroutines a chance to finish before we forcibly stop them.
runtime.Gosched()
}
}

func (srv *Server) fork() error {
runningServerReg.Lock()
defer runningServerReg.Unlock()

// only one server instance should fork!
if runningServersForked {
return errors.New("another process already forked. Ignoring this one")
}

runningServersForked = true

// We need to move the file logs to append pids
setting.RestartLogsWithPIDSuffix()

_, err := RestartProcess()

return err
}

// RegisterPreSignalHook registers a function to be run before the signal handler for
// a given signal. These are not mutex locked and should therefore be only called before Serve.
func (srv *Server) RegisterPreSignalHook(sig os.Signal, f func()) (err error) {
for _, s := range hookableSignals {
if s == sig {
srv.PreSignalHooks[sig] = append(srv.PreSignalHooks[sig], f)
return
}
}
err = fmt.Errorf("Signal %v is not supported", sig)
return
}

// RegisterPostSignalHook registers a function to be run after the signal handler for
// a given signal. These are not mutex locked and should therefore be only called before Serve.
func (srv *Server) RegisterPostSignalHook(sig os.Signal, f func()) (err error) {
for _, s := range hookableSignals {
if s == sig {
srv.PostSignalHooks[sig] = append(srv.PostSignalHooks[sig], f)
return
}
}
err = fmt.Errorf("Signal %v is not supported", sig)
return
}

+ 45
- 0
modules/graceful/server_http.go View File

@@ -0,0 +1,45 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package graceful

import (
"crypto/tls"
"net/http"
)

func newHTTPServer(network, address string, handler http.Handler) (*Server, ServeFunction) {
server := NewServer(network, address)
httpServer := http.Server{
ReadTimeout: DefaultReadTimeOut,
WriteTimeout: DefaultWriteTimeOut,
MaxHeaderBytes: DefaultMaxHeaderBytes,
Handler: handler,
}
server.OnShutdown = func() {
httpServer.SetKeepAlivesEnabled(false)
}
return server, httpServer.Serve
}

// HTTPListenAndServe listens on the provided network address and then calls Serve
// to handle requests on incoming connections.
func HTTPListenAndServe(network, address string, handler http.Handler) error {
server, lHandler := newHTTPServer(network, address, handler)
return server.ListenAndServe(lHandler)
}

// HTTPListenAndServeTLS listens on the provided network address and then calls Serve
// to handle requests on incoming connections.
func HTTPListenAndServeTLS(network, address, certFile, keyFile string, handler http.Handler) error {
server, lHandler := newHTTPServer(network, address, handler)
return server.ListenAndServeTLS(certFile, keyFile, lHandler)
}

// HTTPListenAndServeTLSConfig listens on the provided network address and then calls Serve
// to handle requests on incoming connections.
func HTTPListenAndServeTLSConfig(network, address string, tlsConfig *tls.Config, handler http.Handler) error {
server, lHandler := newHTTPServer(network, address, handler)
return server.ListenAndServeTLSConfig(tlsConfig, lHandler)
}

+ 93
- 0
modules/graceful/server_signals.go View File

@@ -0,0 +1,93 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package graceful

import (
"os"
"os/signal"
"syscall"
"time"

"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)

var hookableSignals []os.Signal

func init() {
hookableSignals = []os.Signal{
syscall.SIGHUP,
syscall.SIGUSR1,
syscall.SIGUSR2,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGTSTP,
}
}

// handleSignals listens for os Signals and calls any hooked in function that the
// user had registered with the signal.
func (srv *Server) handleSignals() {
var sig os.Signal

signal.Notify(
srv.sigChan,
hookableSignals...,
)

pid := syscall.Getpid()
for {
sig = <-srv.sigChan
srv.preSignalHooks(sig)
switch sig {
case syscall.SIGHUP:
if setting.GracefulRestartable {
log.Info("PID: %d. Received SIGHUP. Forking...", pid)
err := srv.fork()
if err != nil {
log.Error("Error whilst forking from PID: %d : %v", pid, err)
}
} else {
log.Info("PID: %d. Received SIGHUP. Not set restartable. Shutting down...", pid)

srv.shutdown()
}
case syscall.SIGUSR1:
log.Info("PID %d. Received SIGUSR1.", pid)
case syscall.SIGUSR2:
log.Warn("PID %d. Received SIGUSR2. Hammering...", pid)
srv.hammerTime(0 * time.Second)
case syscall.SIGINT:
log.Warn("PID %d. Received SIGINT. Shutting down...", pid)
srv.shutdown()
case syscall.SIGTERM:
log.Warn("PID %d. Received SIGTERM. Shutting down...", pid)
srv.shutdown()
case syscall.SIGTSTP:
log.Info("PID %d. Received SIGTSTP.")
default:
log.Info("PID %d. Received %v.", sig)
}
srv.postSignalHooks(sig)
}
}

func (srv *Server) preSignalHooks(sig os.Signal) {
if _, notSet := srv.PreSignalHooks[sig]; !notSet {
return
}
for _, f := range srv.PreSignalHooks[sig] {
f()
}
}

func (srv *Server) postSignalHooks(sig os.Signal) {
if _, notSet := srv.PostSignalHooks[sig]; !notSet {
return
}
for _, f := range srv.PostSignalHooks[sig] {
f()
}
}

+ 130
- 63
modules/indexer/issues/indexer.go View File

@@ -5,9 +5,11 @@
package issues

import (
"fmt"
"sync"
"time"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@@ -45,78 +47,143 @@ type Indexer interface {
Search(kw string, repoID int64, limit, start int) (*SearchResult, error)
}

type indexerHolder struct {
indexer Indexer
mutex sync.RWMutex
cond *sync.Cond
}

func newIndexerHolder() *indexerHolder {
h := &indexerHolder{}
h.cond = sync.NewCond(h.mutex.RLocker())
return h
}

func (h *indexerHolder) set(indexer Indexer) {
h.mutex.Lock()
defer h.mutex.Unlock()
h.indexer = indexer
h.cond.Broadcast()
}

func (h *indexerHolder) get() Indexer {
h.mutex.RLock()
defer h.mutex.RUnlock()
if h.indexer == nil {
h.cond.Wait()
}
return h.indexer
}

var (
issueIndexerChannel = make(chan *IndexerData, setting.Indexer.UpdateQueueLength)
// issueIndexerQueue queue of issue ids to be updated
issueIndexerQueue Queue
issueIndexer Indexer
holder = newIndexerHolder()
)

// InitIssueIndexer initialize issue indexer, syncReindex is true then reindex until
// all issue index done.
func InitIssueIndexer(syncReindex bool) error {
var populate bool
var dummyQueue bool
switch setting.Indexer.IssueType {
case "bleve":
issueIndexer = NewBleveIndexer(setting.Indexer.IssuePath)
exist, err := issueIndexer.Init()
if err != nil {
return err
func InitIssueIndexer(syncReindex bool) {
waitChannel := make(chan time.Duration)
go func() {
start := time.Now()
log.Info("Initializing Issue Indexer")
var populate bool
var dummyQueue bool
switch setting.Indexer.IssueType {
case "bleve":
issueIndexer := NewBleveIndexer(setting.Indexer.IssuePath)
exist, err := issueIndexer.Init()
if err != nil {
log.Fatal("Unable to initialize Bleve Issue Indexer: %v", err)
}
populate = !exist
holder.set(issueIndexer)
case "db":
issueIndexer := &DBIndexer{}
holder.set(issueIndexer)
dummyQueue = true
default:
log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType)
}
populate = !exist
case "db":
issueIndexer = &DBIndexer{}
dummyQueue = true
default:
return fmt.Errorf("unknow issue indexer type: %s", setting.Indexer.IssueType)
}

if dummyQueue {
issueIndexerQueue = &DummyQueue{}
return nil
}
if dummyQueue {
issueIndexerQueue = &DummyQueue{}
} else {
var err error
switch setting.Indexer.IssueQueueType {
case setting.LevelQueueType:
issueIndexerQueue, err = NewLevelQueue(
holder.get(),
setting.Indexer.IssueQueueDir,
setting.Indexer.IssueQueueBatchNumber)
if err != nil {
log.Fatal(
"Unable create level queue for issue queue dir: %s batch number: %d : %v",
setting.Indexer.IssueQueueDir,
setting.Indexer.IssueQueueBatchNumber,
err)
}
case setting.ChannelQueueType:
issueIndexerQueue = NewChannelQueue(holder.get(), setting.Indexer.IssueQueueBatchNumber)
case setting.RedisQueueType:
addrs, pass, idx, err := parseConnStr(setting.Indexer.IssueQueueConnStr)
if err != nil {
log.Fatal("Unable to parse connection string for RedisQueueType: %s : %v",
setting.Indexer.IssueQueueConnStr,
err)
}
issueIndexerQueue, err = NewRedisQueue(addrs, pass, idx, holder.get(), setting.Indexer.IssueQueueBatchNumber)
if err != nil {
log.Fatal("Unable to create RedisQueue: %s : %v",
setting.Indexer.IssueQueueConnStr,
err)
}
default:
log.Fatal("Unsupported indexer queue type: %v",
setting.Indexer.IssueQueueType)
}

var err error
switch setting.Indexer.IssueQueueType {
case setting.LevelQueueType:
issueIndexerQueue, err = NewLevelQueue(
issueIndexer,
setting.Indexer.IssueQueueDir,
setting.Indexer.IssueQueueBatchNumber)
if err != nil {
return err
go func() {
err = issueIndexerQueue.Run()
if err != nil {
log.Error("issueIndexerQueue.Run: %v", err)
}
}()
}
case setting.ChannelQueueType:
issueIndexerQueue = NewChannelQueue(issueIndexer, setting.Indexer.IssueQueueBatchNumber)
case setting.RedisQueueType:
addrs, pass, idx, err := parseConnStr(setting.Indexer.IssueQueueConnStr)
if err != nil {
return err
}
issueIndexerQueue, err = NewRedisQueue(addrs, pass, idx, issueIndexer, setting.Indexer.IssueQueueBatchNumber)
if err != nil {
return err
}
default:
return fmt.Errorf("Unsupported indexer queue type: %v", setting.Indexer.IssueQueueType)
}

go func() {
err = issueIndexerQueue.Run()
if err != nil {
log.Error("issueIndexerQueue.Run: %v", err)
}
}()
go func() {
for data := range issueIndexerChannel {
_ = issueIndexerQueue.Push(data)
}
}()

if populate {
if syncReindex {
populateIssueIndexer()
} else {
go populateIssueIndexer()
if populate {
if syncReindex {
populateIssueIndexer()
} else {
go populateIssueIndexer()
}
}
waitChannel <- time.Since(start)
}()
if syncReindex {
<-waitChannel
} else if setting.Indexer.StartupTimeout > 0 {
go func() {
timeout := setting.Indexer.StartupTimeout
if graceful.IsChild && setting.GracefulHammerTime > 0 {
timeout += setting.GracefulHammerTime
}
select {
case duration := <-waitChannel:
log.Info("Issue Indexer Initialization took %v", duration)
case <-time.After(timeout):
log.Fatal("Issue Indexer Initialization timed-out after: %v", timeout)
}
}()
}

return nil
}

// populateIssueIndexer populate the issue indexer with issue data
@@ -166,13 +233,13 @@ func UpdateIssueIndexer(issue *models.Issue) {
comments = append(comments, comment.Content)
}
}
_ = issueIndexerQueue.Push(&IndexerData{
issueIndexerChannel <- &IndexerData{
ID: issue.ID,
RepoID: issue.RepoID,
Title: issue.Title,
Content: issue.Content,
Comments: comments,
})
}
}

// DeleteRepoIssueIndexer deletes repo's all issues indexes
@@ -188,16 +255,16 @@ func DeleteRepoIssueIndexer(repo *models.Repository) {
return
}

_ = issueIndexerQueue.Push(&IndexerData{
issueIndexerChannel <- &IndexerData{
IDs: ids,
IsDelete: true,
})
}
}

// SearchIssuesByKeyword search issue ids by keywords and repo id
func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
var issueIDs []int64
res, err := issueIndexer.Search(keyword, repoID, 1000, 0)
res, err := holder.get().Search(keyword, repoID, 1000, 0)
if err != nil {
return nil, err
}


+ 2
- 12
modules/indexer/issues/indexer_test.go View File

@@ -5,7 +5,6 @@
package issues

import (
"fmt"
"os"
"path/filepath"
"testing"
@@ -17,11 +16,6 @@ import (
"github.com/stretchr/testify/assert"
)

func fatalTestError(fmtStr string, args ...interface{}) {
fmt.Fprintf(os.Stderr, fmtStr, args...)
os.Exit(1)
}

func TestMain(m *testing.M) {
models.MainTest(m, filepath.Join("..", "..", ".."))
}
@@ -32,9 +26,7 @@ func TestBleveSearchIssues(t *testing.T) {
os.RemoveAll(setting.Indexer.IssueQueueDir)
os.RemoveAll(setting.Indexer.IssuePath)
setting.Indexer.IssueType = "bleve"
if err := InitIssueIndexer(true); err != nil {
fatalTestError("Error InitIssueIndexer: %v\n", err)
}
InitIssueIndexer(true)

time.Sleep(5 * time.Second)

@@ -59,9 +51,7 @@ func TestDBSearchIssues(t *testing.T) {
assert.NoError(t, models.PrepareTestDatabase())

setting.Indexer.IssueType = "db"
if err := InitIssueIndexer(true); err != nil {
fatalTestError("Error InitIssueIndexer: %v\n", err)
}
InitIssueIndexer(true)

ids, err := SearchIssuesByKeyword(1, "issue2")
assert.NoError(t, err)


+ 41
- 11
modules/indexer/repo.go View File

@@ -6,6 +6,7 @@ package indexer

import (
"strings"
"sync"

"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
@@ -25,8 +26,36 @@ const (
repoIndexerLatestVersion = 4
)

type bleveIndexerHolder struct {
index bleve.Index
mutex sync.RWMutex
cond *sync.Cond
}

func newBleveIndexerHolder() *bleveIndexerHolder {
b := &bleveIndexerHolder{}
b.cond = sync.NewCond(b.mutex.RLocker())
return b
}

func (r *bleveIndexerHolder) set(index bleve.Index) {
r.mutex.Lock()
defer r.mutex.Unlock()
r.index = index
r.cond.Broadcast()
}

func (r *bleveIndexerHolder) get() bleve.Index {
r.mutex.RLock()
defer r.mutex.RUnlock()
if r.index == nil {
r.cond.Wait()
}
return r.index
}

// repoIndexer (thread-safe) index for repository contents
var repoIndexer bleve.Index
var indexerHolder = newBleveIndexerHolder()

// RepoIndexerOp type of operation to perform on repo indexer
type RepoIndexerOp int
@@ -73,12 +102,12 @@ func (update RepoIndexerUpdate) AddToFlushingBatch(batch rupture.FlushingBatch)

// InitRepoIndexer initialize repo indexer
func InitRepoIndexer(populateIndexer func() error) {
var err error
repoIndexer, err = openIndexer(setting.Indexer.RepoPath, repoIndexerLatestVersion)
indexer, err := openIndexer(setting.Indexer.RepoPath, repoIndexerLatestVersion)
if err != nil {
log.Fatal("InitRepoIndexer: %v", err)
}
if repoIndexer != nil {
if indexer != nil {
indexerHolder.set(indexer)
return
}

@@ -92,7 +121,6 @@ func InitRepoIndexer(populateIndexer func() error) {

// createRepoIndexer create a repo indexer if one does not already exist
func createRepoIndexer(path string, latestVersion int) error {
var err error
docMapping := bleve.NewDocumentMapping()
numericFieldMapping := bleve.NewNumericFieldMapping()
numericFieldMapping.IncludeInAll = false
@@ -103,9 +131,9 @@ func createRepoIndexer(path string, latestVersion int) error {
docMapping.AddFieldMappingsAt("Content", textFieldMapping)

mapping := bleve.NewIndexMapping()
if err = addUnicodeNormalizeTokenFilter(mapping); err != nil {
if err := addUnicodeNormalizeTokenFilter(mapping); err != nil {
return err
} else if err = mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]interface{}{
} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]interface{}{
"type": custom.Name,
"char_filters": []string{},
"tokenizer": unicode.Name,
@@ -117,10 +145,12 @@ func createRepoIndexer(path string, latestVersion int) error {
mapping.AddDocumentMapping(repoIndexerDocType, docMapping)
mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping())

repoIndexer, err = bleve.New(path, mapping)
indexer, err := bleve.New(path, mapping)
if err != nil {
return err
}
indexerHolder.set(indexer)

return rupture.WriteIndexMetadata(path, &rupture.IndexMetadata{
Version: latestVersion,
})
@@ -140,14 +170,14 @@ func filenameOfIndexerID(indexerID string) string {

// RepoIndexerBatch batch to add updates to
func RepoIndexerBatch() rupture.FlushingBatch {
return rupture.NewFlushingBatch(repoIndexer, maxBatchSize)
return rupture.NewFlushingBatch(indexerHolder.get(), maxBatchSize)
}

// DeleteRepoFromIndexer delete all of a repo's files from indexer
func DeleteRepoFromIndexer(repoID int64) error {
query := numericEqualityQuery(repoID, "RepoID")
searchRequest := bleve.NewSearchRequestOptions(query, 2147483647, 0, false)
result, err := repoIndexer.Search(searchRequest)
result, err := indexerHolder.get().Search(searchRequest)
if err != nil {
return err
}
@@ -196,7 +226,7 @@ func SearchRepoByKeyword(repoIDs []int64, keyword string, page, pageSize int) (i
searchRequest.Fields = []string{"Content", "RepoID"}
searchRequest.IncludeLocations = true

result, err := repoIndexer.Search(searchRequest)
result, err := indexerHolder.get().Search(searchRequest)
if err != nil {
return 0, nil, err
}


+ 3
- 0
modules/setting/indexer.go View File

@@ -8,6 +8,7 @@ import (
"path"
"path/filepath"
"strings"
"time"

"code.gitea.io/gitea/modules/log"

@@ -34,6 +35,7 @@ var (
IssueQueueDir string
IssueQueueConnStr string
IssueQueueBatchNumber int
StartupTimeout time.Duration
IncludePatterns []glob.Glob
ExcludePatterns []glob.Glob
}{
@@ -67,6 +69,7 @@ func newIndexerService() {
Indexer.IssueQueueDir = sec.Key("ISSUE_INDEXER_QUEUE_DIR").MustString(path.Join(AppDataPath, "indexers/issues.queue"))
Indexer.IssueQueueConnStr = sec.Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString(path.Join(AppDataPath, ""))
Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(20)
Indexer.StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(30 * time.Second)
}

// IndexerGlobFromString parses a comma separated list of patterns and returns a glob.Glob slice suited for repo indexing


+ 10
- 1
modules/setting/log.go View File

@@ -6,6 +6,7 @@ package setting

import (
"encoding/json"
"fmt"
golog "log"
"os"
"path"
@@ -17,6 +18,8 @@ import (
ini "gopkg.in/ini.v1"
)

var filenameSuffix = ""

type defaultLogOptions struct {
levelName string // LogLevel
flags string
@@ -112,7 +115,7 @@ func generateLogConfig(sec *ini.Section, name string, defaults defaultLogOptions
panic(err.Error())
}

logConfig["filename"] = logPath
logConfig["filename"] = logPath + filenameSuffix
logConfig["rotate"] = sec.Key("LOG_ROTATE").MustBool(true)
logConfig["maxsize"] = 1 << uint(sec.Key("MAX_SIZE_SHIFT").MustInt(28))
logConfig["daily"] = sec.Key("DAILY_ROTATE").MustBool(true)
@@ -277,6 +280,12 @@ func newLogService() {
golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT)))
}

// RestartLogsWithPIDSuffix restarts the logs with a PID suffix on files
func RestartLogsWithPIDSuffix() {
filenameSuffix = fmt.Sprintf(".%d", os.Getpid())
NewLogServices(false)
}

// NewLogServices creates all the log services
func NewLogServices(disableConsole bool) {
newLogService()


+ 4
- 0
modules/setting/setting.go View File

@@ -97,6 +97,8 @@ var (
LetsEncryptTOS bool
LetsEncryptDirectory string
LetsEncryptEmail string
GracefulRestartable bool
GracefulHammerTime time.Duration

SSH = struct {
Disabled bool `ini:"DISABLE_SSH"`
@@ -563,6 +565,8 @@ func NewContext() {
Domain = sec.Key("DOMAIN").MustString("localhost")
HTTPAddr = sec.Key("HTTP_ADDR").MustString("0.0.0.0")
HTTPPort = sec.Key("HTTP_PORT").MustString("3000")
GracefulRestartable = sec.Key("ALLOW_GRACEFUL_RESTARTS").MustBool(true)
GracefulHammerTime = sec.Key("GRACEFUL_HAMMER_TIME").MustDuration(60 * time.Second)

defaultAppURL := string(Protocol) + "://" + Domain
if (Protocol == HTTP && HTTPPort != "80") || (Protocol == HTTPS && HTTPPort != "443") {


+ 1
- 6
modules/ssh/ssh.go View File

@@ -183,12 +183,7 @@ func Listen(host string, port int, ciphers []string, keyExchanges []string, macs
log.Error("Failed to set Host Key. %s", err)
}

go func() {
err := srv.ListenAndServe()
if err != nil {
log.Error("Failed to serve with builtin SSH server. %s", err)
}
}()
go listen(&srv)

}



+ 30
- 0
modules/ssh/ssh_graceful.go View File

@@ -0,0 +1,30 @@
// +build !windows

// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package ssh

import (
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"

"github.com/gliderlabs/ssh"
)

func listen(server *ssh.Server) {
gracefulServer := graceful.NewServer("tcp", server.Addr)

err := gracefulServer.ListenAndServe(server.Serve)
if err != nil {
log.Critical("Failed to start SSH server: %v", err)
}
log.Info("SSH Listener: %s Closed", server.Addr)

}

// Unused informs our cleanup routine that we will not be using a ssh port
func Unused() {
graceful.InformCleanup()
}

+ 24
- 0
modules/ssh/ssh_windows.go View File

@@ -0,0 +1,24 @@
// +build windows

// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package ssh

import (
"code.gitea.io/gitea/modules/log"
"github.com/gliderlabs/ssh"
)

func listen(server *ssh.Server) {
err := server.ListenAndServe()
if err != nil {
log.Critical("Failed to serve with builtin SSH server. %s", err)
}
}

// Unused does nothing on windows
func Unused() {
// Do nothing
}

+ 11
- 6
routers/init.go View File

@@ -96,9 +96,7 @@ func GlobalInit() {

// Booting long running goroutines.
cron.NewContext()
if err := issue_indexer.InitIssueIndexer(false); err != nil {
log.Fatal("Failed to initialize issue indexer: %v", err)
}
issue_indexer.InitIssueIndexer(false)
models.InitRepoIndexer()
mirror_service.InitSyncMirrors()
models.InitDeliverHooks()
@@ -112,8 +110,15 @@ func GlobalInit() {
}
checkRunMode()

if setting.InstallLock && setting.SSH.StartBuiltinServer {
ssh.Listen(setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs)
log.Info("SSH server started on %s:%d. Cipher list (%v), key exchange algorithms (%v), MACs (%v)", setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs)
// Now because Install will re-run GlobalInit once it has set InstallLock
// we can't tell if the ssh port will remain unused until that's done.
// However, see FIXME comment in install.go
if setting.InstallLock {
if setting.SSH.StartBuiltinServer {
ssh.Listen(setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs)
log.Info("SSH server started on %s:%d. Cipher list (%v), key exchange algorithms (%v), MACs (%v)", setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs)
} else {
ssh.Unused()
}
}
}

+ 6
- 0
routers/install.go View File

@@ -386,6 +386,12 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) {
}

log.Info("First-time run install finished!")
// FIXME: This isn't really enough to completely take account of new configuration
// We should really be restarting:
// - On windows this is probably just a simple restart
// - On linux we can't just use graceful.RestartProcess() everything that was passed in on LISTEN_FDS
// (active or not) needs to be passed out and everything new passed out too.
// This means we need to prevent the cleanup goroutine from running prior to the second GlobalInit
ctx.Flash.Success(ctx.Tr("install.install_success"))
ctx.Redirect(form.AppURL + "user/login")
}

+ 0
- 21
vendor/github.com/facebookgo/clock/LICENSE View File

@@ -1,21 +0,0 @@
The MIT License (MIT)

Copyright (c) 2014 Ben Johnson

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

+ 0
- 104
vendor/github.com/facebookgo/clock/README.md View File

@@ -1,104 +0,0 @@
clock [![Build Status](https://drone.io/github.com/benbjohnson/clock/status.png)](https://drone.io/github.com/benbjohnson/clock/latest) [![Coverage Status](https://coveralls.io/repos/benbjohnson/clock/badge.png?branch=master)](https://coveralls.io/r/benbjohnson/clock?branch=master) [![GoDoc](https://godoc.org/github.com/benbjohnson/clock?status.png)](https://godoc.org/github.com/benbjohnson/clock) ![Project status](http://img.shields.io/status/experimental.png?color=red)
=====

Clock is a small library for mocking time in Go. It provides an interface
around the standard library's [`time`][time] package so that the application
can use the realtime clock while tests can use the mock clock.

[time]: http://golang.org/pkg/time/


## Usage

### Realtime Clock

Your application can maintain a `Clock` variable that will allow realtime and
mock clocks to be interchangable. For example, if you had an `Application` type:

```go
import "github.com/benbjohnson/clock"

type Application struct {
Clock clock.Clock
}
```

You could initialize it to use the realtime clock like this:

```go
var app Application
app.Clock = clock.New()
...
```

Then all timers and time-related functionality should be performed from the
`Clock` variable.


### Mocking time

In your tests, you will want to use a `Mock` clock:

```go
import (
"testing"

"github.com/benbjohnson/clock"
)

func TestApplication_DoSomething(t *testing.T) {
mock := clock.NewMock()
app := Application{Clock: mock}
...
}
```

Now that you've initialized your application to use the mock clock, you can
adjust the time programmatically. The mock clock always starts from the Unix
epoch (midnight, Jan 1, 1970 UTC).


### Controlling time

The mock clock provides the same functions that the standard library's `time`
package provides. For example, to find the current time, you use the `Now()`
function:

```go
mock := clock.NewMock()

// Find the current time.
mock.Now().UTC() // 1970-01-01 00:00:00 +0000 UTC

// Move the clock forward.
mock.Add(2 * time.Hour)

// Check the time again. It's 2 hours later!
mock.Now().UTC() // 1970-01-01 02:00:00 +0000 UTC
```

Timers and Tickers are also controlled by this same mock clock. They will only
execute when the clock is moved forward:

```
mock := clock.NewMock()
count := 0

// Kick off a timer to increment every 1 mock second.
go func() {
ticker := clock.Ticker(1 * time.Second)
for {
<-ticker.C
count++
}
}()
runtime.Gosched()

// Move the clock forward 10 second.
mock.Add(10 * time.Second)

// This prints 10.
fmt.Println(count)
```



+ 0
- 363
vendor/github.com/facebookgo/clock/clock.go View File

@@ -1,363 +0,0 @@
package clock

import (
"runtime"
"sort"
"sync"
"time"
)

// Clock represents an interface to the functions in the standard library time
// package. Two implementations are available in the clock package. The first
// is a real-time clock which simply wraps the time package's functions. The
// second is a mock clock which will only make forward progress when
// programmatically adjusted.
type Clock interface {
After(d time.Duration) <-chan time.Time
AfterFunc(d time.Duration, f func()) *Timer
Now() time.Time
Sleep(d time.Duration)
Tick(d time.Duration) <-chan time.Time
Ticker(d time.Duration) *Ticker
Timer(d time.Duration) *Timer
}

// New returns an instance of a real-time clock.
func New() Clock {
return &clock{}
}

// clock implements a real-time clock by simply wrapping the time package functions.
type clock struct{}

func (c *clock) After(d time.Duration) <-chan time.Time { return time.After(d) }

func (c *clock) AfterFunc(d time.Duration, f func()) *Timer {
return &Timer{timer: time.AfterFunc(d, f)}
}

func (c *clock) Now() time.Time { return time.Now() }

func (c *clock) Sleep(d time.Duration) { time.Sleep(d) }

func (c *clock) Tick(d time.Duration) <-chan time.Time { return time.Tick(d) }

func (c *clock) Ticker(d time.Duration) *Ticker {
t := time.NewTicker(d)
return &Ticker{C: t.C, ticker: t}
}

func (c *clock) Timer(d time.Duration) *Timer {
t := time.NewTimer(d)
return &Timer{C: t.C, timer: t}
}

// Mock represents a mock clock that only moves forward programmically.
// It can be preferable to a real-time clock when testing time-based functionality.
type Mock struct {
mu sync.Mutex
now time.Time // current time
timers clockTimers // tickers & timers

calls Calls
waiting []waiting
callsMutex sync.Mutex
}

// NewMock returns an instance of a mock clock.
// The current time of the mock clock on initialization is the Unix epoch.
func NewMock() *Mock {
return &Mock{now: time.Unix(0, 0)}
}

// Add moves the current time of the mock clock forward by the duration.
// This should only be called from a single goroutine at a time.
func (m *Mock) Add(d time.Duration) {
// Calculate the final current time.
t := m.now.Add(d)

// Continue to execute timers until there are no more before the new time.
for {
if !m.runNextTimer(t) {
break
}
}

// Ensure that we end with the new time.
m.mu.Lock()
m.now = t
m.mu.Unlock()

// Give a small buffer to make sure the other goroutines get handled.
gosched()
}

// runNextTimer executes the next timer in chronological order and moves the
// current time to the timer's next tick time. The next time is not executed if
// it's next time if after the max time. Returns true if a timer is executed.
func (m *Mock) runNextTimer(max time.Time) bool {
m.mu.Lock()

// Sort timers by time.
sort.Sort(m.timers)

// If we have no more timers then exit.
if len(m.timers) == 0 {
m.mu.Unlock()
return false
}

// Retrieve next timer. Exit if next tick is after new time.
t := m.timers[0]
if t.Next().After(max) {
m.mu.Unlock()
return false
}

// Move "now" forward and unlock clock.
m.now = t.Next()
m.mu.Unlock()

// Execute timer.
t.Tick(m.now)