Use buffersize to reduce database connection when iterate (#2724)

* use buffersize to reduce database connection when iterate

* fix typo

* add default value on app.ini comment
This commit is contained in:
Lunny Xiao 2017-10-21 14:26:23 +08:00 committed by GitHub
parent 2112eb8741
commit 985a39590b
7 changed files with 19 additions and 15 deletions

2
conf/app.ini vendored
View File

@ -184,6 +184,8 @@ SSL_MODE = disable
PATH = data/gitea.db PATH = data/gitea.db
; For "sqlite3" only. Query timeout ; For "sqlite3" only. Query timeout
SQLITE_TIMEOUT = 500 SQLITE_TIMEOUT = 500
; For iterate buffer, default is 50
ITERATE_BUFFER_SIZE = 50
[indexer] [indexer]
ISSUE_INDEXER_PATH = indexers/issues.bleve ISSUE_INDEXER_PATH = indexers/issues.bleve

View File

@ -42,7 +42,7 @@ func generateAndMigrateGitHooks(x *xorm.Engine) (err error) {
} }
) )
return x.Where("id > 0").Iterate(new(Repository), return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
func(idx int, bean interface{}) error { func(idx int, bean interface{}) error {
repo := bean.(*Repository) repo := bean.(*Repository)
user := new(User) user := new(User)

View File

@ -42,7 +42,7 @@ func generateAndMigrateWikiGitHooks(x *xorm.Engine) (err error) {
} }
) )
return x.Where("id > 0").Iterate(new(Repository), return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
func(idx int, bean interface{}) error { func(idx int, bean interface{}) error {
repo := bean.(*Repository) repo := bean.(*Repository)
user := new(User) user := new(User)

View File

@ -36,7 +36,7 @@ func generateAndMigrateGitHookChains(x *xorm.Engine) (err error) {
hookTpl = fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType) hookTpl = fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType)
) )
return x.Where("id > 0").Iterate(new(Repository), return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
func(idx int, bean interface{}) error { func(idx int, bean interface{}) error {
repo := bean.(*Repository) repo := bean.(*Repository)
user := new(User) user := new(User)

View File

@ -2145,7 +2145,7 @@ func GitFsck() {
log.Trace("Doing: GitFsck") log.Trace("Doing: GitFsck")
if err := x. if err := x.
Where("id>0"). Where("id>0").BufferSize(setting.IterateBufferSize).
Iterate(new(Repository), Iterate(new(Repository),
func(idx int, bean interface{}) error { func(idx int, bean interface{}) error {
repo := bean.(*Repository) repo := bean.(*Repository)
@ -2167,7 +2167,7 @@ func GitFsck() {
func GitGcRepos() error { func GitGcRepos() error {
args := append([]string{"gc"}, setting.Git.GCArgs...) args := append([]string{"gc"}, setting.Git.GCArgs...)
return x. return x.
Where("id > 0"). Where("id > 0").BufferSize(setting.IterateBufferSize).
Iterate(new(Repository), Iterate(new(Repository),
func(idx int, bean interface{}) error { func(idx int, bean interface{}) error {
repo := bean.(*Repository) repo := bean.(*Repository)

View File

@ -816,7 +816,7 @@ func ChangeUserName(u *User, newUserName string) (err error) {
} }
// Delete all local copies of repository wiki that user owns. // Delete all local copies of repository wiki that user owns.
if err = x. if err = x.BufferSize(setting.IterateBufferSize).
Where("owner_id=?", u.ID). Where("owner_id=?", u.ID).
Iterate(new(Repository), func(idx int, bean interface{}) error { Iterate(new(Repository), func(idx int, bean interface{}) error {
repo := bean.(*Repository) repo := bean.(*Repository)

View File

@ -489,6 +489,7 @@ var (
IsWindows bool IsWindows bool
HasRobotsTxt bool HasRobotsTxt bool
InternalToken string // internal access token InternalToken string // internal access token
IterateBufferSize int
) )
// DateLang transforms standard language locale name to corresponding value in datetime plugin. // DateLang transforms standard language locale name to corresponding value in datetime plugin.
@ -873,6 +874,7 @@ func NewContext() {
log.Fatal(4, "Error saving generated JWT Secret to custom config: %v", err) log.Fatal(4, "Error saving generated JWT Secret to custom config: %v", err)
} }
} }
IterateBufferSize = Cfg.Section("database").Key("ITERATE_BUFFER_SIZE").MustInt(50)
sec = Cfg.Section("attachment") sec = Cfg.Section("attachment")
AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments")) AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))