#1984 Better mirror repo management
This commit is contained in:
parent
1cbd4c01fb
commit
120cd4e471
|
@ -17,7 +17,7 @@ github.com/go-sql-driver/mysql = commit:d512f20
|
||||||
github.com/go-xorm/core = commit:acb6f00
|
github.com/go-xorm/core = commit:acb6f00
|
||||||
github.com/go-xorm/xorm = commit:a8fba4d
|
github.com/go-xorm/xorm = commit:a8fba4d
|
||||||
github.com/gogits/chardet = commit:2404f77725
|
github.com/gogits/chardet = commit:2404f77725
|
||||||
github.com/gogits/git-shell = commit:de77627
|
github.com/gogits/git-shell =
|
||||||
github.com/gogits/go-gogs-client = commit:4b541fa
|
github.com/gogits/go-gogs-client = commit:4b541fa
|
||||||
github.com/issue9/identicon = commit:f8c0d2c
|
github.com/issue9/identicon = commit:f8c0d2c
|
||||||
github.com/klauspost/compress = commit:bcd0709
|
github.com/klauspost/compress = commit:bcd0709
|
||||||
|
|
|
@ -5,7 +5,7 @@ Gogs - Go Git Service [![Build Status](https://travis-ci.org/gogits/gogs.svg?bra
|
||||||
|
|
||||||
![](public/img/gogs-large-resize.png)
|
![](public/img/gogs-large-resize.png)
|
||||||
|
|
||||||
##### Current version: 0.7.34 Beta
|
##### Current version: 0.7.35 Beta
|
||||||
|
|
||||||
| Web | UI | Preview |
|
| Web | UI | Preview |
|
||||||
|:-------------:|:-------:|:-------:|
|
|:-------------:|:-------:|:-------:|
|
||||||
|
|
|
@ -352,6 +352,8 @@ auto_init = Initialize this repository with selected files and template
|
||||||
create_repo = Create Repository
|
create_repo = Create Repository
|
||||||
default_branch = Default Branch
|
default_branch = Default Branch
|
||||||
mirror_interval = Mirror Interval (hour)
|
mirror_interval = Mirror Interval (hour)
|
||||||
|
mirror_address = Mirror Address
|
||||||
|
mirror_address_desc = Please include necessary user credentials in the address.
|
||||||
watchers = Watchers
|
watchers = Watchers
|
||||||
stargazers = Stargazers
|
stargazers = Stargazers
|
||||||
forks = Forks
|
forks = Forks
|
||||||
|
@ -369,6 +371,7 @@ migrate.permission_denied = You are not allowed to import local repositories.
|
||||||
migrate.invalid_local_path = Invalid local path, it does not exist or not a directory.
|
migrate.invalid_local_path = Invalid local path, it does not exist or not a directory.
|
||||||
migrate.failed = Migration failed: %v
|
migrate.failed = Migration failed: %v
|
||||||
|
|
||||||
|
mirror_from = mirror from
|
||||||
forked_from = forked from
|
forked_from = forked from
|
||||||
fork_from_self = You cannot fork a repository you already own!
|
fork_from_self = You cannot fork a repository you already own!
|
||||||
copy_link = Copy
|
copy_link = Copy
|
||||||
|
|
2
gogs.go
2
gogs.go
|
@ -17,7 +17,7 @@ import (
|
||||||
"github.com/gogits/gogs/modules/setting"
|
"github.com/gogits/gogs/modules/setting"
|
||||||
)
|
)
|
||||||
|
|
||||||
const APP_VER = "0.7.34.1208 Beta"
|
const APP_VER = "0.7.35.1208 Beta"
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
runtime.GOMAXPROCS(runtime.NumCPU())
|
runtime.GOMAXPROCS(runtime.NumCPU())
|
||||||
|
|
124
models/repo.go
124
models/repo.go
|
@ -301,6 +301,10 @@ func (repo *Repository) RepoPath() string {
|
||||||
return repo.repoPath(x)
|
return repo.repoPath(x)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (repo *Repository) GitConfigPath() string {
|
||||||
|
return filepath.Join(repo.RepoPath(), "config")
|
||||||
|
}
|
||||||
|
|
||||||
func (repo *Repository) RepoLink() string {
|
func (repo *Repository) RepoLink() string {
|
||||||
return setting.AppSubUrl + "/" + repo.MustOwner().Name + "/" + repo.Name
|
return setting.AppSubUrl + "/" + repo.MustOwner().Name + "/" + repo.Name
|
||||||
}
|
}
|
||||||
|
@ -345,7 +349,7 @@ func (repo *Repository) LocalCopyPath() string {
|
||||||
|
|
||||||
func updateLocalCopy(repoPath, localPath string) error {
|
func updateLocalCopy(repoPath, localPath string) error {
|
||||||
if !com.IsExist(localPath) {
|
if !com.IsExist(localPath) {
|
||||||
if err := git.Clone(repoPath, localPath); err != nil {
|
if err := git.Clone(repoPath, localPath, git.CloneRepoOptions{}); err != nil {
|
||||||
return fmt.Errorf("Clone: %v", err)
|
return fmt.Errorf("Clone: %v", err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
@ -484,6 +488,8 @@ type Mirror struct {
|
||||||
Interval int // Hour.
|
Interval int // Hour.
|
||||||
Updated time.Time `xorm:"UPDATED"`
|
Updated time.Time `xorm:"UPDATED"`
|
||||||
NextUpdate time.Time
|
NextUpdate time.Time
|
||||||
|
|
||||||
|
address string `xorm:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
|
func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
|
||||||
|
@ -497,6 +503,61 @@ func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *Mirror) readAddress() {
|
||||||
|
if len(m.address) > 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := ini.Load(m.Repo.GitConfigPath())
|
||||||
|
if err != nil {
|
||||||
|
log.Error(4, "Load: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m.address = cfg.Section("remote \"origin\"").Key("url").Value()
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandleCloneUserCredentials replaces user credentials from HTTP/HTTPS URL
|
||||||
|
// with placeholder <credentials>.
|
||||||
|
// It will fail for any other forms of clone addresses.
|
||||||
|
func HandleCloneUserCredentials(url string, mosaics bool) string {
|
||||||
|
i := strings.Index(url, "@")
|
||||||
|
if i == -1 {
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
start := strings.Index(url, "://")
|
||||||
|
if start == -1 {
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
if mosaics {
|
||||||
|
return url[:start+3] + "<credentials>" + url[i:]
|
||||||
|
}
|
||||||
|
return url[:start+3] + url[i+1:]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Address returns mirror address from Git repository config without credentials.
|
||||||
|
func (m *Mirror) Address() string {
|
||||||
|
m.readAddress()
|
||||||
|
return HandleCloneUserCredentials(m.address, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// FullAddress returns mirror address from Git repository config.
|
||||||
|
func (m *Mirror) FullAddress() string {
|
||||||
|
m.readAddress()
|
||||||
|
return m.address
|
||||||
|
}
|
||||||
|
|
||||||
|
// SaveAddress writes new address to Git repository config.
|
||||||
|
func (m *Mirror) SaveAddress(addr string) error {
|
||||||
|
configPath := m.Repo.GitConfigPath()
|
||||||
|
cfg, err := ini.Load(configPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Load: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Section("remote \"origin\"").Key("url").SetValue(addr)
|
||||||
|
return cfg.SaveToIndent(configPath, "\t")
|
||||||
|
}
|
||||||
|
|
||||||
func getMirror(e Engine, repoId int64) (*Mirror, error) {
|
func getMirror(e Engine, repoId int64) (*Mirror, error) {
|
||||||
m := &Mirror{RepoID: repoId}
|
m := &Mirror{RepoID: repoId}
|
||||||
has, err := e.Get(m)
|
has, err := e.Get(m)
|
||||||
|
@ -527,25 +588,6 @@ func createUpdateHook(repoPath string) error {
|
||||||
fmt.Sprintf(_TPL_UPDATE_HOOK, setting.ScriptType, "\""+setting.AppPath+"\"", setting.CustomConf))
|
fmt.Sprintf(_TPL_UPDATE_HOOK, setting.ScriptType, "\""+setting.AppPath+"\"", setting.CustomConf))
|
||||||
}
|
}
|
||||||
|
|
||||||
// MirrorRepository creates a mirror repository from source.
|
|
||||||
func MirrorRepository(repoId int64, userName, repoName, repoPath, url string) error {
|
|
||||||
_, stderr, err := process.ExecTimeout(10*time.Minute,
|
|
||||||
fmt.Sprintf("MirrorRepository: %s/%s", userName, repoName),
|
|
||||||
"git", "clone", "--mirror", url, repoPath)
|
|
||||||
if err != nil {
|
|
||||||
return errors.New("git clone --mirror: " + stderr)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err = x.InsertOne(&Mirror{
|
|
||||||
RepoID: repoId,
|
|
||||||
Interval: 24,
|
|
||||||
NextUpdate: time.Now().Add(24 * time.Hour),
|
|
||||||
}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type MigrateRepoOptions struct {
|
type MigrateRepoOptions struct {
|
||||||
Name string
|
Name string
|
||||||
Description string
|
Description string
|
||||||
|
@ -582,29 +624,35 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
|
||||||
repo.NumWatches = 1
|
repo.NumWatches = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
repo.IsBare = false
|
|
||||||
if opts.IsMirror {
|
|
||||||
if err = MirrorRepository(repo.ID, u.Name, repo.Name, repoPath, opts.RemoteAddr); err != nil {
|
|
||||||
return repo, err
|
|
||||||
}
|
|
||||||
repo.IsMirror = true
|
|
||||||
return repo, UpdateRepository(repo, false)
|
|
||||||
} else {
|
|
||||||
os.RemoveAll(repoPath)
|
os.RemoveAll(repoPath)
|
||||||
|
if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{
|
||||||
|
Mirror: true,
|
||||||
|
Quiet: true,
|
||||||
|
Timeout: 10 * time.Minute,
|
||||||
|
}); err != nil {
|
||||||
|
return repo, fmt.Errorf("Clone: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// FIXME: this command could for both migrate and mirror
|
if opts.IsMirror {
|
||||||
_, stderr, err := process.ExecTimeout(10*time.Minute,
|
if _, err = x.InsertOne(&Mirror{
|
||||||
fmt.Sprintf("MigrateRepository: %s", repoPath),
|
RepoID: repo.ID,
|
||||||
"git", "clone", "--mirror", "--bare", "--quiet", opts.RemoteAddr, repoPath)
|
Interval: 24,
|
||||||
if err != nil {
|
NextUpdate: time.Now().Add(24 * time.Hour),
|
||||||
return repo, fmt.Errorf("git clone --mirror --bare --quiet: %v", stderr)
|
}); err != nil {
|
||||||
} else if err = createUpdateHook(repoPath); err != nil {
|
return repo, fmt.Errorf("InsertOne: %v", err)
|
||||||
return repo, fmt.Errorf("create update hook: %v", err)
|
}
|
||||||
|
|
||||||
|
repo.IsMirror = true
|
||||||
|
return repo, UpdateRepository(repo, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = createUpdateHook(repoPath); err != nil {
|
||||||
|
return repo, fmt.Errorf("createUpdateHook: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clean up mirror info which prevents "push --all".
|
// Clean up mirror info which prevents "push --all".
|
||||||
configPath := filepath.Join(repoPath, "/config")
|
// This also removes possible user credentials.
|
||||||
|
configPath := repo.GitConfigPath()
|
||||||
cfg, err := ini.Load(configPath)
|
cfg, err := ini.Load(configPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return repo, fmt.Errorf("open config file: %v", err)
|
return repo, fmt.Errorf("open config file: %v", err)
|
||||||
|
@ -615,7 +663,7 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if repository is empty.
|
// Check if repository is empty.
|
||||||
_, stderr, err = com.ExecCmdDir(repoPath, "git", "log", "-1")
|
_, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") {
|
if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") {
|
||||||
repo.IsBare = true
|
repo.IsBare = true
|
||||||
|
|
|
@ -69,6 +69,9 @@ func (f MigrateRepoForm) ParseRemoteAddr(user *models.User) (string, error) {
|
||||||
}
|
}
|
||||||
if len(f.AuthUsername)+len(f.AuthPassword) > 0 {
|
if len(f.AuthUsername)+len(f.AuthPassword) > 0 {
|
||||||
u.User = url.UserPassword(f.AuthUsername, f.AuthPassword)
|
u.User = url.UserPassword(f.AuthUsername, f.AuthPassword)
|
||||||
|
} else {
|
||||||
|
// Fake user name and password to prevent prompt and fail quick.
|
||||||
|
u.User = url.UserPassword("fake_user", "")
|
||||||
}
|
}
|
||||||
remoteAddr = u.String()
|
remoteAddr = u.String()
|
||||||
} else if !user.CanImportLocal() {
|
} else if !user.CanImportLocal() {
|
||||||
|
@ -86,6 +89,7 @@ type RepoSettingForm struct {
|
||||||
Website string `binding:"Url;MaxSize(100)"`
|
Website string `binding:"Url;MaxSize(100)"`
|
||||||
Branch string
|
Branch string
|
||||||
Interval int
|
Interval int
|
||||||
|
MirrorAddress string
|
||||||
Private bool
|
Private bool
|
||||||
|
|
||||||
// Advanced settings
|
// Advanced settings
|
||||||
|
|
File diff suppressed because one or more lines are too long
|
@ -116,6 +116,7 @@ func Toggle(options *ToggleOptions) macaron.Handler {
|
||||||
ctx.Handle(500, "AutoSignIn", err)
|
ctx.Handle(500, "AutoSignIn", err)
|
||||||
return
|
return
|
||||||
} else if succeed {
|
} else if succeed {
|
||||||
|
log.Trace("Auto-login succeed: %s", ctx.Session.Get("uname"))
|
||||||
ctx.Redirect(setting.AppSubUrl + ctx.Req.RequestURI)
|
ctx.Redirect(setting.AppSubUrl + ctx.Req.RequestURI)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -129,6 +129,7 @@ func RepoAssignment(args ...bool) macaron.Handler {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
ctx.Data["MirrorInterval"] = ctx.Repo.Mirror.Interval
|
ctx.Data["MirrorInterval"] = ctx.Repo.Mirror.Interval
|
||||||
|
ctx.Data["Mirror"] = ctx.Repo.Mirror
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.Repo.Repository = repo
|
ctx.Repo.Repository = repo
|
||||||
|
|
|
@ -234,7 +234,7 @@ func Migrate(ctx *middleware.Context, form auth.MigrateRepoForm) {
|
||||||
log.Error(4, "DeleteRepository: %v", errDelete)
|
log.Error(4, "DeleteRepository: %v", errDelete)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ctx.APIError(500, "MigrateRepository", err)
|
ctx.APIError(500, "MigrateRepository", models.HandleCloneUserCredentials(err.Error(), true))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -41,7 +41,8 @@ func checkRunMode() {
|
||||||
macaron.Env = macaron.PROD
|
macaron.Env = macaron.PROD
|
||||||
macaron.ColorLog = false
|
macaron.ColorLog = false
|
||||||
setting.ProdMode = true
|
setting.ProdMode = true
|
||||||
git.Debug = false
|
default:
|
||||||
|
git.Debug = true
|
||||||
}
|
}
|
||||||
log.Info("Run Mode: %s", strings.Title(macaron.Env))
|
log.Info("Run Mode: %s", strings.Title(macaron.Env))
|
||||||
}
|
}
|
||||||
|
|
|
@ -206,11 +206,11 @@ func MigratePost(ctx *middleware.Context, form auth.MigrateRepoForm) {
|
||||||
if strings.Contains(err.Error(), "Authentication failed") ||
|
if strings.Contains(err.Error(), "Authentication failed") ||
|
||||||
strings.Contains(err.Error(), "could not read Username") {
|
strings.Contains(err.Error(), "could not read Username") {
|
||||||
ctx.Data["Err_Auth"] = true
|
ctx.Data["Err_Auth"] = true
|
||||||
ctx.RenderWithErr(ctx.Tr("form.auth_failed", strings.Replace(err.Error(), ":"+form.AuthPassword+"@", ":<password>@", 1)), MIGRATE, &form)
|
ctx.RenderWithErr(ctx.Tr("form.auth_failed", models.HandleCloneUserCredentials(err.Error(), true)), MIGRATE, &form)
|
||||||
return
|
return
|
||||||
} else if strings.Contains(err.Error(), "fatal:") {
|
} else if strings.Contains(err.Error(), "fatal:") {
|
||||||
ctx.Data["Err_CloneAddr"] = true
|
ctx.Data["Err_CloneAddr"] = true
|
||||||
ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", strings.Replace(err.Error(), ":"+form.AuthPassword+"@", ":<password>@", 1)), MIGRATE, &form)
|
ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", models.HandleCloneUserCredentials(err.Error(), true)), MIGRATE, &form)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -109,9 +109,14 @@ func SettingsPost(ctx *middleware.Context, form auth.RepoSettingForm) {
|
||||||
ctx.Repo.Mirror.Interval = form.Interval
|
ctx.Repo.Mirror.Interval = form.Interval
|
||||||
ctx.Repo.Mirror.NextUpdate = time.Now().Add(time.Duration(form.Interval) * time.Hour)
|
ctx.Repo.Mirror.NextUpdate = time.Now().Add(time.Duration(form.Interval) * time.Hour)
|
||||||
if err := models.UpdateMirror(ctx.Repo.Mirror); err != nil {
|
if err := models.UpdateMirror(ctx.Repo.Mirror); err != nil {
|
||||||
log.Error(4, "UpdateMirror: %v", err)
|
ctx.Handle(500, "UpdateMirror", err)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := ctx.Repo.Mirror.SaveAddress(form.MirrorAddress); err != nil {
|
||||||
|
ctx.Handle(500, "SaveAddress", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success"))
|
ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success"))
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
0.7.34.1208 Beta
|
0.7.35.1208 Beta
|
|
@ -8,7 +8,7 @@
|
||||||
<a href="{{AppSubUrl}}/{{.Owner.Name}}">{{.Owner.Name}}</a>
|
<a href="{{AppSubUrl}}/{{.Owner.Name}}">{{.Owner.Name}}</a>
|
||||||
<div class="divider"> / </div>
|
<div class="divider"> / </div>
|
||||||
<a href="{{$.RepoLink}}">{{.Name}}</a>
|
<a href="{{$.RepoLink}}">{{.Name}}</a>
|
||||||
{{if .IsMirror}}<div class="ui label">{{$.i18n.Tr "mirror"}}</div>{{end}}
|
{{if .IsMirror}}<div class="fork-flag">{{$.i18n.Tr "repo.mirror_from"}} <a target="_blank" href="{{$.MirrorAddress}}">{{$.Mirror.Address}}</a></div>{{end}}
|
||||||
{{if .IsFork}}<div class="fork-flag">{{$.i18n.Tr "repo.forked_from"}} <a href="{{.BaseRepo.RepoLink}}">{{SubStr .BaseRepo.RepoLink 1 -1}}</a></div>{{end}}
|
{{if .IsFork}}<div class="fork-flag">{{$.i18n.Tr "repo.forked_from"}} <a href="{{.BaseRepo.RepoLink}}">{{SubStr .BaseRepo.RepoLink 1 -1}}</a></div>{{end}}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
|
@ -55,6 +55,11 @@
|
||||||
<label for="interval">{{.i18n.Tr "repo.mirror_interval"}}</label>
|
<label for="interval">{{.i18n.Tr "repo.mirror_interval"}}</label>
|
||||||
<input id="interval" name="interval" type="number" value="{{.MirrorInterval}}">
|
<input id="interval" name="interval" type="number" value="{{.MirrorInterval}}">
|
||||||
</div>
|
</div>
|
||||||
|
<div class="field">
|
||||||
|
<label for="mirror_address">{{.i18n.Tr "repo.mirror_address"}}</label>
|
||||||
|
<input id="mirror_address" name="mirror_address" value="{{.Mirror.FullAddress}}">
|
||||||
|
<p class="help">{{.i18n.Tr "repo.mirror_address_desc"}}</p>
|
||||||
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
<div class="ui divider"></div>
|
<div class="ui divider"></div>
|
||||||
|
|
Loading…
Reference in New Issue