From 095fb9f2e32fe78acb9bc034f0e1d60922ff1e60 Mon Sep 17 00:00:00 2001 From: Antoine GIRARD Date: Tue, 16 Jan 2018 12:07:47 +0100 Subject: [PATCH] Add integrations tests from git cli (#3377) * test: integration add git cli tests Extracted form for easing review process and debug #3152 * test: integration add git cli big file commit * fix: Don't rewrite key if internal server --- integrations/git_test.go | 344 +++++++++++++++++++++++------------- integrations/mysql.ini.tmpl | 7 +- integrations/pgsql.ini.tmpl | 29 +-- integrations/repo_test.go | 2 +- integrations/sqlite.ini | 8 +- models/ssh_key.go | 10 ++ 6 files changed, 258 insertions(+), 142 deletions(-) diff --git a/integrations/git_test.go b/integrations/git_test.go index 53814cf41..05b2c366b 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -6,8 +6,9 @@ package integrations import ( "context" + "crypto/rand" + "fmt" "io/ioutil" - "math/rand" "net" "net/http" "net/url" @@ -17,6 +18,7 @@ import ( "time" "code.gitea.io/git" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/sdk/gitea" @@ -24,7 +26,13 @@ import ( "github.com/stretchr/testify/assert" ) -func onGiteaWebRun(t *testing.T, callback func(*testing.T, *url.URL)) { +const ( + littleSize = 1024 //1ko + bigSize = 128 * 1024 * 1024 //128Mo +) + +func onGiteaRun(t *testing.T, callback func(*testing.T, *url.URL)) { + prepareTestEnv(t) s := http.Server{ Handler: mac, } @@ -35,151 +43,241 @@ func onGiteaWebRun(t *testing.T, callback func(*testing.T, *url.URL)) { assert.NoError(t, err) defer func() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) s.Shutdown(ctx) cancel() }() go s.Serve(listener) + //Started by config go ssh.Listen(setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs) callback(t, u) } func TestGit(t *testing.T) { - prepareTestEnv(t) - - onGiteaWebRun(t, func(t *testing.T, u *url.URL) { - dstPath, err := ioutil.TempDir("", "repo-tmp-17") - assert.NoError(t, err) - defer os.RemoveAll(dstPath) + onGiteaRun(t, func(t *testing.T, u *url.URL) { u.Path = "user2/repo1.git" - t.Run("Standard", func(t *testing.T) { - - t.Run("CloneNoLogin", func(t *testing.T) { - dstLocalPath, err := ioutil.TempDir("", "repo1") - assert.NoError(t, err) - defer os.RemoveAll(dstLocalPath) - err = git.Clone(u.String(), dstLocalPath, git.CloneRepoOptions{}) - assert.NoError(t, err) - assert.True(t, com.IsExist(filepath.Join(dstLocalPath, "README.md"))) - }) - - t.Run("CreateRepo", func(t *testing.T) { - session := loginUser(t, "user2") - req := NewRequestWithJSON(t, "POST", "/api/v1/user/repos", &api.CreateRepoOption{ - AutoInit: true, - Description: "Temporary repo", - Name: "repo-tmp-17", - Private: false, - Gitignores: "", - License: "WTFPL", - Readme: "Default", + t.Run("HTTP", func(t *testing.T) { + dstPath, err := ioutil.TempDir("", "repo-tmp-17") + assert.NoError(t, err) + defer os.RemoveAll(dstPath) + t.Run("Standard", func(t *testing.T) { + t.Run("CloneNoLogin", func(t *testing.T) { + dstLocalPath, err := ioutil.TempDir("", "repo1") + assert.NoError(t, err) + defer os.RemoveAll(dstLocalPath) + err = git.Clone(u.String(), dstLocalPath, git.CloneRepoOptions{}) + assert.NoError(t, err) + assert.True(t, com.IsExist(filepath.Join(dstLocalPath, "README.md"))) }) - session.MakeRequest(t, req, http.StatusCreated) - }) - u.Path = "user2/repo-tmp-17.git" - u.User = url.UserPassword("user2", userPassword) - t.Run("Clone", func(t *testing.T) { - err = git.Clone(u.String(), dstPath, git.CloneRepoOptions{}) - assert.NoError(t, err) - assert.True(t, com.IsExist(filepath.Join(dstPath, "README.md"))) - }) - - t.Run("PushCommit", func(t *testing.T) { - data := make([]byte, 1024) - _, err := rand.Read(data) - assert.NoError(t, err) - tmpFile, err := ioutil.TempFile(dstPath, "data-file-") - defer tmpFile.Close() - _, err = tmpFile.Write(data) - assert.NoError(t, err) - - //Commit - err = git.AddChanges(dstPath, false, filepath.Base(tmpFile.Name())) - assert.NoError(t, err) - err = git.CommitChanges(dstPath, git.CommitChangesOptions{ - Committer: &git.Signature{ - Email: "user2@example.com", - Name: "User Two", - When: time.Now(), - }, - Author: &git.Signature{ - Email: "user2@example.com", - Name: "User Two", - When: time.Now(), - }, - Message: "Testing commit", + t.Run("CreateRepo", func(t *testing.T) { + session := loginUser(t, "user2") + req := NewRequestWithJSON(t, "POST", "/api/v1/user/repos", &api.CreateRepoOption{ + AutoInit: true, + Description: "Temporary repo", + Name: "repo-tmp-17", + Private: false, + Gitignores: "", + License: "WTFPL", + Readme: "Default", + }) + session.MakeRequest(t, req, http.StatusCreated) }) - assert.NoError(t, err) - //Push - err = git.Push(dstPath, git.PushOptions{ - Branch: "master", - Remote: u.String(), - Force: false, + u.Path = "user2/repo-tmp-17.git" + u.User = url.UserPassword("user2", userPassword) + t.Run("Clone", func(t *testing.T) { + err = git.Clone(u.String(), dstPath, git.CloneRepoOptions{}) + assert.NoError(t, err) + assert.True(t, com.IsExist(filepath.Join(dstPath, "README.md"))) + }) + + t.Run("PushCommit", func(t *testing.T) { + t.Run("Little", func(t *testing.T) { + commitAndPush(t, littleSize, dstPath) + }) + t.Run("Big", func(t *testing.T) { + commitAndPush(t, bigSize, dstPath) + }) + }) + }) + t.Run("LFS", func(t *testing.T) { + t.Run("PushCommit", func(t *testing.T) { + //Setup git LFS + _, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath) + assert.NoError(t, err) + _, err = git.NewCommand("lfs").AddArguments("track", "data-file-*").RunInDir(dstPath) + assert.NoError(t, err) + err = git.AddChanges(dstPath, false, ".gitattributes") + assert.NoError(t, err) + + t.Run("Little", func(t *testing.T) { + commitAndPush(t, littleSize, dstPath) + }) + t.Run("Big", func(t *testing.T) { + commitAndPush(t, bigSize, dstPath) + }) + }) + t.Run("Locks", func(t *testing.T) { + lockTest(t, u.String(), dstPath) }) - assert.NoError(t, err) }) }) - t.Run("LFS", func(t *testing.T) { - t.Run("PushCommit", func(t *testing.T) { - /* Generate random file */ - data := make([]byte, 1024) - _, err := rand.Read(data) - assert.NoError(t, err) - tmpFile, err := ioutil.TempFile(dstPath, "data-file-") - defer tmpFile.Close() - _, err = tmpFile.Write(data) - assert.NoError(t, err) + t.Run("SSH", func(t *testing.T) { + //Setup remote link + u.Scheme = "ssh" + u.User = url.User("git") + u.Host = fmt.Sprintf("%s:%d", setting.SSH.ListenHost, setting.SSH.ListenPort) + u.Path = "user2/repo-tmp-18.git" - //Setup git LFS - _, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath) - assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("track", "data-file-*").RunInDir(dstPath) - assert.NoError(t, err) + //Setup key + keyFile := filepath.Join(setting.AppDataPath, "my-testing-key") + _, _, err := com.ExecCmd("ssh-keygen", "-f", keyFile, "-t", "rsa", "-N", "") + assert.NoError(t, err) + defer os.RemoveAll(keyFile) + defer os.RemoveAll(keyFile + ".pub") - //Commit - err = git.AddChanges(dstPath, false, ".gitattributes", filepath.Base(tmpFile.Name())) - assert.NoError(t, err) - err = git.CommitChanges(dstPath, git.CommitChangesOptions{ - Committer: &git.Signature{ - Email: "user2@example.com", - Name: "User Two", - When: time.Now(), - }, - Author: &git.Signature{ - Email: "user2@example.com", - Name: "User Two", - When: time.Now(), - }, - Message: "Testing LFS ", - }) - assert.NoError(t, err) + session := loginUser(t, "user1") + keyOwner := models.AssertExistsAndLoadBean(t, &models.User{Name: "user2"}).(*models.User) + urlStr := fmt.Sprintf("/api/v1/admin/users/%s/keys", keyOwner.Name) - //Push - u.User = url.UserPassword("user2", userPassword) - err = git.Push(dstPath, git.PushOptions{ - Branch: "master", - Remote: u.String(), - Force: false, - }) - assert.NoError(t, err) - }) - t.Run("Locks", func(t *testing.T) { - _, err = git.NewCommand("remote").AddArguments("set-url", "origin", u.String()).RunInDir(dstPath) //TODO add test ssh git-lfs-creds - assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("locks").RunInDir(dstPath) - assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("lock", "README.md").RunInDir(dstPath) - assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("locks").RunInDir(dstPath) - assert.NoError(t, err) - _, err = git.NewCommand("lfs").AddArguments("unlock", "README.md").RunInDir(dstPath) - assert.NoError(t, err) + dataPubKey, err := ioutil.ReadFile(keyFile + ".pub") + assert.NoError(t, err) + req := NewRequestWithValues(t, "POST", urlStr, map[string]string{ + "key": string(dataPubKey), + "title": "test-key", }) + session.MakeRequest(t, req, http.StatusCreated) + //Setup ssh wrapper + sshWrapper, err := ioutil.TempFile(setting.AppDataPath, "tmp-ssh-wrapper") + sshWrapper.WriteString("#!/bin/sh\n\n") + sshWrapper.WriteString("ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i \"" + filepath.Join(setting.AppWorkPath, keyFile) + "\" $* \n\n") + err = sshWrapper.Chmod(os.ModePerm) + assert.NoError(t, err) + sshWrapper.Close() + defer os.RemoveAll(sshWrapper.Name()) + + //Setup clone folder + dstPath, err := ioutil.TempDir("", "repo-tmp-18") + assert.NoError(t, err) + defer os.RemoveAll(dstPath) + + t.Run("Standard", func(t *testing.T) { + t.Run("CreateRepo", func(t *testing.T) { + session := loginUser(t, "user2") + req := NewRequestWithJSON(t, "POST", "/api/v1/user/repos", &api.CreateRepoOption{ + AutoInit: true, + Description: "Temporary repo", + Name: "repo-tmp-18", + Private: false, + Gitignores: "", + License: "WTFPL", + Readme: "Default", + }) + session.MakeRequest(t, req, http.StatusCreated) + }) + //TODO get url from api + t.Run("Clone", func(t *testing.T) { + _, err = git.NewCommand("clone").AddArguments("--config", "core.sshCommand="+filepath.Join(setting.AppWorkPath, sshWrapper.Name()), u.String(), dstPath).Run() + assert.NoError(t, err) + assert.True(t, com.IsExist(filepath.Join(dstPath, "README.md"))) + }) + //time.Sleep(5 * time.Minute) + t.Run("PushCommit", func(t *testing.T) { + t.Run("Little", func(t *testing.T) { + commitAndPush(t, littleSize, dstPath) + }) + t.Run("Big", func(t *testing.T) { + commitAndPush(t, bigSize, dstPath) + }) + }) + }) + t.Run("LFS", func(t *testing.T) { + os.Setenv("GIT_SSH_COMMAND", filepath.Join(setting.AppWorkPath, sshWrapper.Name())) //TODO remove when fixed https://github.com/git-lfs/git-lfs/issues/2215 + defer os.Unsetenv("GIT_SSH_COMMAND") + t.Run("PushCommit", func(t *testing.T) { + //Setup git LFS + _, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath) + assert.NoError(t, err) + _, err = git.NewCommand("lfs").AddArguments("track", "data-file-*").RunInDir(dstPath) + assert.NoError(t, err) + err = git.AddChanges(dstPath, false, ".gitattributes") + assert.NoError(t, err) + + t.Run("Little", func(t *testing.T) { + commitAndPush(t, littleSize, dstPath) + }) + t.Run("Big", func(t *testing.T) { + commitAndPush(t, bigSize, dstPath) + }) + }) + /* Failed without #3152. TODO activate with fix. + t.Run("Locks", func(t *testing.T) { + lockTest(t, u.String(), dstPath) + }) + */ + }) }) }) } + +func lockTest(t *testing.T, remote, repoPath string) { + _, err := git.NewCommand("remote").AddArguments("set-url", "origin", remote).RunInDir(repoPath) //TODO add test ssh git-lfs-creds + assert.NoError(t, err) + _, err = git.NewCommand("lfs").AddArguments("locks").RunInDir(repoPath) + assert.NoError(t, err) + _, err = git.NewCommand("lfs").AddArguments("lock", "README.md").RunInDir(repoPath) + assert.NoError(t, err) + _, err = git.NewCommand("lfs").AddArguments("locks").RunInDir(repoPath) + assert.NoError(t, err) + _, err = git.NewCommand("lfs").AddArguments("unlock", "README.md").RunInDir(repoPath) + assert.NoError(t, err) +} + +func commitAndPush(t *testing.T, size int, repoPath string) { + err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two") + assert.NoError(t, err) + _, err = git.NewCommand("push").RunInDir(repoPath) //Push + assert.NoError(t, err) +} + +func generateCommitWithNewData(size int, repoPath, email, fullName string) error { + //Generate random file + data := make([]byte, size) + _, err := rand.Read(data) + if err != nil { + return err + } + tmpFile, err := ioutil.TempFile(repoPath, "data-file-") + if err != nil { + return err + } + defer tmpFile.Close() + _, err = tmpFile.Write(data) + if err != nil { + return err + } + + //Commit + err = git.AddChanges(repoPath, false, filepath.Base(tmpFile.Name())) + if err != nil { + return err + } + err = git.CommitChanges(repoPath, git.CommitChangesOptions{ + Committer: &git.Signature{ + Email: email, + Name: fullName, + When: time.Now(), + }, + Author: &git.Signature{ + Email: email, + Name: fullName, + When: time.Now(), + }, + Message: fmt.Sprintf("Testing commit @ %v", time.Now()), + }) + return err +} diff --git a/integrations/mysql.ini.tmpl b/integrations/mysql.ini.tmpl index 08f8ddfa5..e75df81bd 100644 --- a/integrations/mysql.ini.tmpl +++ b/integrations/mysql.ini.tmpl @@ -8,7 +8,6 @@ NAME = {{TEST_MYSQL_DBNAME}} USER = {{TEST_MYSQL_USERNAME}} PASSWD = {{TEST_MYSQL_PASSWORD}} SSL_MODE = disable -PATH = data/gitea.db [indexer] ISSUE_INDEXER_PATH = integrations/indexers-mysql/issues.bleve @@ -27,10 +26,14 @@ SSH_DOMAIN = localhost HTTP_PORT = 3001 ROOT_URL = http://localhost:3001/ DISABLE_SSH = false -SSH_PORT = 22 +SSH_LISTEN_HOST = localhost +SSH_PORT = 2201 +START_SSH_SERVER = true LFS_START_SERVER = true LFS_CONTENT_PATH = data/lfs-mysql OFFLINE_MODE = false +LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w +APP_DATA_PATH = integrations/gitea-integration-mysql/data [mailer] ENABLED = false diff --git a/integrations/pgsql.ini.tmpl b/integrations/pgsql.ini.tmpl index 17ecacec5..923608235 100644 --- a/integrations/pgsql.ini.tmpl +++ b/integrations/pgsql.ini.tmpl @@ -8,7 +8,6 @@ NAME = {{TEST_PGSQL_DBNAME}} USER = {{TEST_PGSQL_USERNAME}} PASSWD = {{TEST_PGSQL_PASSWORD}} SSL_MODE = disable -PATH = data/gitea.db [indexer] ISSUE_INDEXER_PATH = integrations/indexers-pgsql/issues.bleve @@ -27,23 +26,27 @@ SSH_DOMAIN = localhost HTTP_PORT = 3002 ROOT_URL = http://localhost:3002/ DISABLE_SSH = false -SSH_PORT = 22 +SSH_LISTEN_HOST = localhost +SSH_PORT = 2202 +START_SSH_SERVER = true LFS_START_SERVER = true LFS_CONTENT_PATH = data/lfs-pgsql OFFLINE_MODE = false +LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w +APP_DATA_PATH = integrations/gitea-integration-pgsql/data [mailer] ENABLED = false [service] -REGISTER_EMAIL_CONFIRM = false -ENABLE_NOTIFY_MAIL = false -DISABLE_REGISTRATION = false -ENABLE_CAPTCHA = false -REQUIRE_SIGNIN_VIEW = false -DEFAULT_KEEP_EMAIL_PRIVATE = false +REGISTER_EMAIL_CONFIRM = false +ENABLE_NOTIFY_MAIL = false +DISABLE_REGISTRATION = false +ENABLE_CAPTCHA = false +REQUIRE_SIGNIN_VIEW = false +DEFAULT_KEEP_EMAIL_PRIVATE = false DEFAULT_ALLOW_CREATE_ORGANIZATION = true -NO_REPLY_ADDRESS = noreply.example.org +NO_REPLY_ADDRESS = noreply.example.org [picture] DISABLE_GRAVATAR = false @@ -54,7 +57,7 @@ PROVIDER = file PROVIDER_CONFIG = data/sessions-pgsql [log] -MODE = console,file +MODE = console,file ROOT_PATH = pgsql-log [log.console] @@ -64,6 +67,6 @@ LEVEL = Warn LEVEL = Debug [security] -INSTALL_LOCK = true -SECRET_KEY = 9pCviYTWSb -INTERNAL_TOKEN = test +INSTALL_LOCK = true +SECRET_KEY = 9pCviYTWSb +INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ diff --git a/integrations/repo_test.go b/integrations/repo_test.go index 8465bbdc5..a18bfabdf 100644 --- a/integrations/repo_test.go +++ b/integrations/repo_test.go @@ -71,6 +71,6 @@ func TestViewRepo1CloneLinkAuthorized(t *testing.T) { assert.Equal(t, setting.AppURL+"user2/repo1.git", link) link, exists = htmlDoc.doc.Find("#repo-clone-ssh").Attr("data-link") assert.True(t, exists, "The template has changed") - sshURL := fmt.Sprintf("%s@%s:user2/repo1.git", setting.RunUser, setting.SSH.Domain) + sshURL := fmt.Sprintf("ssh://%s@%s:%d/user2/repo1.git", setting.RunUser, setting.SSH.Domain, setting.SSH.Port) assert.Equal(t, sshURL, link) } diff --git a/integrations/sqlite.ini b/integrations/sqlite.ini index 7c9cd09b4..857cfe461 100644 --- a/integrations/sqlite.ini +++ b/integrations/sqlite.ini @@ -3,7 +3,7 @@ RUN_MODE = prod [database] DB_TYPE = sqlite3 -PATH = :memory: +PATH = integrations/gitea-integration-sqlite/gitea.db [indexer] ISSUE_INDEXER_PATH = integrations/indexers-sqlite/issues.bleve @@ -22,11 +22,14 @@ SSH_DOMAIN = localhost HTTP_PORT = 3003 ROOT_URL = http://localhost:3003/ DISABLE_SSH = false -SSH_PORT = 22 +SSH_LISTEN_HOST = localhost +SSH_PORT = 2203 +START_SSH_SERVER = true LFS_START_SERVER = true LFS_CONTENT_PATH = data/lfs-sqlite OFFLINE_MODE = false LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w +APP_DATA_PATH = integrations/gitea-integration-sqlite/data [mailer] ENABLED = false @@ -62,4 +65,3 @@ LEVEL = Debug INSTALL_LOCK = true SECRET_KEY = 9pCviYTWSb INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTI3OTU5ODN9.OQkH5UmzID2XBdwQ9TAI6Jj2t1X-wElVTjbE7aoN4I8 - diff --git a/models/ssh_key.go b/models/ssh_key.go index 2878177d4..e121b0d5b 100644 --- a/models/ssh_key.go +++ b/models/ssh_key.go @@ -304,6 +304,11 @@ func CheckPublicKeyString(content string) (_ string, err error) { // appendAuthorizedKeysToFile appends new SSH keys' content to authorized_keys file. func appendAuthorizedKeysToFile(keys ...*PublicKey) error { + // Don't need to rewrite this file if builtin SSH server is enabled. + if setting.SSH.StartBuiltinServer { + return nil + } + sshOpLocker.Lock() defer sshOpLocker.Unlock() @@ -532,6 +537,11 @@ func DeletePublicKey(doer *User, id int64) (err error) { // Note: x.Iterate does not get latest data after insert/delete, so we have to call this function // outside any session scope independently. func RewriteAllPublicKeys() error { + //Don't rewrite key if internal server + if setting.SSH.StartBuiltinServer { + return nil + } + sshOpLocker.Lock() defer sshOpLocker.Unlock()