fix bugs when too many IN variables (#4594)
This commit is contained in:
parent
c40f5d2d4c
commit
9c354a539a
|
@ -9,6 +9,11 @@ import "fmt"
|
||||||
// IssueList defines a list of issues
|
// IssueList defines a list of issues
|
||||||
type IssueList []*Issue
|
type IssueList []*Issue
|
||||||
|
|
||||||
|
const (
|
||||||
|
// default variables number on IN () in SQL
|
||||||
|
defaultMaxInSize = 50
|
||||||
|
)
|
||||||
|
|
||||||
func (issues IssueList) getRepoIDs() []int64 {
|
func (issues IssueList) getRepoIDs() []int64 {
|
||||||
repoIDs := make(map[int64]struct{}, len(issues))
|
repoIDs := make(map[int64]struct{}, len(issues))
|
||||||
for _, issue := range issues {
|
for _, issue := range issues {
|
||||||
|
@ -26,11 +31,20 @@ func (issues IssueList) loadRepositories(e Engine) ([]*Repository, error) {
|
||||||
|
|
||||||
repoIDs := issues.getRepoIDs()
|
repoIDs := issues.getRepoIDs()
|
||||||
repoMaps := make(map[int64]*Repository, len(repoIDs))
|
repoMaps := make(map[int64]*Repository, len(repoIDs))
|
||||||
err := e.
|
var left = len(repoIDs)
|
||||||
In("id", repoIDs).
|
for left > 0 {
|
||||||
Find(&repoMaps)
|
var limit = defaultMaxInSize
|
||||||
if err != nil {
|
if left < limit {
|
||||||
return nil, fmt.Errorf("find repository: %v", err)
|
limit = left
|
||||||
|
}
|
||||||
|
err := e.
|
||||||
|
In("id", repoIDs[:limit]).
|
||||||
|
Find(&repoMaps)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("find repository: %v", err)
|
||||||
|
}
|
||||||
|
left = left - limit
|
||||||
|
repoIDs = repoIDs[limit:]
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, issue := range issues {
|
for _, issue := range issues {
|
||||||
|
@ -61,11 +75,20 @@ func (issues IssueList) loadPosters(e Engine) error {
|
||||||
|
|
||||||
posterIDs := issues.getPosterIDs()
|
posterIDs := issues.getPosterIDs()
|
||||||
posterMaps := make(map[int64]*User, len(posterIDs))
|
posterMaps := make(map[int64]*User, len(posterIDs))
|
||||||
err := e.
|
var left = len(posterIDs)
|
||||||
In("id", posterIDs).
|
for left > 0 {
|
||||||
Find(&posterMaps)
|
var limit = defaultMaxInSize
|
||||||
if err != nil {
|
if left < limit {
|
||||||
return err
|
limit = left
|
||||||
|
}
|
||||||
|
err := e.
|
||||||
|
In("id", posterIDs[:limit]).
|
||||||
|
Find(&posterMaps)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
left = left - limit
|
||||||
|
posterIDs = posterIDs[limit:]
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, issue := range issues {
|
for _, issue := range issues {
|
||||||
|
@ -99,23 +122,34 @@ func (issues IssueList) loadLabels(e Engine) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
var issueLabels = make(map[int64][]*Label, len(issues)*3)
|
var issueLabels = make(map[int64][]*Label, len(issues)*3)
|
||||||
rows, err := e.Table("label").
|
var issueIDs = issues.getIssueIDs()
|
||||||
Join("LEFT", "issue_label", "issue_label.label_id = label.id").
|
var left = len(issueIDs)
|
||||||
In("issue_label.issue_id", issues.getIssueIDs()).
|
for left > 0 {
|
||||||
Asc("label.name").
|
var limit = defaultMaxInSize
|
||||||
Rows(new(LabelIssue))
|
if left < limit {
|
||||||
if err != nil {
|
limit = left
|
||||||
return err
|
}
|
||||||
}
|
rows, err := e.Table("label").
|
||||||
defer rows.Close()
|
Join("LEFT", "issue_label", "issue_label.label_id = label.id").
|
||||||
|
In("issue_label.issue_id", issueIDs[:limit]).
|
||||||
for rows.Next() {
|
Asc("label.name").
|
||||||
var labelIssue LabelIssue
|
Rows(new(LabelIssue))
|
||||||
err = rows.Scan(&labelIssue)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
issueLabels[labelIssue.IssueLabel.IssueID] = append(issueLabels[labelIssue.IssueLabel.IssueID], labelIssue.Label)
|
|
||||||
|
for rows.Next() {
|
||||||
|
var labelIssue LabelIssue
|
||||||
|
err = rows.Scan(&labelIssue)
|
||||||
|
if err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
issueLabels[labelIssue.IssueLabel.IssueID] = append(issueLabels[labelIssue.IssueLabel.IssueID], labelIssue.Label)
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
left = left - limit
|
||||||
|
issueIDs = issueIDs[limit:]
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, issue := range issues {
|
for _, issue := range issues {
|
||||||
|
@ -141,11 +175,20 @@ func (issues IssueList) loadMilestones(e Engine) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
milestoneMaps := make(map[int64]*Milestone, len(milestoneIDs))
|
milestoneMaps := make(map[int64]*Milestone, len(milestoneIDs))
|
||||||
err := e.
|
var left = len(milestoneIDs)
|
||||||
In("id", milestoneIDs).
|
for left > 0 {
|
||||||
Find(&milestoneMaps)
|
var limit = defaultMaxInSize
|
||||||
if err != nil {
|
if left < limit {
|
||||||
return err
|
limit = left
|
||||||
|
}
|
||||||
|
err := e.
|
||||||
|
In("id", milestoneIDs[:limit]).
|
||||||
|
Find(&milestoneMaps)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
left = left - limit
|
||||||
|
milestoneIDs = milestoneIDs[limit:]
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, issue := range issues {
|
for _, issue := range issues {
|
||||||
|
@ -165,23 +208,35 @@ func (issues IssueList) loadAssignees(e Engine) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
var assignees = make(map[int64][]*User, len(issues))
|
var assignees = make(map[int64][]*User, len(issues))
|
||||||
rows, err := e.Table("issue_assignees").
|
var issueIDs = issues.getIssueIDs()
|
||||||
Join("INNER", "`user`", "`user`.id = `issue_assignees`.assignee_id").
|
var left = len(issueIDs)
|
||||||
In("`issue_assignees`.issue_id", issues.getIssueIDs()).
|
for left > 0 {
|
||||||
Rows(new(AssigneeIssue))
|
var limit = defaultMaxInSize
|
||||||
if err != nil {
|
if left < limit {
|
||||||
return err
|
limit = left
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
rows, err := e.Table("issue_assignees").
|
||||||
|
Join("INNER", "`user`", "`user`.id = `issue_assignees`.assignee_id").
|
||||||
for rows.Next() {
|
In("`issue_assignees`.issue_id", issueIDs[:limit]).
|
||||||
var assigneeIssue AssigneeIssue
|
Rows(new(AssigneeIssue))
|
||||||
err = rows.Scan(&assigneeIssue)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
assignees[assigneeIssue.IssueAssignee.IssueID] = append(assignees[assigneeIssue.IssueAssignee.IssueID], assigneeIssue.Assignee)
|
for rows.Next() {
|
||||||
|
var assigneeIssue AssigneeIssue
|
||||||
|
err = rows.Scan(&assigneeIssue)
|
||||||
|
if err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
assignees[assigneeIssue.IssueAssignee.IssueID] = append(assignees[assigneeIssue.IssueAssignee.IssueID], assigneeIssue.Assignee)
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
|
||||||
|
left = left - limit
|
||||||
|
issueIDs = issueIDs[limit:]
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, issue := range issues {
|
for _, issue := range issues {
|
||||||
|
@ -207,21 +262,32 @@ func (issues IssueList) loadPullRequests(e Engine) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
pullRequestMaps := make(map[int64]*PullRequest, len(issuesIDs))
|
pullRequestMaps := make(map[int64]*PullRequest, len(issuesIDs))
|
||||||
rows, err := e.
|
var left = len(issuesIDs)
|
||||||
In("issue_id", issuesIDs).
|
for left > 0 {
|
||||||
Rows(new(PullRequest))
|
var limit = defaultMaxInSize
|
||||||
if err != nil {
|
if left < limit {
|
||||||
return err
|
limit = left
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
rows, err := e.
|
||||||
|
In("issue_id", issuesIDs[:limit]).
|
||||||
for rows.Next() {
|
Rows(new(PullRequest))
|
||||||
var pr PullRequest
|
|
||||||
err = rows.Scan(&pr)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
pullRequestMaps[pr.IssueID] = &pr
|
|
||||||
|
for rows.Next() {
|
||||||
|
var pr PullRequest
|
||||||
|
err = rows.Scan(&pr)
|
||||||
|
if err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pullRequestMaps[pr.IssueID] = &pr
|
||||||
|
}
|
||||||
|
|
||||||
|
rows.Close()
|
||||||
|
left = left - limit
|
||||||
|
issuesIDs = issuesIDs[limit:]
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, issue := range issues {
|
for _, issue := range issues {
|
||||||
|
@ -236,22 +302,34 @@ func (issues IssueList) loadAttachments(e Engine) (err error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
var attachments = make(map[int64][]*Attachment, len(issues))
|
var attachments = make(map[int64][]*Attachment, len(issues))
|
||||||
rows, err := e.Table("attachment").
|
var issuesIDs = issues.getIssueIDs()
|
||||||
Join("INNER", "issue", "issue.id = attachment.issue_id").
|
var left = len(issuesIDs)
|
||||||
In("issue.id", issues.getIssueIDs()).
|
for left > 0 {
|
||||||
Rows(new(Attachment))
|
var limit = defaultMaxInSize
|
||||||
if err != nil {
|
if left < limit {
|
||||||
return err
|
limit = left
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
rows, err := e.Table("attachment").
|
||||||
|
Join("INNER", "issue", "issue.id = attachment.issue_id").
|
||||||
for rows.Next() {
|
In("issue.id", issuesIDs[:limit]).
|
||||||
var attachment Attachment
|
Rows(new(Attachment))
|
||||||
err = rows.Scan(&attachment)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
attachments[attachment.IssueID] = append(attachments[attachment.IssueID], &attachment)
|
|
||||||
|
for rows.Next() {
|
||||||
|
var attachment Attachment
|
||||||
|
err = rows.Scan(&attachment)
|
||||||
|
if err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
attachments[attachment.IssueID] = append(attachments[attachment.IssueID], &attachment)
|
||||||
|
}
|
||||||
|
|
||||||
|
rows.Close()
|
||||||
|
left = left - limit
|
||||||
|
issuesIDs = issuesIDs[limit:]
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, issue := range issues {
|
for _, issue := range issues {
|
||||||
|
@ -266,22 +344,33 @@ func (issues IssueList) loadComments(e Engine) (err error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
var comments = make(map[int64][]*Comment, len(issues))
|
var comments = make(map[int64][]*Comment, len(issues))
|
||||||
rows, err := e.Table("comment").
|
var issuesIDs = issues.getIssueIDs()
|
||||||
Join("INNER", "issue", "issue.id = comment.issue_id").
|
var left = len(issuesIDs)
|
||||||
In("issue.id", issues.getIssueIDs()).
|
for left > 0 {
|
||||||
Rows(new(Comment))
|
var limit = defaultMaxInSize
|
||||||
if err != nil {
|
if left < limit {
|
||||||
return err
|
limit = left
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
rows, err := e.Table("comment").
|
||||||
|
Join("INNER", "issue", "issue.id = comment.issue_id").
|
||||||
for rows.Next() {
|
In("issue.id", issuesIDs[:limit]).
|
||||||
var comment Comment
|
Rows(new(Comment))
|
||||||
err = rows.Scan(&comment)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
comments[comment.IssueID] = append(comments[comment.IssueID], &comment)
|
|
||||||
|
for rows.Next() {
|
||||||
|
var comment Comment
|
||||||
|
err = rows.Scan(&comment)
|
||||||
|
if err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
comments[comment.IssueID] = append(comments[comment.IssueID], &comment)
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
left = left - limit
|
||||||
|
issuesIDs = issuesIDs[limit:]
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, issue := range issues {
|
for _, issue := range issues {
|
||||||
|
@ -307,25 +396,35 @@ func (issues IssueList) loadTotalTrackedTimes(e Engine) (err error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// select issue_id, sum(time) from tracked_time where issue_id in (<issue ids in current page>) group by issue_id
|
var left = len(ids)
|
||||||
rows, err := e.Table("tracked_time").
|
for left > 0 {
|
||||||
Select("issue_id, sum(time) as time").
|
var limit = defaultMaxInSize
|
||||||
In("issue_id", ids).
|
if left < limit {
|
||||||
GroupBy("issue_id").
|
limit = left
|
||||||
Rows(new(totalTimesByIssue))
|
}
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer rows.Close()
|
// select issue_id, sum(time) from tracked_time where issue_id in (<issue ids in current page>) group by issue_id
|
||||||
|
rows, err := e.Table("tracked_time").
|
||||||
for rows.Next() {
|
Select("issue_id, sum(time) as time").
|
||||||
var totalTime totalTimesByIssue
|
In("issue_id", ids[:limit]).
|
||||||
err = rows.Scan(&totalTime)
|
GroupBy("issue_id").
|
||||||
|
Rows(new(totalTimesByIssue))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
trackedTimes[totalTime.IssueID] = totalTime.Time
|
|
||||||
|
for rows.Next() {
|
||||||
|
var totalTime totalTimesByIssue
|
||||||
|
err = rows.Scan(&totalTime)
|
||||||
|
if err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
trackedTimes[totalTime.IssueID] = totalTime.Time
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
left = left - limit
|
||||||
|
ids = ids[limit:]
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, issue := range issues {
|
for _, issue := range issues {
|
||||||
|
|
Loading…
Reference in New Issue