Nemcio/Gitea
1
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2025-10-31 02:46:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: decouple context from migration structs (#33399)

Browse Source 
Use context as much as possible.

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
TheFox0x7
2025-02-07 06:37:32 +01:00
committed by GitHub
parent 466cc725bc
commit 1ec8d80fa3
23 changed files with 455 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
modules/migration/downloader.go
View File

@@ -12,18 +12,17 @@ import (
// Downloader downloads the site repo information // Downloader downloads the site repo information
type Downloader interface { type Downloader interface {
SetContext(context.Context) GetRepoInfo(ctx context.Context) (*Repository, error)
GetRepoInfo() (*Repository, error) GetTopics(ctx context.Context) ([]string, error)
GetTopics() ([]string, error) GetMilestones(ctx context.Context) ([]*Milestone, error)
GetMilestones() ([]*Milestone, error) GetReleases(ctx context.Context) ([]*Release, error)
GetReleases() ([]*Release, error) GetLabels(ctx context.Context) ([]*Label, error)
GetLabels() ([]*Label, error) GetIssues(ctx context.Context, page, perPage int) ([]*Issue, bool, error)
GetIssues(page, perPage int) ([]*Issue, bool, error) GetComments(ctx context.Context, commentable Commentable) ([]*Comment, bool, error)
GetComments(commentable Commentable) ([]*Comment, bool, error) GetAllComments(ctx context.Context, page, perPage int) ([]*Comment, bool, error)
GetAllComments(page, perPage int) ([]*Comment, bool, error)
SupportGetRepoComments() bool SupportGetRepoComments() bool
GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) GetPullRequests(ctx context.Context, page, perPage int) ([]*PullRequest, bool, error)
GetReviews(reviewable Reviewable) ([]*Review, error) GetReviews(ctx context.Context, reviewable Reviewable) ([]*Review, error)
FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
} }

23
modules/migration/null_downloader.go
View File

@@ -13,56 +13,53 @@ type NullDownloader struct{}
var _ Downloader = &NullDownloader{} var _ Downloader = &NullDownloader{}
// SetContext set context
func (n NullDownloader) SetContext(_ context.Context) {}
// GetRepoInfo returns a repository information // GetRepoInfo returns a repository information
func (n NullDownloader) GetRepoInfo() (*Repository, error) { func (n NullDownloader) GetRepoInfo(_ context.Context) (*Repository, error) {
return nil, ErrNotSupported{Entity: "RepoInfo"} return nil, ErrNotSupported{Entity: "RepoInfo"}
} }
// GetTopics return repository topics // GetTopics return repository topics
func (n NullDownloader) GetTopics() ([]string, error) { func (n NullDownloader) GetTopics(_ context.Context) ([]string, error) {
return nil, ErrNotSupported{Entity: "Topics"} return nil, ErrNotSupported{Entity: "Topics"}
} }
// GetMilestones returns milestones // GetMilestones returns milestones
func (n NullDownloader) GetMilestones() ([]*Milestone, error) { func (n NullDownloader) GetMilestones(_ context.Context) ([]*Milestone, error) {
return nil, ErrNotSupported{Entity: "Milestones"} return nil, ErrNotSupported{Entity: "Milestones"}
} }
// GetReleases returns releases // GetReleases returns releases
func (n NullDownloader) GetReleases() ([]*Release, error) { func (n NullDownloader) GetReleases(_ context.Context) ([]*Release, error) {
return nil, ErrNotSupported{Entity: "Releases"} return nil, ErrNotSupported{Entity: "Releases"}
} }
// GetLabels returns labels // GetLabels returns labels
func (n NullDownloader) GetLabels() ([]*Label, error) { func (n NullDownloader) GetLabels(_ context.Context) ([]*Label, error) {
return nil, ErrNotSupported{Entity: "Labels"} return nil, ErrNotSupported{Entity: "Labels"}
} }
// GetIssues returns issues according start and limit // GetIssues returns issues according start and limit
func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { func (n NullDownloader) GetIssues(_ context.Context, page, perPage int) ([]*Issue, bool, error) {
return nil, false, ErrNotSupported{Entity: "Issues"} return nil, false, ErrNotSupported{Entity: "Issues"}
} }
// GetComments returns comments of an issue or PR // GetComments returns comments of an issue or PR
func (n NullDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { func (n NullDownloader) GetComments(_ context.Context, commentable Commentable) ([]*Comment, bool, error) {
return nil, false, ErrNotSupported{Entity: "Comments"} return nil, false, ErrNotSupported{Entity: "Comments"}
} }
// GetAllComments returns paginated comments // GetAllComments returns paginated comments
func (n NullDownloader) GetAllComments(page, perPage int) ([]*Comment, bool, error) { func (n NullDownloader) GetAllComments(_ context.Context, page, perPage int) ([]*Comment, bool, error) {
return nil, false, ErrNotSupported{Entity: "AllComments"} return nil, false, ErrNotSupported{Entity: "AllComments"}
} }
// GetPullRequests returns pull requests according page and perPage // GetPullRequests returns pull requests according page and perPage
func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) { func (n NullDownloader) GetPullRequests(_ context.Context, page, perPage int) ([]*PullRequest, bool, error) {
return nil, false, ErrNotSupported{Entity: "PullRequests"} return nil, false, ErrNotSupported{Entity: "PullRequests"}
} }
// GetReviews returns pull requests review // GetReviews returns pull requests review
func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { func (n NullDownloader) GetReviews(_ context.Context, reviewable Reviewable) ([]*Review, error) {
return nil, ErrNotSupported{Entity: "Reviews"} return nil, ErrNotSupported{Entity: "Reviews"}
} }

43
modules/migration/retry_downloader.go
View File

@@ -49,21 +49,15 @@ func (d *RetryDownloader) retry(work func() error) error {
return err return err
} }
// SetContext set context
func (d *RetryDownloader) SetContext(ctx context.Context) {
d.ctx = ctx
d.Downloader.SetContext(ctx)
}
// GetRepoInfo returns a repository information with retry // GetRepoInfo returns a repository information with retry
func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { func (d *RetryDownloader) GetRepoInfo(ctx context.Context) (*Repository, error) {
var ( var (
repo *Repository repo *Repository
err error err error
) )
err = d.retry(func() error { err = d.retry(func() error {
repo, err = d.Downloader.GetRepoInfo() repo, err = d.Downloader.GetRepoInfo(ctx)
return err return err
}) })
@@ -71,14 +65,14 @@ func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
} }
// GetTopics returns a repository's topics with retry // GetTopics returns a repository's topics with retry
func (d *RetryDownloader) GetTopics() ([]string, error) { func (d *RetryDownloader) GetTopics(ctx context.Context) ([]string, error) {
var ( var (
topics []string topics []string
err error err error
) )
err = d.retry(func() error { err = d.retry(func() error {
topics, err = d.Downloader.GetTopics() topics, err = d.Downloader.GetTopics(ctx)
return err return err
}) })
@@ -86,14 +80,14 @@ func (d *RetryDownloader) GetTopics() ([]string, error) {
} }
// GetMilestones returns a repository's milestones with retry // GetMilestones returns a repository's milestones with retry
func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) { func (d *RetryDownloader) GetMilestones(ctx context.Context) ([]*Milestone, error) {
var ( var (
milestones []*Milestone milestones []*Milestone
err error err error
) )
err = d.retry(func() error { err = d.retry(func() error {
milestones, err = d.Downloader.GetMilestones() milestones, err = d.Downloader.GetMilestones(ctx)
return err return err
}) })
@@ -101,14 +95,14 @@ func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) {
} }
// GetReleases returns a repository's releases with retry // GetReleases returns a repository's releases with retry
func (d *RetryDownloader) GetReleases() ([]*Release, error) { func (d *RetryDownloader) GetReleases(ctx context.Context) ([]*Release, error) {
var ( var (
releases []*Release releases []*Release
err error err error
) )
err = d.retry(func() error { err = d.retry(func() error {
releases, err = d.Downloader.GetReleases() releases, err = d.Downloader.GetReleases(ctx)
return err return err
}) })
@@ -116,14 +110,14 @@ func (d *RetryDownloader) GetReleases() ([]*Release, error) {
} }
// GetLabels returns a repository's labels with retry // GetLabels returns a repository's labels with retry
func (d *RetryDownloader) GetLabels() ([]*Label, error) { func (d *RetryDownloader) GetLabels(ctx context.Context) ([]*Label, error) {
var ( var (
labels []*Label labels []*Label
err error err error
) )
err = d.retry(func() error { err = d.retry(func() error {
labels, err = d.Downloader.GetLabels() labels, err = d.Downloader.GetLabels(ctx)
return err return err
}) })
@@ -131,7 +125,7 @@ func (d *RetryDownloader) GetLabels() ([]*Label, error) {
} }
// GetIssues returns a repository's issues with retry // GetIssues returns a repository's issues with retry
func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { func (d *RetryDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*Issue, bool, error) {
var ( var (
issues []*Issue issues []*Issue
isEnd bool isEnd bool
@@ -139,7 +133,7 @@ func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
) )
err = d.retry(func() error { err = d.retry(func() error {
issues, isEnd, err = d.Downloader.GetIssues(page, perPage) issues, isEnd, err = d.Downloader.GetIssues(ctx, page, perPage)
return err return err
}) })
@@ -147,7 +141,7 @@ func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
} }
// GetComments returns a repository's comments with retry // GetComments returns a repository's comments with retry
func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { func (d *RetryDownloader) GetComments(ctx context.Context, commentable Commentable) ([]*Comment, bool, error) {
var ( var (
comments []*Comment comments []*Comment
isEnd bool isEnd bool
@@ -155,7 +149,7 @@ func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool
) )
err = d.retry(func() error { err = d.retry(func() error {
comments, isEnd, err = d.Downloader.GetComments(commentable) comments, isEnd, err = d.Downloader.GetComments(ctx, commentable)
return err return err
}) })
@@ -163,7 +157,7 @@ func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool
} }
// GetPullRequests returns a repository's pull requests with retry // GetPullRequests returns a repository's pull requests with retry
func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) { func (d *RetryDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*PullRequest, bool, error) {
var ( var (
prs []*PullRequest prs []*PullRequest
err error err error
@@ -171,7 +165,7 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo
) )
err = d.retry(func() error { err = d.retry(func() error {
prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage) prs, isEnd, err = d.Downloader.GetPullRequests(ctx, page, perPage)
return err return err
}) })
@@ -179,14 +173,13 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo
} }
// GetReviews returns pull requests reviews // GetReviews returns pull requests reviews
func (d *RetryDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { func (d *RetryDownloader) GetReviews(ctx context.Context, reviewable Reviewable) ([]*Review, error) {
var ( var (
reviews []*Review reviews []*Review
err error err error
) )
err = d.retry(func() error { err = d.retry(func() error {
reviews, err = d.Downloader.GetReviews(reviewable) reviews, err = d.Downloader.GetReviews(ctx, reviewable)
return err return err
}) })

24
modules/migration/uploader.go
View File

@@ -4,20 +4,22 @@
package migration package migration
import "context"
// Uploader uploads all the information of one repository // Uploader uploads all the information of one repository
type Uploader interface { type Uploader interface {
MaxBatchInsertSize(tp string) int MaxBatchInsertSize(tp string) int
CreateRepo(repo *Repository, opts MigrateOptions) error CreateRepo(ctx context.Context, repo *Repository, opts MigrateOptions) error
CreateTopics(topic ...string) error CreateTopics(ctx context.Context, topic ...string) error
CreateMilestones(milestones ...*Milestone) error CreateMilestones(ctx context.Context, milestones ...*Milestone) error
CreateReleases(releases ...*Release) error CreateReleases(ctx context.Context, releases ...*Release) error
SyncTags() error SyncTags(ctx context.Context) error
CreateLabels(labels ...*Label) error CreateLabels(ctx context.Context, labels ...*Label) error
CreateIssues(issues ...*Issue) error CreateIssues(ctx context.Context, issues ...*Issue) error
CreateComments(comments ...*Comment) error CreateComments(ctx context.Context, comments ...*Comment) error
CreatePullRequests(prs ...*PullRequest) error CreatePullRequests(ctx context.Context, prs ...*PullRequest) error
CreateReviews(reviews ...*Review) error CreateReviews(ctx context.Context, reviews ...*Review) error
Rollback() error Rollback() error
Finish() error Finish(ctx context.Context) error
Close() Close()
} }

50
services/migrations/codebase.go
View File

@@ -66,7 +66,6 @@ type codebaseUser struct {
// from Codebase // from Codebase
type CodebaseDownloader struct { type CodebaseDownloader struct {
base.NullDownloader base.NullDownloader
ctx context.Context
client *http.Client client *http.Client
baseURL *url.URL baseURL *url.URL
projectURL *url.URL projectURL *url.URL
@@ -77,17 +76,11 @@ type CodebaseDownloader struct {
commitMap map[string]string commitMap map[string]string
} }
// SetContext set context
func (d *CodebaseDownloader) SetContext(ctx context.Context) {
d.ctx = ctx
}
// NewCodebaseDownloader creates a new downloader // NewCodebaseDownloader creates a new downloader
func NewCodebaseDownloader(ctx context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader { func NewCodebaseDownloader(_ context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader {
baseURL, _ := url.Parse("https://api3.codebasehq.com") baseURL, _ := url.Parse("https://api3.codebasehq.com")
downloader := &CodebaseDownloader{ downloader := &CodebaseDownloader{
ctx: ctx,
baseURL: baseURL, baseURL: baseURL,
projectURL: projectURL, projectURL: projectURL,
project: project, project: project,
@@ -127,7 +120,7 @@ func (d *CodebaseDownloader) FormatCloneURL(opts base.MigrateOptions, remoteAddr
return opts.CloneAddr, nil return opts.CloneAddr, nil
} }
func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]string, result any) error { func (d *CodebaseDownloader) callAPI(ctx context.Context, endpoint string, parameter map[string]string, result any) error {
u, err := d.baseURL.Parse(endpoint) u, err := d.baseURL.Parse(endpoint)
if err != nil { if err != nil {
return err return err
@@ -141,7 +134,7 @@ func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]strin
u.RawQuery = query.Encode() u.RawQuery = query.Encode()
} }
req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil) req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
if err != nil { if err != nil {
return err return err
} }
@@ -158,7 +151,7 @@ func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]strin
// GetRepoInfo returns repository information // GetRepoInfo returns repository information
// https://support.codebasehq.com/kb/projects // https://support.codebasehq.com/kb/projects
func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) { func (d *CodebaseDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) {
var rawRepository struct { var rawRepository struct {
XMLName xml.Name `xml:"repository"` XMLName xml.Name `xml:"repository"`
Name string `xml:"name"` Name string `xml:"name"`
@@ -169,6 +162,7 @@ func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) {
} }
err := d.callAPI( err := d.callAPI(
ctx,
fmt.Sprintf("/%s/%s", d.project, d.repoName), fmt.Sprintf("/%s/%s", d.project, d.repoName),
nil, nil,
&rawRepository, &rawRepository,
@@ -187,7 +181,7 @@ func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) {
// GetMilestones returns milestones // GetMilestones returns milestones
// https://support.codebasehq.com/kb/tickets-and-milestones/milestones // https://support.codebasehq.com/kb/tickets-and-milestones/milestones
func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) { func (d *CodebaseDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) {
var rawMilestones struct { var rawMilestones struct {
XMLName xml.Name `xml:"ticketing-milestone"` XMLName xml.Name `xml:"ticketing-milestone"`
Type string `xml:"type,attr"` Type string `xml:"type,attr"`
@@ -209,6 +203,7 @@ func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) {
} }
err := d.callAPI( err := d.callAPI(
ctx,
fmt.Sprintf("/%s/milestones", d.project), fmt.Sprintf("/%s/milestones", d.project),
nil, nil,
&rawMilestones, &rawMilestones,
@@ -245,7 +240,7 @@ func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) {
// GetLabels returns labels // GetLabels returns labels
// https://support.codebasehq.com/kb/tickets-and-milestones/statuses-priorities-and-categories // https://support.codebasehq.com/kb/tickets-and-milestones/statuses-priorities-and-categories
func (d *CodebaseDownloader) GetLabels() ([]*base.Label, error) { func (d *CodebaseDownloader) GetLabels(ctx context.Context) ([]*base.Label, error) {
var rawTypes struct { var rawTypes struct {
XMLName xml.Name `xml:"ticketing-types"` XMLName xml.Name `xml:"ticketing-types"`
Type string `xml:"type,attr"` Type string `xml:"type,attr"`
@@ -259,6 +254,7 @@ func (d *CodebaseDownloader) GetLabels() ([]*base.Label, error) {
} }
err := d.callAPI( err := d.callAPI(
ctx,
fmt.Sprintf("/%s/tickets/types", d.project), fmt.Sprintf("/%s/tickets/types", d.project),
nil, nil,
&rawTypes, &rawTypes,
@@ -284,7 +280,7 @@ type codebaseIssueContext struct {
// GetIssues returns issues, limits are not supported // GetIssues returns issues, limits are not supported
// https://support.codebasehq.com/kb/tickets-and-milestones // https://support.codebasehq.com/kb/tickets-and-milestones
// https://support.codebasehq.com/kb/tickets-and-milestones/updating-tickets // https://support.codebasehq.com/kb/tickets-and-milestones/updating-tickets
func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { func (d *CodebaseDownloader) GetIssues(ctx context.Context, _, _ int) ([]*base.Issue, bool, error) {
var rawIssues struct { var rawIssues struct {
XMLName xml.Name `xml:"tickets"` XMLName xml.Name `xml:"tickets"`
Type string `xml:"type,attr"` Type string `xml:"type,attr"`
@@ -324,6 +320,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool,
} }
err := d.callAPI( err := d.callAPI(
ctx,
fmt.Sprintf("/%s/tickets", d.project), fmt.Sprintf("/%s/tickets", d.project),
nil, nil,
&rawIssues, &rawIssues,
@@ -358,6 +355,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool,
} `xml:"ticket-note"` } `xml:"ticket-note"`
} }
err := d.callAPI( err := d.callAPI(
ctx,
fmt.Sprintf("/%s/tickets/%d/notes", d.project, issue.TicketID.Value), fmt.Sprintf("/%s/tickets/%d/notes", d.project, issue.TicketID.Value),
nil, nil,
&notes, &notes,
@@ -370,7 +368,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool,
if len(note.Content) == 0 { if len(note.Content) == 0 {
continue continue
} }
poster := d.tryGetUser(note.UserID.Value) poster := d.tryGetUser(ctx, note.UserID.Value)
comments = append(comments, &base.Comment{ comments = append(comments, &base.Comment{
IssueIndex: issue.TicketID.Value, IssueIndex: issue.TicketID.Value,
Index: note.ID.Value, Index: note.ID.Value,
@@ -390,7 +388,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool,
if issue.Status.TreatAsClosed.Value { if issue.Status.TreatAsClosed.Value {
state = "closed" state = "closed"
} }
poster := d.tryGetUser(issue.ReporterID.Value) poster := d.tryGetUser(ctx, issue.ReporterID.Value)
issues = append(issues, &base.Issue{ issues = append(issues, &base.Issue{
Title: issue.Summary, Title: issue.Summary,
Number: issue.TicketID.Value, Number: issue.TicketID.Value,
@@ -419,7 +417,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool,
} }
// GetComments returns comments // GetComments returns comments
func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { func (d *CodebaseDownloader) GetComments(_ context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
context, ok := commentable.GetContext().(codebaseIssueContext) context, ok := commentable.GetContext().(codebaseIssueContext)
if !ok { if !ok {
return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext())
@@ -430,7 +428,7 @@ func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base.
// GetPullRequests returns pull requests // GetPullRequests returns pull requests
// https://support.codebasehq.com/kb/repositories/merge-requests // https://support.codebasehq.com/kb/repositories/merge-requests
func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { func (d *CodebaseDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
var rawMergeRequests struct { var rawMergeRequests struct {
XMLName xml.Name `xml:"merge-requests"` XMLName xml.Name `xml:"merge-requests"`
Type string `xml:"type,attr"` Type string `xml:"type,attr"`
@@ -443,6 +441,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq
} }
err := d.callAPI( err := d.callAPI(
ctx,
fmt.Sprintf("/%s/%s/merge_requests", d.project, d.repoName), fmt.Sprintf("/%s/%s/merge_requests", d.project, d.repoName),
map[string]string{ map[string]string{
"query": `"Target Project" is "` + d.repoName + `"`, "query": `"Target Project" is "` + d.repoName + `"`,
@@ -503,6 +502,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq
} `xml:"comments"` } `xml:"comments"`
} }
err := d.callAPI( err := d.callAPI(
ctx,
fmt.Sprintf("/%s/%s/merge_requests/%d", d.project, d.repoName, mr.ID.Value), fmt.Sprintf("/%s/%s/merge_requests/%d", d.project, d.repoName, mr.ID.Value),
nil, nil,
&rawMergeRequest, &rawMergeRequest,
@@ -531,7 +531,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq
} }
continue continue
} }
poster := d.tryGetUser(comment.UserID.Value) poster := d.tryGetUser(ctx, comment.UserID.Value)
comments = append(comments, &base.Comment{ comments = append(comments, &base.Comment{
IssueIndex: number, IssueIndex: number,
Index: comment.ID.Value, Index: comment.ID.Value,
@@ -547,7 +547,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq
comments = append(comments, &base.Comment{}) comments = append(comments, &base.Comment{})
} }
poster := d.tryGetUser(rawMergeRequest.UserID.Value) poster := d.tryGetUser(ctx, rawMergeRequest.UserID.Value)
pullRequests = append(pullRequests, &base.PullRequest{ pullRequests = append(pullRequests, &base.PullRequest{
Title: rawMergeRequest.Subject, Title: rawMergeRequest.Subject,
@@ -563,12 +563,12 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq
MergedTime: mergedTime, MergedTime: mergedTime,
Head: base.PullRequestBranch{ Head: base.PullRequestBranch{
Ref: rawMergeRequest.SourceRef, Ref: rawMergeRequest.SourceRef,
SHA: d.getHeadCommit(rawMergeRequest.SourceRef), SHA: d.getHeadCommit(ctx, rawMergeRequest.SourceRef),
RepoName: d.repoName, RepoName: d.repoName,
}, },
Base: base.PullRequestBranch{ Base: base.PullRequestBranch{
Ref: rawMergeRequest.TargetRef, Ref: rawMergeRequest.TargetRef,
SHA: d.getHeadCommit(rawMergeRequest.TargetRef), SHA: d.getHeadCommit(ctx, rawMergeRequest.TargetRef),
RepoName: d.repoName, RepoName: d.repoName,
}, },
ForeignIndex: rawMergeRequest.ID.Value, ForeignIndex: rawMergeRequest.ID.Value,
@@ -584,7 +584,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq
return pullRequests, true, nil return pullRequests, true, nil
} }
func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser { func (d *CodebaseDownloader) tryGetUser(ctx context.Context, userID int64) *codebaseUser {
if len(d.userMap) == 0 { if len(d.userMap) == 0 {
var rawUsers struct { var rawUsers struct {
XMLName xml.Name `xml:"users"` XMLName xml.Name `xml:"users"`
@@ -602,6 +602,7 @@ func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser {
} }
err := d.callAPI( err := d.callAPI(
ctx,
"/users", "/users",
nil, nil,
&rawUsers, &rawUsers,
@@ -627,7 +628,7 @@ func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser {
return user return user
} }
func (d *CodebaseDownloader) getHeadCommit(ref string) string { func (d *CodebaseDownloader) getHeadCommit(ctx context.Context, ref string) string {
commitRef, ok := d.commitMap[ref] commitRef, ok := d.commitMap[ref]
if !ok { if !ok {
var rawCommits struct { var rawCommits struct {
@@ -638,6 +639,7 @@ func (d *CodebaseDownloader) getHeadCommit(ref string) string {
} `xml:"commit"` } `xml:"commit"`
} }
err := d.callAPI( err := d.callAPI(
ctx,
fmt.Sprintf("/%s/%s/commits/%s", d.project, d.repoName, ref), fmt.Sprintf("/%s/%s/commits/%s", d.project, d.repoName, ref),
nil, nil,
&rawCommits, &rawCommits,

18
services/migrations/codebase_test.go
View File

@@ -30,9 +30,9 @@ func TestCodebaseDownloadRepo(t *testing.T) {
if cloneUser != "" { if cloneUser != "" {
u.User = url.UserPassword(cloneUser, clonePassword) u.User = url.UserPassword(cloneUser, clonePassword)
} }
ctx := context.Background()
factory := &CodebaseDownloaderFactory{} factory := &CodebaseDownloaderFactory{}
downloader, err := factory.New(context.Background(), base.MigrateOptions{ downloader, err := factory.New(ctx, base.MigrateOptions{
CloneAddr: u.String(), CloneAddr: u.String(),
AuthUsername: apiUser, AuthUsername: apiUser,
AuthPassword: apiPassword, AuthPassword: apiPassword,
@@ -40,7 +40,7 @@ func TestCodebaseDownloadRepo(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Error creating Codebase downloader: %v", err) t.Fatalf("Error creating Codebase downloader: %v", err)
} }
repo, err := downloader.GetRepoInfo() repo, err := downloader.GetRepoInfo(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertRepositoryEqual(t, &base.Repository{ assertRepositoryEqual(t, &base.Repository{
Name: "test", Name: "test",
@@ -50,7 +50,7 @@ func TestCodebaseDownloadRepo(t *testing.T) {
OriginalURL: cloneAddr, OriginalURL: cloneAddr,
}, repo) }, repo)
milestones, err := downloader.GetMilestones() milestones, err := downloader.GetMilestones(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertMilestonesEqual(t, []*base.Milestone{ assertMilestonesEqual(t, []*base.Milestone{
{ {
@@ -65,11 +65,11 @@ func TestCodebaseDownloadRepo(t *testing.T) {
}, },
}, milestones) }, milestones)
labels, err := downloader.GetLabels() labels, err := downloader.GetLabels(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assert.Len(t, labels, 4) assert.Len(t, labels, 4)
issues, isEnd, err := downloader.GetIssues(1, 2) issues, isEnd, err := downloader.GetIssues(ctx, 1, 2)
assert.NoError(t, err) assert.NoError(t, err)
assert.True(t, isEnd) assert.True(t, isEnd)
assertIssuesEqual(t, []*base.Issue{ assertIssuesEqual(t, []*base.Issue{
@@ -106,7 +106,7 @@ func TestCodebaseDownloadRepo(t *testing.T) {
}, },
}, issues) }, issues)
comments, _, err := downloader.GetComments(issues[0]) comments, _, err := downloader.GetComments(ctx, issues[0])
assert.NoError(t, err) assert.NoError(t, err)
assertCommentsEqual(t, []*base.Comment{ assertCommentsEqual(t, []*base.Comment{
{ {
@@ -119,7 +119,7 @@ func TestCodebaseDownloadRepo(t *testing.T) {
}, },
}, comments) }, comments)
prs, _, err := downloader.GetPullRequests(1, 1) prs, _, err := downloader.GetPullRequests(ctx, 1, 1)
assert.NoError(t, err) assert.NoError(t, err)
assertPullRequestsEqual(t, []*base.PullRequest{ assertPullRequestsEqual(t, []*base.PullRequest{
{ {
@@ -144,7 +144,7 @@ func TestCodebaseDownloadRepo(t *testing.T) {
}, },
}, prs) }, prs)
rvs, err := downloader.GetReviews(prs[0]) rvs, err := downloader.GetReviews(ctx, prs[0])
assert.NoError(t, err) assert.NoError(t, err)
assert.Empty(t, rvs) assert.Empty(t, rvs)
} }

27
services/migrations/codecommit.go
View File

@@ -62,9 +62,8 @@ func (c *CodeCommitDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.CodeCommitService return structs.CodeCommitService
} }
func NewCodeCommitDownloader(ctx context.Context, repoName, baseURL, accessKeyID, secretAccessKey, region string) *CodeCommitDownloader { func NewCodeCommitDownloader(_ context.Context, repoName, baseURL, accessKeyID, secretAccessKey, region string) *CodeCommitDownloader {
downloader := CodeCommitDownloader{ downloader := CodeCommitDownloader{
ctx: ctx,
repoName: repoName, repoName: repoName,
baseURL: baseURL, baseURL: baseURL,
codeCommitClient: codecommit.New(codecommit.Options{ codeCommitClient: codecommit.New(codecommit.Options{
@@ -79,21 +78,15 @@ func NewCodeCommitDownloader(ctx context.Context, repoName, baseURL, accessKeyID
// CodeCommitDownloader implements a downloader for AWS CodeCommit // CodeCommitDownloader implements a downloader for AWS CodeCommit
type CodeCommitDownloader struct { type CodeCommitDownloader struct {
base.NullDownloader base.NullDownloader
ctx context.Context
codeCommitClient *codecommit.Client codeCommitClient *codecommit.Client
repoName string repoName string
baseURL string baseURL string
allPullRequestIDs []string allPullRequestIDs []string
} }
// SetContext set context
func (c *CodeCommitDownloader) SetContext(ctx context.Context) {
c.ctx = ctx
}
// GetRepoInfo returns a repository information // GetRepoInfo returns a repository information
func (c *CodeCommitDownloader) GetRepoInfo() (*base.Repository, error) { func (c *CodeCommitDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) {
output, err := c.codeCommitClient.GetRepository(c.ctx, &codecommit.GetRepositoryInput{ output, err := c.codeCommitClient.GetRepository(ctx, &codecommit.GetRepositoryInput{
RepositoryName: util.ToPointer(c.repoName), RepositoryName: util.ToPointer(c.repoName),
}) })
if err != nil { if err != nil {
@@ -117,14 +110,14 @@ func (c *CodeCommitDownloader) GetRepoInfo() (*base.Repository, error) {
} }
// GetComments returns comments of an issue or PR // GetComments returns comments of an issue or PR
func (c *CodeCommitDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { func (c *CodeCommitDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
var ( var (
nextToken *string nextToken *string
comments []*base.Comment comments []*base.Comment
) )
for { for {
resp, err := c.codeCommitClient.GetCommentsForPullRequest(c.ctx, &codecommit.GetCommentsForPullRequestInput{ resp, err := c.codeCommitClient.GetCommentsForPullRequest(ctx, &codecommit.GetCommentsForPullRequestInput{
NextToken: nextToken, NextToken: nextToken,
PullRequestId: util.ToPointer(strconv.FormatInt(commentable.GetForeignIndex(), 10)), PullRequestId: util.ToPointer(strconv.FormatInt(commentable.GetForeignIndex(), 10)),
}) })
@@ -155,8 +148,8 @@ func (c *CodeCommitDownloader) GetComments(commentable base.Commentable) ([]*bas
} }
// GetPullRequests returns pull requests according page and perPage // GetPullRequests returns pull requests according page and perPage
func (c *CodeCommitDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { func (c *CodeCommitDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
allPullRequestIDs, err := c.getAllPullRequestIDs() allPullRequestIDs, err := c.getAllPullRequestIDs(ctx)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }
@@ -170,7 +163,7 @@ func (c *CodeCommitDownloader) GetPullRequests(page, perPage int) ([]*base.PullR
prs := make([]*base.PullRequest, 0, len(batch)) prs := make([]*base.PullRequest, 0, len(batch))
for _, id := range batch { for _, id := range batch {
output, err := c.codeCommitClient.GetPullRequest(c.ctx, &codecommit.GetPullRequestInput{ output, err := c.codeCommitClient.GetPullRequest(ctx, &codecommit.GetPullRequestInput{
PullRequestId: util.ToPointer(id), PullRequestId: util.ToPointer(id),
}) })
if err != nil { if err != nil {
@@ -231,7 +224,7 @@ func (c *CodeCommitDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr st
return u.String(), nil return u.String(), nil
} }
func (c *CodeCommitDownloader) getAllPullRequestIDs() ([]string, error) { func (c *CodeCommitDownloader) getAllPullRequestIDs(ctx context.Context) ([]string, error) {
if len(c.allPullRequestIDs) > 0 { if len(c.allPullRequestIDs) > 0 {
return c.allPullRequestIDs, nil return c.allPullRequestIDs, nil
} }
@@ -242,7 +235,7 @@ func (c *CodeCommitDownloader) getAllPullRequestIDs() ([]string, error) {
) )
for { for {
output, err := c.codeCommitClient.ListPullRequests(c.ctx, &codecommit.ListPullRequestsInput{ output, err := c.codeCommitClient.ListPullRequests(ctx, &codecommit.ListPullRequestsInput{
RepositoryName: util.ToPointer(c.repoName), RepositoryName: util.ToPointer(c.repoName),
NextToken: nextToken, NextToken: nextToken,
}) })

48
services/migrations/dump.go
View File

@@ -32,7 +32,6 @@ var _ base.Uploader = &RepositoryDumper{}
// RepositoryDumper implements an Uploader to the local directory // RepositoryDumper implements an Uploader to the local directory
type RepositoryDumper struct { type RepositoryDumper struct {
ctx context.Context
baseDir string baseDir string
repoOwner string repoOwner string
repoName string repoName string
@@ -56,7 +55,6 @@ func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName strin
return nil, err return nil, err
} }
return &RepositoryDumper{ return &RepositoryDumper{
ctx: ctx,
opts: opts, opts: opts,
baseDir: baseDir, baseDir: baseDir,
repoOwner: repoOwner, repoOwner: repoOwner,
@@ -105,7 +103,7 @@ func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) {
} }
// CreateRepo creates a repository // CreateRepo creates a repository
func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { func (g *RepositoryDumper) CreateRepo(ctx context.Context, repo *base.Repository, opts base.MigrateOptions) error {
f, err := os.Create(filepath.Join(g.baseDir, "repo.yml")) f, err := os.Create(filepath.Join(g.baseDir, "repo.yml"))
if err != nil { if err != nil {
return err return err
@@ -149,7 +147,7 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp
return err return err
} }
err = git.Clone(g.ctx, remoteAddr, repoPath, git.CloneRepoOptions{ err = git.Clone(ctx, remoteAddr, repoPath, git.CloneRepoOptions{
Mirror: true, Mirror: true,
Quiet: true, Quiet: true,
Timeout: migrateTimeout, Timeout: migrateTimeout,
@@ -158,19 +156,19 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp
if err != nil { if err != nil {
return fmt.Errorf("Clone: %w", err) return fmt.Errorf("Clone: %w", err)
} }
if err := git.WriteCommitGraph(g.ctx, repoPath); err != nil { if err := git.WriteCommitGraph(ctx, repoPath); err != nil {
return err return err
} }
if opts.Wiki { if opts.Wiki {
wikiPath := g.wikiPath() wikiPath := g.wikiPath()
wikiRemotePath := repository.WikiRemoteURL(g.ctx, remoteAddr) wikiRemotePath := repository.WikiRemoteURL(ctx, remoteAddr)
if len(wikiRemotePath) > 0 { if len(wikiRemotePath) > 0 {
if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil { if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil {
return fmt.Errorf("Failed to remove %s: %w", wikiPath, err) return fmt.Errorf("Failed to remove %s: %w", wikiPath, err)
} }
if err := git.Clone(g.ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ if err := git.Clone(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{
Mirror: true, Mirror: true,
Quiet: true, Quiet: true,
Timeout: migrateTimeout, Timeout: migrateTimeout,
@@ -181,13 +179,13 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp
if err := os.RemoveAll(wikiPath); err != nil { if err := os.RemoveAll(wikiPath); err != nil {
return fmt.Errorf("Failed to remove %s: %w", wikiPath, err) return fmt.Errorf("Failed to remove %s: %w", wikiPath, err)
} }
} else if err := git.WriteCommitGraph(g.ctx, wikiPath); err != nil { } else if err := git.WriteCommitGraph(ctx, wikiPath); err != nil {
return err return err
} }
} }
} }
g.gitRepo, err = git.OpenRepository(g.ctx, g.gitPath()) g.gitRepo, err = git.OpenRepository(ctx, g.gitPath())
return err return err
} }
@@ -220,7 +218,7 @@ func (g *RepositoryDumper) Close() {
} }
// CreateTopics creates topics // CreateTopics creates topics
func (g *RepositoryDumper) CreateTopics(topics ...string) error { func (g *RepositoryDumper) CreateTopics(_ context.Context, topics ...string) error {
f, err := os.Create(filepath.Join(g.baseDir, "topic.yml")) f, err := os.Create(filepath.Join(g.baseDir, "topic.yml"))
if err != nil { if err != nil {
return err return err
@@ -242,7 +240,7 @@ func (g *RepositoryDumper) CreateTopics(topics ...string) error {
} }
// CreateMilestones creates milestones // CreateMilestones creates milestones
func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error { func (g *RepositoryDumper) CreateMilestones(_ context.Context, milestones ...*base.Milestone) error {
var err error var err error
if g.milestoneFile == nil { if g.milestoneFile == nil {
g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml")) g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml"))
@@ -264,7 +262,7 @@ func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error
} }
// CreateLabels creates labels // CreateLabels creates labels
func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { func (g *RepositoryDumper) CreateLabels(_ context.Context, labels ...*base.Label) error {
var err error var err error
if g.labelFile == nil { if g.labelFile == nil {
g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml")) g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml"))
@@ -286,7 +284,7 @@ func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error {
} }
// CreateReleases creates releases // CreateReleases creates releases
func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { func (g *RepositoryDumper) CreateReleases(_ context.Context, releases ...*base.Release) error {
if g.opts.ReleaseAssets { if g.opts.ReleaseAssets {
for _, release := range releases { for _, release := range releases {
attachDir := filepath.Join("release_assets", release.TagName) attachDir := filepath.Join("release_assets", release.TagName)
@@ -354,12 +352,12 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error {
} }
// SyncTags syncs releases with tags in the database // SyncTags syncs releases with tags in the database
func (g *RepositoryDumper) SyncTags() error { func (g *RepositoryDumper) SyncTags(ctx context.Context) error {
return nil return nil
} }
// CreateIssues creates issues // CreateIssues creates issues
func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error { func (g *RepositoryDumper) CreateIssues(_ context.Context, issues ...*base.Issue) error {
var err error var err error
if g.issueFile == nil { if g.issueFile == nil {
g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml")) g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml"))
@@ -412,7 +410,7 @@ func (g *RepositoryDumper) encodeItems(number int64, items []any, dir string, it
} }
// CreateComments creates comments of issues // CreateComments creates comments of issues
func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { func (g *RepositoryDumper) CreateComments(_ context.Context, comments ...*base.Comment) error {
commentsMap := make(map[int64][]any, len(comments)) commentsMap := make(map[int64][]any, len(comments))
for _, comment := range comments { for _, comment := range comments {
commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment) commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment)
@@ -421,7 +419,7 @@ func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error {
return g.createItems(g.commentDir(), g.commentFiles, commentsMap) return g.createItems(g.commentDir(), g.commentFiles, commentsMap)
} }
func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { func (g *RepositoryDumper) handlePullRequest(ctx context.Context, pr *base.PullRequest) error {
// SECURITY: this pr must have been ensured safe // SECURITY: this pr must have been ensured safe
if !pr.EnsuredSafe { if !pr.EnsuredSafe {
log.Error("PR #%d in %s/%s has not been checked for safety ... We will ignore this.", pr.Number, g.repoOwner, g.repoName) log.Error("PR #%d in %s/%s has not been checked for safety ... We will ignore this.", pr.Number, g.repoOwner, g.repoName)
@@ -490,7 +488,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error {
if pr.Head.CloneURL == "" || pr.Head.Ref == "" { if pr.Head.CloneURL == "" || pr.Head.Ref == "" {
// Set head information if pr.Head.SHA is available // Set head information if pr.Head.SHA is available
if pr.Head.SHA != "" { if pr.Head.SHA != "" {
_, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()}) _, _, err = git.NewCommand(ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()})
if err != nil { if err != nil {
log.Error("PR #%d in %s/%s unable to update-ref for pr HEAD: %v", pr.Number, g.repoOwner, g.repoName, err) log.Error("PR #%d in %s/%s unable to update-ref for pr HEAD: %v", pr.Number, g.repoOwner, g.repoName, err)
} }
@@ -520,7 +518,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error {
if !ok { if !ok {
// Set head information if pr.Head.SHA is available // Set head information if pr.Head.SHA is available
if pr.Head.SHA != "" { if pr.Head.SHA != "" {
_, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()}) _, _, err = git.NewCommand(ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()})
if err != nil { if err != nil {
log.Error("PR #%d in %s/%s unable to update-ref for pr HEAD: %v", pr.Number, g.repoOwner, g.repoName, err) log.Error("PR #%d in %s/%s unable to update-ref for pr HEAD: %v", pr.Number, g.repoOwner, g.repoName, err)
} }
@@ -555,7 +553,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error {
fetchArg = git.BranchPrefix + fetchArg fetchArg = git.BranchPrefix + fetchArg
} }
_, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags").AddDashesAndList(remote, fetchArg).RunStdString(&git.RunOpts{Dir: g.gitPath()}) _, _, err = git.NewCommand(ctx, "fetch", "--no-tags").AddDashesAndList(remote, fetchArg).RunStdString(&git.RunOpts{Dir: g.gitPath()})
if err != nil { if err != nil {
log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
// We need to continue here so that the Head.Ref is reset and we attempt to set the gitref for the PR // We need to continue here so that the Head.Ref is reset and we attempt to set the gitref for the PR
@@ -579,7 +577,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error {
pr.Head.SHA = headSha pr.Head.SHA = headSha
} }
if pr.Head.SHA != "" { if pr.Head.SHA != "" {
_, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()}) _, _, err = git.NewCommand(ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()})
if err != nil { if err != nil {
log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err) log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err)
} }
@@ -589,7 +587,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error {
} }
// CreatePullRequests creates pull requests // CreatePullRequests creates pull requests
func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { func (g *RepositoryDumper) CreatePullRequests(ctx context.Context, prs ...*base.PullRequest) error {
var err error var err error
if g.pullrequestFile == nil { if g.pullrequestFile == nil {
if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil { if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil {
@@ -607,7 +605,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
count := 0 count := 0
for i := 0; i < len(prs); i++ { for i := 0; i < len(prs); i++ {
pr := prs[i] pr := prs[i]
if err := g.handlePullRequest(pr); err != nil { if err := g.handlePullRequest(ctx, pr); err != nil {
log.Error("PR #%d in %s/%s failed - skipping", pr.Number, g.repoOwner, g.repoName, err) log.Error("PR #%d in %s/%s failed - skipping", pr.Number, g.repoOwner, g.repoName, err)
continue continue
} }
@@ -620,7 +618,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
} }
// CreateReviews create pull request reviews // CreateReviews create pull request reviews
func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error { func (g *RepositoryDumper) CreateReviews(_ context.Context, reviews ...*base.Review) error {
reviewsMap := make(map[int64][]any, len(reviews)) reviewsMap := make(map[int64][]any, len(reviews))
for _, review := range reviews { for _, review := range reviews {
reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review) reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review)
@@ -636,7 +634,7 @@ func (g *RepositoryDumper) Rollback() error {
} }
// Finish when migrating succeed, this will update something. // Finish when migrating succeed, this will update something.
func (g *RepositoryDumper) Finish() error { func (g *RepositoryDumper) Finish(_ context.Context) error {
return nil return nil
} }

8
services/migrations/git.go
View File

@@ -28,12 +28,8 @@ func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownl
} }
} }
// SetContext set context
func (g *PlainGitDownloader) SetContext(ctx context.Context) {
}
// GetRepoInfo returns a repository information // GetRepoInfo returns a repository information
func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) { func (g *PlainGitDownloader) GetRepoInfo(_ context.Context) (*base.Repository, error) {
// convert github repo to stand Repo // convert github repo to stand Repo
return &base.Repository{ return &base.Repository{
Owner: g.ownerName, Owner: g.ownerName,
@@ -43,6 +39,6 @@ func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) {
} }
// GetTopics return empty string slice // GetTopics return empty string slice
func (g PlainGitDownloader) GetTopics() ([]string, error) { func (g PlainGitDownloader) GetTopics(_ context.Context) ([]string, error) {
return []string{}, nil return []string{}, nil
} }

35
services/migrations/gitea_downloader.go
View File

@@ -67,7 +67,6 @@ func (f *GiteaDownloaderFactory) GitServiceType() structs.GitServiceType {
// GiteaDownloader implements a Downloader interface to get repository information's // GiteaDownloader implements a Downloader interface to get repository information's
type GiteaDownloader struct { type GiteaDownloader struct {
base.NullDownloader base.NullDownloader
ctx context.Context
client *gitea_sdk.Client client *gitea_sdk.Client
baseURL string baseURL string
repoOwner string repoOwner string
@@ -114,7 +113,6 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
} }
return &GiteaDownloader{ return &GiteaDownloader{
ctx: ctx,
client: giteaClient, client: giteaClient,
baseURL: baseURL, baseURL: baseURL,
repoOwner: path[0], repoOwner: path[0],
@@ -124,11 +122,6 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
}, nil }, nil
} }
// SetContext set context
func (g *GiteaDownloader) SetContext(ctx context.Context) {
g.ctx = ctx
}
// String implements Stringer // String implements Stringer
func (g *GiteaDownloader) String() string { func (g *GiteaDownloader) String() string {
return fmt.Sprintf("migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) return fmt.Sprintf("migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
@@ -142,7 +135,7 @@ func (g *GiteaDownloader) LogString() string {
} }
// GetRepoInfo returns a repository information // GetRepoInfo returns a repository information
func (g *GiteaDownloader) GetRepoInfo() (*base.Repository, error) { func (g *GiteaDownloader) GetRepoInfo(_ context.Context) (*base.Repository, error) {
if g == nil { if g == nil {
return nil, errors.New("error: GiteaDownloader is nil") return nil, errors.New("error: GiteaDownloader is nil")
} }
@@ -164,19 +157,19 @@ func (g *GiteaDownloader) GetRepoInfo() (*base.Repository, error) {
} }
// GetTopics return gitea topics // GetTopics return gitea topics
func (g *GiteaDownloader) GetTopics() ([]string, error) { func (g *GiteaDownloader) GetTopics(_ context.Context) ([]string, error) {
topics, _, err := g.client.ListRepoTopics(g.repoOwner, g.repoName, gitea_sdk.ListRepoTopicsOptions{}) topics, _, err := g.client.ListRepoTopics(g.repoOwner, g.repoName, gitea_sdk.ListRepoTopicsOptions{})
return topics, err return topics, err
} }
// GetMilestones returns milestones // GetMilestones returns milestones
func (g *GiteaDownloader) GetMilestones() ([]*base.Milestone, error) { func (g *GiteaDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) {
milestones := make([]*base.Milestone, 0, g.maxPerPage) milestones := make([]*base.Milestone, 0, g.maxPerPage)
for i := 1; ; i++ { for i := 1; ; i++ {
// make sure gitea can shutdown gracefully // make sure gitea can shutdown gracefully
select { select {
case <-g.ctx.Done(): case <-ctx.Done():
return nil, nil return nil, nil
default: default:
} }
@@ -235,13 +228,13 @@ func (g *GiteaDownloader) convertGiteaLabel(label *gitea_sdk.Label) *base.Label
} }
// GetLabels returns labels // GetLabels returns labels
func (g *GiteaDownloader) GetLabels() ([]*base.Label, error) { func (g *GiteaDownloader) GetLabels(ctx context.Context) ([]*base.Label, error) {
labels := make([]*base.Label, 0, g.maxPerPage) labels := make([]*base.Label, 0, g.maxPerPage)
for i := 1; ; i++ { for i := 1; ; i++ {
// make sure gitea can shutdown gracefully // make sure gitea can shutdown gracefully
select { select {
case <-g.ctx.Done(): case <-ctx.Done():
return nil, nil return nil, nil
default: default:
} }
@@ -323,13 +316,13 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
} }
// GetReleases returns releases // GetReleases returns releases
func (g *GiteaDownloader) GetReleases() ([]*base.Release, error) { func (g *GiteaDownloader) GetReleases(ctx context.Context) ([]*base.Release, error) {
releases := make([]*base.Release, 0, g.maxPerPage) releases := make([]*base.Release, 0, g.maxPerPage)
for i := 1; ; i++ { for i := 1; ; i++ {
// make sure gitea can shutdown gracefully // make sure gitea can shutdown gracefully
select { select {
case <-g.ctx.Done(): case <-ctx.Done():
return nil, nil return nil, nil
default: default:
} }
@@ -395,7 +388,7 @@ func (g *GiteaDownloader) getCommentReactions(commentID int64) ([]*base.Reaction
} }
// GetIssues returns issues according start and limit // GetIssues returns issues according start and limit
func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { func (g *GiteaDownloader) GetIssues(_ context.Context, page, perPage int) ([]*base.Issue, bool, error) {
if perPage > g.maxPerPage { if perPage > g.maxPerPage {
perPage = g.maxPerPage perPage = g.maxPerPage
} }
@@ -458,13 +451,13 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err
} }
// GetComments returns comments according issueNumber // GetComments returns comments according issueNumber
func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { func (g *GiteaDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
allComments := make([]*base.Comment, 0, g.maxPerPage) allComments := make([]*base.Comment, 0, g.maxPerPage)
for i := 1; ; i++ { for i := 1; ; i++ {
// make sure gitea can shutdown gracefully // make sure gitea can shutdown gracefully
select { select {
case <-g.ctx.Done(): case <-ctx.Done():
return nil, false, nil return nil, false, nil
default: default:
} }
@@ -504,7 +497,7 @@ func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Com
} }
// GetPullRequests returns pull requests according page and perPage // GetPullRequests returns pull requests according page and perPage
func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { func (g *GiteaDownloader) GetPullRequests(_ context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
if perPage > g.maxPerPage { if perPage > g.maxPerPage {
perPage = g.maxPerPage perPage = g.maxPerPage
} }
@@ -624,7 +617,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques
} }
// GetReviews returns pull requests review // GetReviews returns pull requests review
func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { func (g *GiteaDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) {
if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil { if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil {
log.Info("GiteaDownloader: instance to old, skip GetReviews") log.Info("GiteaDownloader: instance to old, skip GetReviews")
return nil, nil return nil, nil
@@ -635,7 +628,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review
for i := 1; ; i++ { for i := 1; ; i++ {
// make sure gitea can shutdown gracefully // make sure gitea can shutdown gracefully
select { select {
case <-g.ctx.Done(): case <-ctx.Done():
return nil, nil return nil, nil
default: default:
} }

26
services/migrations/gitea_downloader_test.go
View File

@@ -28,12 +28,12 @@ func TestGiteaDownloadRepo(t *testing.T) {
if err != nil || resp.StatusCode != http.StatusOK { if err != nil || resp.StatusCode != http.StatusOK {
t.Skipf("Can't reach https://gitea.com, skipping %s", t.Name()) t.Skipf("Can't reach https://gitea.com, skipping %s", t.Name())
} }
ctx := context.Background()
downloader, err := NewGiteaDownloader(context.Background(), "https://gitea.com", "gitea/test_repo", "", "", giteaToken) downloader, err := NewGiteaDownloader(ctx, "https://gitea.com", "gitea/test_repo", "", "", giteaToken)
require.NoError(t, err, "NewGiteaDownloader error occur") require.NoError(t, err, "NewGiteaDownloader error occur")
require.NotNil(t, downloader, "NewGiteaDownloader is nil") require.NotNil(t, downloader, "NewGiteaDownloader is nil")
repo, err := downloader.GetRepoInfo() repo, err := downloader.GetRepoInfo(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertRepositoryEqual(t, &base.Repository{ assertRepositoryEqual(t, &base.Repository{
Name: "test_repo", Name: "test_repo",
@@ -45,12 +45,12 @@ func TestGiteaDownloadRepo(t *testing.T) {
DefaultBranch: "master", DefaultBranch: "master",
}, repo) }, repo)
topics, err := downloader.GetTopics() topics, err := downloader.GetTopics(ctx)
assert.NoError(t, err) assert.NoError(t, err)
sort.Strings(topics) sort.Strings(topics)
assert.EqualValues(t, []string{"ci", "gitea", "migration", "test"}, topics) assert.EqualValues(t, []string{"ci", "gitea", "migration", "test"}, topics)
labels, err := downloader.GetLabels() labels, err := downloader.GetLabels(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertLabelsEqual(t, []*base.Label{ assertLabelsEqual(t, []*base.Label{
{ {
@@ -80,7 +80,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
}, },
}, labels) }, labels)
milestones, err := downloader.GetMilestones() milestones, err := downloader.GetMilestones(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertMilestonesEqual(t, []*base.Milestone{ assertMilestonesEqual(t, []*base.Milestone{
{ {
@@ -100,7 +100,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
}, },
}, milestones) }, milestones)
releases, err := downloader.GetReleases() releases, err := downloader.GetReleases(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertReleasesEqual(t, []*base.Release{ assertReleasesEqual(t, []*base.Release{
{ {
@@ -131,13 +131,13 @@ func TestGiteaDownloadRepo(t *testing.T) {
}, },
}, releases) }, releases)
issues, isEnd, err := downloader.GetIssues(1, 50) issues, isEnd, err := downloader.GetIssues(ctx, 1, 50)
assert.NoError(t, err) assert.NoError(t, err)
assert.True(t, isEnd) assert.True(t, isEnd)
assert.Len(t, issues, 7) assert.Len(t, issues, 7)
assert.EqualValues(t, "open", issues[0].State) assert.EqualValues(t, "open", issues[0].State)
issues, isEnd, err = downloader.GetIssues(3, 2) issues, isEnd, err = downloader.GetIssues(ctx, 3, 2)
assert.NoError(t, err) assert.NoError(t, err)
assert.False(t, isEnd) assert.False(t, isEnd)
@@ -194,7 +194,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
}, },
}, issues) }, issues)
comments, _, err := downloader.GetComments(&base.Issue{Number: 4, ForeignIndex: 4}) comments, _, err := downloader.GetComments(ctx, &base.Issue{Number: 4, ForeignIndex: 4})
assert.NoError(t, err) assert.NoError(t, err)
assertCommentsEqual(t, []*base.Comment{ assertCommentsEqual(t, []*base.Comment{
{ {
@@ -217,11 +217,11 @@ func TestGiteaDownloadRepo(t *testing.T) {
}, },
}, comments) }, comments)
prs, isEnd, err := downloader.GetPullRequests(1, 50) prs, isEnd, err := downloader.GetPullRequests(ctx, 1, 50)
assert.NoError(t, err) assert.NoError(t, err)
assert.True(t, isEnd) assert.True(t, isEnd)
assert.Len(t, prs, 6) assert.Len(t, prs, 6)
prs, isEnd, err = downloader.GetPullRequests(1, 3) prs, isEnd, err = downloader.GetPullRequests(ctx, 1, 3)
assert.NoError(t, err) assert.NoError(t, err)
assert.False(t, isEnd) assert.False(t, isEnd)
assert.Len(t, prs, 3) assert.Len(t, prs, 3)
@@ -259,7 +259,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch", PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch",
}, prs[1]) }, prs[1])
reviews, err := downloader.GetReviews(&base.Issue{Number: 7, ForeignIndex: 7}) reviews, err := downloader.GetReviews(ctx, &base.Issue{Number: 7, ForeignIndex: 7})
assert.NoError(t, err) assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{ assertReviewsEqual(t, []*base.Review{
{ {

116
services/migrations/gitea_uploader.go
View File

@@ -41,7 +41,6 @@ var _ base.Uploader = &GiteaLocalUploader{}
// GiteaLocalUploader implements an Uploader to gitea sites // GiteaLocalUploader implements an Uploader to gitea sites
type GiteaLocalUploader struct { type GiteaLocalUploader struct {
ctx context.Context
doer *user_model.User doer *user_model.User
repoOwner string repoOwner string
repoName string repoName string
@@ -58,9 +57,8 @@ type GiteaLocalUploader struct {
} }
// NewGiteaLocalUploader creates an gitea Uploader via gitea API v1 // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
func NewGiteaLocalUploader(ctx context.Context, doer *user_model.User, repoOwner, repoName string) *GiteaLocalUploader { func NewGiteaLocalUploader(_ context.Context, doer *user_model.User, repoOwner, repoName string) *GiteaLocalUploader {
return &GiteaLocalUploader{ return &GiteaLocalUploader{
ctx: ctx,
doer: doer, doer: doer,
repoOwner: repoOwner, repoOwner: repoOwner,
repoName: repoName, repoName: repoName,
@@ -93,15 +91,15 @@ func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int {
} }
// CreateRepo creates a repository // CreateRepo creates a repository
func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { func (g *GiteaLocalUploader) CreateRepo(ctx context.Context, repo *base.Repository, opts base.MigrateOptions) error {
owner, err := user_model.GetUserByName(g.ctx, g.repoOwner) owner, err := user_model.GetUserByName(ctx, g.repoOwner)
if err != nil { if err != nil {
return err return err
} }
var r *repo_model.Repository var r *repo_model.Repository
if opts.MigrateToRepoID <= 0 { if opts.MigrateToRepoID <= 0 {
r, err = repo_service.CreateRepositoryDirectly(g.ctx, g.doer, owner, repo_service.CreateRepoOptions{ r, err = repo_service.CreateRepositoryDirectly(ctx, g.doer, owner, repo_service.CreateRepoOptions{
Name: g.repoName, Name: g.repoName,
Description: repo.Description, Description: repo.Description,
OriginalURL: repo.OriginalURL, OriginalURL: repo.OriginalURL,
@@ -111,7 +109,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
Status: repo_model.RepositoryBeingMigrated, Status: repo_model.RepositoryBeingMigrated,
}) })
} else { } else {
r, err = repo_model.GetRepositoryByID(g.ctx, opts.MigrateToRepoID) r, err = repo_model.GetRepositoryByID(ctx, opts.MigrateToRepoID)
} }
if err != nil { if err != nil {
return err return err
@@ -119,7 +117,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
r.DefaultBranch = repo.DefaultBranch r.DefaultBranch = repo.DefaultBranch
r.Description = repo.Description r.Description = repo.Description
r, err = repo_service.MigrateRepositoryGitData(g.ctx, owner, r, base.MigrateOptions{ r, err = repo_service.MigrateRepositoryGitData(ctx, owner, r, base.MigrateOptions{
RepoName: g.repoName, RepoName: g.repoName,
Description: repo.Description, Description: repo.Description,
OriginalURL: repo.OriginalURL, OriginalURL: repo.OriginalURL,
@@ -139,7 +137,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
if err != nil { if err != nil {
return err return err
} }
g.gitRepo, err = gitrepo.OpenRepository(g.ctx, g.repo) g.gitRepo, err = gitrepo.OpenRepository(ctx, g.repo)
if err != nil { if err != nil {
return err return err
} }
@@ -150,7 +148,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
return err return err
} }
g.repo.ObjectFormatName = objectFormat.Name() g.repo.ObjectFormatName = objectFormat.Name()
return repo_model.UpdateRepositoryCols(g.ctx, g.repo, "object_format_name") return repo_model.UpdateRepositoryCols(ctx, g.repo, "object_format_name")
} }
// Close closes this uploader // Close closes this uploader
@@ -161,7 +159,7 @@ func (g *GiteaLocalUploader) Close() {
} }
// CreateTopics creates topics // CreateTopics creates topics
func (g *GiteaLocalUploader) CreateTopics(topics ...string) error { func (g *GiteaLocalUploader) CreateTopics(ctx context.Context, topics ...string) error {
// Ignore topics too long for the db // Ignore topics too long for the db
c := 0 c := 0
for _, topic := range topics { for _, topic := range topics {
@@ -173,11 +171,11 @@ func (g *GiteaLocalUploader) CreateTopics(topics ...string) error {
c++ c++
} }
topics = topics[:c] topics = topics[:c]
return repo_model.SaveTopics(g.ctx, g.repo.ID, topics...) return repo_model.SaveTopics(ctx, g.repo.ID, topics...)
} }
// CreateMilestones creates milestones // CreateMilestones creates milestones
func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) error { func (g *GiteaLocalUploader) CreateMilestones(ctx context.Context, milestones ...*base.Milestone) error {
mss := make([]*issues_model.Milestone, 0, len(milestones)) mss := make([]*issues_model.Milestone, 0, len(milestones))
for _, milestone := range milestones { for _, milestone := range milestones {
var deadline timeutil.TimeStamp var deadline timeutil.TimeStamp
@@ -216,7 +214,7 @@ func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) err
mss = append(mss, &ms) mss = append(mss, &ms)
} }
err := issues_model.InsertMilestones(g.ctx, mss...) err := issues_model.InsertMilestones(ctx, mss...)
if err != nil { if err != nil {
return err return err
} }
@@ -228,7 +226,7 @@ func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) err
} }
// CreateLabels creates labels // CreateLabels creates labels
func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { func (g *GiteaLocalUploader) CreateLabels(ctx context.Context, labels ...*base.Label) error {
lbs := make([]*issues_model.Label, 0, len(labels)) lbs := make([]*issues_model.Label, 0, len(labels))
for _, l := range labels { for _, l := range labels {
if color, err := label.NormalizeColor(l.Color); err != nil { if color, err := label.NormalizeColor(l.Color); err != nil {
@@ -247,7 +245,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
}) })
} }
err := issues_model.NewLabels(g.ctx, lbs...) err := issues_model.NewLabels(ctx, lbs...)
if err != nil { if err != nil {
return err return err
} }
@@ -258,7 +256,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
} }
// CreateReleases creates releases // CreateReleases creates releases
func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { func (g *GiteaLocalUploader) CreateReleases(ctx context.Context, releases ...*base.Release) error {
rels := make([]*repo_model.Release, 0, len(releases)) rels := make([]*repo_model.Release, 0, len(releases))
for _, release := range releases { for _, release := range releases {
if release.Created.IsZero() { if release.Created.IsZero() {
@@ -292,7 +290,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
CreatedUnix: timeutil.TimeStamp(release.Created.Unix()), CreatedUnix: timeutil.TimeStamp(release.Created.Unix()),
} }
if err := g.remapUser(release, &rel); err != nil { if err := g.remapUser(ctx, release, &rel); err != nil {
return err return err
} }
@@ -361,16 +359,16 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
rels = append(rels, &rel) rels = append(rels, &rel)
} }
return repo_model.InsertReleases(g.ctx, rels...) return repo_model.InsertReleases(ctx, rels...)
} }
// SyncTags syncs releases with tags in the database // SyncTags syncs releases with tags in the database
func (g *GiteaLocalUploader) SyncTags() error { func (g *GiteaLocalUploader) SyncTags(ctx context.Context) error {
return repo_module.SyncReleasesWithTags(g.ctx, g.repo, g.gitRepo) return repo_module.SyncReleasesWithTags(ctx, g.repo, g.gitRepo)
} }
// CreateIssues creates issues // CreateIssues creates issues
func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { func (g *GiteaLocalUploader) CreateIssues(ctx context.Context, issues ...*base.Issue) error {
iss := make([]*issues_model.Issue, 0, len(issues)) iss := make([]*issues_model.Issue, 0, len(issues))
for _, issue := range issues { for _, issue := range issues {
var labels []*issues_model.Label var labels []*issues_model.Label
@@ -419,7 +417,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()), UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()),
} }
if err := g.remapUser(issue, &is); err != nil { if err := g.remapUser(ctx, issue, &is); err != nil {
return err return err
} }
@@ -432,7 +430,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
Type: reaction.Content, Type: reaction.Content,
CreatedUnix: timeutil.TimeStampNow(), CreatedUnix: timeutil.TimeStampNow(),
} }
if err := g.remapUser(reaction, &res); err != nil { if err := g.remapUser(ctx, reaction, &res); err != nil {
return err return err
} }
is.Reactions = append(is.Reactions, &res) is.Reactions = append(is.Reactions, &res)
@@ -441,7 +439,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
} }
if len(iss) > 0 { if len(iss) > 0 {
if err := issues_model.InsertIssues(g.ctx, iss...); err != nil { if err := issues_model.InsertIssues(ctx, iss...); err != nil {
return err return err
} }
@@ -454,7 +452,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
} }
// CreateComments creates comments of issues // CreateComments creates comments of issues
func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { func (g *GiteaLocalUploader) CreateComments(ctx context.Context, comments ...*base.Comment) error {
cms := make([]*issues_model.Comment, 0, len(comments)) cms := make([]*issues_model.Comment, 0, len(comments))
for _, comment := range comments { for _, comment := range comments {
var issue *issues_model.Issue var issue *issues_model.Issue
@@ -513,7 +511,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
default: default:
} }
if err := g.remapUser(comment, &cm); err != nil { if err := g.remapUser(ctx, comment, &cm); err != nil {
return err return err
} }
@@ -523,7 +521,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
Type: reaction.Content, Type: reaction.Content,
CreatedUnix: timeutil.TimeStampNow(), CreatedUnix: timeutil.TimeStampNow(),
} }
if err := g.remapUser(reaction, &res); err != nil { if err := g.remapUser(ctx, reaction, &res); err != nil {
return err return err
} }
cm.Reactions = append(cm.Reactions, &res) cm.Reactions = append(cm.Reactions, &res)
@@ -535,35 +533,35 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
if len(cms) == 0 { if len(cms) == 0 {
return nil return nil
} }
return issues_model.InsertIssueComments(g.ctx, cms) return issues_model.InsertIssueComments(ctx, cms)
} }
// CreatePullRequests creates pull requests // CreatePullRequests creates pull requests
func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error { func (g *GiteaLocalUploader) CreatePullRequests(ctx context.Context, prs ...*base.PullRequest) error {
gprs := make([]*issues_model.PullRequest, 0, len(prs)) gprs := make([]*issues_model.PullRequest, 0, len(prs))
for _, pr := range prs { for _, pr := range prs {
gpr, err := g.newPullRequest(pr) gpr, err := g.newPullRequest(ctx, pr)
if err != nil { if err != nil {
return err return err
} }
if err := g.remapUser(pr, gpr.Issue); err != nil { if err := g.remapUser(ctx, pr, gpr.Issue); err != nil {
return err return err
} }
gprs = append(gprs, gpr) gprs = append(gprs, gpr)
} }
if err := issues_model.InsertPullRequests(g.ctx, gprs...); err != nil { if err := issues_model.InsertPullRequests(ctx, gprs...); err != nil {
return err return err
} }
for _, pr := range gprs { for _, pr := range gprs {
g.issues[pr.Issue.Index] = pr.Issue g.issues[pr.Issue.Index] = pr.Issue
pull.AddToTaskQueue(g.ctx, pr) pull.AddToTaskQueue(ctx, pr)
} }
return nil return nil
} }
func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) { func (g *GiteaLocalUploader) updateGitForPullRequest(ctx context.Context, pr *base.PullRequest) (head string, err error) {
// SECURITY: this pr must have been must have been ensured safe // SECURITY: this pr must have been must have been ensured safe
if !pr.EnsuredSafe { if !pr.EnsuredSafe {
log.Error("PR #%d in %s/%s has not been checked for safety.", pr.Number, g.repoOwner, g.repoName) log.Error("PR #%d in %s/%s has not been checked for safety.", pr.Number, g.repoOwner, g.repoName)
@@ -664,7 +662,7 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head
fetchArg = git.BranchPrefix + fetchArg fetchArg = git.BranchPrefix + fetchArg
} }
_, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags").AddDashesAndList(remote, fetchArg).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) _, _, err = git.NewCommand(ctx, "fetch", "--no-tags").AddDashesAndList(remote, fetchArg).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()})
if err != nil { if err != nil {
log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
return head, nil return head, nil
@@ -683,7 +681,7 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head
pr.Head.SHA = headSha pr.Head.SHA = headSha
} }
_, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) _, _, err = git.NewCommand(ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()})
if err != nil { if err != nil {
return "", err return "", err
} }
@@ -700,13 +698,13 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head
// The SHA is empty // The SHA is empty
log.Warn("Empty reference, no pull head for PR #%d in %s/%s", pr.Number, g.repoOwner, g.repoName) log.Warn("Empty reference, no pull head for PR #%d in %s/%s", pr.Number, g.repoOwner, g.repoName)
} else { } else {
_, _, err = git.NewCommand(g.ctx, "rev-list", "--quiet", "-1").AddDynamicArguments(pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) _, _, err = git.NewCommand(ctx, "rev-list", "--quiet", "-1").AddDynamicArguments(pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()})
if err != nil { if err != nil {
// Git update-ref remove bad references with a relative path // Git update-ref remove bad references with a relative path
log.Warn("Deprecated local head %s for PR #%d in %s/%s, removing %s", pr.Head.SHA, pr.Number, g.repoOwner, g.repoName, pr.GetGitRefName()) log.Warn("Deprecated local head %s for PR #%d in %s/%s, removing %s", pr.Head.SHA, pr.Number, g.repoOwner, g.repoName, pr.GetGitRefName())
} else { } else {
// set head information // set head information
_, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) _, _, err = git.NewCommand(ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()})
if err != nil { if err != nil {
log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err) log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err)
} }
@@ -716,7 +714,7 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head
return head, nil return head, nil
} }
func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model.PullRequest, error) { func (g *GiteaLocalUploader) newPullRequest(ctx context.Context, pr *base.PullRequest) (*issues_model.PullRequest, error) {
var labels []*issues_model.Label var labels []*issues_model.Label
for _, label := range pr.Labels { for _, label := range pr.Labels {
lb, ok := g.labels[label.Name] lb, ok := g.labels[label.Name]
@@ -727,7 +725,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model
milestoneID := g.milestones[pr.Milestone] milestoneID := g.milestones[pr.Milestone]
head, err := g.updateGitForPullRequest(pr) head, err := g.updateGitForPullRequest(ctx, pr)
if err != nil { if err != nil {
return nil, fmt.Errorf("updateGitForPullRequest: %w", err) return nil, fmt.Errorf("updateGitForPullRequest: %w", err)
} }
@@ -779,7 +777,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model
UpdatedUnix: timeutil.TimeStamp(pr.Updated.Unix()), UpdatedUnix: timeutil.TimeStamp(pr.Updated.Unix()),
} }
if err := g.remapUser(pr, &issue); err != nil { if err := g.remapUser(ctx, pr, &issue); err != nil {
return nil, err return nil, err
} }
@@ -789,7 +787,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model
Type: reaction.Content, Type: reaction.Content,
CreatedUnix: timeutil.TimeStampNow(), CreatedUnix: timeutil.TimeStampNow(),
} }
if err := g.remapUser(reaction, &res); err != nil { if err := g.remapUser(ctx, reaction, &res); err != nil {
return nil, err return nil, err
} }
issue.Reactions = append(issue.Reactions, &res) issue.Reactions = append(issue.Reactions, &res)
@@ -839,7 +837,7 @@ func convertReviewState(state string) issues_model.ReviewType {
} }
// CreateReviews create pull request reviews of currently migrated issues // CreateReviews create pull request reviews of currently migrated issues
func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { func (g *GiteaLocalUploader) CreateReviews(ctx context.Context, reviews ...*base.Review) error {
cms := make([]*issues_model.Review, 0, len(reviews)) cms := make([]*issues_model.Review, 0, len(reviews))
for _, review := range reviews { for _, review := range reviews {
var issue *issues_model.Issue var issue *issues_model.Issue
@@ -860,7 +858,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()), UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
} }
if err := g.remapUser(review, &cm); err != nil { if err := g.remapUser(ctx, review, &cm); err != nil {
return err return err
} }
@@ -870,7 +868,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
pr, ok := g.prCache[issue.ID] pr, ok := g.prCache[issue.ID]
if !ok { if !ok {
var err error var err error
pr, err = issues_model.GetPullRequestByIssueIDWithNoAttributes(g.ctx, issue.ID) pr, err = issues_model.GetPullRequestByIssueIDWithNoAttributes(ctx, issue.ID)
if err != nil { if err != nil {
return err return err
} }
@@ -940,7 +938,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
UpdatedUnix: timeutil.TimeStamp(comment.UpdatedAt.Unix()), UpdatedUnix: timeutil.TimeStamp(comment.UpdatedAt.Unix()),
} }
if err := g.remapUser(review, &c); err != nil { if err := g.remapUser(ctx, review, &c); err != nil {
return err return err
} }
@@ -948,7 +946,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
} }
} }
return issues_model.InsertReviews(g.ctx, cms) return issues_model.InsertReviews(ctx, cms)
} }
// Rollback when migrating failed, this will rollback all the changes. // Rollback when migrating failed, this will rollback all the changes.
@@ -962,31 +960,31 @@ func (g *GiteaLocalUploader) Rollback() error {
} }
// Finish when migrating success, this will do some status update things. // Finish when migrating success, this will do some status update things.
func (g *GiteaLocalUploader) Finish() error { func (g *GiteaLocalUploader) Finish(ctx context.Context) error {
if g.repo == nil || g.repo.ID <= 0 { if g.repo == nil || g.repo.ID <= 0 {
return ErrRepoNotCreated return ErrRepoNotCreated
} }
// update issue_index // update issue_index
if err := issues_model.RecalculateIssueIndexForRepo(g.ctx, g.repo.ID); err != nil { if err := issues_model.RecalculateIssueIndexForRepo(ctx, g.repo.ID); err != nil {
return err return err
} }
if err := models.UpdateRepoStats(g.ctx, g.repo.ID); err != nil { if err := models.UpdateRepoStats(ctx, g.repo.ID); err != nil {
return err return err
} }
g.repo.Status = repo_model.RepositoryReady g.repo.Status = repo_model.RepositoryReady
return repo_model.UpdateRepositoryCols(g.ctx, g.repo, "status") return repo_model.UpdateRepositoryCols(ctx, g.repo, "status")
} }
func (g *GiteaLocalUploader) remapUser(source user_model.ExternalUserMigrated, target user_model.ExternalUserRemappable) error { func (g *GiteaLocalUploader) remapUser(ctx context.Context, source user_model.ExternalUserMigrated, target user_model.ExternalUserRemappable) error {
var userID int64 var userID int64
var err error var err error
if g.sameApp { if g.sameApp {
userID, err = g.remapLocalUser(source) userID, err = g.remapLocalUser(ctx, source)
} else { } else {
userID, err = g.remapExternalUser(source) userID, err = g.remapExternalUser(ctx, source)
} }
if err != nil { if err != nil {
return err return err
@@ -998,10 +996,10 @@ func (g *GiteaLocalUploader) remapUser(source user_model.ExternalUserMigrated, t
return target.RemapExternalUser(source.GetExternalName(), source.GetExternalID(), g.doer.ID) return target.RemapExternalUser(source.GetExternalName(), source.GetExternalID(), g.doer.ID)
} }
func (g *GiteaLocalUploader) remapLocalUser(source user_model.ExternalUserMigrated) (int64, error) { func (g *GiteaLocalUploader) remapLocalUser(ctx context.Context, source user_model.ExternalUserMigrated) (int64, error) {
userid, ok := g.userMap[source.GetExternalID()] userid, ok := g.userMap[source.GetExternalID()]
if !ok { if !ok {
name, err := user_model.GetUserNameByID(g.ctx, source.GetExternalID()) name, err := user_model.GetUserNameByID(ctx, source.GetExternalID())
if err != nil { if err != nil {
return 0, err return 0, err
} }
@@ -1016,10 +1014,10 @@ func (g *GiteaLocalUploader) remapLocalUser(source user_model.ExternalUserMigrat
return userid, nil return userid, nil
} }
func (g *GiteaLocalUploader) remapExternalUser(source user_model.ExternalUserMigrated) (userid int64, err error) { func (g *GiteaLocalUploader) remapExternalUser(ctx context.Context, source user_model.ExternalUserMigrated) (userid int64, err error) {
userid, ok := g.userMap[source.GetExternalID()] userid, ok := g.userMap[source.GetExternalID()]
if !ok { if !ok {
userid, err = user_model.GetUserIDByExternalUserID(g.ctx, g.gitServiceType.Name(), fmt.Sprintf("%d", source.GetExternalID())) userid, err = user_model.GetUserIDByExternalUserID(ctx, g.gitServiceType.Name(), fmt.Sprintf("%d", source.GetExternalID()))
if err != nil { if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err) log.Error("GetUserIDByExternalUserID: %v", err)
return 0, err return 0, err

24
services/migrations/gitea_uploader_test.go
View File

@@ -132,8 +132,9 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) {
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
ctx := context.Background()
repoName := "migrated" repoName := "migrated"
uploader := NewGiteaLocalUploader(context.Background(), doer, doer.Name, repoName) uploader := NewGiteaLocalUploader(ctx, doer, doer.Name, repoName)
// call remapLocalUser // call remapLocalUser
uploader.sameApp = true uploader.sameApp = true
@@ -150,7 +151,7 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) {
// //
target := repo_model.Release{} target := repo_model.Release{}
uploader.userMap = make(map[int64]int64) uploader.userMap = make(map[int64]int64)
err := uploader.remapUser(&source, &target) err := uploader.remapUser(ctx, &source, &target)
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, doer.ID, target.GetUserID()) assert.EqualValues(t, doer.ID, target.GetUserID())
@@ -161,7 +162,7 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) {
source.PublisherID = user.ID source.PublisherID = user.ID
target = repo_model.Release{} target = repo_model.Release{}
uploader.userMap = make(map[int64]int64) uploader.userMap = make(map[int64]int64)
err = uploader.remapUser(&source, &target) err = uploader.remapUser(ctx, &source, &target)
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, doer.ID, target.GetUserID()) assert.EqualValues(t, doer.ID, target.GetUserID())
@@ -172,7 +173,7 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) {
source.PublisherName = user.Name source.PublisherName = user.Name
target = repo_model.Release{} target = repo_model.Release{}
uploader.userMap = make(map[int64]int64) uploader.userMap = make(map[int64]int64)
err = uploader.remapUser(&source, &target) err = uploader.remapUser(ctx, &source, &target)
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, user.ID, target.GetUserID()) assert.EqualValues(t, user.ID, target.GetUserID())
} }
@@ -180,9 +181,9 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) {
func TestGiteaUploadRemapExternalUser(t *testing.T) { func TestGiteaUploadRemapExternalUser(t *testing.T) {
unittest.PrepareTestEnv(t) unittest.PrepareTestEnv(t)
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
ctx := context.Background()
repoName := "migrated" repoName := "migrated"
uploader := NewGiteaLocalUploader(context.Background(), doer, doer.Name, repoName) uploader := NewGiteaLocalUploader(ctx, doer, doer.Name, repoName)
uploader.gitServiceType = structs.GiteaService uploader.gitServiceType = structs.GiteaService
// call remapExternalUser // call remapExternalUser
uploader.sameApp = false uploader.sameApp = false
@@ -200,7 +201,7 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) {
// //
uploader.userMap = make(map[int64]int64) uploader.userMap = make(map[int64]int64)
target := repo_model.Release{} target := repo_model.Release{}
err := uploader.remapUser(&source, &target) err := uploader.remapUser(ctx, &source, &target)
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, doer.ID, target.GetUserID()) assert.EqualValues(t, doer.ID, target.GetUserID())
@@ -223,7 +224,7 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) {
// //
uploader.userMap = make(map[int64]int64) uploader.userMap = make(map[int64]int64)
target = repo_model.Release{} target = repo_model.Release{}
err = uploader.remapUser(&source, &target) err = uploader.remapUser(ctx, &source, &target)
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, linkedUser.ID, target.GetUserID()) assert.EqualValues(t, linkedUser.ID, target.GetUserID())
} }
@@ -301,11 +302,12 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
toRepoName := "migrated" toRepoName := "migrated"
uploader := NewGiteaLocalUploader(context.Background(), fromRepoOwner, fromRepoOwner.Name, toRepoName) ctx := context.Background()
uploader := NewGiteaLocalUploader(ctx, fromRepoOwner, fromRepoOwner.Name, toRepoName)
uploader.gitServiceType = structs.GiteaService uploader.gitServiceType = structs.GiteaService
assert.NoError(t, repo_service.Init(context.Background())) assert.NoError(t, repo_service.Init(context.Background()))
assert.NoError(t, uploader.CreateRepo(&base.Repository{ assert.NoError(t, uploader.CreateRepo(ctx, &base.Repository{
Description: "description", Description: "description",
OriginalURL: fromRepo.RepoPath(), OriginalURL: fromRepo.RepoPath(),
CloneURL: fromRepo.RepoPath(), CloneURL: fromRepo.RepoPath(),
@@ -505,7 +507,7 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) {
testCase.pr.EnsuredSafe = true testCase.pr.EnsuredSafe = true
head, err := uploader.updateGitForPullRequest(&testCase.pr) head, err := uploader.updateGitForPullRequest(ctx, &testCase.pr)
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, testCase.head, head) assert.EqualValues(t, testCase.head, head)

133
services/migrations/github.go
View File

@@ -64,7 +64,6 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
// from github via APIv3 // from github via APIv3
type GithubDownloaderV3 struct { type GithubDownloaderV3 struct {
base.NullDownloader base.NullDownloader
ctx context.Context
clients []*github.Client clients []*github.Client
baseURL string baseURL string
repoOwner string repoOwner string
@@ -79,12 +78,11 @@ type GithubDownloaderV3 struct {
} }
// NewGithubDownloaderV3 creates a github Downloader via github v3 API // NewGithubDownloaderV3 creates a github Downloader via github v3 API
func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 { func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
downloader := GithubDownloaderV3{ downloader := GithubDownloaderV3{
userName: userName, userName: userName,
baseURL: baseURL, baseURL: baseURL,
password: password, password: password,
ctx: ctx,
repoOwner: repoOwner, repoOwner: repoOwner,
repoName: repoName, repoName: repoName,
maxPerPage: 100, maxPerPage: 100,
@@ -141,12 +139,7 @@ func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) {
g.rates = append(g.rates, nil) g.rates = append(g.rates, nil)
} }
// SetContext set context func (g *GithubDownloaderV3) waitAndPickClient(ctx context.Context) {
func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
g.ctx = ctx
}
func (g *GithubDownloaderV3) waitAndPickClient() {
var recentIdx int var recentIdx int
var maxRemaining int var maxRemaining int
for i := 0; i < len(g.clients); i++ { for i := 0; i < len(g.clients); i++ {
@@ -160,13 +153,13 @@ func (g *GithubDownloaderV3) waitAndPickClient() {
for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining { for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining {
timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time)) timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time))
select { select {
case <-g.ctx.Done(): case <-ctx.Done():
timer.Stop() timer.Stop()
return return
case <-timer.C: case <-timer.C:
} }
err := g.RefreshRate() err := g.RefreshRate(ctx)
if err != nil { if err != nil {
log.Error("g.getClient().RateLimit.Get: %s", err) log.Error("g.getClient().RateLimit.Get: %s", err)
} }
@@ -174,8 +167,8 @@ func (g *GithubDownloaderV3) waitAndPickClient() {
} }
// RefreshRate update the current rate (doesn't count in rate limit) // RefreshRate update the current rate (doesn't count in rate limit)
func (g *GithubDownloaderV3) RefreshRate() error { func (g *GithubDownloaderV3) RefreshRate(ctx context.Context) error {
rates, _, err := g.getClient().RateLimit.Get(g.ctx) rates, _, err := g.getClient().RateLimit.Get(ctx)
if err != nil { if err != nil {
// if rate limit is not enabled, ignore it // if rate limit is not enabled, ignore it
if strings.Contains(err.Error(), "404") { if strings.Contains(err.Error(), "404") {
@@ -198,9 +191,9 @@ func (g *GithubDownloaderV3) setRate(rate *github.Rate) {
} }
// GetRepoInfo returns a repository information // GetRepoInfo returns a repository information
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { func (g *GithubDownloaderV3) GetRepoInfo(ctx context.Context) (*base.Repository, error) {
g.waitAndPickClient() g.waitAndPickClient(ctx)
gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName) gr, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -219,9 +212,9 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
} }
// GetTopics return github topics // GetTopics return github topics
func (g *GithubDownloaderV3) GetTopics() ([]string, error) { func (g *GithubDownloaderV3) GetTopics(ctx context.Context) ([]string, error) {
g.waitAndPickClient() g.waitAndPickClient(ctx)
r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName) r, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -230,12 +223,12 @@ func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
} }
// GetMilestones returns milestones // GetMilestones returns milestones
func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { func (g *GithubDownloaderV3) GetMilestones(ctx context.Context) ([]*base.Milestone, error) {
perPage := g.maxPerPage perPage := g.maxPerPage
milestones := make([]*base.Milestone, 0, perPage) milestones := make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
g.waitAndPickClient() g.waitAndPickClient(ctx)
ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, ms, resp, err := g.getClient().Issues.ListMilestones(ctx, g.repoOwner, g.repoName,
&github.MilestoneListOptions{ &github.MilestoneListOptions{
State: "all", State: "all",
ListOptions: github.ListOptions{ ListOptions: github.ListOptions{
@@ -279,12 +272,12 @@ func convertGithubLabel(label *github.Label) *base.Label {
} }
// GetLabels returns labels // GetLabels returns labels
func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { func (g *GithubDownloaderV3) GetLabels(ctx context.Context) ([]*base.Label, error) {
perPage := g.maxPerPage perPage := g.maxPerPage
labels := make([]*base.Label, 0, perPage) labels := make([]*base.Label, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
g.waitAndPickClient() g.waitAndPickClient(ctx)
ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, ls, resp, err := g.getClient().Issues.ListLabels(ctx, g.repoOwner, g.repoName,
&github.ListOptions{ &github.ListOptions{
Page: i, Page: i,
PerPage: perPage, PerPage: perPage,
@@ -304,7 +297,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
return labels, nil return labels, nil
} }
func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) *base.Release { func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *github.RepositoryRelease) *base.Release {
// GitHub allows commitish to be a reference. // GitHub allows commitish to be a reference.
// In this case, we need to remove the prefix, i.e. convert "refs/heads/main" to "main". // In this case, we need to remove the prefix, i.e. convert "refs/heads/main" to "main".
targetCommitish := strings.TrimPrefix(rel.GetTargetCommitish(), git.BranchPrefix) targetCommitish := strings.TrimPrefix(rel.GetTargetCommitish(), git.BranchPrefix)
@@ -339,12 +332,12 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
Created: asset.CreatedAt.Time, Created: asset.CreatedAt.Time,
Updated: asset.UpdatedAt.Time, Updated: asset.UpdatedAt.Time,
DownloadFunc: func() (io.ReadCloser, error) { DownloadFunc: func() (io.ReadCloser, error) {
g.waitAndPickClient() g.waitAndPickClient(ctx)
readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil) readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(ctx, g.repoOwner, g.repoName, assetID, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if err := g.RefreshRate(); err != nil { if err := g.RefreshRate(ctx); err != nil {
log.Error("g.getClient().RateLimits: %s", err) log.Error("g.getClient().RateLimits: %s", err)
} }
@@ -364,13 +357,13 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
return io.NopCloser(strings.NewReader(redirectURL)), nil return io.NopCloser(strings.NewReader(redirectURL)), nil
} }
g.waitAndPickClient() g.waitAndPickClient(ctx)
req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil) req, err := http.NewRequestWithContext(ctx, "GET", redirectURL, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
resp, err := httpClient.Do(req) resp, err := httpClient.Do(req)
err1 := g.RefreshRate() err1 := g.RefreshRate(ctx)
if err1 != nil { if err1 != nil {
log.Error("g.RefreshRate(): %s", err1) log.Error("g.RefreshRate(): %s", err1)
} }
@@ -385,12 +378,12 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
} }
// GetReleases returns releases // GetReleases returns releases
func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { func (g *GithubDownloaderV3) GetReleases(ctx context.Context) ([]*base.Release, error) {
perPage := g.maxPerPage perPage := g.maxPerPage
releases := make([]*base.Release, 0, perPage) releases := make([]*base.Release, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
g.waitAndPickClient() g.waitAndPickClient(ctx)
ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, ls, resp, err := g.getClient().Repositories.ListReleases(ctx, g.repoOwner, g.repoName,
&github.ListOptions{ &github.ListOptions{
Page: i, Page: i,
PerPage: perPage, PerPage: perPage,
@@ -401,7 +394,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
g.setRate(&resp.Rate) g.setRate(&resp.Rate)
for _, release := range ls { for _, release := range ls {
releases = append(releases, g.convertGithubRelease(release)) releases = append(releases, g.convertGithubRelease(ctx, release))
} }
if len(ls) < perPage { if len(ls) < perPage {
break break
@@ -411,7 +404,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
} }
// GetIssues returns issues according start and limit // GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) {
if perPage > g.maxPerPage { if perPage > g.maxPerPage {
perPage = g.maxPerPage perPage = g.maxPerPage
} }
@@ -426,8 +419,8 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
} }
allIssues := make([]*base.Issue, 0, perPage) allIssues := make([]*base.Issue, 0, perPage)
g.waitAndPickClient() g.waitAndPickClient(ctx)
issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) issues, resp, err := g.getClient().Issues.ListByRepo(ctx, g.repoOwner, g.repoName, opt)
if err != nil { if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %w", err) return nil, false, fmt.Errorf("error while listing repos: %w", err)
} }
@@ -447,8 +440,8 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
var reactions []*base.Reaction var reactions []*base.Reaction
if !g.SkipReactions { if !g.SkipReactions {
for i := 1; ; i++ { for i := 1; ; i++ {
g.waitAndPickClient() g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{ res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{
Page: i, Page: i,
PerPage: perPage, PerPage: perPage,
}) })
@@ -503,12 +496,12 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool {
} }
// GetComments returns comments according issueNumber // GetComments returns comments according issueNumber
func (g *GithubDownloaderV3) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { func (g *GithubDownloaderV3) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
comments, err := g.getComments(commentable) comments, err := g.getComments(ctx, commentable)
return comments, false, err return comments, false, err
} }
func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.Comment, error) { func (g *GithubDownloaderV3) getComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, error) {
var ( var (
allComments = make([]*base.Comment, 0, g.maxPerPage) allComments = make([]*base.Comment, 0, g.maxPerPage)
created = "created" created = "created"
@@ -522,8 +515,8 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.
}, },
} }
for { for {
g.waitAndPickClient() g.waitAndPickClient(ctx)
comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt) comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt)
if err != nil { if err != nil {
return nil, fmt.Errorf("error while listing repos: %w", err) return nil, fmt.Errorf("error while listing repos: %w", err)
} }
@@ -533,8 +526,8 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.
var reactions []*base.Reaction var reactions []*base.Reaction
if !g.SkipReactions { if !g.SkipReactions {
for i := 1; ; i++ { for i := 1; ; i++ {
g.waitAndPickClient() g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
Page: i, Page: i,
PerPage: g.maxPerPage, PerPage: g.maxPerPage,
}) })
@@ -576,7 +569,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.
} }
// GetAllComments returns repository comments according page and perPageSize // GetAllComments returns repository comments according page and perPageSize
func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) { func (g *GithubDownloaderV3) GetAllComments(ctx context.Context, page, perPage int) ([]*base.Comment, bool, error) {
var ( var (
allComments = make([]*base.Comment, 0, perPage) allComments = make([]*base.Comment, 0, perPage)
created = "created" created = "created"
@@ -594,8 +587,8 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment,
}, },
} }
g.waitAndPickClient() g.waitAndPickClient(ctx)
comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt) comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, 0, opt)
if err != nil { if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %w", err) return nil, false, fmt.Errorf("error while listing repos: %w", err)
} }
@@ -608,8 +601,8 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment,
var reactions []*base.Reaction var reactions []*base.Reaction
if !g.SkipReactions { if !g.SkipReactions {
for i := 1; ; i++ { for i := 1; ; i++ {
g.waitAndPickClient() g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
Page: i, Page: i,
PerPage: g.maxPerPage, PerPage: g.maxPerPage,
}) })
@@ -648,7 +641,7 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment,
} }
// GetPullRequests returns pull requests according page and perPage // GetPullRequests returns pull requests according page and perPage
func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { func (g *GithubDownloaderV3) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
if perPage > g.maxPerPage { if perPage > g.maxPerPage {
perPage = g.maxPerPage perPage = g.maxPerPage
} }
@@ -662,8 +655,8 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
}, },
} }
allPRs := make([]*base.PullRequest, 0, perPage) allPRs := make([]*base.PullRequest, 0, perPage)
g.waitAndPickClient() g.waitAndPickClient(ctx)
prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) prs, resp, err := g.getClient().PullRequests.List(ctx, g.repoOwner, g.repoName, opt)
if err != nil { if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %w", err) return nil, false, fmt.Errorf("error while listing repos: %w", err)
} }
@@ -679,8 +672,8 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
var reactions []*base.Reaction var reactions []*base.Reaction
if !g.SkipReactions { if !g.SkipReactions {
for i := 1; ; i++ { for i := 1; ; i++ {
g.waitAndPickClient() g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{ res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{
Page: i, Page: i,
PerPage: perPage, PerPage: perPage,
}) })
@@ -702,7 +695,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
} }
// download patch and saved as tmp file // download patch and saved as tmp file
g.waitAndPickClient() g.waitAndPickClient(ctx)
allPRs = append(allPRs, &base.PullRequest{ allPRs = append(allPRs, &base.PullRequest{
Title: pr.GetTitle(), Title: pr.GetTitle(),
@@ -759,15 +752,15 @@ func convertGithubReview(r *github.PullRequestReview) *base.Review {
} }
} }
func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullRequestComment) ([]*base.ReviewComment, error) { func (g *GithubDownloaderV3) convertGithubReviewComments(ctx context.Context, cs []*github.PullRequestComment) ([]*base.ReviewComment, error) {
rcs := make([]*base.ReviewComment, 0, len(cs)) rcs := make([]*base.ReviewComment, 0, len(cs))
for _, c := range cs { for _, c := range cs {
// get reactions // get reactions
var reactions []*base.Reaction var reactions []*base.Reaction
if !g.SkipReactions { if !g.SkipReactions {
for i := 1; ; i++ { for i := 1; ; i++ {
g.waitAndPickClient() g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{ res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
Page: i, Page: i,
PerPage: g.maxPerPage, PerPage: g.maxPerPage,
}) })
@@ -806,7 +799,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
} }
// GetReviews returns pull requests review // GetReviews returns pull requests review
func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) {
allReviews := make([]*base.Review, 0, g.maxPerPage) allReviews := make([]*base.Review, 0, g.maxPerPage)
if g.SkipReviews { if g.SkipReviews {
return allReviews, nil return allReviews, nil
@@ -816,8 +809,8 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev
} }
// Get approve/request change reviews // Get approve/request change reviews
for { for {
g.waitAndPickClient() g.waitAndPickClient(ctx)
reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) reviews, resp, err := g.getClient().PullRequests.ListReviews(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
if err != nil { if err != nil {
return nil, fmt.Errorf("error while listing repos: %w", err) return nil, fmt.Errorf("error while listing repos: %w", err)
} }
@@ -830,14 +823,14 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev
PerPage: g.maxPerPage, PerPage: g.maxPerPage,
} }
for { for {
g.waitAndPickClient() g.waitAndPickClient(ctx)
reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2) reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2)
if err != nil { if err != nil {
return nil, fmt.Errorf("error while listing repos: %w", err) return nil, fmt.Errorf("error while listing repos: %w", err)
} }
g.setRate(&resp.Rate) g.setRate(&resp.Rate)
cs, err := g.convertGithubReviewComments(reviewComments) cs, err := g.convertGithubReviewComments(ctx, reviewComments)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -856,8 +849,8 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev
} }
// Get requested reviews // Get requested reviews
for { for {
g.waitAndPickClient() g.waitAndPickClient(ctx)
reviewers, resp, err := g.getClient().PullRequests.ListReviewers(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) reviewers, resp, err := g.getClient().PullRequests.ListReviewers(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
if err != nil { if err != nil {
return nil, fmt.Errorf("error while listing repos: %w", err) return nil, fmt.Errorf("error while listing repos: %w", err)
} }

25
services/migrations/github_test.go
View File

@@ -21,11 +21,12 @@ func TestGitHubDownloadRepo(t *testing.T) {
if token == "" { if token == "" {
t.Skip("Skipping GitHub migration test because GITHUB_READ_TOKEN is empty") t.Skip("Skipping GitHub migration test because GITHUB_READ_TOKEN is empty")
} }
downloader := NewGithubDownloaderV3(context.Background(), "https://github.com", "", "", token, "go-gitea", "test_repo") ctx := context.Background()
err := downloader.RefreshRate() downloader := NewGithubDownloaderV3(ctx, "https://github.com", "", "", token, "go-gitea", "test_repo")
err := downloader.RefreshRate(ctx)
assert.NoError(t, err) assert.NoError(t, err)
repo, err := downloader.GetRepoInfo() repo, err := downloader.GetRepoInfo(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertRepositoryEqual(t, &base.Repository{ assertRepositoryEqual(t, &base.Repository{
Name: "test_repo", Name: "test_repo",
@@ -36,11 +37,11 @@ func TestGitHubDownloadRepo(t *testing.T) {
DefaultBranch: "master", DefaultBranch: "master",
}, repo) }, repo)
topics, err := downloader.GetTopics() topics, err := downloader.GetTopics(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assert.Contains(t, topics, "gitea") assert.Contains(t, topics, "gitea")
milestones, err := downloader.GetMilestones() milestones, err := downloader.GetMilestones(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertMilestonesEqual(t, []*base.Milestone{ assertMilestonesEqual(t, []*base.Milestone{
{ {
@@ -63,7 +64,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, },
}, milestones) }, milestones)
labels, err := downloader.GetLabels() labels, err := downloader.GetLabels(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertLabelsEqual(t, []*base.Label{ assertLabelsEqual(t, []*base.Label{
{ {
@@ -113,7 +114,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, },
}, labels) }, labels)
releases, err := downloader.GetReleases() releases, err := downloader.GetReleases(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertReleasesEqual(t, []*base.Release{ assertReleasesEqual(t, []*base.Release{
{ {
@@ -129,7 +130,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, releases) }, releases)
// downloader.GetIssues() // downloader.GetIssues()
issues, isEnd, err := downloader.GetIssues(1, 2) issues, isEnd, err := downloader.GetIssues(ctx, 1, 2)
assert.NoError(t, err) assert.NoError(t, err)
assert.False(t, isEnd) assert.False(t, isEnd)
assertIssuesEqual(t, []*base.Issue{ assertIssuesEqual(t, []*base.Issue{
@@ -218,7 +219,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, issues) }, issues)
// downloader.GetComments() // downloader.GetComments()
comments, _, err := downloader.GetComments(&base.Issue{Number: 2, ForeignIndex: 2}) comments, _, err := downloader.GetComments(ctx, &base.Issue{Number: 2, ForeignIndex: 2})
assert.NoError(t, err) assert.NoError(t, err)
assertCommentsEqual(t, []*base.Comment{ assertCommentsEqual(t, []*base.Comment{
{ {
@@ -248,7 +249,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, comments) }, comments)
// downloader.GetPullRequests() // downloader.GetPullRequests()
prs, _, err := downloader.GetPullRequests(1, 2) prs, _, err := downloader.GetPullRequests(ctx, 1, 2)
assert.NoError(t, err) assert.NoError(t, err)
assertPullRequestsEqual(t, []*base.PullRequest{ assertPullRequestsEqual(t, []*base.PullRequest{
{ {
@@ -338,7 +339,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, },
}, prs) }, prs)
reviews, err := downloader.GetReviews(&base.PullRequest{Number: 3, ForeignIndex: 3}) reviews, err := downloader.GetReviews(ctx, &base.PullRequest{Number: 3, ForeignIndex: 3})
assert.NoError(t, err) assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{ assertReviewsEqual(t, []*base.Review{
{ {
@@ -370,7 +371,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, },
}, reviews) }, reviews)
reviews, err = downloader.GetReviews(&base.PullRequest{Number: 4, ForeignIndex: 4}) reviews, err = downloader.GetReviews(ctx, &base.PullRequest{Number: 4, ForeignIndex: 4})
assert.NoError(t, err) assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{ assertReviewsEqual(t, []*base.Review{
{ {

61
services/migrations/gitlab.go
View File

@@ -80,7 +80,6 @@ func (r *gitlabIIDResolver) generatePullRequestNumber(mrIID int) int64 {
// because Gitlab has individual Issue and Pull Request numbers. // because Gitlab has individual Issue and Pull Request numbers.
type GitlabDownloader struct { type GitlabDownloader struct {
base.NullDownloader base.NullDownloader
ctx context.Context
client *gitlab.Client client *gitlab.Client
baseURL string baseURL string
repoID int repoID int
@@ -143,7 +142,6 @@ func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, passw
} }
return &GitlabDownloader{ return &GitlabDownloader{
ctx: ctx,
client: gitlabClient, client: gitlabClient,
baseURL: baseURL, baseURL: baseURL,
repoID: gr.ID, repoID: gr.ID,
@@ -164,14 +162,9 @@ func (g *GitlabDownloader) LogString() string {
return fmt.Sprintf("<GitlabDownloader %s [%d]/%s>", g.baseURL, g.repoID, g.repoName) return fmt.Sprintf("<GitlabDownloader %s [%d]/%s>", g.baseURL, g.repoID, g.repoName)
} }
// SetContext set context
func (g *GitlabDownloader) SetContext(ctx context.Context) {
g.ctx = ctx
}
// GetRepoInfo returns a repository information // GetRepoInfo returns a repository information
func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) { func (g *GitlabDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) {
gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx)) gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(ctx))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -207,8 +200,8 @@ func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) {
} }
// GetTopics return gitlab topics // GetTopics return gitlab topics
func (g *GitlabDownloader) GetTopics() ([]string, error) { func (g *GitlabDownloader) GetTopics(ctx context.Context) ([]string, error) {
gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx)) gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(ctx))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -216,7 +209,7 @@ func (g *GitlabDownloader) GetTopics() ([]string, error) {
} }
// GetMilestones returns milestones // GetMilestones returns milestones
func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) { func (g *GitlabDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) {
perPage := g.maxPerPage perPage := g.maxPerPage
state := "all" state := "all"
milestones := make([]*base.Milestone, 0, perPage) milestones := make([]*base.Milestone, 0, perPage)
@@ -227,7 +220,7 @@ func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
Page: i, Page: i,
PerPage: perPage, PerPage: perPage,
}, },
}, nil, gitlab.WithContext(g.ctx)) }, nil, gitlab.WithContext(ctx))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -288,14 +281,14 @@ func (g *GitlabDownloader) normalizeColor(val string) string {
} }
// GetLabels returns labels // GetLabels returns labels
func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) { func (g *GitlabDownloader) GetLabels(ctx context.Context) ([]*base.Label, error) {
perPage := g.maxPerPage perPage := g.maxPerPage
labels := make([]*base.Label, 0, perPage) labels := make([]*base.Label, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{ ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{
Page: i, Page: i,
PerPage: perPage, PerPage: perPage,
}}, nil, gitlab.WithContext(g.ctx)) }}, nil, gitlab.WithContext(ctx))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -314,7 +307,7 @@ func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
return labels, nil return labels, nil
} }
func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Release { func (g *GitlabDownloader) convertGitlabRelease(ctx context.Context, rel *gitlab.Release) *base.Release {
var zero int var zero int
r := &base.Release{ r := &base.Release{
TagName: rel.TagName, TagName: rel.TagName,
@@ -337,7 +330,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
Size: &zero, Size: &zero,
DownloadCount: &zero, DownloadCount: &zero,
DownloadFunc: func() (io.ReadCloser, error) { DownloadFunc: func() (io.ReadCloser, error) {
link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, assetID, gitlab.WithContext(g.ctx)) link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, assetID, gitlab.WithContext(ctx))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -351,7 +344,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
if err != nil { if err != nil {
return nil, err return nil, err
} }
req = req.WithContext(g.ctx) req = req.WithContext(ctx)
resp, err := httpClient.Do(req) resp, err := httpClient.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -366,7 +359,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
} }
// GetReleases returns releases // GetReleases returns releases
func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { func (g *GitlabDownloader) GetReleases(ctx context.Context) ([]*base.Release, error) {
perPage := g.maxPerPage perPage := g.maxPerPage
releases := make([]*base.Release, 0, perPage) releases := make([]*base.Release, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
@@ -375,13 +368,13 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
Page: i, Page: i,
PerPage: perPage, PerPage: perPage,
}, },
}, nil, gitlab.WithContext(g.ctx)) }, nil, gitlab.WithContext(ctx))
if err != nil { if err != nil {
return nil, err return nil, err
} }
for _, release := range ls { for _, release := range ls {
releases = append(releases, g.convertGitlabRelease(release)) releases = append(releases, g.convertGitlabRelease(ctx, release))
} }
if len(ls) < perPage { if len(ls) < perPage {
break break
@@ -397,7 +390,7 @@ type gitlabIssueContext struct {
// GetIssues returns issues according start and limit // GetIssues returns issues according start and limit
// //
// Note: issue label description and colors are not supported by the go-gitlab library at this time // Note: issue label description and colors are not supported by the go-gitlab library at this time
func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { func (g *GitlabDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) {
state := "all" state := "all"
sort := "asc" sort := "asc"
@@ -416,7 +409,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
allIssues := make([]*base.Issue, 0, perPage) allIssues := make([]*base.Issue, 0, perPage)
issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil, gitlab.WithContext(g.ctx)) issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil, gitlab.WithContext(ctx))
if err != nil { if err != nil {
return nil, false, fmt.Errorf("error while listing issues: %w", err) return nil, false, fmt.Errorf("error while listing issues: %w", err)
} }
@@ -436,7 +429,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
var reactions []*gitlab.AwardEmoji var reactions []*gitlab.AwardEmoji
awardPage := 1 awardPage := 1
for { for {
awards, _, err := g.client.AwardEmoji.ListIssueAwardEmoji(g.repoID, issue.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx)) awards, _, err := g.client.AwardEmoji.ListIssueAwardEmoji(g.repoID, issue.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(ctx))
if err != nil { if err != nil {
return nil, false, fmt.Errorf("error while listing issue awards: %w", err) return nil, false, fmt.Errorf("error while listing issue awards: %w", err)
} }
@@ -477,7 +470,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
// GetComments returns comments according issueNumber // GetComments returns comments according issueNumber
// TODO: figure out how to transfer comment reactions // TODO: figure out how to transfer comment reactions
func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { func (g *GitlabDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
context, ok := commentable.GetContext().(gitlabIssueContext) context, ok := commentable.GetContext().(gitlabIssueContext)
if !ok { if !ok {
return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext())
@@ -495,12 +488,12 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co
comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListIssueDiscussionsOptions{ comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListIssueDiscussionsOptions{
Page: page, Page: page,
PerPage: g.maxPerPage, PerPage: g.maxPerPage,
}, nil, gitlab.WithContext(g.ctx)) }, nil, gitlab.WithContext(ctx))
} else { } else {
comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListMergeRequestDiscussionsOptions{ comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListMergeRequestDiscussionsOptions{
Page: page, Page: page,
PerPage: g.maxPerPage, PerPage: g.maxPerPage,
}, nil, gitlab.WithContext(g.ctx)) }, nil, gitlab.WithContext(ctx))
} }
if err != nil { if err != nil {
@@ -528,14 +521,14 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co
Page: page, Page: page,
PerPage: g.maxPerPage, PerPage: g.maxPerPage,
}, },
}, nil, gitlab.WithContext(g.ctx)) }, nil, gitlab.WithContext(ctx))
} else { } else {
stateEvents, resp, err = g.client.ResourceStateEvents.ListIssueStateEvents(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListStateEventsOptions{ stateEvents, resp, err = g.client.ResourceStateEvents.ListIssueStateEvents(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListStateEventsOptions{
ListOptions: gitlab.ListOptions{ ListOptions: gitlab.ListOptions{
Page: page, Page: page,
PerPage: g.maxPerPage, PerPage: g.maxPerPage,
}, },
}, nil, gitlab.WithContext(g.ctx)) }, nil, gitlab.WithContext(ctx))
} }
if err != nil { if err != nil {
return nil, false, fmt.Errorf("error while listing state events: %v %w", g.repoID, err) return nil, false, fmt.Errorf("error while listing state events: %v %w", g.repoID, err)
@@ -604,7 +597,7 @@ func (g *GitlabDownloader) convertNoteToComment(localIndex int64, note *gitlab.N
} }
// GetPullRequests returns pull requests according page and perPage // GetPullRequests returns pull requests according page and perPage
func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { func (g *GitlabDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
if perPage > g.maxPerPage { if perPage > g.maxPerPage {
perPage = g.maxPerPage perPage = g.maxPerPage
} }
@@ -620,7 +613,7 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque
allPRs := make([]*base.PullRequest, 0, perPage) allPRs := make([]*base.PullRequest, 0, perPage)
prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil, gitlab.WithContext(g.ctx)) prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil, gitlab.WithContext(ctx))
if err != nil { if err != nil {
return nil, false, fmt.Errorf("error while listing merge requests: %w", err) return nil, false, fmt.Errorf("error while listing merge requests: %w", err)
} }
@@ -673,7 +666,7 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque
var reactions []*gitlab.AwardEmoji var reactions []*gitlab.AwardEmoji
awardPage := 1 awardPage := 1
for { for {
awards, _, err := g.client.AwardEmoji.ListMergeRequestAwardEmoji(g.repoID, pr.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx)) awards, _, err := g.client.AwardEmoji.ListMergeRequestAwardEmoji(g.repoID, pr.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(ctx))
if err != nil { if err != nil {
return nil, false, fmt.Errorf("error while listing merge requests awards: %w", err) return nil, false, fmt.Errorf("error while listing merge requests awards: %w", err)
} }
@@ -733,8 +726,8 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque
} }
// GetReviews returns pull requests review // GetReviews returns pull requests review
func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { func (g *GitlabDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) {
approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(reviewable.GetForeignIndex()), gitlab.WithContext(g.ctx)) approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(reviewable.GetForeignIndex()), gitlab.WithContext(ctx))
if err != nil { if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound { if resp != nil && resp.StatusCode == http.StatusNotFound {
log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error())) log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error()))

29
services/migrations/gitlab_test.go
View File

@@ -31,12 +31,12 @@ func TestGitlabDownloadRepo(t *testing.T) {
if err != nil || resp.StatusCode != http.StatusOK { if err != nil || resp.StatusCode != http.StatusOK {
t.Skipf("Can't access test repo, skipping %s", t.Name()) t.Skipf("Can't access test repo, skipping %s", t.Name())
} }
ctx := context.Background()
downloader, err := NewGitlabDownloader(context.Background(), "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken) downloader, err := NewGitlabDownloader(ctx, "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken)
if err != nil { if err != nil {
t.Fatalf("NewGitlabDownloader is nil: %v", err) t.Fatalf("NewGitlabDownloader is nil: %v", err)
} }
repo, err := downloader.GetRepoInfo() repo, err := downloader.GetRepoInfo(ctx)
assert.NoError(t, err) assert.NoError(t, err)
// Repo Owner is blank in Gitlab Group repos // Repo Owner is blank in Gitlab Group repos
assertRepositoryEqual(t, &base.Repository{ assertRepositoryEqual(t, &base.Repository{
@@ -48,12 +48,12 @@ func TestGitlabDownloadRepo(t *testing.T) {
DefaultBranch: "master", DefaultBranch: "master",
}, repo) }, repo)
topics, err := downloader.GetTopics() topics, err := downloader.GetTopics(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assert.Len(t, topics, 2) assert.Len(t, topics, 2)
assert.EqualValues(t, []string{"migration", "test"}, topics) assert.EqualValues(t, []string{"migration", "test"}, topics)
milestones, err := downloader.GetMilestones() milestones, err := downloader.GetMilestones(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertMilestonesEqual(t, []*base.Milestone{ assertMilestonesEqual(t, []*base.Milestone{
{ {
@@ -71,7 +71,7 @@ func TestGitlabDownloadRepo(t *testing.T) {
}, },
}, milestones) }, milestones)
labels, err := downloader.GetLabels() labels, err := downloader.GetLabels(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertLabelsEqual(t, []*base.Label{ assertLabelsEqual(t, []*base.Label{
{ {
@@ -112,7 +112,7 @@ func TestGitlabDownloadRepo(t *testing.T) {
}, },
}, labels) }, labels)
releases, err := downloader.GetReleases() releases, err := downloader.GetReleases(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertReleasesEqual(t, []*base.Release{ assertReleasesEqual(t, []*base.Release{
{ {
@@ -126,7 +126,7 @@ func TestGitlabDownloadRepo(t *testing.T) {
}, },
}, releases) }, releases)
issues, isEnd, err := downloader.GetIssues(1, 2) issues, isEnd, err := downloader.GetIssues(ctx, 1, 2)
assert.NoError(t, err) assert.NoError(t, err)
assert.False(t, isEnd) assert.False(t, isEnd)
@@ -214,7 +214,7 @@ func TestGitlabDownloadRepo(t *testing.T) {
}, },
}, issues) }, issues)
comments, _, err := downloader.GetComments(&base.Issue{ comments, _, err := downloader.GetComments(ctx, &base.Issue{
Number: 2, Number: 2,
ForeignIndex: 2, ForeignIndex: 2,
Context: gitlabIssueContext{IsMergeRequest: false}, Context: gitlabIssueContext{IsMergeRequest: false},
@@ -255,7 +255,7 @@ func TestGitlabDownloadRepo(t *testing.T) {
}, },
}, comments) }, comments)
prs, _, err := downloader.GetPullRequests(1, 1) prs, _, err := downloader.GetPullRequests(ctx, 1, 1)
assert.NoError(t, err) assert.NoError(t, err)
assertPullRequestsEqual(t, []*base.PullRequest{ assertPullRequestsEqual(t, []*base.PullRequest{
{ {
@@ -304,7 +304,7 @@ func TestGitlabDownloadRepo(t *testing.T) {
}, },
}, prs) }, prs)
rvs, err := downloader.GetReviews(&base.PullRequest{Number: 1, ForeignIndex: 1}) rvs, err := downloader.GetReviews(ctx, &base.PullRequest{Number: 1, ForeignIndex: 1})
assert.NoError(t, err) assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{ assertReviewsEqual(t, []*base.Review{
{ {
@@ -323,7 +323,7 @@ func TestGitlabDownloadRepo(t *testing.T) {
}, },
}, rvs) }, rvs)
rvs, err = downloader.GetReviews(&base.PullRequest{Number: 2, ForeignIndex: 2}) rvs, err = downloader.GetReviews(ctx, &base.PullRequest{Number: 2, ForeignIndex: 2})
assert.NoError(t, err) assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{ assertReviewsEqual(t, []*base.Review{
{ {
@@ -423,9 +423,8 @@ func TestGitlabGetReviews(t *testing.T) {
defer gitlabClientMockTeardown(server) defer gitlabClientMockTeardown(server)
repoID := 1324 repoID := 1324
ctx := context.Background()
downloader := &GitlabDownloader{ downloader := &GitlabDownloader{
ctx: context.Background(),
client: client, client: client,
repoID: repoID, repoID: repoID,
} }
@@ -465,7 +464,7 @@ func TestGitlabGetReviews(t *testing.T) {
mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/merge_requests/%d/approvals", testCase.repoID, testCase.prID), mock) mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/merge_requests/%d/approvals", testCase.repoID, testCase.prID), mock)
id := int64(testCase.prID) id := int64(testCase.prID)
rvs, err := downloader.GetReviews(&base.Issue{Number: id, ForeignIndex: id}) rvs, err := downloader.GetReviews(ctx, &base.Issue{Number: id, ForeignIndex: id})
assert.NoError(t, err) assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{&review}, rvs) assertReviewsEqual(t, []*base.Review{&review}, rvs)
} }

85
services/migrations/gogs.go
View File

@@ -13,7 +13,6 @@ import (
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration" base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
"github.com/gogs/go-gogs-client" "github.com/gogs/go-gogs-client"
@@ -60,16 +59,14 @@ func (f *GogsDownloaderFactory) GitServiceType() structs.GitServiceType {
// from gogs via API // from gogs via API
type GogsDownloader struct { type GogsDownloader struct {
base.NullDownloader base.NullDownloader
ctx context.Context
client *gogs.Client
baseURL string baseURL string
repoOwner string repoOwner string
repoName string repoName string
userName string userName string
password string password string
token string
openIssuesFinished bool openIssuesFinished bool
openIssuesPages int openIssuesPages int
transport http.RoundTripper
} }
// String implements Stringer // String implements Stringer
@@ -84,53 +81,45 @@ func (g *GogsDownloader) LogString() string {
return fmt.Sprintf("<GogsDownloader %s %s/%s>", g.baseURL, g.repoOwner, g.repoName) return fmt.Sprintf("<GogsDownloader %s %s/%s>", g.baseURL, g.repoOwner, g.repoName)
} }
// SetContext set context
func (g *GogsDownloader) SetContext(ctx context.Context) {
g.ctx = ctx
}
// NewGogsDownloader creates a gogs Downloader via gogs API // NewGogsDownloader creates a gogs Downloader via gogs API
func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader { func NewGogsDownloader(_ context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader {
downloader := GogsDownloader{ downloader := GogsDownloader{
ctx: ctx,
baseURL: baseURL, baseURL: baseURL,
userName: userName, userName: userName,
password: password, password: password,
token: token,
repoOwner: repoOwner, repoOwner: repoOwner,
repoName: repoName, repoName: repoName,
} }
var client *gogs.Client
if len(token) != 0 {
client = gogs.NewClient(baseURL, token)
downloader.userName = token
} else {
transport := NewMigrationHTTPTransport()
transport.Proxy = func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return proxy.Proxy()(req)
}
downloader.transport = transport
client = gogs.NewClient(baseURL, "")
client.SetHTTPClient(&http.Client{
Transport: &downloader,
})
}
downloader.client = client
return &downloader return &downloader
} }
// RoundTrip wraps the provided request within this downloader's context and passes it to our internal http.Transport. type roundTripperFunc func(req *http.Request) (*http.Response, error)
// This implements http.RoundTripper and makes the gogs client requests cancellable even though it is not cancellable itself
func (g *GogsDownloader) RoundTrip(req *http.Request) (*http.Response, error) { func (rt roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return g.transport.RoundTrip(req.WithContext(g.ctx)) return rt(r)
}
func (g *GogsDownloader) client(ctx context.Context) *gogs.Client {
// Gogs client lacks the context support, so we use a custom transport
// Then each request uses a dedicated client with its own context
httpTransport := NewMigrationHTTPTransport()
gogsClient := gogs.NewClient(g.baseURL, g.token)
gogsClient.SetHTTPClient(&http.Client{
Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
if g.password != "" {
// Gogs client lacks the support for basic auth, this is the only way to set it
req.SetBasicAuth(g.userName, g.password)
}
return httpTransport.RoundTrip(req.WithContext(ctx))
}),
})
return gogsClient
} }
// GetRepoInfo returns a repository information // GetRepoInfo returns a repository information
func (g *GogsDownloader) GetRepoInfo() (*base.Repository, error) { func (g *GogsDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) {
gr, err := g.client.GetRepo(g.repoOwner, g.repoName) gr, err := g.client(ctx).GetRepo(g.repoOwner, g.repoName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -148,11 +137,11 @@ func (g *GogsDownloader) GetRepoInfo() (*base.Repository, error) {
} }
// GetMilestones returns milestones // GetMilestones returns milestones
func (g *GogsDownloader) GetMilestones() ([]*base.Milestone, error) { func (g *GogsDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) {
perPage := 100 perPage := 100
milestones := make([]*base.Milestone, 0, perPage) milestones := make([]*base.Milestone, 0, perPage)
ms, err := g.client.ListRepoMilestones(g.repoOwner, g.repoName) ms, err := g.client(ctx).ListRepoMilestones(g.repoOwner, g.repoName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -171,10 +160,10 @@ func (g *GogsDownloader) GetMilestones() ([]*base.Milestone, error) {
} }
// GetLabels returns labels // GetLabels returns labels
func (g *GogsDownloader) GetLabels() ([]*base.Label, error) { func (g *GogsDownloader) GetLabels(ctx context.Context) ([]*base.Label, error) {
perPage := 100 perPage := 100
labels := make([]*base.Label, 0, perPage) labels := make([]*base.Label, 0, perPage)
ls, err := g.client.ListRepoLabels(g.repoOwner, g.repoName) ls, err := g.client(ctx).ListRepoLabels(g.repoOwner, g.repoName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -187,7 +176,7 @@ func (g *GogsDownloader) GetLabels() ([]*base.Label, error) {
} }
// GetIssues returns issues according start and limit, perPage is not supported // GetIssues returns issues according start and limit, perPage is not supported
func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) { func (g *GogsDownloader) GetIssues(ctx context.Context, page, _ int) ([]*base.Issue, bool, error) {
var state string var state string
if g.openIssuesFinished { if g.openIssuesFinished {
state = string(gogs.STATE_CLOSED) state = string(gogs.STATE_CLOSED)
@@ -197,7 +186,7 @@ func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) {
g.openIssuesPages = page g.openIssuesPages = page
} }
issues, isEnd, err := g.getIssues(page, state) issues, isEnd, err := g.getIssues(ctx, page, state)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }
@@ -212,10 +201,10 @@ func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) {
return issues, false, nil return issues, false, nil
} }
func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, error) { func (g *GogsDownloader) getIssues(ctx context.Context, page int, state string) ([]*base.Issue, bool, error) {
allIssues := make([]*base.Issue, 0, 10) allIssues := make([]*base.Issue, 0, 10)
issues, err := g.client.ListRepoIssues(g.repoOwner, g.repoName, gogs.ListIssueOption{ issues, err := g.client(ctx).ListRepoIssues(g.repoOwner, g.repoName, gogs.ListIssueOption{
Page: page, Page: page,
State: state, State: state,
}) })
@@ -234,10 +223,10 @@ func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool,
} }
// GetComments returns comments according issueNumber // GetComments returns comments according issueNumber
func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { func (g *GogsDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
allComments := make([]*base.Comment, 0, 100) allComments := make([]*base.Comment, 0, 100)
comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex()) comments, err := g.client(ctx).ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex())
if err != nil { if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %w", err) return nil, false, fmt.Errorf("error while listing repos: %w", err)
} }
@@ -261,7 +250,7 @@ func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comm
} }
// GetTopics return repository topics // GetTopics return repository topics
func (g *GogsDownloader) GetTopics() ([]string, error) { func (g *GogsDownloader) GetTopics(_ context.Context) ([]string, error) {
return []string{}, nil return []string{}, nil
} }

16
services/migrations/gogs_test.go
View File

@@ -28,9 +28,9 @@ func TestGogsDownloadRepo(t *testing.T) {
t.Skipf("visit test repo failed, ignored") t.Skipf("visit test repo failed, ignored")
return return
} }
ctx := context.Background()
downloader := NewGogsDownloader(context.Background(), "https://try.gogs.io", "", "", gogsPersonalAccessToken, "lunnytest", "TESTREPO") downloader := NewGogsDownloader(ctx, "https://try.gogs.io", "", "", gogsPersonalAccessToken, "lunnytest", "TESTREPO")
repo, err := downloader.GetRepoInfo() repo, err := downloader.GetRepoInfo(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertRepositoryEqual(t, &base.Repository{ assertRepositoryEqual(t, &base.Repository{
@@ -42,7 +42,7 @@ func TestGogsDownloadRepo(t *testing.T) {
DefaultBranch: "master", DefaultBranch: "master",
}, repo) }, repo)
milestones, err := downloader.GetMilestones() milestones, err := downloader.GetMilestones(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertMilestonesEqual(t, []*base.Milestone{ assertMilestonesEqual(t, []*base.Milestone{
{ {
@@ -51,7 +51,7 @@ func TestGogsDownloadRepo(t *testing.T) {
}, },
}, milestones) }, milestones)
labels, err := downloader.GetLabels() labels, err := downloader.GetLabels(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertLabelsEqual(t, []*base.Label{ assertLabelsEqual(t, []*base.Label{
{ {
@@ -85,7 +85,7 @@ func TestGogsDownloadRepo(t *testing.T) {
}, labels) }, labels)
// downloader.GetIssues() // downloader.GetIssues()
issues, isEnd, err := downloader.GetIssues(1, 8) issues, isEnd, err := downloader.GetIssues(ctx, 1, 8)
assert.NoError(t, err) assert.NoError(t, err)
assert.False(t, isEnd) assert.False(t, isEnd)
assertIssuesEqual(t, []*base.Issue{ assertIssuesEqual(t, []*base.Issue{
@@ -110,7 +110,7 @@ func TestGogsDownloadRepo(t *testing.T) {
}, issues) }, issues)
// downloader.GetComments() // downloader.GetComments()
comments, _, err := downloader.GetComments(&base.Issue{Number: 1, ForeignIndex: 1}) comments, _, err := downloader.GetComments(ctx, &base.Issue{Number: 1, ForeignIndex: 1})
assert.NoError(t, err) assert.NoError(t, err)
assertCommentsEqual(t, []*base.Comment{ assertCommentsEqual(t, []*base.Comment{
{ {
@@ -134,6 +134,6 @@ func TestGogsDownloadRepo(t *testing.T) {
}, comments) }, comments)
// downloader.GetPullRequests() // downloader.GetPullRequests()
_, _, err = downloader.GetPullRequests(1, 3) _, _, err = downloader.GetPullRequests(ctx, 1, 3)
assert.Error(t, err) assert.Error(t, err)
} }

56
services/migrations/migrate.go
View File

@@ -176,12 +176,12 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio
// migrateRepository will download information and then upload it to Uploader, this is a simple // migrateRepository will download information and then upload it to Uploader, this is a simple
// process for small repository. For a big repository, save all the data to disk // process for small repository. For a big repository, save all the data to disk
// before upload is better // before upload is better
func migrateRepository(_ context.Context, doer *user_model.User, downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error { func migrateRepository(ctx context.Context, doer *user_model.User, downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error {
if messenger == nil { if messenger == nil {
messenger = base.NilMessenger messenger = base.NilMessenger
} }
repo, err := downloader.GetRepoInfo() repo, err := downloader.GetRepoInfo(ctx)
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
return err return err
@@ -220,14 +220,14 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
log.Trace("migrating git data from %s", repo.CloneURL) log.Trace("migrating git data from %s", repo.CloneURL)
messenger("repo.migrate.migrating_git") messenger("repo.migrate.migrating_git")
if err = uploader.CreateRepo(repo, opts); err != nil { if err = uploader.CreateRepo(ctx, repo, opts); err != nil {
return err return err
} }
defer uploader.Close() defer uploader.Close()
log.Trace("migrating topics") log.Trace("migrating topics")
messenger("repo.migrate.migrating_topics") messenger("repo.migrate.migrating_topics")
topics, err := downloader.GetTopics() topics, err := downloader.GetTopics(ctx)
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
return err return err
@@ -235,7 +235,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
log.Warn("migrating topics is not supported, ignored") log.Warn("migrating topics is not supported, ignored")
} }
if len(topics) != 0 { if len(topics) != 0 {
if err = uploader.CreateTopics(topics...); err != nil { if err = uploader.CreateTopics(ctx, topics...); err != nil {
return err return err
} }
} }
@@ -243,7 +243,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
if opts.Milestones { if opts.Milestones {
log.Trace("migrating milestones") log.Trace("migrating milestones")
messenger("repo.migrate.migrating_milestones") messenger("repo.migrate.migrating_milestones")
milestones, err := downloader.GetMilestones() milestones, err := downloader.GetMilestones(ctx)
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
return err return err
@@ -256,7 +256,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
msBatchSize = len(milestones) msBatchSize = len(milestones)
} }
if err := uploader.CreateMilestones(milestones[:msBatchSize]...); err != nil { if err := uploader.CreateMilestones(ctx, milestones[:msBatchSize]...); err != nil {
return err return err
} }
milestones = milestones[msBatchSize:] milestones = milestones[msBatchSize:]
@@ -266,7 +266,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
if opts.Labels { if opts.Labels {
log.Trace("migrating labels") log.Trace("migrating labels")
messenger("repo.migrate.migrating_labels") messenger("repo.migrate.migrating_labels")
labels, err := downloader.GetLabels() labels, err := downloader.GetLabels(ctx)
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
return err return err
@@ -280,7 +280,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
lbBatchSize = len(labels) lbBatchSize = len(labels)
} }
if err := uploader.CreateLabels(labels[:lbBatchSize]...); err != nil { if err := uploader.CreateLabels(ctx, labels[:lbBatchSize]...); err != nil {
return err return err
} }
labels = labels[lbBatchSize:] labels = labels[lbBatchSize:]
@@ -290,7 +290,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
if opts.Releases { if opts.Releases {
log.Trace("migrating releases") log.Trace("migrating releases")
messenger("repo.migrate.migrating_releases") messenger("repo.migrate.migrating_releases")
releases, err := downloader.GetReleases() releases, err := downloader.GetReleases(ctx)
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
return err return err
@@ -304,14 +304,14 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
relBatchSize = len(releases) relBatchSize = len(releases)
} }
if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil { if err = uploader.CreateReleases(ctx, releases[:relBatchSize]...); err != nil {
return err return err
} }
releases = releases[relBatchSize:] releases = releases[relBatchSize:]
} }
// Once all releases (if any) are inserted, sync any remaining non-release tags // Once all releases (if any) are inserted, sync any remaining non-release tags
if err = uploader.SyncTags(); err != nil { if err = uploader.SyncTags(ctx); err != nil {
return err return err
} }
} }
@@ -329,7 +329,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
issueBatchSize := uploader.MaxBatchInsertSize("issue") issueBatchSize := uploader.MaxBatchInsertSize("issue")
for i := 1; ; i++ { for i := 1; ; i++ {
issues, isEnd, err := downloader.GetIssues(i, issueBatchSize) issues, isEnd, err := downloader.GetIssues(ctx, i, issueBatchSize)
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
return err return err
@@ -338,7 +338,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
break break
} }
if err := uploader.CreateIssues(issues...); err != nil { if err := uploader.CreateIssues(ctx, issues...); err != nil {
return err return err
} }
@@ -346,7 +346,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
allComments := make([]*base.Comment, 0, commentBatchSize) allComments := make([]*base.Comment, 0, commentBatchSize)
for _, issue := range issues { for _, issue := range issues {
log.Trace("migrating issue %d's comments", issue.Number) log.Trace("migrating issue %d's comments", issue.Number)
comments, _, err := downloader.GetComments(issue) comments, _, err := downloader.GetComments(ctx, issue)
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
return err return err
@@ -357,7 +357,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
allComments = append(allComments, comments...) allComments = append(allComments, comments...)
if len(allComments) >= commentBatchSize { if len(allComments) >= commentBatchSize {
if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { if err = uploader.CreateComments(ctx, allComments[:commentBatchSize]...); err != nil {
return err return err
} }
@@ -366,7 +366,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
} }
if len(allComments) > 0 { if len(allComments) > 0 {
if err = uploader.CreateComments(allComments...); err != nil { if err = uploader.CreateComments(ctx, allComments...); err != nil {
return err return err
} }
} }
@@ -383,7 +383,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
messenger("repo.migrate.migrating_pulls") messenger("repo.migrate.migrating_pulls")
prBatchSize := uploader.MaxBatchInsertSize("pullrequest") prBatchSize := uploader.MaxBatchInsertSize("pullrequest")
for i := 1; ; i++ { for i := 1; ; i++ {
prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize) prs, isEnd, err := downloader.GetPullRequests(ctx, i, prBatchSize)
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
return err return err
@@ -392,7 +392,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
break break
} }
if err := uploader.CreatePullRequests(prs...); err != nil { if err := uploader.CreatePullRequests(ctx, prs...); err != nil {
return err return err
} }
@@ -402,7 +402,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
allComments := make([]*base.Comment, 0, commentBatchSize) allComments := make([]*base.Comment, 0, commentBatchSize)
for _, pr := range prs { for _, pr := range prs {
log.Trace("migrating pull request %d's comments", pr.Number) log.Trace("migrating pull request %d's comments", pr.Number)
comments, _, err := downloader.GetComments(pr) comments, _, err := downloader.GetComments(ctx, pr)
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
return err return err
@@ -413,14 +413,14 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
allComments = append(allComments, comments...) allComments = append(allComments, comments...)
if len(allComments) >= commentBatchSize { if len(allComments) >= commentBatchSize {
if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { if err = uploader.CreateComments(ctx, allComments[:commentBatchSize]...); err != nil {
return err return err
} }
allComments = allComments[commentBatchSize:] allComments = allComments[commentBatchSize:]
} }
} }
if len(allComments) > 0 { if len(allComments) > 0 {
if err = uploader.CreateComments(allComments...); err != nil { if err = uploader.CreateComments(ctx, allComments...); err != nil {
return err return err
} }
} }
@@ -429,7 +429,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
// migrate reviews // migrate reviews
allReviews := make([]*base.Review, 0, reviewBatchSize) allReviews := make([]*base.Review, 0, reviewBatchSize)
for _, pr := range prs { for _, pr := range prs {
reviews, err := downloader.GetReviews(pr) reviews, err := downloader.GetReviews(ctx, pr)
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
return err return err
@@ -441,14 +441,14 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
allReviews = append(allReviews, reviews...) allReviews = append(allReviews, reviews...)
if len(allReviews) >= reviewBatchSize { if len(allReviews) >= reviewBatchSize {
if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { if err = uploader.CreateReviews(ctx, allReviews[:reviewBatchSize]...); err != nil {
return err return err
} }
allReviews = allReviews[reviewBatchSize:] allReviews = allReviews[reviewBatchSize:]
} }
} }
if len(allReviews) > 0 { if len(allReviews) > 0 {
if err = uploader.CreateReviews(allReviews...); err != nil { if err = uploader.CreateReviews(ctx, allReviews...); err != nil {
return err return err
} }
} }
@@ -463,12 +463,12 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
if opts.Comments && supportAllComments { if opts.Comments && supportAllComments {
log.Trace("migrating comments") log.Trace("migrating comments")
for i := 1; ; i++ { for i := 1; ; i++ {
comments, isEnd, err := downloader.GetAllComments(i, commentBatchSize) comments, isEnd, err := downloader.GetAllComments(ctx, i, commentBatchSize)
if err != nil { if err != nil {
return err return err
} }
if err := uploader.CreateComments(comments...); err != nil { if err := uploader.CreateComments(ctx, comments...); err != nil {
return err return err
} }
@@ -478,7 +478,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base
} }
} }
return uploader.Finish() return uploader.Finish(ctx)
} }
// Init migrations service // Init migrations service

52
services/migrations/onedev.go
View File

@@ -71,7 +71,6 @@ type onedevUser struct {
// from OneDev // from OneDev
type OneDevDownloader struct { type OneDevDownloader struct {
base.NullDownloader base.NullDownloader
ctx context.Context
client *http.Client client *http.Client
baseURL *url.URL baseURL *url.URL
repoName string repoName string
@@ -81,15 +80,9 @@ type OneDevDownloader struct {
milestoneMap map[int64]string milestoneMap map[int64]string
} }
// SetContext set context
func (d *OneDevDownloader) SetContext(ctx context.Context) {
d.ctx = ctx
}
// NewOneDevDownloader creates a new downloader // NewOneDevDownloader creates a new downloader
func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader { func NewOneDevDownloader(_ context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader {
downloader := &OneDevDownloader{ downloader := &OneDevDownloader{
ctx: ctx,
baseURL: baseURL, baseURL: baseURL,
repoName: repoName, repoName: repoName,
client: &http.Client{ client: &http.Client{
@@ -121,7 +114,7 @@ func (d *OneDevDownloader) LogString() string {
return fmt.Sprintf("<OneDevDownloader %s [%d]/%s>", d.baseURL, d.repoID, d.repoName) return fmt.Sprintf("<OneDevDownloader %s [%d]/%s>", d.baseURL, d.repoID, d.repoName)
} }
func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result any) error { func (d *OneDevDownloader) callAPI(ctx context.Context, endpoint string, parameter map[string]string, result any) error {
u, err := d.baseURL.Parse(endpoint) u, err := d.baseURL.Parse(endpoint)
if err != nil { if err != nil {
return err return err
@@ -135,7 +128,7 @@ func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string,
u.RawQuery = query.Encode() u.RawQuery = query.Encode()
} }
req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil) req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
if err != nil { if err != nil {
return err return err
} }
@@ -151,7 +144,7 @@ func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string,
} }
// GetRepoInfo returns repository information // GetRepoInfo returns repository information
func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) { func (d *OneDevDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) {
info := make([]struct { info := make([]struct {
ID int64 `json:"id"` ID int64 `json:"id"`
Name string `json:"name"` Name string `json:"name"`
@@ -159,6 +152,7 @@ func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) {
}, 0, 1) }, 0, 1)
err := d.callAPI( err := d.callAPI(
ctx,
"/api/projects", "/api/projects",
map[string]string{ map[string]string{
"query": `"Name" is "` + d.repoName + `"`, "query": `"Name" is "` + d.repoName + `"`,
@@ -194,7 +188,7 @@ func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) {
} }
// GetMilestones returns milestones // GetMilestones returns milestones
func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { func (d *OneDevDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) {
rawMilestones := make([]struct { rawMilestones := make([]struct {
ID int64 `json:"id"` ID int64 `json:"id"`
Name string `json:"name"` Name string `json:"name"`
@@ -209,6 +203,7 @@ func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) {
offset := 0 offset := 0
for { for {
err := d.callAPI( err := d.callAPI(
ctx,
endpoint, endpoint,
map[string]string{ map[string]string{
"offset": strconv.Itoa(offset), "offset": strconv.Itoa(offset),
@@ -243,7 +238,7 @@ func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) {
} }
// GetLabels returns labels // GetLabels returns labels
func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) { func (d *OneDevDownloader) GetLabels(_ context.Context) ([]*base.Label, error) {
return []*base.Label{ return []*base.Label{
{ {
Name: "Bug", Name: "Bug",
@@ -277,7 +272,7 @@ type onedevIssueContext struct {
} }
// GetIssues returns issues // GetIssues returns issues
func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { func (d *OneDevDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) {
rawIssues := make([]struct { rawIssues := make([]struct {
ID int64 `json:"id"` ID int64 `json:"id"`
Number int64 `json:"number"` Number int64 `json:"number"`
@@ -289,6 +284,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
}, 0, perPage) }, 0, perPage)
err := d.callAPI( err := d.callAPI(
ctx,
"/api/issues", "/api/issues",
map[string]string{ map[string]string{
"query": `"Project" is "` + d.repoName + `"`, "query": `"Project" is "` + d.repoName + `"`,
@@ -308,6 +304,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
Value string `json:"value"` Value string `json:"value"`
}, 0, 10) }, 0, 10)
err := d.callAPI( err := d.callAPI(
ctx,
fmt.Sprintf("/api/issues/%d/fields", issue.ID), fmt.Sprintf("/api/issues/%d/fields", issue.ID),
nil, nil,
&fields, &fields,
@@ -329,6 +326,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
Name string `json:"name"` Name string `json:"name"`
}, 0, 10) }, 0, 10)
err = d.callAPI( err = d.callAPI(
ctx,
fmt.Sprintf("/api/issues/%d/milestones", issue.ID), fmt.Sprintf("/api/issues/%d/milestones", issue.ID),
nil, nil,
&milestones, &milestones,
@@ -345,7 +343,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
if state == "released" { if state == "released" {
state = "closed" state = "closed"
} }
poster := d.tryGetUser(issue.SubmitterID) poster := d.tryGetUser(ctx, issue.SubmitterID)
issues = append(issues, &base.Issue{ issues = append(issues, &base.Issue{
Title: issue.Title, Title: issue.Title,
Number: issue.Number, Number: issue.Number,
@@ -370,7 +368,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
} }
// GetComments returns comments // GetComments returns comments
func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { func (d *OneDevDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
context, ok := commentable.GetContext().(onedevIssueContext) context, ok := commentable.GetContext().(onedevIssueContext)
if !ok { if !ok {
return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext())
@@ -391,6 +389,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co
} }
err := d.callAPI( err := d.callAPI(
ctx,
endpoint, endpoint,
nil, nil,
&rawComments, &rawComments,
@@ -412,6 +411,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co
} }
err = d.callAPI( err = d.callAPI(
ctx,
endpoint, endpoint,
nil, nil,
&rawChanges, &rawChanges,
@@ -425,7 +425,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co
if len(comment.Content) == 0 { if len(comment.Content) == 0 {
continue continue
} }
poster := d.tryGetUser(comment.UserID) poster := d.tryGetUser(ctx, comment.UserID)
comments = append(comments, &base.Comment{ comments = append(comments, &base.Comment{
IssueIndex: commentable.GetLocalIndex(), IssueIndex: commentable.GetLocalIndex(),
Index: comment.ID, Index: comment.ID,
@@ -450,7 +450,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co
continue continue
} }
poster := d.tryGetUser(change.UserID) poster := d.tryGetUser(ctx, change.UserID)
comments = append(comments, &base.Comment{ comments = append(comments, &base.Comment{
IssueIndex: commentable.GetLocalIndex(), IssueIndex: commentable.GetLocalIndex(),
PosterID: poster.ID, PosterID: poster.ID,
@@ -466,7 +466,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co
} }
// GetPullRequests returns pull requests // GetPullRequests returns pull requests
func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { func (d *OneDevDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
rawPullRequests := make([]struct { rawPullRequests := make([]struct {
ID int64 `json:"id"` ID int64 `json:"id"`
Number int64 `json:"number"` Number int64 `json:"number"`
@@ -484,6 +484,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque
}, 0, perPage) }, 0, perPage)
err := d.callAPI( err := d.callAPI(
ctx,
"/api/pull-requests", "/api/pull-requests",
map[string]string{ map[string]string{
"query": `"Target Project" is "` + d.repoName + `"`, "query": `"Target Project" is "` + d.repoName + `"`,
@@ -505,6 +506,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque
MergeCommitHash string `json:"mergeCommitHash"` MergeCommitHash string `json:"mergeCommitHash"`
} }
err := d.callAPI( err := d.callAPI(
ctx,
fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID), fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID),
nil, nil,
&mergePreview, &mergePreview,
@@ -525,7 +527,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque
mergedTime = pr.CloseInfo.Date mergedTime = pr.CloseInfo.Date
} }
} }
poster := d.tryGetUser(pr.SubmitterID) poster := d.tryGetUser(ctx, pr.SubmitterID)
number := pr.Number + d.maxIssueIndex number := pr.Number + d.maxIssueIndex
pullRequests = append(pullRequests, &base.PullRequest{ pullRequests = append(pullRequests, &base.PullRequest{
@@ -562,7 +564,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque
} }
// GetReviews returns pull requests reviews // GetReviews returns pull requests reviews
func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { func (d *OneDevDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) {
rawReviews := make([]struct { rawReviews := make([]struct {
ID int64 `json:"id"` ID int64 `json:"id"`
UserID int64 `json:"userId"` UserID int64 `json:"userId"`
@@ -574,6 +576,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie
}, 0, 100) }, 0, 100)
err := d.callAPI( err := d.callAPI(
ctx,
fmt.Sprintf("/api/pull-requests/%d/reviews", reviewable.GetForeignIndex()), fmt.Sprintf("/api/pull-requests/%d/reviews", reviewable.GetForeignIndex()),
nil, nil,
&rawReviews, &rawReviews,
@@ -596,7 +599,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie
} }
} }
poster := d.tryGetUser(review.UserID) poster := d.tryGetUser(ctx, review.UserID)
reviews = append(reviews, &base.Review{ reviews = append(reviews, &base.Review{
IssueIndex: reviewable.GetLocalIndex(), IssueIndex: reviewable.GetLocalIndex(),
ReviewerID: poster.ID, ReviewerID: poster.ID,
@@ -610,14 +613,15 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie
} }
// GetTopics return repository topics // GetTopics return repository topics
func (d *OneDevDownloader) GetTopics() ([]string, error) { func (d *OneDevDownloader) GetTopics(_ context.Context) ([]string, error) {
return []string{}, nil return []string{}, nil
} }
func (d *OneDevDownloader) tryGetUser(userID int64) *onedevUser { func (d *OneDevDownloader) tryGetUser(ctx context.Context, userID int64) *onedevUser {
user, ok := d.userMap[userID] user, ok := d.userMap[userID]
if !ok { if !ok {
err := d.callAPI( err := d.callAPI(
ctx,
fmt.Sprintf("/api/users/%d", userID), fmt.Sprintf("/api/users/%d", userID),
nil, nil,
&user, &user,

17
services/migrations/onedev_test.go
View File

@@ -22,11 +22,12 @@ func TestOneDevDownloadRepo(t *testing.T) {
} }
u, _ := url.Parse("https://code.onedev.io") u, _ := url.Parse("https://code.onedev.io")
downloader := NewOneDevDownloader(context.Background(), u, "", "", "go-gitea-test_repo") ctx := context.Background()
downloader := NewOneDevDownloader(ctx, u, "", "", "go-gitea-test_repo")
if err != nil { if err != nil {
t.Fatalf("NewOneDevDownloader is nil: %v", err) t.Fatalf("NewOneDevDownloader is nil: %v", err)
} }
repo, err := downloader.GetRepoInfo() repo, err := downloader.GetRepoInfo(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assertRepositoryEqual(t, &base.Repository{ assertRepositoryEqual(t, &base.Repository{
Name: "go-gitea-test_repo", Name: "go-gitea-test_repo",
@@ -36,7 +37,7 @@ func TestOneDevDownloadRepo(t *testing.T) {
OriginalURL: "https://code.onedev.io/projects/go-gitea-test_repo", OriginalURL: "https://code.onedev.io/projects/go-gitea-test_repo",
}, repo) }, repo)
milestones, err := downloader.GetMilestones() milestones, err := downloader.GetMilestones(ctx)
assert.NoError(t, err) assert.NoError(t, err)
deadline := time.Unix(1620086400, 0) deadline := time.Unix(1620086400, 0)
assertMilestonesEqual(t, []*base.Milestone{ assertMilestonesEqual(t, []*base.Milestone{
@@ -51,11 +52,11 @@ func TestOneDevDownloadRepo(t *testing.T) {
}, },
}, milestones) }, milestones)
labels, err := downloader.GetLabels() labels, err := downloader.GetLabels(ctx)
assert.NoError(t, err) assert.NoError(t, err)
assert.Len(t, labels, 6) assert.Len(t, labels, 6)
issues, isEnd, err := downloader.GetIssues(1, 2) issues, isEnd, err := downloader.GetIssues(ctx, 1, 2)
assert.NoError(t, err) assert.NoError(t, err)
assert.False(t, isEnd) assert.False(t, isEnd)
assertIssuesEqual(t, []*base.Issue{ assertIssuesEqual(t, []*base.Issue{
@@ -94,7 +95,7 @@ func TestOneDevDownloadRepo(t *testing.T) {
}, },
}, issues) }, issues)
comments, _, err := downloader.GetComments(&base.Issue{ comments, _, err := downloader.GetComments(ctx, &base.Issue{
Number: 4, Number: 4,
ForeignIndex: 398, ForeignIndex: 398,
Context: onedevIssueContext{IsPullRequest: false}, Context: onedevIssueContext{IsPullRequest: false},
@@ -110,7 +111,7 @@ func TestOneDevDownloadRepo(t *testing.T) {
}, },
}, comments) }, comments)
prs, _, err := downloader.GetPullRequests(1, 1) prs, _, err := downloader.GetPullRequests(ctx, 1, 1)
assert.NoError(t, err) assert.NoError(t, err)
assertPullRequestsEqual(t, []*base.PullRequest{ assertPullRequestsEqual(t, []*base.PullRequest{
{ {
@@ -136,7 +137,7 @@ func TestOneDevDownloadRepo(t *testing.T) {
}, },
}, prs) }, prs)
rvs, err := downloader.GetReviews(&base.PullRequest{Number: 5, ForeignIndex: 186}) rvs, err := downloader.GetReviews(ctx, &base.PullRequest{Number: 5, ForeignIndex: 186})
assert.NoError(t, err) assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{ assertReviewsEqual(t, []*base.Review{
{ {

27
services/migrations/restore.go
View File

@@ -18,7 +18,6 @@ import (
// RepositoryRestorer implements an Downloader from the local directory // RepositoryRestorer implements an Downloader from the local directory
type RepositoryRestorer struct { type RepositoryRestorer struct {
base.NullDownloader base.NullDownloader
ctx context.Context
baseDir string baseDir string
repoOwner string repoOwner string
repoName string repoName string
@@ -26,13 +25,12 @@ type RepositoryRestorer struct {
} }
// NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder // NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder
func NewRepositoryRestorer(ctx context.Context, baseDir, owner, repoName string, validation bool) (*RepositoryRestorer, error) { func NewRepositoryRestorer(_ context.Context, baseDir, owner, repoName string, validation bool) (*RepositoryRestorer, error) {
baseDir, err := filepath.Abs(baseDir) baseDir, err := filepath.Abs(baseDir)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &RepositoryRestorer{ return &RepositoryRestorer{
ctx: ctx,
baseDir: baseDir, baseDir: baseDir,
repoOwner: owner, repoOwner: owner,
repoName: repoName, repoName: repoName,
@@ -48,11 +46,6 @@ func (r *RepositoryRestorer) reviewDir() string {
return filepath.Join(r.baseDir, "reviews") return filepath.Join(r.baseDir, "reviews")
} }
// SetContext set context
func (r *RepositoryRestorer) SetContext(ctx context.Context) {
r.ctx = ctx
}
func (r *RepositoryRestorer) getRepoOptions() (map[string]string, error) { func (r *RepositoryRestorer) getRepoOptions() (map[string]string, error) {
p := filepath.Join(r.baseDir, "repo.yml") p := filepath.Join(r.baseDir, "repo.yml")
bs, err := os.ReadFile(p) bs, err := os.ReadFile(p)
@@ -69,7 +62,7 @@ func (r *RepositoryRestorer) getRepoOptions() (map[string]string, error) {
} }
// GetRepoInfo returns a repository information // GetRepoInfo returns a repository information
func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { func (r *RepositoryRestorer) GetRepoInfo(_ context.Context) (*base.Repository, error) {
opts, err := r.getRepoOptions() opts, err := r.getRepoOptions()
if err != nil { if err != nil {
return nil, err return nil, err
@@ -89,7 +82,7 @@ func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) {
} }
// GetTopics return github topics // GetTopics return github topics
func (r *RepositoryRestorer) GetTopics() ([]string, error) { func (r *RepositoryRestorer) GetTopics(_ context.Context) ([]string, error) {
p := filepath.Join(r.baseDir, "topic.yml") p := filepath.Join(r.baseDir, "topic.yml")
topics := struct { topics := struct {
@@ -112,7 +105,7 @@ func (r *RepositoryRestorer) GetTopics() ([]string, error) {
} }
// GetMilestones returns milestones // GetMilestones returns milestones
func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { func (r *RepositoryRestorer) GetMilestones(_ context.Context) ([]*base.Milestone, error) {
milestones := make([]*base.Milestone, 0, 10) milestones := make([]*base.Milestone, 0, 10)
p := filepath.Join(r.baseDir, "milestone.yml") p := filepath.Join(r.baseDir, "milestone.yml")
err := base.Load(p, &milestones, r.validation) err := base.Load(p, &milestones, r.validation)
@@ -127,7 +120,7 @@ func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) {
} }
// GetReleases returns releases // GetReleases returns releases
func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { func (r *RepositoryRestorer) GetReleases(_ context.Context) ([]*base.Release, error) {
releases := make([]*base.Release, 0, 10) releases := make([]*base.Release, 0, 10)
p := filepath.Join(r.baseDir, "release.yml") p := filepath.Join(r.baseDir, "release.yml")
_, err := os.Stat(p) _, err := os.Stat(p)
@@ -158,7 +151,7 @@ func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) {
} }
// GetLabels returns labels // GetLabels returns labels
func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { func (r *RepositoryRestorer) GetLabels(_ context.Context) ([]*base.Label, error) {
labels := make([]*base.Label, 0, 10) labels := make([]*base.Label, 0, 10)
p := filepath.Join(r.baseDir, "label.yml") p := filepath.Join(r.baseDir, "label.yml")
_, err := os.Stat(p) _, err := os.Stat(p)
@@ -182,7 +175,7 @@ func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) {
} }
// GetIssues returns issues according start and limit // GetIssues returns issues according start and limit
func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { func (r *RepositoryRestorer) GetIssues(_ context.Context, _, _ int) ([]*base.Issue, bool, error) {
issues := make([]*base.Issue, 0, 10) issues := make([]*base.Issue, 0, 10)
p := filepath.Join(r.baseDir, "issue.yml") p := filepath.Join(r.baseDir, "issue.yml")
err := base.Load(p, &issues, r.validation) err := base.Load(p, &issues, r.validation)
@@ -196,7 +189,7 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool,
} }
// GetComments returns comments according issueNumber // GetComments returns comments according issueNumber
func (r *RepositoryRestorer) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { func (r *RepositoryRestorer) GetComments(_ context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
comments := make([]*base.Comment, 0, 10) comments := make([]*base.Comment, 0, 10)
p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", commentable.GetForeignIndex())) p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", commentable.GetForeignIndex()))
_, err := os.Stat(p) _, err := os.Stat(p)
@@ -220,7 +213,7 @@ func (r *RepositoryRestorer) GetComments(commentable base.Commentable) ([]*base.
} }
// GetPullRequests returns pull requests according page and perPage // GetPullRequests returns pull requests according page and perPage
func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { func (r *RepositoryRestorer) GetPullRequests(_ context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
pulls := make([]*base.PullRequest, 0, 10) pulls := make([]*base.PullRequest, 0, 10)
p := filepath.Join(r.baseDir, "pull_request.yml") p := filepath.Join(r.baseDir, "pull_request.yml")
_, err := os.Stat(p) _, err := os.Stat(p)
@@ -248,7 +241,7 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq
} }
// GetReviews returns pull requests review // GetReviews returns pull requests review
func (r *RepositoryRestorer) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { func (r *RepositoryRestorer) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) {
reviews := make([]*base.Review, 0, 10) reviews := make([]*base.Review, 0, 10)
p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", reviewable.GetForeignIndex())) p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", reviewable.GetForeignIndex()))
_, err := os.Stat(p) _, err := os.Stat(p)