owen/gitocean
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: cleanup data before migration retry (#12370)

Browse Source 
In the case you hit some API error (Github ratelimit was often a problem) or the instance restarted in the middle of your migration, you would be left with data on the disk and/or database. Upon retrying the migration the migration code would (rightfully) fail because it's trying to migrate stuff that already exists.

This was hit so often on Codeberg it was better to force people to delete and start whole migration process again: https://codeberg.org/Codeberg-Infrastructure/forgejo/commit/28ee60c91f237268aa526086d0cce7a8f5380a9b

Delete the repository data before retrying to solve this.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/12370
Reviewed-by: Mathieu Fenniak <mfenniak@noreply.codeberg.org>
This commit is contained in:
Gusted
2026-05-05 12:41:42 +02:00
committed by Gusted
parent 6f5bef54b0
commit c07ea09050
21 changed files with 191 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/task/fixtures/TestRetryMigrateTask/issue.yml
+16
View File
@@ -0,0 +1,16 @@
-
id: 1001
repo_id: 20
index: 1
poster_id: 1
original_author_id: 0
name: eepy
content: I am part of a migration.
milestone_id: 0
priority: 0
is_closed: false
is_pull: false
num_comments: 0
created_unix: 1777647620
updated_unix: 1777647620
is_locked: false
services/task/fixtures/TestRetryMigrateTask/repo_unit.yml
+34
View File
@@ -0,0 +1,34 @@
-
id: 1001
repo_id: 20
type: 1
config: "{}"
created_unix: 1777646333
-
id: 1002
repo_id: 20
type: 2
config: "{\"EnableTimetracker\":false,\"AllowOnlyContributorsToTrackTime\":false}"
created_unix: 1777646333
-
id: 1003
repo_id: 20
type: 3
config: "{\"IgnoreWhitespaceConflicts\":true,\"AllowMerge\":true,\"AllowRebase\":false,\"AllowRebaseMerge\":true,\"AllowSquash\":false}"
created_unix: 1777646333
-
id: 1004
repo_id: 20
type: 4
config: "{}"
created_unix: 1777646333
-
id: 1005
repo_id: 20
type: 5
config: "{}"
created_unix: 1777646333
services/task/fixtures/TestRetryMigrateTask/task.yml
+22
View File
@@ -0,0 +1,22 @@
-
id: 1001
doer_id: 2
owner_id: 2
repo_id: 1
type: 0
status: 4
start_time: 1777645339
end_time: 1777645339
created: 1777645339
-
id: 1002
doer_id: 2
owner_id: 2
repo_id: 20
type: 0
status: 3
message: 'canceled'
start_time: 1777645339
end_time: 1777645339
created: 1777645339
services/task/task.go
+12 -6
View File
@@ -133,25 +133,31 @@ func CreateMigrateTask(ctx context.Context, doer, u *user_model.User, opts base.
return task, nil
}
// RetryMigrateTask retry a migrate task
// RetryMigrateTask will retry the migration.
// All data, from a previous migration, is deleted before it's retried.
func RetryMigrateTask(ctx context.Context, repoID int64) error {
migratingTask, err := admin_model.GetMigratingTask(ctx, repoID)
if err != nil {
log.Error("GetMigratingTask: %v", err)
return err
return fmt.Errorf("GetMigratingTask: %w", err)
}
if migratingTask.Status == structs.TaskStatusQueued || migratingTask.Status == structs.TaskStatusRunning {
return nil
}
// TODO Need to removing the storage/database garbage brought by the failed task
// The migration is being retried, it could've failed for a variety of cases.
// In most cases however, some data already got uploaded to the disk or
// database. The migration code makes the assumption this is not the case and
// if we do not clean it up, the retry attempt will fail with absolute
// certainty.
if err := repo_service.DeleteRepositoryDirectly(ctx, repoID, repo_service.DeleteRepositoryOpts{IgnoreOrgTeams: true, KeepMigrationBeans: true}); err != nil {
return fmt.Errorf("DeleteRepositoryDirectly: %v", err)
}
// Reset task status and messages
migratingTask.Status = structs.TaskStatusQueued
migratingTask.Message = ""
if err = migratingTask.UpdateCols(ctx, "status", "message"); err != nil {
log.Error("task.UpdateCols failed: %v", err)
return err
return fmt.Errorf("task.UpdateCols: %w", err)
}
return taskQueue.Push(migratingTask)
services/task/task_test.go
+52
View File
@@ -1,12 +1,21 @@
// Copyright 2025 The Forgejo Authors. All rights reserved.
// SPDX-License-Identifier: GPL-3.0-or-later
package task
import (
"testing"
admin_model "forgejo.org/models/admin"
issues_model "forgejo.org/models/issues"
repo_model "forgejo.org/models/repo"
"forgejo.org/models/unittest"
user_model "forgejo.org/models/user"
"forgejo.org/modules/migration"
"forgejo.org/modules/queue"
"forgejo.org/modules/setting"
"forgejo.org/modules/structs"
"forgejo.org/modules/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -50,3 +59,46 @@ func TestCreateMigrateTask(t *testing.T) {
assert.Equal(t, "https://admin:password@example.com", config.CloneAddr)
})
}
func TestRetryMigrateTask(t *testing.T) {
defer unittest.OverrideFixtures("services/task/fixtures/TestRetryMigrateTask/")()
require.NoError(t, unittest.PrepareTestDatabase())
t.Run("Migrate task does not exist", func(t *testing.T) {
err := RetryMigrateTask(t.Context(), 100)
require.ErrorIs(t, err, admin_model.ErrTaskDoesNotExist{RepoID: 100})
})
t.Run("Normal", func(t *testing.T) {
// Override the task queue temporarily.
called := false
testQueue, err := queue.NewWorkerPoolQueueWithContext(t.Context(), "task", setting.QueueSettings{Type: "immediate"}, func(items ...*admin_model.Task) []*admin_model.Task {
if assert.Len(t, items, 1) {
assert.Empty(t, items[0].Message)
assert.Equal(t, structs.TaskStatusQueued, items[0].Status)
assert.EqualValues(t, 1002, items[0].ID)
}
called = true
return nil
}, true)
require.NoError(t, err)
defer test.MockVariableValue(&taskQueue, testQueue)()
// Preconditions.
unittest.AssertExistsIf(t, true, &repo_model.Repository{ID: 20})
unittest.AssertExistsIf(t, true, &admin_model.Task{RepoID: 20, Status: structs.TaskStatusFailed})
unittest.AssertExistsIf(t, true, &issues_model.Issue{RepoID: 20})
unittest.AssertCount(t, &repo_model.RepoUnit{RepoID: 20}, 5)
require.NoError(t, RetryMigrateTask(t.Context(), 20))
// Verify queue was called.
assert.True(t, called)
// Verify some beans were NOT deleted.
unittest.AssertExistsIf(t, true, &repo_model.Repository{ID: 20})
unittest.AssertExistsIf(t, true, &admin_model.Task{RepoID: 20, Status: structs.TaskStatusQueued})
unittest.AssertCount(t, &repo_model.RepoUnit{RepoID: 20}, 5)
// Verify some beans were deleted.
unittest.AssertExistsIf(t, false, &issues_model.Issue{RepoID: 20})
})
}