leggettc18
/
api
forked from vikunja/vikunja
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feat(migration): Trello organization based migration (#2211) 

Migrate Trello organization after organization to limit total memory allocation.
Related discussion: https://community.vikunja.io/t/trello-import-issues/2110

Co-authored-by: Elscrux <nickposer2102@gmail.com>
Co-authored-by: konrad <k@knt.li>
Co-authored-by: kolaente <k@knt.li>
Reviewed-on: vikunja/vikunja#2211
Reviewed-by: konrad <k@knt.li>
Co-authored-by: Elscrux <elscrux@gmail.com>
Co-committed-by: Elscrux <elscrux@gmail.com>
This commit is contained in:
Elscrux 2024-04-09 10:54:38 +00:00 committed by konrad
parent af3b0bbea1
commit 8458e77341
2 changed files with 364 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
pkg/modules/migration/trello/trello.go
View File

@ -107,27 +107,48 @@ func (m *Migration) AuthURL() string {
"&return_url=" + config.MigrationTrelloRedirectURL.GetString()
}
func getTrelloData(token string) (trelloData []*trello.Board, err error) {
allArg := trello.Arguments{"fields": "all"}
client := trello.NewClient(config.MigrationTrelloKey.GetString(), token)
client.Logger = log.GetLogger()
func getTrelloBoards(client *trello.Client) (trelloData []*trello.Board, err error) {
log.Debugf("[Trello Migration] Getting boards...")
trelloData, err = client.GetMyBoards(trello.Defaults())
if err != nil {
return
return nil, err
}
log.Debugf("[Trello Migration] Got %d trello boards", len(trelloData))
for _, board := range trelloData {
return
}
func getTrelloOrganizationsWithBoards(boards []*trello.Board) (boardsByOrg map[string][]*trello.Board) {
boardsByOrg = make(map[string][]*trello.Board)
for _, board := range boards {
// Trello boards without an organization are considered personal boards
if board.IDOrganization == "" {
board.IDOrganization = "Personal"
}
_, has := boardsByOrg[board.IDOrganization]
if !has {
boardsByOrg[board.IDOrganization] = []*trello.Board{}
}
boardsByOrg[board.IDOrganization] = append(boardsByOrg[board.IDOrganization], board)
}
return
}
func fillCardData(client *trello.Client, board *trello.Board) (err error) {
allArg := trello.Arguments{"fields": "all"}
log.Debugf("[Trello Migration] Getting projects for board %s", board.ID)
board.Lists, err = board.GetLists(trello.Defaults())
if err != nil {
return
return err
}
log.Debugf("[Trello Migration] Got %d projects for board %s", len(board.Lists), board.ID)
@ -141,7 +162,7 @@ func getTrelloData(token string) (trelloData []*trello.Board, err error) {
cards, err := board.GetCards(allArg)
if err != nil {
return nil, err
return
}
log.Debugf("[Trello Migration] Got %d cards for board %s", len(cards), board.ID)
@ -154,20 +175,20 @@ func getTrelloData(token string) (trelloData []*trello.Board, err error) {
card.Attachments, err = card.GetAttachments(allArg)
if err != nil {
return nil, err
return
}
if len(card.IDCheckLists) > 0 {
for _, checkListID := range card.IDCheckLists {
checklist, err := client.GetChecklist(checkListID, allArg)
if err != nil {
return nil, err
return err
}
checklist.CheckItems = []trello.CheckItem{}
err = client.Get("checklists/"+checkListID+"/checkItems", allArg, &checklist.CheckItems)
if err != nil {
return nil, err
return err
}
card.Checklists = append(card.Checklists, checklist)
@ -179,7 +200,6 @@ func getTrelloData(token string) (trelloData []*trello.Board, err error) {
}
log.Debugf("[Trello Migration] Looked for attachements on all cards of board %s", board.ID)
}
return
}
@ -196,7 +216,7 @@ func convertMarkdownToHTML(input string) (output string, err error) {
// Converts all previously obtained data from trello into the vikunja format.
// `trelloData` should contain all boards with their projects and cards respectively.
func convertTrelloDataToVikunja(trelloData []*trello.Board, token string) (fullVikunjaHierachie []*models.ProjectWithTasksAndBuckets, err error) {
func convertTrelloDataToVikunja(organizationName string, trelloData []*trello.Board, token string) (fullVikunjaHierachie []*models.ProjectWithTasksAndBuckets, err error) {
log.Debugf("[Trello Migration] ")
@ -205,7 +225,7 @@ func convertTrelloDataToVikunja(trelloData []*trello.Board, token string) (fullV
{
Project: models.Project{
ID: pseudoParentID,
Title: "Imported from Trello",
Title: organizationName,
},
},
}
@ -392,29 +412,58 @@ func (m *Migration) Migrate(u *user.User) (err error) {
log.Debugf("[Trello Migration] Starting migration for user %d", u.ID)
log.Debugf("[Trello Migration] Getting all trello data for user %d", u.ID)
trelloData, err := getTrelloData(m.Token)
client := trello.NewClient(config.MigrationTrelloKey.GetString(), m.Token)
client.Logger = log.GetLogger()
boards, err := getTrelloBoards(client)
if err != nil {
return
}
log.Debugf("[Trello Migration] Got all trello data for user %d", u.ID)
log.Debugf("[Trello Migration] Start converting trello data for user %d", u.ID)
fullVikunjaHierachie, err := convertTrelloDataToVikunja(trelloData, m.Token)
organizationMap := getTrelloOrganizationsWithBoards(boards)
for organizationID, boards := range organizationMap {
log.Debugf("[Trello Migration] Getting organization with id %s for user %d", organizationID, u.ID)
orgName := organizationID
if organizationID != "Personal" {
organization, err := client.GetOrganization(organizationID, trello.Defaults())
if err != nil {
return
return err
}
orgName = organization.DisplayName
}
log.Debugf("[Trello Migration] Done migrating trello data for user %d", u.ID)
log.Debugf("[Trello Migration] Start inserting trello data for user %d", u.ID)
for _, board := range boards {
log.Debugf("[Trello Migration] Getting card data for board %s for user %d for organization %s", board.ID, u.ID, organizationID)
err = migration.InsertFromStructure(fullVikunjaHierachie, u)
err = fillCardData(client, board)
if err != nil {
return
return err
}
log.Debugf("[Trello Migration] Done inserting trello data for user %d", u.ID)
log.Debugf("[Trello Migration] Migration done for user %d", u.ID)
log.Debugf("[Trello Migration] Got card data for board %s for user %d for organization %s", board.ID, u.ID, organizationID)
}
return nil
log.Debugf("[Trello Migration] Start converting trello data for user %d for organization %s", u.ID, organizationID)
hierarchy, err := convertTrelloDataToVikunja(orgName, boards, m.Token)
if err != nil {
return err
}
log.Debugf("[Trello Migration] Done migrating trello data for user %d for organization %s", u.ID, organizationID)
log.Debugf("[Trello Migration] Start inserting trello data for user %d for organization %s", u.ID, organizationID)
err = migration.InsertFromStructure(hierarchy, u)
if err != nil {
return err
}
log.Debugf("[Trello Migration] Done inserting trello data for user %d for organization %s", u.ID, organizationID)
}
log.Debugf("[Trello Migration] Done migrating all trello data for user %d", u.ID)
return
}

164
pkg/modules/migration/trello/trello_test.go
View File

@ -32,18 +32,21 @@ import (
"github.com/stretchr/testify/require"
)
func TestConvertTrelloToVikunja(t *testing.T) {
func getTestBoard(t *testing.T) ([]*trello.Board, time.Time) {
config.InitConfig()
time1, err := time.Parse(time.RFC3339Nano, "2014-09-26T08:25:05Z")
require.NoError(t, err)
exampleFile, err := os.ReadFile(config.ServiceRootpath.GetString() + "/pkg/modules/migration/testimage.jpg")
require.NoError(t, err)
trelloData := []*trello.Board{
{
Name: "TestBoard",
Organization: trello.Organization{
ID: "orgid",
DisplayName: "TestOrg",
},
IDOrganization: "orgid",
Desc: "This is a description",
Closed: false,
Lists: []*trello.List{
@ -168,6 +171,11 @@ func TestConvertTrelloToVikunja(t *testing.T) {
},
},
{
Organization: trello.Organization{
ID: "orgid2",
DisplayName: "TestOrg2",
},
IDOrganization: "orgid2",
Name: "TestBoard 2",
Closed: false,
Lists: []*trello.List{
@ -183,6 +191,11 @@ func TestConvertTrelloToVikunja(t *testing.T) {
},
},
{
Organization: trello.Organization{
ID: "orgid",
DisplayName: "TestOrg",
},
IDOrganization: "orgid",
Name: "TestBoard Archived",
Closed: true,
Lists: []*trello.List{
@ -197,14 +210,38 @@ func TestConvertTrelloToVikunja(t *testing.T) {
},
},
},
{
Name: "Personal Board",
Lists: []*trello.List{
{
Name: "Test Project 6",
Cards: []*trello.Card{
{
Name: "Test Card 5659",
Pos: 123,
},
},
},
},
},
}
trelloData[0].Prefs.BackgroundImage = "https://vikunja.io/testimage.jpg" // Using an image which we are hosting, so it'll still be up
expectedHierachie := []*models.ProjectWithTasksAndBuckets{
return trelloData, time1
}
func TestConvertTrelloToVikunja(t *testing.T) {
trelloData, time1 := getTestBoard(t)
exampleFile, err := os.ReadFile(config.ServiceRootpath.GetString() + "/pkg/modules/migration/testimage.jpg")
require.NoError(t, err)
expectedHierarchyOrg := map[string][]*models.ProjectWithTasksAndBuckets{
"orgid": {
{
Project: models.Project{
ID: 1,
Title: "Imported from Trello",
Title: "orgid",
},
},
{
@ -336,33 +373,12 @@ func TestConvertTrelloToVikunja(t *testing.T) {
Project: models.Project{
ID: 3,
ParentProjectID: 1,
Title: "TestBoard 2",
},
Buckets: []*models.Bucket{
{
ID: 3,
Title: "Test Project 4",
},
},
Tasks: []*models.TaskWithComments{
{
Task: models.Task{
Title: "Test Card 634",
BucketID: 3,
},
},
},
},
{
Project: models.Project{
ID: 4,
ParentProjectID: 1,
Title: "TestBoard Archived",
IsArchived: true,
},
Buckets: []*models.Bucket{
{
ID: 4,
ID: 3,
Title: "Test Project 5",
},
},
@ -370,17 +386,101 @@ func TestConvertTrelloToVikunja(t *testing.T) {
{
Task: models.Task{
Title: "Test Card 63423",
BucketID: 4,
BucketID: 3,
},
},
},
},
},
"orgid2": {
{
Project: models.Project{
ID: 1,
Title: "orgid2",
},
},
{
Project: models.Project{
ID: 2,
ParentProjectID: 1,
Title: "TestBoard 2",
},
Buckets: []*models.Bucket{
{
ID: 1,
Title: "Test Project 4",
},
},
Tasks: []*models.TaskWithComments{
{
Task: models.Task{
Title: "Test Card 634",
BucketID: 1,
},
},
},
},
},
"Personal": {
{
Project: models.Project{
ID: 1,
Title: "Personal",
},
},
{
Project: models.Project{
ID: 2,
ParentProjectID: 1,
Title: "Personal Board",
},
Buckets: []*models.Bucket{
{
ID: 1,
Title: "Test Project 6",
},
},
Tasks: []*models.TaskWithComments{
{
Task: models.Task{
Title: "Test Card 5659",
BucketID: 1,
},
},
},
},
},
}
hierachie, err := convertTrelloDataToVikunja(trelloData, "")
organizationMap := getTrelloOrganizationsWithBoards(trelloData)
for organizationID, boards := range organizationMap {
hierarchy, err := convertTrelloDataToVikunja(organizationID, boards, "")
require.NoError(t, err)
assert.NotNil(t, hierachie)
if diff, equal := messagediff.PrettyDiff(hierachie, expectedHierachie); !equal {
t.Errorf("converted trello data = %v, want %v, diff: %v", hierachie, expectedHierachie, diff)
assert.NotNil(t, hierarchy)
if diff, equal := messagediff.PrettyDiff(hierarchy, expectedHierarchyOrg[organizationID]); !equal {
t.Errorf("converted trello data = %v,\nwant %v,\ndiff: %v", hierarchy, expectedHierarchyOrg[organizationID], diff)
}
}
}
func TestCreateOrganizationMap(t *testing.T) {
trelloData, _ := getTestBoard(t)
organizationMap := getTrelloOrganizationsWithBoards(trelloData)
expectedMap := map[string][]*trello.Board{
"orgid": {
trelloData[0],
trelloData[2],
},
"orgid2": {
trelloData[1],
},
"Personal": {
trelloData[3],
},
}
if diff, equal := messagediff.PrettyDiff(organizationMap, expectedMap); !equal {
t.Errorf("converted trello data = %v,\nwant %v,\ndiff: %v", organizationMap, expectedMap, diff)
}
}