feat(migration): Trello organization based migration (#2211)
Some checks failed
continuous-integration/drone/push Build is failing

Migrate Trello organization after organization to limit total memory allocation.
Related discussion: https://community.vikunja.io/t/trello-import-issues/2110

Co-authored-by: Elscrux <nickposer2102@gmail.com>
Co-authored-by: konrad <k@knt.li>
Co-authored-by: kolaente <k@knt.li>
Reviewed-on: #2211
Reviewed-by: konrad <k@knt.li>
Co-authored-by: Elscrux <elscrux@gmail.com>
Co-committed-by: Elscrux <elscrux@gmail.com>
This commit is contained in:
Elscrux 2024-04-09 10:54:38 +00:00 committed by konrad
parent af3b0bbea1
commit 8458e77341
2 changed files with 364 additions and 215 deletions

View File

@ -107,80 +107,100 @@ func (m *Migration) AuthURL() string {
"&return_url=" + config.MigrationTrelloRedirectURL.GetString()
}
func getTrelloData(token string) (trelloData []*trello.Board, err error) {
allArg := trello.Arguments{"fields": "all"}
client := trello.NewClient(config.MigrationTrelloKey.GetString(), token)
client.Logger = log.GetLogger()
func getTrelloBoards(client *trello.Client) (trelloData []*trello.Board, err error) {
log.Debugf("[Trello Migration] Getting boards...")
trelloData, err = client.GetMyBoards(trello.Defaults())
if err != nil {
return
return nil, err
}
log.Debugf("[Trello Migration] Got %d trello boards", len(trelloData))
for _, board := range trelloData {
log.Debugf("[Trello Migration] Getting projects for board %s", board.ID)
return
}
board.Lists, err = board.GetLists(trello.Defaults())
func getTrelloOrganizationsWithBoards(boards []*trello.Board) (boardsByOrg map[string][]*trello.Board) {
boardsByOrg = make(map[string][]*trello.Board)
for _, board := range boards {
// Trello boards without an organization are considered personal boards
if board.IDOrganization == "" {
board.IDOrganization = "Personal"
}
_, has := boardsByOrg[board.IDOrganization]
if !has {
boardsByOrg[board.IDOrganization] = []*trello.Board{}
}
boardsByOrg[board.IDOrganization] = append(boardsByOrg[board.IDOrganization], board)
}
return
}
func fillCardData(client *trello.Client, board *trello.Board) (err error) {
allArg := trello.Arguments{"fields": "all"}
log.Debugf("[Trello Migration] Getting projects for board %s", board.ID)
board.Lists, err = board.GetLists(trello.Defaults())
if err != nil {
return err
}
log.Debugf("[Trello Migration] Got %d projects for board %s", len(board.Lists), board.ID)
listMap := make(map[string]*trello.List, len(board.Lists))
for _, list := range board.Lists {
listMap[list.ID] = list
}
log.Debugf("[Trello Migration] Getting cards for board %s", board.ID)
cards, err := board.GetCards(allArg)
if err != nil {
return
}
log.Debugf("[Trello Migration] Got %d cards for board %s", len(cards), board.ID)
for _, card := range cards {
list, exists := listMap[card.IDList]
if !exists {
continue
}
card.Attachments, err = card.GetAttachments(allArg)
if err != nil {
return
}
log.Debugf("[Trello Migration] Got %d projects for board %s", len(board.Lists), board.ID)
listMap := make(map[string]*trello.List, len(board.Lists))
for _, list := range board.Lists {
listMap[list.ID] = list
}
log.Debugf("[Trello Migration] Getting cards for board %s", board.ID)
cards, err := board.GetCards(allArg)
if err != nil {
return nil, err
}
log.Debugf("[Trello Migration] Got %d cards for board %s", len(cards), board.ID)
for _, card := range cards {
list, exists := listMap[card.IDList]
if !exists {
continue
}
card.Attachments, err = card.GetAttachments(allArg)
if err != nil {
return nil, err
}
if len(card.IDCheckLists) > 0 {
for _, checkListID := range card.IDCheckLists {
checklist, err := client.GetChecklist(checkListID, allArg)
if err != nil {
return nil, err
}
checklist.CheckItems = []trello.CheckItem{}
err = client.Get("checklists/"+checkListID+"/checkItems", allArg, &checklist.CheckItems)
if err != nil {
return nil, err
}
card.Checklists = append(card.Checklists, checklist)
log.Debugf("Retrieved checklist %s for card %s", checkListID, card.ID)
if len(card.IDCheckLists) > 0 {
for _, checkListID := range card.IDCheckLists {
checklist, err := client.GetChecklist(checkListID, allArg)
if err != nil {
return err
}
}
list.Cards = append(list.Cards, card)
checklist.CheckItems = []trello.CheckItem{}
err = client.Get("checklists/"+checkListID+"/checkItems", allArg, &checklist.CheckItems)
if err != nil {
return err
}
card.Checklists = append(card.Checklists, checklist)
log.Debugf("Retrieved checklist %s for card %s", checkListID, card.ID)
}
}
log.Debugf("[Trello Migration] Looked for attachements on all cards of board %s", board.ID)
list.Cards = append(list.Cards, card)
}
log.Debugf("[Trello Migration] Looked for attachements on all cards of board %s", board.ID)
return
}
@ -196,7 +216,7 @@ func convertMarkdownToHTML(input string) (output string, err error) {
// Converts all previously obtained data from trello into the vikunja format.
// `trelloData` should contain all boards with their projects and cards respectively.
func convertTrelloDataToVikunja(trelloData []*trello.Board, token string) (fullVikunjaHierachie []*models.ProjectWithTasksAndBuckets, err error) {
func convertTrelloDataToVikunja(organizationName string, trelloData []*trello.Board, token string) (fullVikunjaHierachie []*models.ProjectWithTasksAndBuckets, err error) {
log.Debugf("[Trello Migration] ")
@ -205,7 +225,7 @@ func convertTrelloDataToVikunja(trelloData []*trello.Board, token string) (fullV
{
Project: models.Project{
ID: pseudoParentID,
Title: "Imported from Trello",
Title: organizationName,
},
},
}
@ -392,29 +412,58 @@ func (m *Migration) Migrate(u *user.User) (err error) {
log.Debugf("[Trello Migration] Starting migration for user %d", u.ID)
log.Debugf("[Trello Migration] Getting all trello data for user %d", u.ID)
trelloData, err := getTrelloData(m.Token)
client := trello.NewClient(config.MigrationTrelloKey.GetString(), m.Token)
client.Logger = log.GetLogger()
boards, err := getTrelloBoards(client)
if err != nil {
return
}
log.Debugf("[Trello Migration] Got all trello data for user %d", u.ID)
log.Debugf("[Trello Migration] Start converting trello data for user %d", u.ID)
fullVikunjaHierachie, err := convertTrelloDataToVikunja(trelloData, m.Token)
if err != nil {
return
organizationMap := getTrelloOrganizationsWithBoards(boards)
for organizationID, boards := range organizationMap {
log.Debugf("[Trello Migration] Getting organization with id %s for user %d", organizationID, u.ID)
orgName := organizationID
if organizationID != "Personal" {
organization, err := client.GetOrganization(organizationID, trello.Defaults())
if err != nil {
return err
}
orgName = organization.DisplayName
}
for _, board := range boards {
log.Debugf("[Trello Migration] Getting card data for board %s for user %d for organization %s", board.ID, u.ID, organizationID)
err = fillCardData(client, board)
if err != nil {
return err
}
log.Debugf("[Trello Migration] Got card data for board %s for user %d for organization %s", board.ID, u.ID, organizationID)
}
log.Debugf("[Trello Migration] Start converting trello data for user %d for organization %s", u.ID, organizationID)
hierarchy, err := convertTrelloDataToVikunja(orgName, boards, m.Token)
if err != nil {
return err
}
log.Debugf("[Trello Migration] Done migrating trello data for user %d for organization %s", u.ID, organizationID)
log.Debugf("[Trello Migration] Start inserting trello data for user %d for organization %s", u.ID, organizationID)
err = migration.InsertFromStructure(hierarchy, u)
if err != nil {
return err
}
log.Debugf("[Trello Migration] Done inserting trello data for user %d for organization %s", u.ID, organizationID)
}
log.Debugf("[Trello Migration] Done migrating trello data for user %d", u.ID)
log.Debugf("[Trello Migration] Start inserting trello data for user %d", u.ID)
log.Debugf("[Trello Migration] Done migrating all trello data for user %d", u.ID)
err = migration.InsertFromStructure(fullVikunjaHierachie, u)
if err != nil {
return
}
log.Debugf("[Trello Migration] Done inserting trello data for user %d", u.ID)
log.Debugf("[Trello Migration] Migration done for user %d", u.ID)
return nil
return
}

View File

@ -32,20 +32,23 @@ import (
"github.com/stretchr/testify/require"
)
func TestConvertTrelloToVikunja(t *testing.T) {
func getTestBoard(t *testing.T) ([]*trello.Board, time.Time) {
config.InitConfig()
time1, err := time.Parse(time.RFC3339Nano, "2014-09-26T08:25:05Z")
require.NoError(t, err)
exampleFile, err := os.ReadFile(config.ServiceRootpath.GetString() + "/pkg/modules/migration/testimage.jpg")
require.NoError(t, err)
trelloData := []*trello.Board{
{
Name: "TestBoard",
Desc: "This is a description",
Closed: false,
Name: "TestBoard",
Organization: trello.Organization{
ID: "orgid",
DisplayName: "TestOrg",
},
IDOrganization: "orgid",
Desc: "This is a description",
Closed: false,
Lists: []*trello.List{
{
Name: "Test Project 1",
@ -168,8 +171,13 @@ func TestConvertTrelloToVikunja(t *testing.T) {
},
},
{
Name: "TestBoard 2",
Closed: false,
Organization: trello.Organization{
ID: "orgid2",
DisplayName: "TestOrg2",
},
IDOrganization: "orgid2",
Name: "TestBoard 2",
Closed: false,
Lists: []*trello.List{
{
Name: "Test Project 4",
@ -183,8 +191,13 @@ func TestConvertTrelloToVikunja(t *testing.T) {
},
},
{
Name: "TestBoard Archived",
Closed: true,
Organization: trello.Organization{
ID: "orgid",
DisplayName: "TestOrg",
},
IDOrganization: "orgid",
Name: "TestBoard Archived",
Closed: true,
Lists: []*trello.List{
{
Name: "Test Project 5",
@ -197,67 +210,91 @@ func TestConvertTrelloToVikunja(t *testing.T) {
},
},
},
{
Name: "Personal Board",
Lists: []*trello.List{
{
Name: "Test Project 6",
Cards: []*trello.Card{
{
Name: "Test Card 5659",
Pos: 123,
},
},
},
},
},
}
trelloData[0].Prefs.BackgroundImage = "https://vikunja.io/testimage.jpg" // Using an image which we are hosting, so it'll still be up
expectedHierachie := []*models.ProjectWithTasksAndBuckets{
{
Project: models.Project{
ID: 1,
Title: "Imported from Trello",
},
},
{
Project: models.Project{
ID: 2,
ParentProjectID: 1,
Title: "TestBoard",
Description: "This is a description",
BackgroundInformation: bytes.NewBuffer(exampleFile),
},
Buckets: []*models.Bucket{
{
return trelloData, time1
}
func TestConvertTrelloToVikunja(t *testing.T) {
trelloData, time1 := getTestBoard(t)
exampleFile, err := os.ReadFile(config.ServiceRootpath.GetString() + "/pkg/modules/migration/testimage.jpg")
require.NoError(t, err)
expectedHierarchyOrg := map[string][]*models.ProjectWithTasksAndBuckets{
"orgid": {
{
Project: models.Project{
ID: 1,
Title: "Test Project 1",
},
{
ID: 2,
Title: "Test Project 2",
Title: "orgid",
},
},
Tasks: []*models.TaskWithComments{
{
Task: models.Task{
Title: "Test Card 1",
Description: "<p>Card Description <strong>bold</strong></p>\n",
BucketID: 1,
DueDate: time1,
Labels: []*models.Label{
{
Title: "Label 1",
HexColor: trelloColorMap["green"],
{
Project: models.Project{
ID: 2,
ParentProjectID: 1,
Title: "TestBoard",
Description: "This is a description",
BackgroundInformation: bytes.NewBuffer(exampleFile),
},
Buckets: []*models.Bucket{
{
ID: 1,
Title: "Test Project 1",
},
{
ID: 2,
Title: "Test Project 2",
},
},
Tasks: []*models.TaskWithComments{
{
Task: models.Task{
Title: "Test Card 1",
Description: "<p>Card Description <strong>bold</strong></p>\n",
BucketID: 1,
DueDate: time1,
Labels: []*models.Label{
{
Title: "Label 1",
HexColor: trelloColorMap["green"],
},
{
Title: "Label 2",
HexColor: trelloColorMap["orange"],
},
},
{
Title: "Label 2",
HexColor: trelloColorMap["orange"],
},
},
Attachments: []*models.TaskAttachment{
{
File: &files.File{
Name: "Testimage.jpg",
Mime: "image/jpg",
Size: uint64(len(exampleFile)),
FileContent: exampleFile,
Attachments: []*models.TaskAttachment{
{
File: &files.File{
Name: "Testimage.jpg",
Mime: "image/jpg",
Size: uint64(len(exampleFile)),
FileContent: exampleFile,
},
},
},
},
},
},
{
Task: models.Task{
Title: "Test Card 2",
Description: `
{
Task: models.Task{
Title: "Test Card 2",
Description: `
<h2> Checkproject 1</h2>
@ -270,117 +307,180 @@ func TestConvertTrelloToVikunja(t *testing.T) {
<ul data-type="taskList">
<li data-checked="false" data-type="taskItem"><label><input type="checkbox"><span></span></label><div><p>Pending Task</p></div></li>
<li data-checked="false" data-type="taskItem"><label><input type="checkbox"><span></span></label><div><p>Another Pending Task</p></div></li></ul>`,
BucketID: 1,
BucketID: 1,
},
},
},
{
Task: models.Task{
Title: "Test Card 3",
BucketID: 1,
{
Task: models.Task{
Title: "Test Card 3",
BucketID: 1,
},
},
},
{
Task: models.Task{
Title: "Test Card 4",
BucketID: 1,
Labels: []*models.Label{
{
Title: "Label 2",
HexColor: trelloColorMap["orange"],
{
Task: models.Task{
Title: "Test Card 4",
BucketID: 1,
Labels: []*models.Label{
{
Title: "Label 2",
HexColor: trelloColorMap["orange"],
},
},
},
},
},
{
Task: models.Task{
Title: "Test Card 5",
BucketID: 2,
Labels: []*models.Label{
{
Title: "Label 3",
HexColor: trelloColorMap["blue"],
},
{
Title: "Label 4",
HexColor: trelloColorMap["green_dark"],
},
{
Title: "Label 5",
HexColor: trelloColorMap["transparent"],
{
Task: models.Task{
Title: "Test Card 5",
BucketID: 2,
Labels: []*models.Label{
{
Title: "Label 3",
HexColor: trelloColorMap["blue"],
},
{
Title: "Label 4",
HexColor: trelloColorMap["green_dark"],
},
{
Title: "Label 5",
HexColor: trelloColorMap["transparent"],
},
},
},
},
},
{
Task: models.Task{
Title: "Test Card 6",
BucketID: 2,
DueDate: time1,
{
Task: models.Task{
Title: "Test Card 6",
BucketID: 2,
DueDate: time1,
},
},
{
Task: models.Task{
Title: "Test Card 7",
BucketID: 2,
},
},
{
Task: models.Task{
Title: "Test Card 8",
BucketID: 2,
},
},
},
{
Task: models.Task{
Title: "Test Card 7",
BucketID: 2,
},
{
Project: models.Project{
ID: 3,
ParentProjectID: 1,
Title: "TestBoard Archived",
IsArchived: true,
},
Buckets: []*models.Bucket{
{
ID: 3,
Title: "Test Project 5",
},
},
{
Task: models.Task{
Title: "Test Card 8",
BucketID: 2,
Tasks: []*models.TaskWithComments{
{
Task: models.Task{
Title: "Test Card 63423",
BucketID: 3,
},
},
},
},
},
{
Project: models.Project{
ID: 3,
ParentProjectID: 1,
Title: "TestBoard 2",
},
Buckets: []*models.Bucket{
{
ID: 3,
Title: "Test Project 4",
"orgid2": {
{
Project: models.Project{
ID: 1,
Title: "orgid2",
},
},
Tasks: []*models.TaskWithComments{
{
Task: models.Task{
Title: "Test Card 634",
BucketID: 3,
{
Project: models.Project{
ID: 2,
ParentProjectID: 1,
Title: "TestBoard 2",
},
Buckets: []*models.Bucket{
{
ID: 1,
Title: "Test Project 4",
},
},
Tasks: []*models.TaskWithComments{
{
Task: models.Task{
Title: "Test Card 634",
BucketID: 1,
},
},
},
},
},
{
Project: models.Project{
ID: 4,
ParentProjectID: 1,
Title: "TestBoard Archived",
IsArchived: true,
},
Buckets: []*models.Bucket{
{
ID: 4,
Title: "Test Project 5",
"Personal": {
{
Project: models.Project{
ID: 1,
Title: "Personal",
},
},
Tasks: []*models.TaskWithComments{
{
Task: models.Task{
Title: "Test Card 63423",
BucketID: 4,
{
Project: models.Project{
ID: 2,
ParentProjectID: 1,
Title: "Personal Board",
},
Buckets: []*models.Bucket{
{
ID: 1,
Title: "Test Project 6",
},
},
Tasks: []*models.TaskWithComments{
{
Task: models.Task{
Title: "Test Card 5659",
BucketID: 1,
},
},
},
},
},
}
hierachie, err := convertTrelloDataToVikunja(trelloData, "")
require.NoError(t, err)
assert.NotNil(t, hierachie)
if diff, equal := messagediff.PrettyDiff(hierachie, expectedHierachie); !equal {
t.Errorf("converted trello data = %v, want %v, diff: %v", hierachie, expectedHierachie, diff)
organizationMap := getTrelloOrganizationsWithBoards(trelloData)
for organizationID, boards := range organizationMap {
hierarchy, err := convertTrelloDataToVikunja(organizationID, boards, "")
require.NoError(t, err)
assert.NotNil(t, hierarchy)
if diff, equal := messagediff.PrettyDiff(hierarchy, expectedHierarchyOrg[organizationID]); !equal {
t.Errorf("converted trello data = %v,\nwant %v,\ndiff: %v", hierarchy, expectedHierarchyOrg[organizationID], diff)
}
}
}
func TestCreateOrganizationMap(t *testing.T) {
trelloData, _ := getTestBoard(t)
organizationMap := getTrelloOrganizationsWithBoards(trelloData)
expectedMap := map[string][]*trello.Board{
"orgid": {
trelloData[0],
trelloData[2],
},
"orgid2": {
trelloData[1],
},
"Personal": {
trelloData[3],
},
}
if diff, equal := messagediff.PrettyDiff(organizationMap, expectedMap); !equal {
t.Errorf("converted trello data = %v,\nwant %v,\ndiff: %v", organizationMap, expectedMap, diff)
}
}