Migrate Trello organization after organization to limit total memory allocation

pkg/modules/migration/trello/trello.go

@@ -18,7 +18,6 @@ package trello
 
 import (
 	"bytes"
-
 	"code.vikunja.io/api/pkg/config"
 	"code.vikunja.io/api/pkg/files"
 	"code.vikunja.io/api/pkg/log"
@@ -107,80 +106,95 @@ func (m *Migration) AuthURL() string {
 		"&return_url=" + config.MigrationTrelloRedirectURL.GetString()
 }
 
-func getTrelloData(token string) (trelloData []*trello.Board, err error) {
-	allArg := trello.Arguments{"fields": "all"}
-
-	client := trello.NewClient(config.MigrationTrelloKey.GetString(), token)
-	client.Logger = log.GetLogger()
-
+func getTrelloData(client *trello.Client) (trelloData []*trello.Board, err error) {
 	log.Debugf("[Trello Migration] Getting boards...")
 
 	trelloData, err = client.GetMyBoards(trello.Defaults())
 	if err != nil {
-		return
+		return nil, err
 	}
 
 	log.Debugf("[Trello Migration] Got %d trello boards", len(trelloData))
 
-	for _, board := range trelloData {
-		log.Debugf("[Trello Migration] Getting projects for board %s", board.ID)
+	return
+}
 
-		board.Lists, err = board.GetLists(trello.Defaults())
+func createOrganizationMap(trelloData []*trello.Board) map[string][]*trello.Board {
+	boardMap := make(map[string][]*trello.Board)
+
+	for _, board := range trelloData {
+		boards, ok := boardMap[board.IDOrganization]
+		if ok {
+			boards = append(boards, board)
+		} else {
+			boards = []*trello.Board{board}
+		}
+		boardMap[board.IDOrganization] = boards
+	}
+
+	return boardMap
+}
+
+func fillCardData(client *trello.Client, board *trello.Board) (err error) {
+	allArg := trello.Arguments{"fields": "all"}
+
+	log.Debugf("[Trello Migration] Getting projects for board %s", board.ID)
+
+	board.Lists, err = board.GetLists(trello.Defaults())
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("[Trello Migration] Got %d projects for board %s", len(board.Lists), board.ID)
+
+	listMap := make(map[string]*trello.List, len(board.Lists))
+	for _, list := range board.Lists {
+		listMap[list.ID] = list
+	}
+
+	log.Debugf("[Trello Migration] Getting cards for board %s", board.ID)
+
+	cards, err := board.GetCards(allArg)
+	if err != nil {
+		return
+	}
+
+	log.Debugf("[Trello Migration] Got %d cards for board %s", len(cards), board.ID)
+
+	for _, card := range cards {
+		list, exists := listMap[card.IDList]
+		if !exists {
+			continue
+		}
+
+		card.Attachments, err = card.GetAttachments(allArg)
 		if err != nil {
 			return
 		}
 
-		log.Debugf("[Trello Migration] Got %d projects for board %s", len(board.Lists), board.ID)
-
-		listMap := make(map[string]*trello.List, len(board.Lists))
-		for _, list := range board.Lists {
-			listMap[list.ID] = list
-		}
-
-		log.Debugf("[Trello Migration] Getting cards for board %s", board.ID)
-
-		cards, err := board.GetCards(allArg)
-		if err != nil {
-			return nil, err
-		}
-
-		log.Debugf("[Trello Migration] Got %d cards for board %s", len(cards), board.ID)
-
-		for _, card := range cards {
-			list, exists := listMap[card.IDList]
-			if !exists {
-				continue
-			}
-
-			card.Attachments, err = card.GetAttachments(allArg)
-			if err != nil {
-				return nil, err
-			}
-
-			if len(card.IDCheckLists) > 0 {
-				for _, checkListID := range card.IDCheckLists {
-					checklist, err := client.GetChecklist(checkListID, allArg)
-					if err != nil {
-						return nil, err
-					}
-
-					checklist.CheckItems = []trello.CheckItem{}
-					err = client.Get("checklists/"+checkListID+"/checkItems", allArg, &checklist.CheckItems)
-					if err != nil {
-						return nil, err
-					}
-
-					card.Checklists = append(card.Checklists, checklist)
-					log.Debugf("Retrieved checklist %s for card %s", checkListID, card.ID)
+		if len(card.IDCheckLists) > 0 {
+			for _, checkListID := range card.IDCheckLists {
+				checklist, err := client.GetChecklist(checkListID, allArg)
+				if err != nil {
+					return err
 				}
-			}
 
-			list.Cards = append(list.Cards, card)
+				checklist.CheckItems = []trello.CheckItem{}
+				err = client.Get("checklists/"+checkListID+"/checkItems", allArg, &checklist.CheckItems)
+				if err != nil {
+					return err
+				}
+
+				card.Checklists = append(card.Checklists, checklist)
+				log.Debugf("Retrieved checklist %s for card %s", checkListID, card.ID)
+			}
 		}
 
-		log.Debugf("[Trello Migration] Looked for attachements on all cards of board %s", board.ID)
+		list.Cards = append(list.Cards, card)
 	}
 
+	log.Debugf("[Trello Migration] Looked for attachements on all cards of board %s", board.ID)
+
 	return
 }
 
@@ -196,7 +210,7 @@ func convertMarkdownToHTML(input string) (output string, err error) {
 
 // Converts all previously obtained data from trello into the vikunja format.
 // `trelloData` should contain all boards with their projects and cards respectively.
-func convertTrelloDataToVikunja(trelloData []*trello.Board, token string) (fullVikunjaHierachie []*models.ProjectWithTasksAndBuckets, err error) {
+func convertTrelloDataToVikunja(organizationName string, trelloData []*trello.Board, token string) (fullVikunjaHierachie []*models.ProjectWithTasksAndBuckets, err error) {
 
 	log.Debugf("[Trello Migration] ")
 
@@ -205,7 +219,7 @@ func convertTrelloDataToVikunja(trelloData []*trello.Board, token string) (fullV
 		{
 			Project: models.Project{
 				ID:    pseudoParentID,
-				Title: "Imported from Trello",
+				Title: organizationName,
 			},
 		},
 	}
@@ -393,29 +407,54 @@ func (m *Migration) Migrate(u *user.User) (err error) {
 	log.Debugf("[Trello Migration] Starting migration for user %d", u.ID)
 	log.Debugf("[Trello Migration] Getting all trello data for user %d", u.ID)
 
-	trelloData, err := getTrelloData(m.Token)
+	client := trello.NewClient(config.MigrationTrelloKey.GetString(), m.Token)
+	client.Logger = log.GetLogger()
+
+	data, err := getTrelloData(client)
 	if err != nil {
 		return
 	}
 
 	log.Debugf("[Trello Migration] Got all trello data for user %d", u.ID)
-	log.Debugf("[Trello Migration] Start converting trello data for user %d", u.ID)
 
-	fullVikunjaHierachie, err := convertTrelloDataToVikunja(trelloData, m.Token)
-	if err != nil {
-		return
+	organizationMap := createOrganizationMap(data)
+	for organizationID, boards := range organizationMap {
+		log.Debugf("[Trello Migration] Getting organization with id %s for user %d", organizationID, u.ID)
+		organization, err := client.GetOrganization(organizationID, trello.Defaults())
+		if err != nil {
+			return err
+		}
+
+		for _, board := range boards {
+			log.Debugf("[Trello Migration] Getting card data for board %s for user %d for organization %s", board.ID, u.ID, organizationID)
+
+			err = fillCardData(client, board)
+			if err != nil {
+				return err
+			}
+
+			log.Debugf("[Trello Migration] Got card data for board %s for user %d for organization %s", board.ID, u.ID, organizationID)
+		}
+
+		log.Debugf("[Trello Migration] Start converting trello data for user %d for organization %s", u.ID, organizationID)
+
+		hierarchy, err := convertTrelloDataToVikunja(organization.DisplayName, boards, m.Token)
+		if err != nil {
+			return err
+		}
+
+		log.Debugf("[Trello Migration] Done migrating trello data for user %d for organization %s", u.ID, organizationID)
+		log.Debugf("[Trello Migration] Start inserting trello data for user %d for organization %s", u.ID, organizationID)
+
+		err = migration.InsertFromStructure(hierarchy, u)
+		if err != nil {
+			return err
+		}
+
+		log.Debugf("[Trello Migration] Done inserting trello data for user %d for organization %s", u.ID, organizationID)
 	}
 
-	log.Debugf("[Trello Migration] Done migrating trello data for user %d", u.ID)
-	log.Debugf("[Trello Migration] Start inserting trello data for user %d", u.ID)
+	log.Debugf("[Trello Migration] Done migrating all trello data for user %d", u.ID)
 
-	err = migration.InsertFromStructure(fullVikunjaHierachie, u)
-	if err != nil {
-		return
-	}
-
-	log.Debugf("[Trello Migration] Done inserting trello data for user %d", u.ID)
-	log.Debugf("[Trello Migration] Migration done for user %d", u.ID)
-
-	return nil
+	return
 }

pkg/modules/migration/trello/trello_test.go

@@ -38,14 +38,19 @@ func TestConvertTrelloToVikunja(t *testing.T) {
 
 	time1, err := time.Parse(time.RFC3339Nano, "2014-09-26T08:25:05Z")
 	require.NoError(t, err)
-	exampleFile, err := os.ReadFile(config.ServiceRootpath.GetString() + "/pkg/modules/migration/testimage.jpg")
+	exampleFile, err := os.ReadFile("E:\\dev\\vikunja\\pkg\\modules\\migration\\testimage.jpg")
 	require.NoError(t, err)
 
 	trelloData := []*trello.Board{
 		{
-			Name:   "TestBoard",
-			Desc:   "This is a description",
-			Closed: false,
+			Name: "TestBoard",
+			Organization: trello.Organization{
+				ID:          "orgid",
+				DisplayName: "TestOrg",
+			},
+			IDOrganization: "orgid",
+			Desc:           "This is a description",
+			Closed:         false,
 			Lists: []*trello.List{
 				{
 					Name: "Test Project 1",
@@ -168,8 +173,13 @@ func TestConvertTrelloToVikunja(t *testing.T) {
 			},
 		},
 		{
-			Name:   "TestBoard 2",
-			Closed: false,
+			Organization: trello.Organization{
+				ID:          "orgid2",
+				DisplayName: "TestOrg2",
+			},
+			IDOrganization: "orgid2",
+			Name:           "TestBoard 2",
+			Closed:         false,
 			Lists: []*trello.List{
 				{
 					Name: "Test Project 4",
@@ -183,8 +193,13 @@ func TestConvertTrelloToVikunja(t *testing.T) {
 			},
 		},
 		{
-			Name:   "TestBoard Archived",
-			Closed: true,
+			Organization: trello.Organization{
+				ID:          "orgid",
+				DisplayName: "TestOrg",
+			},
+			IDOrganization: "orgid",
+			Name:           "TestBoard Archived",
+			Closed:         true,
 			Lists: []*trello.List{
 				{
 					Name: "Test Project 5",
@@ -200,65 +215,66 @@ func TestConvertTrelloToVikunja(t *testing.T) {
 	}
 	trelloData[0].Prefs.BackgroundImage = "https://vikunja.io/testimage.jpg" // Using an image which we are hosting, so it'll still be up
 
-	expectedHierachie := []*models.ProjectWithTasksAndBuckets{
-		{
-			Project: models.Project{
-				ID:    1,
-				Title: "Imported from Trello",
-			},
-		},
-		{
-			Project: models.Project{
-				ID:                    2,
-				ParentProjectID:       1,
-				Title:                 "TestBoard",
-				Description:           "This is a description",
-				BackgroundInformation: bytes.NewBuffer(exampleFile),
-			},
-			Buckets: []*models.Bucket{
-				{
+	expectedHierarchyOrg := map[string][]*models.ProjectWithTasksAndBuckets{
+		"orgid": {
+			{
+				Project: models.Project{
 					ID:    1,
-					Title: "Test Project 1",
-				},
-				{
-					ID:    2,
-					Title: "Test Project 2",
+					Title: "orgid",
 				},
 			},
-			Tasks: []*models.TaskWithComments{
-				{
-					Task: models.Task{
-						Title:          "Test Card 1",
-						Description:    "<p>Card Description <strong>bold</strong></p>\n",
-						BucketID:       1,
-						KanbanPosition: 123,
-						DueDate:        time1,
-						Labels: []*models.Label{
-							{
-								Title:    "Label 1",
-								HexColor: trelloColorMap["green"],
+			{
+				Project: models.Project{
+					ID:                    2,
+					ParentProjectID:       1,
+					Title:                 "TestBoard",
+					Description:           "This is a description",
+					BackgroundInformation: bytes.NewBuffer(exampleFile),
+				},
+				Buckets: []*models.Bucket{
+					{
+						ID:    1,
+						Title: "Test Project 1",
+					},
+					{
+						ID:    2,
+						Title: "Test Project 2",
+					},
+				},
+				Tasks: []*models.TaskWithComments{
+					{
+						Task: models.Task{
+							Title:          "Test Card 1",
+							Description:    "<p>Card Description <strong>bold</strong></p>\n",
+							BucketID:       1,
+							KanbanPosition: 123,
+							DueDate:        time1,
+							Labels: []*models.Label{
+								{
+									Title:    "Label 1",
+									HexColor: trelloColorMap["green"],
+								},
+								{
+									Title:    "Label 2",
+									HexColor: trelloColorMap["orange"],
+								},
 							},
-							{
-								Title:    "Label 2",
-								HexColor: trelloColorMap["orange"],
-							},
-						},
-						Attachments: []*models.TaskAttachment{
-							{
-								File: &files.File{
-									Name:        "Testimage.jpg",
-									Mime:        "image/jpg",
-									Size:        uint64(len(exampleFile)),
-									FileContent: exampleFile,
+							Attachments: []*models.TaskAttachment{
+								{
+									File: &files.File{
+										Name:        "Testimage.jpg",
+										Mime:        "image/jpg",
+										Size:        uint64(len(exampleFile)),
+										FileContent: exampleFile,
+									},
 								},
 							},
 						},
 					},
-				},
-				{
-					Task: models.Task{
-						Title: "Test Card 2",
-						Description: `
+					{
+						Task: models.Task{
+							Title: "Test Card 2",
+							Description: `
 
 <h2> Checkproject 1</h2>
 
@@ -271,126 +287,139 @@ func TestConvertTrelloToVikunja(t *testing.T) {
 <ul data-type="taskList">
 <li data-checked="false" data-type="taskItem"><label><input type="checkbox"><span></span></label><div><p>Pending Task</p></div></li>
 <li data-checked="false" data-type="taskItem"><label><input type="checkbox"><span></span></label><div><p>Another Pending Task</p></div></li></ul>`,
-						BucketID:       1,
-						KanbanPosition: 124,
+							BucketID:       1,
+							KanbanPosition: 124,
+						},
 					},
-				},
-				{
-					Task: models.Task{
-						Title:          "Test Card 3",
-						BucketID:       1,
-						KanbanPosition: 126,
+					{
+						Task: models.Task{
+							Title:          "Test Card 3",
+							BucketID:       1,
+							KanbanPosition: 126,
+						},
 					},
-				},
-				{
-					Task: models.Task{
-						Title:          "Test Card 4",
-						BucketID:       1,
-						KanbanPosition: 127,
-						Labels: []*models.Label{
-							{
-								Title:    "Label 2",
-								HexColor: trelloColorMap["orange"],
+					{
+						Task: models.Task{
+							Title:          "Test Card 4",
+							BucketID:       1,
+							KanbanPosition: 127,
+							Labels: []*models.Label{
+								{
+									Title:    "Label 2",
+									HexColor: trelloColorMap["orange"],
+								},
 							},
 						},
 					},
-				},
-				{
-					Task: models.Task{
-						Title:          "Test Card 5",
-						BucketID:       2,
-						KanbanPosition: 111,
-						Labels: []*models.Label{
-							{
-								Title:    "Label 3",
-								HexColor: trelloColorMap["blue"],
-							},
-							{
-								Title:    "Label 4",
-								HexColor: trelloColorMap["green_dark"],
-							},
-							{
-								Title:    "Label 5",
-								HexColor: trelloColorMap["transparent"],
+					{
+						Task: models.Task{
+							Title:          "Test Card 5",
+							BucketID:       2,
+							KanbanPosition: 111,
+							Labels: []*models.Label{
+								{
+									Title:    "Label 3",
+									HexColor: trelloColorMap["blue"],
+								},
+								{
+									Title:    "Label 4",
+									HexColor: trelloColorMap["green_dark"],
+								},
+								{
+									Title:    "Label 5",
+									HexColor: trelloColorMap["transparent"],
+								},
 							},
 						},
 					},
-				},
-				{
-					Task: models.Task{
-						Title:          "Test Card 6",
-						BucketID:       2,
-						KanbanPosition: 222,
-						DueDate:        time1,
+					{
+						Task: models.Task{
+							Title:          "Test Card 6",
+							BucketID:       2,
+							KanbanPosition: 222,
+							DueDate:        time1,
+						},
+					},
+					{
+						Task: models.Task{
+							Title:          "Test Card 7",
+							BucketID:       2,
+							KanbanPosition: 333,
+						},
+					},
+					{
+						Task: models.Task{
+							Title:          "Test Card 8",
+							BucketID:       2,
+							KanbanPosition: 444,
+						},
 					},
 				},
-				{
-					Task: models.Task{
-						Title:          "Test Card 7",
-						BucketID:       2,
-						KanbanPosition: 333,
+			},
+			{
+				Project: models.Project{
+					ID:              3,
+					ParentProjectID: 1,
+					Title:           "TestBoard Archived",
+					IsArchived:      true,
+				},
+				Buckets: []*models.Bucket{
+					{
+						ID:    3,
+						Title: "Test Project 5",
 					},
 				},
-				{
-					Task: models.Task{
-						Title:          "Test Card 8",
-						BucketID:       2,
-						KanbanPosition: 444,
+				Tasks: []*models.TaskWithComments{
+					{
+						Task: models.Task{
+							Title:          "Test Card 63423",
+							BucketID:       3,
+							KanbanPosition: 123,
+						},
 					},
 				},
 			},
 		},
-		{
-			Project: models.Project{
-				ID:              3,
-				ParentProjectID: 1,
-				Title:           "TestBoard 2",
-			},
-			Buckets: []*models.Bucket{
-				{
-					ID:    3,
-					Title: "Test Project 4",
+		"orgid2": {
+			{
+				Project: models.Project{
+					ID:    1,
+					Title: "orgid2",
 				},
 			},
-			Tasks: []*models.TaskWithComments{
-				{
-					Task: models.Task{
-						Title:          "Test Card 634",
-						BucketID:       3,
-						KanbanPosition: 123,
+			{
+				Project: models.Project{
+					ID:              2,
+					ParentProjectID: 1,
+					Title:           "TestBoard 2",
+				},
+				Buckets: []*models.Bucket{
+					{
+						ID:    1,
+						Title: "Test Project 4",
 					},
 				},
-			},
-		},
-		{
-			Project: models.Project{
-				ID:              4,
-				ParentProjectID: 1,
-				Title:           "TestBoard Archived",
-				IsArchived:      true,
-			},
-			Buckets: []*models.Bucket{
-				{
-					ID:    4,
-					Title: "Test Project 5",
-				},
-			},
-			Tasks: []*models.TaskWithComments{
-				{
-					Task: models.Task{
-						Title:          "Test Card 63423",
-						BucketID:       4,
-						KanbanPosition: 123,
+				Tasks: []*models.TaskWithComments{
+					{
+						Task: models.Task{
+							Title:          "Test Card 634",
+							BucketID:       1,
+							KanbanPosition: 123,
+						},
 					},
 				},
 			},
 		},
 	}
 
-	hierachie, err := convertTrelloDataToVikunja(trelloData, "")
-	require.NoError(t, err)
-	assert.NotNil(t, hierachie)
-	if diff, equal := messagediff.PrettyDiff(hierachie, expectedHierachie); !equal {
-		t.Errorf("converted trello data = %v, want %v, diff: %v", hierachie, expectedHierachie, diff)
+	organizationMap := createOrganizationMap(trelloData)
+	for organizationID, boards := range organizationMap {
+		hierarchy, err := convertTrelloDataToVikunja(organizationID, boards, "")
+
+		require.NoError(t, err)
+		assert.NotNil(t, hierarchy)
+		if diff, equal := messagediff.PrettyDiff(hierarchy, expectedHierarchyOrg[organizationID]); !equal {
+			t.Errorf("converted trello data = %v,\nwant %v,\ndiff: %v", hierarchy, expectedHierarchyOrg[organizationID], diff)
+		}
 	}
 }