Add support for migrating todoist boards (#732)

Add migrating buckets to converting todoist to vikunja structure

Add buckets migration

Co-authored-by: kolaente <k@knt.li>
Reviewed-on: vikunja/api#732
Co-Authored-By: konrad <konrad@kola-entertainments.de>
Co-Committed-By: konrad <konrad@kola-entertainments.de>

pkg/models/list.go

@@ -50,6 +50,9 @@ type List struct {
 	// Deprecated: you should use the dedicated task list endpoint because it has support for pagination and filtering
 	Tasks []*Task `xorm:"-" json:"-"`
 
+	// Only used for migration.
+	Buckets []*Bucket `xorm:"-" json:"-"`
+
 	// Whether or not a list is archived.
 	IsArchived bool `xorm:"not null default false" json:"is_archived" query:"is_archived"`
 

pkg/modules/migration/create_from_structure.go

@@ -45,9 +45,10 @@ func InsertFromStructure(str []*models.NamespaceWithLists, user *user.User) (err
 
 		// Create all lists
 		for _, l := range n.Lists {
-			// The tasks slice is going to be reset during the creation of the list so we rescue it here to be able
+			// The tasks and bucket slices are going to be reset during the creation of the list so we rescue it here
-			// to still loop over the tasks aftere the list was created.
+			// to be able to still loop over them aftere the list was created.
 			tasks := l.Tasks
+			originalBuckets := l.Buckets
 
 			l.NamespaceID = n.ID
 			err = l.Create(user)
@@ -56,10 +57,36 @@ func InsertFromStructure(str []*models.NamespaceWithLists, user *user.User) (err
 			}
 
 			log.Debugf("[creating structure] Created list %d", l.ID)
+
+			// Create all buckets
+			buckets := make(map[int64]*models.Bucket) // old bucket id is the key
+			if len(l.Buckets) > 0 {
+				log.Debugf("[creating structure] Creating %d buckets", len(l.Buckets))
+			}
+			for _, bucket := range originalBuckets {
+				oldID := bucket.ID
+				bucket.ID = 0 // We want a new id
+				bucket.ListID = l.ID
+				err = bucket.Create(user)
+				if err != nil {
+					return
+				}
+				buckets[oldID] = bucket
+				log.Debugf("[creating structure] Created bucket %d, old ID was %d", bucket.ID, oldID)
+			}
+
 			log.Debugf("[creating structure] Creating %d tasks", len(tasks))
 
 			// Create all tasks
 			for _, t := range tasks {
+				bucket, exists := buckets[t.BucketID]
+				if exists {
+					t.BucketID = bucket.ID
+				} else if t.BucketID > 0 {
+					log.Debugf("[creating structure] No bucket created for original bucket id %d", t.BucketID)
+					t.BucketID = 0
+				}
+
 				t.ListID = l.ID
 				err = t.Create(user)
 				if err != nil {
@@ -150,6 +177,9 @@ func InsertFromStructure(str []*models.NamespaceWithLists, user *user.User) (err
 					log.Debugf("[creating structure] Associated task %d with label %d", t.ID, lb.ID)
 				}
 			}
+
+			l.Tasks = tasks
+			l.Buckets = originalBuckets
 		}
 	}
 

pkg/modules/migration/create_from_structure_test.go

@@ -42,6 +42,12 @@ func TestInsertFromStructure(t *testing.T) {
 					{
 						Title:       "Testlist1",
 						Description: "Something",
+						Buckets: []*models.Bucket{
+							{
+								ID:    1234,
+								Title: "Test Bucket",
+							},
+						},
 						Tasks: []*models.Task{
 							{
 								Title:       "Task1",
@@ -92,6 +98,14 @@ func TestInsertFromStructure(t *testing.T) {
 									},
 								},
 							},
+							{
+								Title:    "Task in a bucket",
+								BucketID: 1234,
+							},
+							{
+								Title:    "Task in a nonexisting bucket",
+								BucketID: 1111,
+							},
 						},
 					},
 				},
@@ -99,5 +113,23 @@ func TestInsertFromStructure(t *testing.T) {
 		}
 		err := InsertFromStructure(testStructure, u)
 		assert.NoError(t, err)
+		db.AssertExists(t, "namespaces", map[string]interface{}{
+			"title":       testStructure[0].Namespace.Title,
+			"description": testStructure[0].Namespace.Description,
+		}, false)
+		db.AssertExists(t, "list", map[string]interface{}{
+			"title":       testStructure[0].Lists[0].Title,
+			"description": testStructure[0].Lists[0].Description,
+		}, false)
+		db.AssertExists(t, "tasks", map[string]interface{}{
+			"title":     testStructure[0].Lists[0].Tasks[5].Title,
+			"bucket_id": testStructure[0].Lists[0].Buckets[0].ID,
+		}, false)
+		db.AssertMissing(t, "tasks", map[string]interface{}{
+			"title":     testStructure[0].Lists[0].Tasks[6].Title,
+			"bucket_id": 1111, // No task with that bucket should exist
+		})
+		assert.NotEqual(t, 0, testStructure[0].Lists[0].Tasks[0].BucketID) // Should get the default bucket
+		assert.NotEqual(t, 0, testStructure[0].Lists[0].Tasks[6].BucketID) // Should get the default bucket
 	})
 }

pkg/modules/migration/todoist/todoist.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"sort"
 	"strings"
 	"time"
 
@@ -151,6 +152,15 @@ type reminder struct {
 	IsDeleted int64    `json:"is_deleted"`
 }
 
+type section struct {
+	ID           int64     `json:"id"`
+	DateAdded    time.Time `json:"date_added"`
+	IsDeleted    bool      `json:"is_deleted"`
+	Name         string    `json:"name"`
+	ProjectID    int64     `json:"project_id"`
+	SectionOrder int64     `json:"section_order"`
+}
+
 type sync struct {
 	Projects     []*project     `json:"projects"`
 	Items        []*item        `json:"items"`
@@ -158,6 +168,7 @@ type sync struct {
 	Notes        []*note        `json:"notes"`
 	ProjectNotes []*projectNote `json:"project_notes"`
 	Reminders    []*reminder    `json:"reminders"`
+	Sections     []*section     `json:"sections"`
 }
 
 var todoistColors = map[int64]string{}
@@ -258,6 +269,22 @@ func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.Namespa
 		newNamespace.Lists = append(newNamespace.Lists, list)
 	}
 
+	sort.Slice(sync.Sections, func(i, j int) bool {
+		return sync.Sections[i].SectionOrder < sync.Sections[j].SectionOrder
+	})
+
+	for _, section := range sync.Sections {
+		if section.IsDeleted || section.ProjectID == 0 {
+			continue
+		}
+
+		lists[section.ProjectID].Buckets = append(lists[section.ProjectID].Buckets, &models.Bucket{
+			ID:      section.ID,
+			Title:   section.Name,
+			Created: section.DateAdded,
+		})
+	}
+
 	for _, label := range sync.Labels {
 		labels[label.ID] = &models.Label{
 			Title:    label.Name,
@@ -267,9 +294,10 @@ func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.Namespa
 
 	for _, i := range sync.Items {
 		task := &models.Task{
-			Title:   i.Content,
+			Title:    i.Content,
-			Created: i.DateAdded.In(config.GetTimeZone()),
+			Created:  i.DateAdded.In(config.GetTimeZone()),
-			Done:    i.Checked == 1,
+			Done:     i.Checked == 1,
+			BucketID: i.SectionID,
 		}
 
 		// Only try to parse the task done at date if the task is actually done

pkg/modules/migration/todoist/todoist_test.go

@@ -143,7 +143,18 @@ func TestConvertTodoistToVikunja(t *testing.T) {
 			makeTestItem(400000106, 396936926, true, true, true),
 			makeTestItem(400000107, 396936926, false, false, true),
 			makeTestItem(400000108, 396936926, false, false, true),
-			makeTestItem(400000109, 396936926, false, false, true),
+			{
+				ID:            400000109,
+				UserID:        1855589,
+				ProjectID:     396936926,
+				Content:       "Task400000109",
+				Priority:      1,
+				ChildOrder:    1,
+				Checked:       1,
+				DateAdded:     time1,
+				DateCompleted: time3,
+				SectionID:     1234,
+			},
 
 			makeTestItem(400000007, 396936927, true, false, false),
 			makeTestItem(400000008, 396936927, true, false, false),
@@ -311,6 +322,13 @@ func TestConvertTodoistToVikunja(t *testing.T) {
 				},
 			},
 		},
+		Sections: []*section{
+			{
+				ID:        1234,
+				Name:      "Some Bucket",
+				ProjectID: 396936926,
+			},
+		},
 	}
 
 	vikunjaLabels := []*models.Label{
@@ -342,6 +360,12 @@ func TestConvertTodoistToVikunja(t *testing.T) {
 					Title:       "Project1",
 					Description: "Lorem Ipsum dolor sit amet\nLorem Ipsum dolor sit amet 2\nLorem Ipsum dolor sit amet 3",
 					HexColor:    todoistColors[30],
+					Buckets: []*models.Bucket{
+						{
+							ID:    1234,
+							Title: "Some Bucket",
+						},
+					},
 					Tasks: []*models.Task{
 						{
 							Title:       "Task400000000",
@@ -434,10 +458,11 @@ func TestConvertTodoistToVikunja(t *testing.T) {
 							DoneAt:  time3,
 						},
 						{
-							Title:   "Task400000109",
+							Title:    "Task400000109",
-							Done:    true,
+							Done:     true,
-							Created: time1,
+							Created:  time1,
-							DoneAt:  time3,
+							DoneAt:   time3,
+							BucketID: 1234,
 						},
 					},
 				},