From 00ed5884b480219ff862075c65c8c46eb579cae0 Mon Sep 17 00:00:00 2001 From: konrad Date: Wed, 16 Dec 2020 14:19:09 +0000 Subject: [PATCH] Add support for migrating todoist boards (#732) Add migrating buckets to converting todoist to vikunja structure Add buckets migration Co-authored-by: kolaente Reviewed-on: https://kolaente.dev/vikunja/api/pulls/732 Co-Authored-By: konrad Co-Committed-By: konrad --- pkg/models/list.go | 3 ++ .../migration/create_from_structure.go | 34 ++++++++++++++++-- .../migration/create_from_structure_test.go | 32 +++++++++++++++++ pkg/modules/migration/todoist/todoist.go | 34 ++++++++++++++++-- pkg/modules/migration/todoist/todoist_test.go | 35 ++++++++++++++++--- 5 files changed, 128 insertions(+), 10 deletions(-) diff --git a/pkg/models/list.go b/pkg/models/list.go index 8c5b13cdab3..d98b5c23ff4 100644 --- a/pkg/models/list.go +++ b/pkg/models/list.go @@ -50,6 +50,9 @@ type List struct { // Deprecated: you should use the dedicated task list endpoint because it has support for pagination and filtering Tasks []*Task `xorm:"-" json:"-"` + // Only used for migration. + Buckets []*Bucket `xorm:"-" json:"-"` + // Whether or not a list is archived. IsArchived bool `xorm:"not null default false" json:"is_archived" query:"is_archived"` diff --git a/pkg/modules/migration/create_from_structure.go b/pkg/modules/migration/create_from_structure.go index 993ff68cad6..eb42055c38c 100644 --- a/pkg/modules/migration/create_from_structure.go +++ b/pkg/modules/migration/create_from_structure.go @@ -45,9 +45,10 @@ func InsertFromStructure(str []*models.NamespaceWithLists, user *user.User) (err // Create all lists for _, l := range n.Lists { - // The tasks slice is going to be reset during the creation of the list so we rescue it here to be able - // to still loop over the tasks aftere the list was created. + // The tasks and bucket slices are going to be reset during the creation of the list so we rescue it here + // to be able to still loop over them aftere the list was created. tasks := l.Tasks + originalBuckets := l.Buckets l.NamespaceID = n.ID err = l.Create(user) @@ -56,10 +57,36 @@ func InsertFromStructure(str []*models.NamespaceWithLists, user *user.User) (err } log.Debugf("[creating structure] Created list %d", l.ID) + + // Create all buckets + buckets := make(map[int64]*models.Bucket) // old bucket id is the key + if len(l.Buckets) > 0 { + log.Debugf("[creating structure] Creating %d buckets", len(l.Buckets)) + } + for _, bucket := range originalBuckets { + oldID := bucket.ID + bucket.ID = 0 // We want a new id + bucket.ListID = l.ID + err = bucket.Create(user) + if err != nil { + return + } + buckets[oldID] = bucket + log.Debugf("[creating structure] Created bucket %d, old ID was %d", bucket.ID, oldID) + } + log.Debugf("[creating structure] Creating %d tasks", len(tasks)) // Create all tasks for _, t := range tasks { + bucket, exists := buckets[t.BucketID] + if exists { + t.BucketID = bucket.ID + } else if t.BucketID > 0 { + log.Debugf("[creating structure] No bucket created for original bucket id %d", t.BucketID) + t.BucketID = 0 + } + t.ListID = l.ID err = t.Create(user) if err != nil { @@ -150,6 +177,9 @@ func InsertFromStructure(str []*models.NamespaceWithLists, user *user.User) (err log.Debugf("[creating structure] Associated task %d with label %d", t.ID, lb.ID) } } + + l.Tasks = tasks + l.Buckets = originalBuckets } } diff --git a/pkg/modules/migration/create_from_structure_test.go b/pkg/modules/migration/create_from_structure_test.go index a94939cb7e1..f58980c22dc 100644 --- a/pkg/modules/migration/create_from_structure_test.go +++ b/pkg/modules/migration/create_from_structure_test.go @@ -42,6 +42,12 @@ func TestInsertFromStructure(t *testing.T) { { Title: "Testlist1", Description: "Something", + Buckets: []*models.Bucket{ + { + ID: 1234, + Title: "Test Bucket", + }, + }, Tasks: []*models.Task{ { Title: "Task1", @@ -92,6 +98,14 @@ func TestInsertFromStructure(t *testing.T) { }, }, }, + { + Title: "Task in a bucket", + BucketID: 1234, + }, + { + Title: "Task in a nonexisting bucket", + BucketID: 1111, + }, }, }, }, @@ -99,5 +113,23 @@ func TestInsertFromStructure(t *testing.T) { } err := InsertFromStructure(testStructure, u) assert.NoError(t, err) + db.AssertExists(t, "namespaces", map[string]interface{}{ + "title": testStructure[0].Namespace.Title, + "description": testStructure[0].Namespace.Description, + }, false) + db.AssertExists(t, "list", map[string]interface{}{ + "title": testStructure[0].Lists[0].Title, + "description": testStructure[0].Lists[0].Description, + }, false) + db.AssertExists(t, "tasks", map[string]interface{}{ + "title": testStructure[0].Lists[0].Tasks[5].Title, + "bucket_id": testStructure[0].Lists[0].Buckets[0].ID, + }, false) + db.AssertMissing(t, "tasks", map[string]interface{}{ + "title": testStructure[0].Lists[0].Tasks[6].Title, + "bucket_id": 1111, // No task with that bucket should exist + }) + assert.NotEqual(t, 0, testStructure[0].Lists[0].Tasks[0].BucketID) // Should get the default bucket + assert.NotEqual(t, 0, testStructure[0].Lists[0].Tasks[6].BucketID) // Should get the default bucket }) } diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index 5a6f3734704..38e42191ffb 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -23,6 +23,7 @@ import ( "fmt" "net/http" "net/url" + "sort" "strings" "time" @@ -151,6 +152,15 @@ type reminder struct { IsDeleted int64 `json:"is_deleted"` } +type section struct { + ID int64 `json:"id"` + DateAdded time.Time `json:"date_added"` + IsDeleted bool `json:"is_deleted"` + Name string `json:"name"` + ProjectID int64 `json:"project_id"` + SectionOrder int64 `json:"section_order"` +} + type sync struct { Projects []*project `json:"projects"` Items []*item `json:"items"` @@ -158,6 +168,7 @@ type sync struct { Notes []*note `json:"notes"` ProjectNotes []*projectNote `json:"project_notes"` Reminders []*reminder `json:"reminders"` + Sections []*section `json:"sections"` } var todoistColors = map[int64]string{} @@ -258,6 +269,22 @@ func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.Namespa newNamespace.Lists = append(newNamespace.Lists, list) } + sort.Slice(sync.Sections, func(i, j int) bool { + return sync.Sections[i].SectionOrder < sync.Sections[j].SectionOrder + }) + + for _, section := range sync.Sections { + if section.IsDeleted || section.ProjectID == 0 { + continue + } + + lists[section.ProjectID].Buckets = append(lists[section.ProjectID].Buckets, &models.Bucket{ + ID: section.ID, + Title: section.Name, + Created: section.DateAdded, + }) + } + for _, label := range sync.Labels { labels[label.ID] = &models.Label{ Title: label.Name, @@ -267,9 +294,10 @@ func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.Namespa for _, i := range sync.Items { task := &models.Task{ - Title: i.Content, - Created: i.DateAdded.In(config.GetTimeZone()), - Done: i.Checked == 1, + Title: i.Content, + Created: i.DateAdded.In(config.GetTimeZone()), + Done: i.Checked == 1, + BucketID: i.SectionID, } // Only try to parse the task done at date if the task is actually done diff --git a/pkg/modules/migration/todoist/todoist_test.go b/pkg/modules/migration/todoist/todoist_test.go index d8d7cdbe685..c69afaddffa 100644 --- a/pkg/modules/migration/todoist/todoist_test.go +++ b/pkg/modules/migration/todoist/todoist_test.go @@ -143,7 +143,18 @@ func TestConvertTodoistToVikunja(t *testing.T) { makeTestItem(400000106, 396936926, true, true, true), makeTestItem(400000107, 396936926, false, false, true), makeTestItem(400000108, 396936926, false, false, true), - makeTestItem(400000109, 396936926, false, false, true), + { + ID: 400000109, + UserID: 1855589, + ProjectID: 396936926, + Content: "Task400000109", + Priority: 1, + ChildOrder: 1, + Checked: 1, + DateAdded: time1, + DateCompleted: time3, + SectionID: 1234, + }, makeTestItem(400000007, 396936927, true, false, false), makeTestItem(400000008, 396936927, true, false, false), @@ -311,6 +322,13 @@ func TestConvertTodoistToVikunja(t *testing.T) { }, }, }, + Sections: []*section{ + { + ID: 1234, + Name: "Some Bucket", + ProjectID: 396936926, + }, + }, } vikunjaLabels := []*models.Label{ @@ -342,6 +360,12 @@ func TestConvertTodoistToVikunja(t *testing.T) { Title: "Project1", Description: "Lorem Ipsum dolor sit amet\nLorem Ipsum dolor sit amet 2\nLorem Ipsum dolor sit amet 3", HexColor: todoistColors[30], + Buckets: []*models.Bucket{ + { + ID: 1234, + Title: "Some Bucket", + }, + }, Tasks: []*models.Task{ { Title: "Task400000000", @@ -434,10 +458,11 @@ func TestConvertTodoistToVikunja(t *testing.T) { DoneAt: time3, }, { - Title: "Task400000109", - Done: true, - Created: time1, - DoneAt: time3, + Title: "Task400000109", + Done: true, + Created: time1, + DoneAt: time3, + BucketID: 1234, }, }, },