From e5394d6d4bec580fbb3dc06ba386392e76a29225 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sun, 9 Oct 2022 18:56:29 +0200 Subject: [PATCH] feat(migration): add TickTick migrator --- pkg/modules/migration/ticktick/ticktick.go | 219 ++++++++++++++++++ .../migration/ticktick/ticktick_test.go | 136 +++++++++++ pkg/routes/routes.go | 2 + 3 files changed, 357 insertions(+) create mode 100644 pkg/modules/migration/ticktick/ticktick.go create mode 100644 pkg/modules/migration/ticktick/ticktick_test.go diff --git a/pkg/modules/migration/ticktick/ticktick.go b/pkg/modules/migration/ticktick/ticktick.go new file mode 100644 index 0000000000..0de11317d4 --- /dev/null +++ b/pkg/modules/migration/ticktick/ticktick.go @@ -0,0 +1,219 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package ticktick + +import ( + "encoding/csv" + "io" + "sort" + "strconv" + "strings" + "time" + + "code.vikunja.io/api/pkg/models" + "code.vikunja.io/api/pkg/modules/migration" + "code.vikunja.io/api/pkg/user" +) + +const timeISO = "2006-01-02T15:04:05-0700" + +type Migrator struct { +} + +type tickTickTask struct { + FolderName string + ListName string + Title string + Tags []string + Content string + IsChecklist bool + StartDate time.Time + DueDate time.Time + Reminder time.Duration + Repeat string + Priority int + Status string + CreatedTime time.Time + CompletedTime time.Time + Order float64 + TaskID int64 + ParentID int64 +} + +func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.NamespaceWithListsAndTasks) { + namespace := &models.NamespaceWithListsAndTasks{ + Namespace: models.Namespace{ + Title: "Migrated from TickTick", + }, + Lists: []*models.ListWithTasksAndBuckets{}, + } + + lists := make(map[string]*models.ListWithTasksAndBuckets) + for _, t := range tasks { + _, has := lists[t.ListName] + if !has { + lists[t.ListName] = &models.ListWithTasksAndBuckets{ + List: models.List{ + Title: t.ListName, + }, + } + } + + labels := make([]*models.Label, 0, len(t.Tags)) + for _, tag := range t.Tags { + labels = append(labels, &models.Label{ + Title: tag, + }) + } + + task := &models.TaskWithComments{ + Task: models.Task{ + ID: t.TaskID, + Title: t.Title, + Description: t.Content, + StartDate: t.StartDate, + EndDate: t.DueDate, + DueDate: t.DueDate, + Reminders: []time.Time{ + t.DueDate.Add(t.Reminder * -1), + }, + Done: t.Status == "1", + DoneAt: t.CompletedTime, + Position: t.Order, + Labels: labels, + }, + } + + if t.ParentID != 0 { + task.RelatedTasks = map[models.RelationKind][]*models.Task{ + models.RelationKindParenttask: {{ID: t.ParentID}}, + } + } + + lists[t.ListName].Tasks = append(lists[t.ListName].Tasks, task) + } + + for _, l := range lists { + namespace.Lists = append(namespace.Lists, l) + } + + sort.Slice(namespace.Lists, func(i, j int) bool { + return namespace.Lists[i].Title < namespace.Lists[j].Title + }) + + return []*models.NamespaceWithListsAndTasks{namespace} +} + +// Name is used to get the name of the ticktick migration - we're using the docs here to annotate the status route. +// @Summary Get migration status +// @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again. +// @tags migration +// @Produce json +// @Security JWTKeyAuth +// @Success 200 {object} migration.Status "The migration status" +// @Failure 500 {object} models.Message "Internal server error" +// @Router /migration/ticktick/status [get] +func (m *Migrator) Name() string { + return "ticktick" +} + +// Migrate takes a ticktick export, parses it and imports everything in it into Vikunja. +// @Summary Import all lists, tasks etc. from a TickTick backup export +// @Description Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja. +// @tags migration +// @Accept json +// @Produce json +// @Security JWTKeyAuth +// @Param import formData string true "The TickTick backup csv file." +// @Success 200 {object} models.Message "A message telling you everything was migrated successfully." +// @Failure 500 {object} models.Message "Internal server error" +// @Router /migration/ticktick/migrate [post] +func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error { + fr := io.NewSectionReader(file, 0, 0) + r := csv.NewReader(fr) + records, err := r.ReadAll() + if err != nil { + return err + } + + allTasks := make([]*tickTickTask, 0, len(records)) + for line, record := range records { + if line <= 3 { + continue + } + startDate, err := time.Parse(timeISO, record[6]) + if err != nil { + return err + } + dueDate, err := time.Parse(timeISO, record[7]) + if err != nil { + return err + } + // TODO: parse properly + reminder, err := time.ParseDuration(record[8]) + if err != nil { + return err + } + priority, err := strconv.Atoi(record[10]) + if err != nil { + return err + } + createdTime, err := time.Parse(timeISO, record[12]) + if err != nil { + return err + } + completedTime, err := time.Parse(timeISO, record[13]) + if err != nil { + return err + } + order, err := strconv.ParseFloat(record[14], 64) + if err != nil { + return err + } + taskID, err := strconv.ParseInt(record[21], 10, 64) + if err != nil { + return err + } + parentID, err := strconv.ParseInt(record[21], 10, 64) + if err != nil { + return err + } + + allTasks = append(allTasks, &tickTickTask{ + ListName: record[1], + Title: record[2], + Tags: strings.Split(record[3], ", "), + Content: record[4], + IsChecklist: record[5] == "Y", + StartDate: startDate, + DueDate: dueDate, + Reminder: reminder, + Repeat: record[9], + Priority: priority, + Status: record[11], + CreatedTime: createdTime, + CompletedTime: completedTime, + Order: order, + TaskID: taskID, + ParentID: parentID, + }) + } + + vikunjaTasks := convertTickTickToVikunja(allTasks) + + return migration.InsertFromStructure(vikunjaTasks, user) +} diff --git a/pkg/modules/migration/ticktick/ticktick_test.go b/pkg/modules/migration/ticktick/ticktick_test.go new file mode 100644 index 0000000000..05fd72739c --- /dev/null +++ b/pkg/modules/migration/ticktick/ticktick_test.go @@ -0,0 +1,136 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package ticktick + +import ( + "testing" + "time" + + "code.vikunja.io/api/pkg/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertTicktickTasksToVikunja(t *testing.T) { + time1, err := time.Parse(time.RFC3339Nano, "2022-11-18T03:00:00.4770000Z") + require.NoError(t, err) + time2, err := time.Parse(time.RFC3339Nano, "2022-12-18T03:00:00.4770000Z") + require.NoError(t, err) + time3, err := time.Parse(time.RFC3339Nano, "2022-12-10T03:00:00.4770000Z") + require.NoError(t, err) + duration, err := time.ParseDuration("24h") + require.NoError(t, err) + + tickTickTasks := []*tickTickTask{ + { + TaskID: 1, + ParentID: 0, + ListName: "List 1", + Title: "Test task 1", + Tags: []string{"label1", "label2"}, + Content: "Lorem Ipsum Dolor sit amet", + StartDate: time1, + DueDate: time2, + Reminder: duration, + Repeat: "FREQ=WEEKLY;INTERVAL=1;UNTIL=20190117T210000Z", + Status: "0", + Order: -1099511627776, + }, + { + TaskID: 2, + ParentID: 1, + ListName: "List 1", + Title: "Test task 2", + Status: "1", + CompletedTime: time3, + Order: -1099511626, + }, + { + TaskID: 3, + ParentID: 0, + ListName: "List 1", + Title: "Test task 3", + Tags: []string{"label1", "label2", "other label"}, + StartDate: time1, + DueDate: time2, + Reminder: duration, + Status: "0", + Order: -109951627776, + }, + { + TaskID: 4, + ParentID: 0, + ListName: "List 2", + Title: "Test task 4", + Status: "0", + Order: -109951627777, + }, + } + + vikunjaTasks := convertTickTickToVikunja(tickTickTasks) + + assert.Len(t, vikunjaTasks, 1) + assert.Len(t, vikunjaTasks[0].Lists, 2) + + assert.Len(t, vikunjaTasks[0].Lists[0].Tasks, 3) + assert.Equal(t, vikunjaTasks[0].Lists[0].Title, tickTickTasks[0].ListName) + + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Title, tickTickTasks[0].Title) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Description, tickTickTasks[0].Content) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].StartDate, tickTickTasks[0].StartDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].EndDate, tickTickTasks[0].DueDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].DueDate, tickTickTasks[0].DueDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Labels, []*models.Label{ + {Title: "label1"}, + {Title: "label2"}, + }) + //assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Reminders, tickTickTasks[0].) // TODO + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Position, tickTickTasks[0].Order) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Done, false) + + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Title, tickTickTasks[1].Title) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Position, tickTickTasks[1].Order) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Done, true) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].DoneAt, tickTickTasks[1].CompletedTime) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].RelatedTasks, models.RelatedTaskMap{ + models.RelationKindParenttask: []*models.Task{ + { + ID: tickTickTasks[1].ParentID, + }, + }, + }) + + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Title, tickTickTasks[2].Title) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Description, tickTickTasks[2].Content) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].StartDate, tickTickTasks[2].StartDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].EndDate, tickTickTasks[2].DueDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].DueDate, tickTickTasks[2].DueDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Labels, []*models.Label{ + {Title: "label1"}, + {Title: "label2"}, + {Title: "other label"}, + }) + //assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Reminders, tickTickTasks[0].) // TODO + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Position, tickTickTasks[2].Order) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Done, false) + + assert.Len(t, vikunjaTasks[0].Lists[1].Tasks, 1) + assert.Equal(t, vikunjaTasks[0].Lists[1].Title, tickTickTasks[3].ListName) + + assert.Equal(t, vikunjaTasks[0].Lists[1].Tasks[0].Title, tickTickTasks[3].Title) + assert.Equal(t, vikunjaTasks[0].Lists[1].Tasks[0].Position, tickTickTasks[3].Order) +} diff --git a/pkg/routes/routes.go b/pkg/routes/routes.go index f7e856bba2..88bc138d59 100644 --- a/pkg/routes/routes.go +++ b/pkg/routes/routes.go @@ -53,6 +53,8 @@ import ( "strings" "time" + "code.vikunja.io/api/pkg/modules/migration/ticktick" + "github.com/ulule/limiter/v3" vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file"