From e5394d6d4bec580fbb3dc06ba386392e76a29225 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sun, 9 Oct 2022 18:56:29 +0200 Subject: [PATCH 1/6] feat(migration): add TickTick migrator --- pkg/modules/migration/ticktick/ticktick.go | 219 ++++++++++++++++++ .../migration/ticktick/ticktick_test.go | 136 +++++++++++ pkg/routes/routes.go | 2 + 3 files changed, 357 insertions(+) create mode 100644 pkg/modules/migration/ticktick/ticktick.go create mode 100644 pkg/modules/migration/ticktick/ticktick_test.go diff --git a/pkg/modules/migration/ticktick/ticktick.go b/pkg/modules/migration/ticktick/ticktick.go new file mode 100644 index 0000000000..0de11317d4 --- /dev/null +++ b/pkg/modules/migration/ticktick/ticktick.go @@ -0,0 +1,219 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package ticktick + +import ( + "encoding/csv" + "io" + "sort" + "strconv" + "strings" + "time" + + "code.vikunja.io/api/pkg/models" + "code.vikunja.io/api/pkg/modules/migration" + "code.vikunja.io/api/pkg/user" +) + +const timeISO = "2006-01-02T15:04:05-0700" + +type Migrator struct { +} + +type tickTickTask struct { + FolderName string + ListName string + Title string + Tags []string + Content string + IsChecklist bool + StartDate time.Time + DueDate time.Time + Reminder time.Duration + Repeat string + Priority int + Status string + CreatedTime time.Time + CompletedTime time.Time + Order float64 + TaskID int64 + ParentID int64 +} + +func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.NamespaceWithListsAndTasks) { + namespace := &models.NamespaceWithListsAndTasks{ + Namespace: models.Namespace{ + Title: "Migrated from TickTick", + }, + Lists: []*models.ListWithTasksAndBuckets{}, + } + + lists := make(map[string]*models.ListWithTasksAndBuckets) + for _, t := range tasks { + _, has := lists[t.ListName] + if !has { + lists[t.ListName] = &models.ListWithTasksAndBuckets{ + List: models.List{ + Title: t.ListName, + }, + } + } + + labels := make([]*models.Label, 0, len(t.Tags)) + for _, tag := range t.Tags { + labels = append(labels, &models.Label{ + Title: tag, + }) + } + + task := &models.TaskWithComments{ + Task: models.Task{ + ID: t.TaskID, + Title: t.Title, + Description: t.Content, + StartDate: t.StartDate, + EndDate: t.DueDate, + DueDate: t.DueDate, + Reminders: []time.Time{ + t.DueDate.Add(t.Reminder * -1), + }, + Done: t.Status == "1", + DoneAt: t.CompletedTime, + Position: t.Order, + Labels: labels, + }, + } + + if t.ParentID != 0 { + task.RelatedTasks = map[models.RelationKind][]*models.Task{ + models.RelationKindParenttask: {{ID: t.ParentID}}, + } + } + + lists[t.ListName].Tasks = append(lists[t.ListName].Tasks, task) + } + + for _, l := range lists { + namespace.Lists = append(namespace.Lists, l) + } + + sort.Slice(namespace.Lists, func(i, j int) bool { + return namespace.Lists[i].Title < namespace.Lists[j].Title + }) + + return []*models.NamespaceWithListsAndTasks{namespace} +} + +// Name is used to get the name of the ticktick migration - we're using the docs here to annotate the status route. +// @Summary Get migration status +// @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again. +// @tags migration +// @Produce json +// @Security JWTKeyAuth +// @Success 200 {object} migration.Status "The migration status" +// @Failure 500 {object} models.Message "Internal server error" +// @Router /migration/ticktick/status [get] +func (m *Migrator) Name() string { + return "ticktick" +} + +// Migrate takes a ticktick export, parses it and imports everything in it into Vikunja. +// @Summary Import all lists, tasks etc. from a TickTick backup export +// @Description Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja. +// @tags migration +// @Accept json +// @Produce json +// @Security JWTKeyAuth +// @Param import formData string true "The TickTick backup csv file." +// @Success 200 {object} models.Message "A message telling you everything was migrated successfully." +// @Failure 500 {object} models.Message "Internal server error" +// @Router /migration/ticktick/migrate [post] +func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error { + fr := io.NewSectionReader(file, 0, 0) + r := csv.NewReader(fr) + records, err := r.ReadAll() + if err != nil { + return err + } + + allTasks := make([]*tickTickTask, 0, len(records)) + for line, record := range records { + if line <= 3 { + continue + } + startDate, err := time.Parse(timeISO, record[6]) + if err != nil { + return err + } + dueDate, err := time.Parse(timeISO, record[7]) + if err != nil { + return err + } + // TODO: parse properly + reminder, err := time.ParseDuration(record[8]) + if err != nil { + return err + } + priority, err := strconv.Atoi(record[10]) + if err != nil { + return err + } + createdTime, err := time.Parse(timeISO, record[12]) + if err != nil { + return err + } + completedTime, err := time.Parse(timeISO, record[13]) + if err != nil { + return err + } + order, err := strconv.ParseFloat(record[14], 64) + if err != nil { + return err + } + taskID, err := strconv.ParseInt(record[21], 10, 64) + if err != nil { + return err + } + parentID, err := strconv.ParseInt(record[21], 10, 64) + if err != nil { + return err + } + + allTasks = append(allTasks, &tickTickTask{ + ListName: record[1], + Title: record[2], + Tags: strings.Split(record[3], ", "), + Content: record[4], + IsChecklist: record[5] == "Y", + StartDate: startDate, + DueDate: dueDate, + Reminder: reminder, + Repeat: record[9], + Priority: priority, + Status: record[11], + CreatedTime: createdTime, + CompletedTime: completedTime, + Order: order, + TaskID: taskID, + ParentID: parentID, + }) + } + + vikunjaTasks := convertTickTickToVikunja(allTasks) + + return migration.InsertFromStructure(vikunjaTasks, user) +} diff --git a/pkg/modules/migration/ticktick/ticktick_test.go b/pkg/modules/migration/ticktick/ticktick_test.go new file mode 100644 index 0000000000..05fd72739c --- /dev/null +++ b/pkg/modules/migration/ticktick/ticktick_test.go @@ -0,0 +1,136 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package ticktick + +import ( + "testing" + "time" + + "code.vikunja.io/api/pkg/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertTicktickTasksToVikunja(t *testing.T) { + time1, err := time.Parse(time.RFC3339Nano, "2022-11-18T03:00:00.4770000Z") + require.NoError(t, err) + time2, err := time.Parse(time.RFC3339Nano, "2022-12-18T03:00:00.4770000Z") + require.NoError(t, err) + time3, err := time.Parse(time.RFC3339Nano, "2022-12-10T03:00:00.4770000Z") + require.NoError(t, err) + duration, err := time.ParseDuration("24h") + require.NoError(t, err) + + tickTickTasks := []*tickTickTask{ + { + TaskID: 1, + ParentID: 0, + ListName: "List 1", + Title: "Test task 1", + Tags: []string{"label1", "label2"}, + Content: "Lorem Ipsum Dolor sit amet", + StartDate: time1, + DueDate: time2, + Reminder: duration, + Repeat: "FREQ=WEEKLY;INTERVAL=1;UNTIL=20190117T210000Z", + Status: "0", + Order: -1099511627776, + }, + { + TaskID: 2, + ParentID: 1, + ListName: "List 1", + Title: "Test task 2", + Status: "1", + CompletedTime: time3, + Order: -1099511626, + }, + { + TaskID: 3, + ParentID: 0, + ListName: "List 1", + Title: "Test task 3", + Tags: []string{"label1", "label2", "other label"}, + StartDate: time1, + DueDate: time2, + Reminder: duration, + Status: "0", + Order: -109951627776, + }, + { + TaskID: 4, + ParentID: 0, + ListName: "List 2", + Title: "Test task 4", + Status: "0", + Order: -109951627777, + }, + } + + vikunjaTasks := convertTickTickToVikunja(tickTickTasks) + + assert.Len(t, vikunjaTasks, 1) + assert.Len(t, vikunjaTasks[0].Lists, 2) + + assert.Len(t, vikunjaTasks[0].Lists[0].Tasks, 3) + assert.Equal(t, vikunjaTasks[0].Lists[0].Title, tickTickTasks[0].ListName) + + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Title, tickTickTasks[0].Title) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Description, tickTickTasks[0].Content) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].StartDate, tickTickTasks[0].StartDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].EndDate, tickTickTasks[0].DueDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].DueDate, tickTickTasks[0].DueDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Labels, []*models.Label{ + {Title: "label1"}, + {Title: "label2"}, + }) + //assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Reminders, tickTickTasks[0].) // TODO + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Position, tickTickTasks[0].Order) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Done, false) + + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Title, tickTickTasks[1].Title) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Position, tickTickTasks[1].Order) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Done, true) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].DoneAt, tickTickTasks[1].CompletedTime) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].RelatedTasks, models.RelatedTaskMap{ + models.RelationKindParenttask: []*models.Task{ + { + ID: tickTickTasks[1].ParentID, + }, + }, + }) + + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Title, tickTickTasks[2].Title) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Description, tickTickTasks[2].Content) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].StartDate, tickTickTasks[2].StartDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].EndDate, tickTickTasks[2].DueDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].DueDate, tickTickTasks[2].DueDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Labels, []*models.Label{ + {Title: "label1"}, + {Title: "label2"}, + {Title: "other label"}, + }) + //assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Reminders, tickTickTasks[0].) // TODO + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Position, tickTickTasks[2].Order) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Done, false) + + assert.Len(t, vikunjaTasks[0].Lists[1].Tasks, 1) + assert.Equal(t, vikunjaTasks[0].Lists[1].Title, tickTickTasks[3].ListName) + + assert.Equal(t, vikunjaTasks[0].Lists[1].Tasks[0].Title, tickTickTasks[3].Title) + assert.Equal(t, vikunjaTasks[0].Lists[1].Tasks[0].Position, tickTickTasks[3].Order) +} diff --git a/pkg/routes/routes.go b/pkg/routes/routes.go index f7e856bba2..88bc138d59 100644 --- a/pkg/routes/routes.go +++ b/pkg/routes/routes.go @@ -53,6 +53,8 @@ import ( "strings" "time" + "code.vikunja.io/api/pkg/modules/migration/ticktick" + "github.com/ulule/limiter/v3" vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file" From 3af98551481d95cf6507c79823b45d824df02cf9 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sun, 9 Oct 2022 18:56:37 +0200 Subject: [PATCH 2/6] feat(migration): add routes for TickTick migrator --- pkg/routes/routes.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/routes/routes.go b/pkg/routes/routes.go index 88bc138d59..ee85e2265d 100644 --- a/pkg/routes/routes.go +++ b/pkg/routes/routes.go @@ -47,6 +47,7 @@ package routes import ( + "code.vikunja.io/api/pkg/modules/migration/ticktick" "errors" "fmt" "net/url" @@ -655,12 +656,21 @@ func registerMigrations(m *echo.Group) { microsoftTodoMigrationHandler.RegisterRoutes(m) } + // Vikunja File Migrator vikunjaFileMigrationHandler := &migrationHandler.FileMigratorWeb{ MigrationStruct: func() migration.FileMigrator { return &vikunja_file.FileMigrator{} }, } vikunjaFileMigrationHandler.RegisterRoutes(m) + + // TickTick File Migrator + tickTickFileMigrator := migrationHandler.FileMigratorWeb{ + MigrationStruct: func() migration.FileMigrator { + return &ticktick.Migrator{} + }, + } + tickTickFileMigrator.RegisterRoutes(m) } func registerCalDavRoutes(c *echo.Group) { From 5871d32c2db4e05599062688c53265c4b7e973de Mon Sep 17 00:00:00 2001 From: kolaente Date: Sun, 9 Oct 2022 18:59:03 +0200 Subject: [PATCH 3/6] feat(migration): generate swagger docs --- pkg/swagger/docs.go | 74 ++++++++++++++++++++++++++++++++++++++++ pkg/swagger/swagger.json | 74 ++++++++++++++++++++++++++++++++++++++++ pkg/swagger/swagger.yaml | 49 ++++++++++++++++++++++++++ 3 files changed, 197 insertions(+) diff --git a/pkg/swagger/docs.go b/pkg/swagger/docs.go index fec1f2452f..0c2e6c95f1 100644 --- a/pkg/swagger/docs.go +++ b/pkg/swagger/docs.go @@ -2699,6 +2699,80 @@ const docTemplate = `{ } } }, + "/migration/ticktick/migrate": { + "post": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Import all lists, tasks etc. from a TickTick backup export", + "parameters": [ + { + "type": "string", + "description": "The TickTick backup csv file.", + "name": "import", + "in": "formData", + "required": true + } + ], + "responses": { + "200": { + "description": "A message telling you everything was migrated successfully.", + "schema": { + "$ref": "#/definitions/models.Message" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, + "/migration/ticktick/status": { + "get": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.", + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Get migration status", + "responses": { + "200": { + "description": "The migration status", + "schema": { + "$ref": "#/definitions/migration.Status" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, "/migration/todoist/auth": { "get": { "security": [ diff --git a/pkg/swagger/swagger.json b/pkg/swagger/swagger.json index 30eb40d18c..6b6e43c6bc 100644 --- a/pkg/swagger/swagger.json +++ b/pkg/swagger/swagger.json @@ -2690,6 +2690,80 @@ } } }, + "/migration/ticktick/migrate": { + "post": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Import all lists, tasks etc. from a TickTick backup export", + "parameters": [ + { + "type": "string", + "description": "The TickTick backup csv file.", + "name": "import", + "in": "formData", + "required": true + } + ], + "responses": { + "200": { + "description": "A message telling you everything was migrated successfully.", + "schema": { + "$ref": "#/definitions/models.Message" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, + "/migration/ticktick/status": { + "get": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.", + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Get migration status", + "responses": { + "200": { + "description": "The migration status", + "schema": { + "$ref": "#/definitions/migration.Status" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, "/migration/todoist/auth": { "get": { "security": [ diff --git a/pkg/swagger/swagger.yaml b/pkg/swagger/swagger.yaml index 34e6fc3bf9..d7cb7c064d 100644 --- a/pkg/swagger/swagger.yaml +++ b/pkg/swagger/swagger.yaml @@ -3250,6 +3250,55 @@ paths: summary: Get migration status tags: - migration + /migration/ticktick/migrate: + post: + consumes: + - application/json + description: Imports all projects, tasks, notes, reminders, subtasks and files + from a TickTick backup export into Vikunja. + parameters: + - description: The TickTick backup csv file. + in: formData + name: import + required: true + type: string + produces: + - application/json + responses: + "200": + description: A message telling you everything was migrated successfully. + schema: + $ref: '#/definitions/models.Message' + "500": + description: Internal server error + schema: + $ref: '#/definitions/models.Message' + security: + - JWTKeyAuth: [] + summary: Import all lists, tasks etc. from a TickTick backup export + tags: + - migration + /migration/ticktick/status: + get: + description: Returns if the current user already did the migation or not. This + is useful to show a confirmation message in the frontend if the user is trying + to do the same migration again. + produces: + - application/json + responses: + "200": + description: The migration status + schema: + $ref: '#/definitions/migration.Status' + "500": + description: Internal server error + schema: + $ref: '#/definitions/models.Message' + security: + - JWTKeyAuth: [] + summary: Get migration status + tags: + - migration /migration/todoist/auth: get: description: Returns the auth url where the user needs to get its auth code. From 5e40f4ec89a96f1326dfaf8efd1218d48c265c32 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sun, 9 Oct 2022 19:03:00 +0200 Subject: [PATCH 4/6] fix(migration): properly parse duration --- pkg/modules/migration/ticktick/ticktick.go | 34 ++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/pkg/modules/migration/ticktick/ticktick.go b/pkg/modules/migration/ticktick/ticktick.go index 0de11317d4..e67bf28cc1 100644 --- a/pkg/modules/migration/ticktick/ticktick.go +++ b/pkg/modules/migration/ticktick/ticktick.go @@ -19,6 +19,7 @@ package ticktick import ( "encoding/csv" "io" + "regexp" "sort" "strconv" "strings" @@ -54,6 +55,32 @@ type tickTickTask struct { ParentID int64 } +// Copied from https://stackoverflow.com/a/57617885 +var durationRegex = regexp.MustCompile(`P([\d\.]+Y)?([\d\.]+M)?([\d\.]+D)?T?([\d\.]+H)?([\d\.]+M)?([\d\.]+?S)?`) + +// ParseDuration converts a ISO8601 duration into a time.Duration +func parseDuration(str string) time.Duration { + matches := durationRegex.FindStringSubmatch(str) + + years := parseDurationPart(matches[1], time.Hour*24*365) + months := parseDurationPart(matches[2], time.Hour*24*30) + days := parseDurationPart(matches[3], time.Hour*24) + hours := parseDurationPart(matches[4], time.Hour) + minutes := parseDurationPart(matches[5], time.Second*60) + seconds := parseDurationPart(matches[6], time.Second) + + return time.Duration(years + months + days + hours + minutes + seconds) +} + +func parseDurationPart(value string, unit time.Duration) time.Duration { + if len(value) != 0 { + if parsed, err := strconv.ParseFloat(value[:len(value)-1], 64); err == nil { + return time.Duration(float64(unit) * parsed) + } + } + return 0 +} + func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.NamespaceWithListsAndTasks) { namespace := &models.NamespaceWithListsAndTasks{ Namespace: models.Namespace{ @@ -163,11 +190,6 @@ func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error if err != nil { return err } - // TODO: parse properly - reminder, err := time.ParseDuration(record[8]) - if err != nil { - return err - } priority, err := strconv.Atoi(record[10]) if err != nil { return err @@ -193,6 +215,8 @@ func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error return err } + reminder := parseDuration(record[8]) + allTasks = append(allTasks, &tickTickTask{ ListName: record[1], Title: record[2], From 0d044997dfd318d8a1932519134abbdcc5d42383 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sun, 9 Oct 2022 19:04:41 +0200 Subject: [PATCH 5/6] fix(migration): expose ticktick migrator to /info --- pkg/routes/api/v1/info.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/routes/api/v1/info.go b/pkg/routes/api/v1/info.go index 737abaef56..63d822e21c 100644 --- a/pkg/routes/api/v1/info.go +++ b/pkg/routes/api/v1/info.go @@ -17,6 +17,7 @@ package v1 import ( + "code.vikunja.io/api/pkg/modules/migration/ticktick" "net/http" vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file" @@ -97,6 +98,7 @@ func Info(c echo.Context) error { TaskCommentsEnabled: config.ServiceEnableTaskComments.GetBool(), AvailableMigrators: []string{ (&vikunja_file.FileMigrator{}).Name(), + (&ticktick.Migrator{}).Name(), }, Legal: legalInfo{ ImprintURL: config.LegalImprintURL.GetString(), From f5a33478f28ae9996011eaf156eafd279684bf86 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sun, 9 Oct 2022 19:23:23 +0200 Subject: [PATCH 6/6] fix(migration): make sure importing works when the csv file has errors and don't try to parse empty values as dates --- pkg/modules/migration/ticktick/ticktick.go | 108 +++++++++++++-------- pkg/routes/api/v1/info.go | 15 ++- pkg/routes/routes.go | 10 +- 3 files changed, 74 insertions(+), 59 deletions(-) diff --git a/pkg/modules/migration/ticktick/ticktick.go b/pkg/modules/migration/ticktick/ticktick.go index e67bf28cc1..b86594568d 100644 --- a/pkg/modules/migration/ticktick/ticktick.go +++ b/pkg/modules/migration/ticktick/ticktick.go @@ -18,6 +18,7 @@ package ticktick import ( "encoding/csv" + "errors" "io" "regexp" "sort" @@ -25,6 +26,8 @@ import ( "strings" "time" + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/models" "code.vikunja.io/api/pkg/modules/migration" "code.vikunja.io/api/pkg/user" @@ -62,6 +65,10 @@ var durationRegex = regexp.MustCompile(`P([\d\.]+Y)?([\d\.]+M)?([\d\.]+D)?T?([\d func parseDuration(str string) time.Duration { matches := durationRegex.FindStringSubmatch(str) + if len(matches) == 0 { + return 0 + } + years := parseDurationPart(matches[1], time.Hour*24*365) months := parseDurationPart(matches[2], time.Hour*24*30) days := parseDurationPart(matches[3], time.Hour*24) @@ -69,7 +76,7 @@ func parseDuration(str string) time.Duration { minutes := parseDurationPart(matches[5], time.Second*60) seconds := parseDurationPart(matches[6], time.Second) - return time.Duration(years + months + days + hours + minutes + seconds) + return years + months + days + hours + minutes + seconds } func parseDurationPart(value string, unit time.Duration) time.Duration { @@ -170,38 +177,30 @@ func (m *Migrator) Name() string { // @Failure 500 {object} models.Message "Internal server error" // @Router /migration/ticktick/migrate [post] func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error { - fr := io.NewSectionReader(file, 0, 0) + fr := io.NewSectionReader(file, 0, size) r := csv.NewReader(fr) - records, err := r.ReadAll() - if err != nil { - return err - } - allTasks := make([]*tickTickTask, 0, len(records)) - for line, record := range records { - if line <= 3 { + allTasks := []*tickTickTask{} + line := 0 + for { + + record, err := r.Read() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + log.Debugf("[TickTick Migration] CSV parse error: %s", err) + } + + line++ + if line <= 4 { continue } - startDate, err := time.Parse(timeISO, record[6]) - if err != nil { - return err - } - dueDate, err := time.Parse(timeISO, record[7]) - if err != nil { - return err - } + priority, err := strconv.Atoi(record[10]) if err != nil { return err } - createdTime, err := time.Parse(timeISO, record[12]) - if err != nil { - return err - } - completedTime, err := time.Parse(timeISO, record[13]) - if err != nil { - return err - } order, err := strconv.ParseFloat(record[14], 64) if err != nil { return err @@ -217,24 +216,47 @@ func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error reminder := parseDuration(record[8]) - allTasks = append(allTasks, &tickTickTask{ - ListName: record[1], - Title: record[2], - Tags: strings.Split(record[3], ", "), - Content: record[4], - IsChecklist: record[5] == "Y", - StartDate: startDate, - DueDate: dueDate, - Reminder: reminder, - Repeat: record[9], - Priority: priority, - Status: record[11], - CreatedTime: createdTime, - CompletedTime: completedTime, - Order: order, - TaskID: taskID, - ParentID: parentID, - }) + t := &tickTickTask{ + ListName: record[1], + Title: record[2], + Tags: strings.Split(record[3], ", "), + Content: record[4], + IsChecklist: record[5] == "Y", + Reminder: reminder, + Repeat: record[9], + Priority: priority, + Status: record[11], + Order: order, + TaskID: taskID, + ParentID: parentID, + } + + if record[6] != "" { + t.StartDate, err = time.Parse(timeISO, record[6]) + if err != nil { + return err + } + } + if record[7] != "" { + t.DueDate, err = time.Parse(timeISO, record[7]) + if err != nil { + return err + } + } + if record[12] != "" { + t.StartDate, err = time.Parse(timeISO, record[12]) + if err != nil { + return err + } + } + if record[13] != "" { + t.CompletedTime, err = time.Parse(timeISO, record[13]) + if err != nil { + return err + } + } + + allTasks = append(allTasks, t) } vikunjaTasks := convertTickTickToVikunja(allTasks) diff --git a/pkg/routes/api/v1/info.go b/pkg/routes/api/v1/info.go index 63d822e21c..37f63b7f76 100644 --- a/pkg/routes/api/v1/info.go +++ b/pkg/routes/api/v1/info.go @@ -17,22 +17,19 @@ package v1 import ( - "code.vikunja.io/api/pkg/modules/migration/ticktick" "net/http" - vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file" - - microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo" - - "code.vikunja.io/api/pkg/modules/migration/trello" - - "code.vikunja.io/api/pkg/log" - "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/modules/auth/openid" + microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo" + "code.vikunja.io/api/pkg/modules/migration/ticktick" "code.vikunja.io/api/pkg/modules/migration/todoist" + "code.vikunja.io/api/pkg/modules/migration/trello" + vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file" "code.vikunja.io/api/pkg/modules/migration/wunderlist" "code.vikunja.io/api/pkg/version" + "github.com/labstack/echo/v4" ) diff --git a/pkg/routes/routes.go b/pkg/routes/routes.go index ee85e2265d..1f98a9fd63 100644 --- a/pkg/routes/routes.go +++ b/pkg/routes/routes.go @@ -47,19 +47,12 @@ package routes import ( - "code.vikunja.io/api/pkg/modules/migration/ticktick" "errors" "fmt" "net/url" "strings" "time" - "code.vikunja.io/api/pkg/modules/migration/ticktick" - - "github.com/ulule/limiter/v3" - - vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file" - "code.vikunja.io/api/pkg/config" "code.vikunja.io/api/pkg/db" "code.vikunja.io/api/pkg/log" @@ -73,8 +66,10 @@ import ( "code.vikunja.io/api/pkg/modules/migration" migrationHandler "code.vikunja.io/api/pkg/modules/migration/handler" microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo" + "code.vikunja.io/api/pkg/modules/migration/ticktick" "code.vikunja.io/api/pkg/modules/migration/todoist" "code.vikunja.io/api/pkg/modules/migration/trello" + vikunja_file "code.vikunja.io/api/pkg/modules/migration/vikunja-file" "code.vikunja.io/api/pkg/modules/migration/wunderlist" apiv1 "code.vikunja.io/api/pkg/routes/api/v1" "code.vikunja.io/api/pkg/routes/caldav" @@ -89,6 +84,7 @@ import ( "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" elog "github.com/labstack/gommon/log" + "github.com/ulule/limiter/v3" ) // NewEcho registers a new Echo instance