diff --git a/go.mod b/go.mod index 246956ef4a..b178a3ef5b 100644 --- a/go.mod +++ b/go.mod @@ -98,6 +98,7 @@ require ( github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/spec v0.20.4 // indirect github.com/go-openapi/swag v0.19.15 // indirect + github.com/gocarina/gocsv v0.0.0-20221216233619-1fea7ae8d380 // indirect github.com/goccy/go-json v0.9.11 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/protobuf v1.5.2 // indirect diff --git a/go.sum b/go.sum index 9f817ca7d4..0bf2193056 100644 --- a/go.sum +++ b/go.sum @@ -249,6 +249,8 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-testfixtures/testfixtures/v3 v3.8.1 h1:uonwvepqRvSgddcrReZQhojTlWlmOlHkYAb9ZaOMWgU= github.com/go-testfixtures/testfixtures/v3 v3.8.1/go.mod h1:Kdu7YeMC0KRXVHdaQ91Vmx3pcjoTF63h4f1qTJDdXLA= +github.com/gocarina/gocsv v0.0.0-20221216233619-1fea7ae8d380 h1:JJq8YZiS07gFIMYZxkbbiMrXIglG3k5JPPtdvckcnfQ= +github.com/gocarina/gocsv v0.0.0-20221216233619-1fea7ae8d380/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= github.com/goccy/go-json v0.8.1/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= diff --git a/pkg/modules/migration/ticktick/ticktick.go b/pkg/modules/migration/ticktick/ticktick.go index d055a010c1..cdf6f2b0ff 100644 --- a/pkg/modules/migration/ticktick/ticktick.go +++ b/pkg/modules/migration/ticktick/ticktick.go @@ -17,8 +17,10 @@ package ticktick import ( + "code.vikunja.io/api/pkg/log" "encoding/csv" "errors" + "github.com/gocarina/gocsv" "io" "regexp" "sort" @@ -26,8 +28,6 @@ import ( "strings" "time" - "code.vikunja.io/api/pkg/log" - "code.vikunja.io/api/pkg/models" "code.vikunja.io/api/pkg/modules/migration" "code.vikunja.io/api/pkg/user" @@ -39,23 +39,39 @@ type Migrator struct { } type tickTickTask struct { - FolderName string - ListName string - Title string - Tags []string - Content string - IsChecklist bool - StartDate time.Time - DueDate time.Time - Reminder time.Duration - Repeat string - Priority int - Status string - CreatedTime time.Time - CompletedTime time.Time - Order float64 - TaskID int64 - ParentID int64 + FolderName string `csv:"Folder Name"` + ListName string `csv:"List Name"` + Title string `csv:"Title"` + TagsList string `csv:"Tags"` + Tags []string `csv:"-"` + Content string `csv:"Content"` + IsChecklistString string `csv:"Is Check list"` + IsChecklist bool `csv:"-"` + StartDate tickTickTime `csv:"Start Date"` + DueDate tickTickTime `csv:"Due Date"` + ReminderDuration string `csv:"Reminder"` + Reminder time.Duration `csv:"-"` + Repeat string `csv:"Repeat"` + Priority int `csv:"Priority"` + Status string `csv:"Status"` + CreatedTime tickTickTime `csv:"Created Time"` + CompletedTime tickTickTime `csv:"Completed Time"` + Order float64 `csv:"Order"` + TaskID int64 `csv:"taskId"` + ParentID int64 `csv:"parentId"` +} + +type tickTickTime struct { + time.Time +} + +func (date *tickTickTime) UnmarshalCSV(csv string) (err error) { + date.Time = time.Time{} + if csv == "" { + return nil + } + date.Time, err = time.Parse(timeISO, csv) + return err } // Copied from https://stackoverflow.com/a/57617885 @@ -119,19 +135,22 @@ func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.Namespace ID: t.TaskID, Title: t.Title, Description: t.Content, - StartDate: t.StartDate, - EndDate: t.DueDate, - DueDate: t.DueDate, - Reminders: []time.Time{ - t.DueDate.Add(t.Reminder * -1), - }, - Done: t.Status == "1", - DoneAt: t.CompletedTime, - Position: t.Order, - Labels: labels, + StartDate: t.StartDate.Time, + EndDate: t.DueDate.Time, + DueDate: t.DueDate.Time, + Done: t.Status == "1", + DoneAt: t.CompletedTime.Time, + Position: t.Order, + Labels: labels, }, } + if !t.DueDate.IsZero() && t.Reminder > 0 { + task.Task.Reminders = []time.Time{ + t.DueDate.Add(t.Reminder * -1), + } + } + if t.ParentID != 0 { task.RelatedTasks = map[models.RelationKind][]*models.Task{ models.RelationKindParenttask: {{ID: t.ParentID}}, @@ -165,6 +184,22 @@ func (m *Migrator) Name() string { return "ticktick" } +func newLineSkipDecoder(r io.Reader, LinesToSkip int) (gocsv.SimpleDecoder, error) { + reader := csv.NewReader(r) + // reader.FieldsPerRecord = -1 + for i := 0; i < LinesToSkip; i++ { + _, err := reader.Read() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + log.Debugf("[TickTick Migration] CSV parse error: %s", err) + } + } + reader.FieldsPerRecord = 0 + return gocsv.NewSimpleDecoderFromCSVReader(reader), nil +} + // Migrate takes a ticktick export, parses it and imports everything in it into Vikunja. // @Summary Import all lists, tasks etc. from a TickTick backup export // @Description Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja. @@ -178,85 +213,29 @@ func (m *Migrator) Name() string { // @Router /migration/ticktick/migrate [post] func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error { fr := io.NewSectionReader(file, 0, size) - r := csv.NewReader(fr) + //r := csv.NewReader(fr) allTasks := []*tickTickTask{} - line := 0 - for { + decode, err := newLineSkipDecoder(fr, 3) + if err != nil { + return err + } + err = gocsv.UnmarshalDecoder(decode, &allTasks) + if err != nil { + return err + } - record, err := r.Read() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - log.Debugf("[TickTick Migration] CSV parse error: %s", err) + for _, task := range allTasks { + if task.IsChecklistString == "Y" { + task.IsChecklist = true } - line++ - if line <= 4 { - continue + reminder := parseDuration(task.ReminderDuration) + if reminder > 0 { + task.Reminder = reminder } - priority, err := strconv.Atoi(record[11]) - if err != nil { - return err - } - order, err := strconv.ParseFloat(record[15], 64) - if err != nil { - return err - } - taskID, err := strconv.ParseInt(record[22], 10, 64) - if err != nil { - return err - } - parentID, err := strconv.ParseInt(record[23], 10, 64) - if err != nil { - parentID = 0 - } - - reminder := parseDuration(record[9]) - - t := &tickTickTask{ - ListName: record[1], - Title: record[2], - Tags: strings.Split(record[4], ", "), - Content: record[5], - IsChecklist: record[6] == "Y", - Reminder: reminder, - Repeat: record[7], - Priority: priority, - Status: record[12], - Order: order, - TaskID: taskID, - ParentID: parentID, - } - - if record[7] != "" { - t.StartDate, err = time.Parse(timeISO, record[7]) - if err != nil { - return err - } - } - if record[8] != "" { - t.DueDate, err = time.Parse(timeISO, record[8]) - if err != nil { - return err - } - } - if record[13] != "" { - t.StartDate, err = time.Parse(timeISO, record[13]) - if err != nil { - return err - } - } - if record[14] != "" { - t.CompletedTime, err = time.Parse(timeISO, record[14]) - if err != nil { - return err - } - } - - allTasks = append(allTasks, t) + task.Tags = strings.Split(task.TagsList, ", ") } vikunjaTasks := convertTickTickToVikunja(allTasks) diff --git a/pkg/modules/migration/ticktick/ticktick_test.go b/pkg/modules/migration/ticktick/ticktick_test.go index 05fd72739c..a6834442ed 100644 --- a/pkg/modules/migration/ticktick/ticktick_test.go +++ b/pkg/modules/migration/ticktick/ticktick_test.go @@ -26,12 +26,15 @@ import ( ) func TestConvertTicktickTasksToVikunja(t *testing.T) { - time1, err := time.Parse(time.RFC3339Nano, "2022-11-18T03:00:00.4770000Z") + t1, err := time.Parse(time.RFC3339Nano, "2022-11-18T03:00:00.4770000Z") require.NoError(t, err) - time2, err := time.Parse(time.RFC3339Nano, "2022-12-18T03:00:00.4770000Z") + time1 := tickTickTime{Time: t1} + t2, err := time.Parse(time.RFC3339Nano, "2022-12-18T03:00:00.4770000Z") require.NoError(t, err) - time3, err := time.Parse(time.RFC3339Nano, "2022-12-10T03:00:00.4770000Z") + time2 := tickTickTime{Time: t2} + t3, err := time.Parse(time.RFC3339Nano, "2022-12-10T03:00:00.4770000Z") require.NoError(t, err) + time3 := tickTickTime{Time: t3} duration, err := time.ParseDuration("24h") require.NoError(t, err) @@ -91,9 +94,9 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) { assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Title, tickTickTasks[0].Title) assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Description, tickTickTasks[0].Content) - assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].StartDate, tickTickTasks[0].StartDate) - assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].EndDate, tickTickTasks[0].DueDate) - assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].DueDate, tickTickTasks[0].DueDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].StartDate, tickTickTasks[0].StartDate.Time) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].EndDate, tickTickTasks[0].DueDate.Time) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].DueDate, tickTickTasks[0].DueDate.Time) assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Labels, []*models.Label{ {Title: "label1"}, {Title: "label2"}, @@ -105,7 +108,7 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) { assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Title, tickTickTasks[1].Title) assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Position, tickTickTasks[1].Order) assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Done, true) - assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].DoneAt, tickTickTasks[1].CompletedTime) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].DoneAt, tickTickTasks[1].CompletedTime.Time) assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].RelatedTasks, models.RelatedTaskMap{ models.RelationKindParenttask: []*models.Task{ { @@ -116,9 +119,9 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) { assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Title, tickTickTasks[2].Title) assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Description, tickTickTasks[2].Content) - assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].StartDate, tickTickTasks[2].StartDate) - assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].EndDate, tickTickTasks[2].DueDate) - assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].DueDate, tickTickTasks[2].DueDate) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].StartDate, tickTickTasks[2].StartDate.Time) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].EndDate, tickTickTasks[2].DueDate.Time) + assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].DueDate, tickTickTasks[2].DueDate.Time) assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Labels, []*models.Label{ {Title: "label1"}, {Title: "label2"},