kooshi
/
api
forked from vikunja/vikunja
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(migration): bullet-proof csv parsing for TickTick import

This commit is contained in:
kolaente 2023-01-23 19:13:07 +01:00
parent af9faad25a
commit b96b10dc70
Signed by untrusted user: konrad
GPG Key ID: F40E70337AB24C9B
4 changed files with 96 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -98,6 +98,7 @@ require (
github.com/go-openapi/jsonreference v0.19.6 // indirect
github.com/go-openapi/spec v0.20.4 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/gocarina/gocsv v0.0.0-20221216233619-1fea7ae8d380 // indirect
github.com/goccy/go-json v0.9.11 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang/protobuf v1.5.2 // indirect

2
go.sum
View File

@ -249,6 +249,8 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-testfixtures/testfixtures/v3 v3.8.1 h1:uonwvepqRvSgddcrReZQhojTlWlmOlHkYAb9ZaOMWgU=
github.com/go-testfixtures/testfixtures/v3 v3.8.1/go.mod h1:Kdu7YeMC0KRXVHdaQ91Vmx3pcjoTF63h4f1qTJDdXLA=
github.com/gocarina/gocsv v0.0.0-20221216233619-1fea7ae8d380 h1:JJq8YZiS07gFIMYZxkbbiMrXIglG3k5JPPtdvckcnfQ=
github.com/gocarina/gocsv v0.0.0-20221216233619-1fea7ae8d380/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
github.com/goccy/go-json v0.8.1/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=

181
pkg/modules/migration/ticktick/ticktick.go
View File

@ -17,8 +17,10 @@
package ticktick
import (
"code.vikunja.io/api/pkg/log"
"encoding/csv"
"errors"
"github.com/gocarina/gocsv"
"io"
"regexp"
"sort"
@ -26,8 +28,6 @@ import (
"strings"
"time"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/user"
@ -39,23 +39,39 @@ type Migrator struct {
}
type tickTickTask struct {
FolderName string
ListName string
Title string
Tags []string
Content string
IsChecklist bool
StartDate time.Time
DueDate time.Time
Reminder time.Duration
Repeat string
Priority int
Status string
CreatedTime time.Time
CompletedTime time.Time
Order float64
TaskID int64
ParentID int64
FolderName string `csv:"Folder Name"`
ListName string `csv:"List Name"`
Title string `csv:"Title"`
TagsList string `csv:"Tags"`
Tags []string `csv:"-"`
Content string `csv:"Content"`
IsChecklistString string `csv:"Is Check list"`
IsChecklist bool `csv:"-"`
StartDate tickTickTime `csv:"Start Date"`
DueDate tickTickTime `csv:"Due Date"`
ReminderDuration string `csv:"Reminder"`
Reminder time.Duration `csv:"-"`
Repeat string `csv:"Repeat"`
Priority int `csv:"Priority"`
Status string `csv:"Status"`
CreatedTime tickTickTime `csv:"Created Time"`
CompletedTime tickTickTime `csv:"Completed Time"`
Order float64 `csv:"Order"`
TaskID int64 `csv:"taskId"`
ParentID int64 `csv:"parentId"`
}
type tickTickTime struct {
time.Time
}
func (date *tickTickTime) UnmarshalCSV(csv string) (err error) {
date.Time = time.Time{}
if csv == "" {
return nil
}
date.Time, err = time.Parse(timeISO, csv)
return err
}
// Copied from https://stackoverflow.com/a/57617885
@ -119,19 +135,22 @@ func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.Namespace
ID: t.TaskID,
Title: t.Title,
Description: t.Content,
StartDate: t.StartDate,
EndDate: t.DueDate,
DueDate: t.DueDate,
Reminders: []time.Time{
t.DueDate.Add(t.Reminder * -1),
},
Done: t.Status == "1",
DoneAt: t.CompletedTime,
Position: t.Order,
Labels: labels,
StartDate: t.StartDate.Time,
EndDate: t.DueDate.Time,
DueDate: t.DueDate.Time,
Done: t.Status == "1",
DoneAt: t.CompletedTime.Time,
Position: t.Order,
Labels: labels,
},
}
if !t.DueDate.IsZero() && t.Reminder > 0 {
task.Task.Reminders = []time.Time{
t.DueDate.Add(t.Reminder * -1),
}
}
if t.ParentID != 0 {
task.RelatedTasks = map[models.RelationKind][]*models.Task{
models.RelationKindParenttask: {{ID: t.ParentID}},
@ -165,6 +184,22 @@ func (m *Migrator) Name() string {
return "ticktick"
}
func newLineSkipDecoder(r io.Reader, LinesToSkip int) (gocsv.SimpleDecoder, error) {
reader := csv.NewReader(r)
// reader.FieldsPerRecord = -1
for i := 0; i < LinesToSkip; i++ {
_, err := reader.Read()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
log.Debugf("[TickTick Migration] CSV parse error: %s", err)
}
}
reader.FieldsPerRecord = 0
return gocsv.NewSimpleDecoderFromCSVReader(reader), nil
}
// Migrate takes a ticktick export, parses it and imports everything in it into Vikunja.
// @Summary Import all lists, tasks etc. from a TickTick backup export
// @Description Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja.
@ -178,85 +213,29 @@ func (m *Migrator) Name() string {
// @Router /migration/ticktick/migrate [post]
func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error {
fr := io.NewSectionReader(file, 0, size)
r := csv.NewReader(fr)
//r := csv.NewReader(fr)
allTasks := []*tickTickTask{}
line := 0
for {
decode, err := newLineSkipDecoder(fr, 3)
if err != nil {
return err
}
err = gocsv.UnmarshalDecoder(decode, &allTasks)
if err != nil {
return err
}
record, err := r.Read()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
log.Debugf("[TickTick Migration] CSV parse error: %s", err)
for _, task := range allTasks {
if task.IsChecklistString == "Y" {
task.IsChecklist = true
}
line++
if line <= 4 {
continue
reminder := parseDuration(task.ReminderDuration)
if reminder > 0 {
task.Reminder = reminder
}
priority, err := strconv.Atoi(record[11])
if err != nil {
return err
}
order, err := strconv.ParseFloat(record[15], 64)
if err != nil {
return err
}
taskID, err := strconv.ParseInt(record[22], 10, 64)
if err != nil {
return err
}
parentID, err := strconv.ParseInt(record[23], 10, 64)
if err != nil {
parentID = 0
}
reminder := parseDuration(record[9])
t := &tickTickTask{
ListName: record[1],
Title: record[2],
Tags: strings.Split(record[4], ", "),
Content: record[5],
IsChecklist: record[6] == "Y",
Reminder: reminder,
Repeat: record[7],
Priority: priority,
Status: record[12],
Order: order,
TaskID: taskID,
ParentID: parentID,
}
if record[7] != "" {
t.StartDate, err = time.Parse(timeISO, record[7])
if err != nil {
return err
}
}
if record[8] != "" {
t.DueDate, err = time.Parse(timeISO, record[8])
if err != nil {
return err
}
}
if record[13] != "" {
t.StartDate, err = time.Parse(timeISO, record[13])
if err != nil {
return err
}
}
if record[14] != "" {
t.CompletedTime, err = time.Parse(timeISO, record[14])
if err != nil {
return err
}
}
allTasks = append(allTasks, t)
task.Tags = strings.Split(task.TagsList, ", ")
}
vikunjaTasks := convertTickTickToVikunja(allTasks)

23
pkg/modules/migration/ticktick/ticktick_test.go
View File

@ -26,12 +26,15 @@ import (
)
func TestConvertTicktickTasksToVikunja(t *testing.T) {
time1, err := time.Parse(time.RFC3339Nano, "2022-11-18T03:00:00.4770000Z")
t1, err := time.Parse(time.RFC3339Nano, "2022-11-18T03:00:00.4770000Z")
require.NoError(t, err)
time2, err := time.Parse(time.RFC3339Nano, "2022-12-18T03:00:00.4770000Z")
time1 := tickTickTime{Time: t1}
t2, err := time.Parse(time.RFC3339Nano, "2022-12-18T03:00:00.4770000Z")
require.NoError(t, err)
time3, err := time.Parse(time.RFC3339Nano, "2022-12-10T03:00:00.4770000Z")
time2 := tickTickTime{Time: t2}
t3, err := time.Parse(time.RFC3339Nano, "2022-12-10T03:00:00.4770000Z")
require.NoError(t, err)
time3 := tickTickTime{Time: t3}
duration, err := time.ParseDuration("24h")
require.NoError(t, err)
@ -91,9 +94,9 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) {
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Title, tickTickTasks[0].Title)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Description, tickTickTasks[0].Content)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].StartDate, tickTickTasks[0].StartDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].EndDate, tickTickTasks[0].DueDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].DueDate, tickTickTasks[0].DueDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].StartDate, tickTickTasks[0].StartDate.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].EndDate, tickTickTasks[0].DueDate.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].DueDate, tickTickTasks[0].DueDate.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[0].Labels, []*models.Label{
{Title: "label1"},
{Title: "label2"},
@ -105,7 +108,7 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) {
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Title, tickTickTasks[1].Title)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Position, tickTickTasks[1].Order)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].Done, true)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].DoneAt, tickTickTasks[1].CompletedTime)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].DoneAt, tickTickTasks[1].CompletedTime.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[1].RelatedTasks, models.RelatedTaskMap{
models.RelationKindParenttask: []*models.Task{
{
@ -116,9 +119,9 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) {
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Title, tickTickTasks[2].Title)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Description, tickTickTasks[2].Content)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].StartDate, tickTickTasks[2].StartDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].EndDate, tickTickTasks[2].DueDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].DueDate, tickTickTasks[2].DueDate)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].StartDate, tickTickTasks[2].StartDate.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].EndDate, tickTickTasks[2].DueDate.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].DueDate, tickTickTasks[2].DueDate.Time)
assert.Equal(t, vikunjaTasks[0].Lists[0].Tasks[2].Labels, []*models.Label{
{Title: "label1"},
{Title: "label2"},