From 1888a5e25729f80c293f32838d0815e960534b89 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 16 May 2020 13:52:44 +0200 Subject: [PATCH 01/16] Add basic structure for todoist migration --- pkg/config/config.go | 4 ++ pkg/modules/migration/todoist/todoist.go | 83 ++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 pkg/modules/migration/todoist/todoist.go diff --git a/pkg/config/config.go b/pkg/config/config.go index f1708f522..f2a5c42bf 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -101,6 +101,9 @@ const ( MigrationWunderlistClientID Key = `migration.wunderlist.clientid` MigrationWunderlistClientSecret Key = `migration.wunderlist.clientsecret` MigrationWunderlistRedirectURL Key = `migration.wunderlist.redirecturl` + MigrationTodoistEnable Key = `migration.todoist.enable` + MigrationTodoistClientID Key = `migration.todoist.clientid` + MigrationTodoistClientSecret Key = `migration.todoist.clientsecret` CorsEnable Key = `cors.enable` CorsOrigins Key = `cors.origins` @@ -235,6 +238,7 @@ func InitDefaultConfig() { CorsMaxAge.setDefault(0) // Migration MigrationWunderlistEnable.setDefault(false) + MigrationTodoistEnable.setDefault(false) // Avatar AvatarProvider.setDefault("gravatar") AvatarGravaterExpiration.setDefault(3600) diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go new file mode 100644 index 000000000..d1871bfa6 --- /dev/null +++ b/pkg/modules/migration/todoist/todoist.go @@ -0,0 +1,83 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2020 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package todoist + +import ( + "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/user" + "code.vikunja.io/api/pkg/utils" +) + +type Migration struct { +} + +var todoistColors = map[int64]string{} + +func init() { + todoistColors = make(map[int64]string, 19) + // The todoists colors are static, taken from https://developer.todoist.com/sync/v8/#colors + todoistColors = map[int64]string{ + 30: "b8256f", + 31: "db4035", + 32: "ff9933", + 33: "fad000", + 34: "afb83b", + 35: "7ecc49", + 36: "299438", + 37: "6accbc", + 38: "158fad", + 39: "14aaf5", + 40: "96c3eb", + 41: "4073ff", + 42: "884dff", + 43: "af38eb", + 44: "eb96eb", + 45: "e05194", + 46: "ff8d85", + 47: "808080", + 48: "b8b8b8", + 49: "ccac93", + } +} + +func (m *Migration) Name() string { + return "todoist" +} + +func (m *Migration) AuthURL() string { + return "https://todoist.com/oauth/authorize" + + "?client_id=" + config.MigrationTodoistClientID.GetString() + + "&scope=data:read" + + "&state=" + utils.MakeRandomString(32) +} + +func (m *Migration) Migrate(u *user.User) error { + panic("implement me") + + // 0. Get an api token from the obtained api token + // 1. Get all labels and put them in a map for easier retrieval later on + // 2. Get all projects + // 3. Get all lists + // 4. Get all tasks + // Put all root tasks in a seperate list? + // 5. Get all notes for all tasks and set them as description + // -> Notes contain file attachments, these need to also be added to the task + // 6. Get all reminders + // 7. Get all due dates? + // -> Reccurring also happens here + +} -- 2.47.2 From ac055b9746050da1589c779915af1a872c3bb73a Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 20 May 2020 22:04:28 +0200 Subject: [PATCH 02/16] Add api token exchange --- pkg/config/config.go | 1 + pkg/modules/migration/todoist/todoist.go | 38 ++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index f2a5c42bf..e1c38e992 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -104,6 +104,7 @@ const ( MigrationTodoistEnable Key = `migration.todoist.enable` MigrationTodoistClientID Key = `migration.todoist.clientid` MigrationTodoistClientSecret Key = `migration.todoist.clientsecret` + MigrationTodoistRedirectURL Key = `migration.todoist.redirecturl` CorsEnable Key = `cors.enable` CorsOrigins Key = `cors.origins` diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index d1871bfa6..80e65e522 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -20,9 +20,19 @@ import ( "code.vikunja.io/api/pkg/config" "code.vikunja.io/api/pkg/user" "code.vikunja.io/api/pkg/utils" + "encoding/json" + "net/http" + "net/url" + "strings" ) type Migration struct { + Code string `json:"code"` +} + +type apiTokenResponse struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` } var todoistColors = map[int64]string{} @@ -65,10 +75,32 @@ func (m *Migration) AuthURL() string { "&state=" + utils.MakeRandomString(32) } -func (m *Migration) Migrate(u *user.User) error { - panic("implement me") +func getAccessTokenFromAuthToken(authToken string) (accessToken string, err error) { + + form := url.Values{ + "client_id": []string{config.MigrationTodoistClientID.GetString()}, + "client_secret": []string{config.MigrationTodoistClientSecret.GetString()}, + "code": []string{authToken}, + "redirect_uri": []string{config.MigrationTodoistRedirectURL.GetString()}, + } + resp, err := http.NewRequest("POST", "https://todoist.com/oauth/access_token", strings.NewReader(form.Encode())) + if err != nil { + return + } + + token := &apiTokenResponse{} + err = json.NewDecoder(resp.Body).Decode(token) + return token.AccessToken, err +} + +func (m *Migration) Migrate(u *user.User) (err error) { + + // 0. Get an api token from the obtained auth token + token, err := getAccessTokenFromAuthToken(m.Code) + if err != nil { + return + } - // 0. Get an api token from the obtained api token // 1. Get all labels and put them in a map for easier retrieval later on // 2. Get all projects // 3. Get all lists -- 2.47.2 From 2790bdaaa4ae6c63329ecfbc96a91d27d218fd15 Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 20 May 2020 22:44:54 +0200 Subject: [PATCH 03/16] Add routes for todoist migrator --- pkg/routes/routes.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/routes/routes.go b/pkg/routes/routes.go index af5c53a04..5fe208151 100644 --- a/pkg/routes/routes.go +++ b/pkg/routes/routes.go @@ -49,6 +49,7 @@ import ( "code.vikunja.io/api/pkg/models" "code.vikunja.io/api/pkg/modules/migration" migrationHandler "code.vikunja.io/api/pkg/modules/migration/handler" + "code.vikunja.io/api/pkg/modules/migration/todoist" "code.vikunja.io/api/pkg/modules/migration/wunderlist" apiv1 "code.vikunja.io/api/pkg/routes/api/v1" "code.vikunja.io/api/pkg/routes/caldav" @@ -433,6 +434,16 @@ func registerAPIRoutes(a *echo.Group) { } wunderlistMigrationHandler.RegisterRoutes(m) } + + // Todoist + if config.MigrationTodoistEnable.GetBool() { + todoistMigrationHandler := &migrationHandler.MigrationWeb{ + MigrationStruct: func() migration.Migrator { + return &todoist.Migration{} + }, + } + todoistMigrationHandler.RegisterRoutes(m) + } } func registerCalDavRoutes(c *echo.Group) { -- 2.47.2 From 8d9505f43c1a429bdb6bedebfc0c07acc5b27699 Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 20 May 2020 22:45:28 +0200 Subject: [PATCH 04/16] Add docs for config options --- config.yml.sample | 14 ++++++++++++++ docs/content/doc/setup/config.md | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/config.yml.sample b/config.yml.sample index a5c99f018..dfe765b6e 100644 --- a/config.yml.sample +++ b/config.yml.sample @@ -156,6 +156,20 @@ migration: # with the code obtained from the wunderlist api. # Note that the vikunja frontend expects this to be /migrate/wunderlist redirecturl: + todoist: + # Wheter to enable the todoist migrator or not + enable: false + # The client id, required for making requests to the wunderlist api + # You need to register your vikunja instance at https://developer.todoist.com/appconsole.html to get this + clientid: + # The client secret, also required for making requests to the todoist api + clientsecret: + # The url where clients are redirected after they authorized Vikunja to access their todoist items. + # This needs to match the url you entered when registering your Vikunja instance at todoist. + # This is usually the frontend url where the frontend then makes a request to /migration/todoist/migrate + # with the code obtained from the todoist api. + # Note that the vikunja frontend expects this to be /migrate/todoist + redirecturl: avatar: # Switch between avatar providers. Possible values are gravatar and default. diff --git a/docs/content/doc/setup/config.md b/docs/content/doc/setup/config.md index 105b6ec22..927046142 100644 --- a/docs/content/doc/setup/config.md +++ b/docs/content/doc/setup/config.md @@ -199,6 +199,20 @@ migration: # with the code obtained from the wunderlist api. # Note that the vikunja frontend expects this to be /migrate/wunderlist redirecturl: + todoist: + # Wheter to enable the todoist migrator or not + enable: false + # The client id, required for making requests to the wunderlist api + # You need to register your vikunja instance at https://developer.todoist.com/appconsole.html to get this + clientid: + # The client secret, also required for making requests to the todoist api + clientsecret: + # The url where clients are redirected after they authorized Vikunja to access their todoist items. + # This needs to match the url you entered when registering your Vikunja instance at todoist. + # This is usually the frontend url where the frontend then makes a request to /migration/todoist/migrate + # with the code obtained from the todoist api. + # Note that the vikunja frontend expects this to be /migrate/todoist + redirecturl: avatar: # Switch between avatar providers. Possible values are gravatar and default. -- 2.47.2 From d1f85e717dd79030f14530fc813fbe972e2966bd Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 20 May 2020 22:50:16 +0200 Subject: [PATCH 05/16] Add all structs for todoist api --- pkg/modules/migration/todoist/todoist.go | 157 ++++++++++++++++++++++- 1 file changed, 152 insertions(+), 5 deletions(-) diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index 80e65e522..3c45318c1 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -24,6 +24,7 @@ import ( "net/http" "net/url" "strings" + "time" ) type Migration struct { @@ -35,12 +36,124 @@ type apiTokenResponse struct { TokenType string `json:"token_type"` } -var todoistColors = map[int64]string{} +type label struct { + ID int `json:"id"` + Name string `json:"name"` + Color int `json:"color"` + ItemOrder int `json:"item_order"` + IsDeleted int `json:"is_deleted"` + IsFavorite int `json:"is_favorite"` +} + +type project struct { + ID int `json:"id"` + LegacyID int `json:"legacy_id"` + Name string `json:"name"` + Color int `json:"color"` + ParentID int `json:"parent_id"` + ChildOrder int `json:"child_order"` + Collapsed int `json:"collapsed"` + Shared bool `json:"shared"` + LegacyParentID int `json:"legacy_parent_id"` + SyncID int `json:"sync_id"` + IsDeleted int `json:"is_deleted"` + IsArchived int `json:"is_archived"` + IsFavorite int `json:"is_favorite"` +} + +type dueDate struct { + Date string `json:"date"` + Timezone interface{} `json:"timezone"` + String string `json:"string"` + Lang string `json:"lang"` + IsRecurring bool `json:"is_recurring"` +} + +type item struct { + DayOrder int `json:"day_order"` + AssignedByUID int `json:"assigned_by_uid"` + Labels []int `json:"labels"` + SyncID int `json:"sync_id"` + SectionID int `json:"section_id"` + InHistory int `json:"in_history"` + ChildOrder int `json:"child_order"` + DateAdded time.Time `json:"date_added"` + ID int64 `json:"id"` + Content string `json:"content"` + Checked int `json:"checked"` + AddedByUID int `json:"added_by_uid"` + UserID int `json:"user_id"` + HasMoreNotes bool `json:"has_more_notes"` + Due *dueDate `json:"due"` + Priority int `json:"priority"` + ParentID int `json:"parent_id"` + IsDeleted int `json:"is_deleted"` + ResponsibleUID int `json:"responsible_uid"` + ProjectID int64 `json:"project_id"` + DateCompleted time.Time `json:"date_completed"` + Collapsed int `json:"collapsed"` +} + +type fileAttachment struct { + FileType string `json:"file_type"` + FileName string `json:"file_name"` + FileSize int `json:"file_size"` + FileURL string `json:"file_url"` + UploadState string `json:"upload_state"` +} + +type note struct { + ID int `json:"id"` + LegacyID int `json:"legacy_id"` + PostedUID int `json:"posted_uid"` + ProjectID int `json:"project_id"` + LegacyProjectID int `json:"legacy_project_id"` + ItemID int `json:"item_id"` + LegacyItemID int `json:"legacy_item_id"` + Content string `json:"content"` + FileAttachment *fileAttachment `json:"file_attachment"` + UidsToNotify []int `json:"uids_to_notify"` + IsDeleted int `json:"is_deleted"` + Posted time.Time `json:"posted"` +} + +type projectNote struct { + Content string `json:"content"` + FileAttachment *fileAttachment `json:"file_attachment"` + ID int64 `json:"id"` + IsDeleted int `json:"is_deleted"` + Posted time.Time `json:"posted"` + PostedUID int `json:"posted_uid"` + ProjectID int64 `json:"project_id"` + UidsToNotify []int `json:"uids_to_notify"` +} + +type reminder struct { + ID int `json:"id"` + NotifyUID int `json:"notify_uid"` + ItemID int `json:"item_id"` + Service string `json:"service"` + Type string `json:"type"` + Due *dueDate `json:"due"` + MmOffset int `json:"mm_offset"` + IsDeleted int `json:"is_deleted"` +} + +type sync struct { + Projects []*project `json:"projects"` + Items []*item `json:"items"` + Labels []*label `json:"labels"` + Notes []*note `json:"notes"` + ProjectNotes []*projectNote `json:"project_notes"` + Reminders []*reminder `json:"reminders"` +} + +var todoistColors = map[int]string{} func init() { - todoistColors = make(map[int64]string, 19) + todoistColors = make(map[int]string, 19) // The todoists colors are static, taken from https://developer.todoist.com/sync/v8/#colors - todoistColors = map[int64]string{ + todoistColors = map[int]string{ 30: "b8256f", 31: "db4035", 32: "ff9933", @@ -75,6 +188,17 @@ func (m *Migration) AuthURL() string { "&state=" + utils.MakeRandomString(32) } +func doPost(url string, form url.Values) (resp *http.Response, err error) { + req, err := http.NewRequest("POST", url, strings.NewReader(form.Encode())) + if err != nil { + return + } + + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + hc := http.Client{} + return hc.Do(req) +} + func getAccessTokenFromAuthToken(authToken string) (accessToken string, err error) { form := url.Values{ @@ -83,7 +207,7 @@ func getAccessTokenFromAuthToken(authToken string) (accessToken string, err erro "code": []string{authToken}, "redirect_uri": []string{config.MigrationTodoistRedirectURL.GetString()}, } - resp, err := http.NewRequest("POST", "https://todoist.com/oauth/access_token", strings.NewReader(form.Encode())) + resp, err := doPost("https://todoist.com/oauth/access_token", form) if err != nil { return } @@ -101,15 +225,38 @@ func (m *Migration) Migrate(u *user.User) (err error) { return } - // 1. Get all labels and put them in a map for easier retrieval later on + // Get everything with the sync api + form := url.Values{ + "token": []string{token}, + "sync_token": []string{"*"}, + "resource_types": []string{"[\"all\"]"}, + } + resp, err := doPost("https://api.todoist.com/sync/v8/sync", form) + if err != nil { + return + } + + syncResponse := &sync{} + err = json.NewDecoder(resp.Body).Decode(syncResponse) + if err != nil { + return + } + + //// 1. Get all labels and put them in a map for easier retrieval later on + //labels := make(map[int64]*label) + // + // 2. Get all projects // 3. Get all lists // 4. Get all tasks + // Put all root tasks in a seperate list? + // 5. Get all notes for all tasks and set them as description // -> Notes contain file attachments, these need to also be added to the task // 6. Get all reminders // 7. Get all due dates? // -> Reccurring also happens here + return nil } -- 2.47.2 From 433286ad4d523ecdd826dedf24fd6fad3c326c9a Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 20 May 2020 23:05:15 +0200 Subject: [PATCH 06/16] Add basic structure to migrate everything --- pkg/modules/migration/todoist/todoist.go | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index 3c45318c1..abc47e594 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -17,10 +17,15 @@ package todoist import ( + "bytes" "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/models" + "code.vikunja.io/api/pkg/modules/migration" "code.vikunja.io/api/pkg/user" "code.vikunja.io/api/pkg/utils" "encoding/json" + "fmt" "net/http" "net/url" "strings" @@ -212,6 +217,12 @@ func getAccessTokenFromAuthToken(authToken string) (accessToken string, err erro return } + if resp.StatusCode > 399 { + buf := &bytes.Buffer{} + _, _ = buf.ReadFrom(resp.Body) + return "", fmt.Errorf("got http status %d while trying to get token, error was %s", resp.StatusCode, buf.String()) + } + token := &apiTokenResponse{} err = json.NewDecoder(resp.Body).Decode(token) return token.AccessToken, err @@ -219,12 +230,22 @@ func getAccessTokenFromAuthToken(authToken string) (accessToken string, err erro func (m *Migration) Migrate(u *user.User) (err error) { + log.Debugf("[Todoist Migration] Starting migration for user %d", u.ID) + // 0. Get an api token from the obtained auth token token, err := getAccessTokenFromAuthToken(m.Code) if err != nil { return } + if token == "" { + log.Debugf("[Todoist Migration] Could not get token") + return + } + + log.Debugf("[Todoist Migration] Got user token for user %d", u.ID) + log.Debugf("[Todoist Migration] Getting todoist data for user %d", u.ID) + // Get everything with the sync api form := url.Values{ "token": []string{token}, @@ -242,6 +263,25 @@ func (m *Migration) Migrate(u *user.User) (err error) { return } + log.Debugf("[Todoist Migration] Got all todoist user data for user %d", u.ID) + log.Debugf("[Todoist Migration] Start converting data for user %d", u.ID) + + fullVikunjaHierachie, err := convertTodoistToVikunja(syncResponse) + if err != nil { + return + } + + log.Debugf("[Todoist Migration] Done converting data for user %d", u.ID) + log.Debugf("[Todoist Migration] Start inserting data for user %d", u.ID) + + err = migration.InsertFromStructure(fullVikunjaHierachie, u) + if err != nil { + return + } + + log.Debugf("[Todoist Migration] Done inserting data for user %d", u.ID) + log.Debugf("[Todoist Migration] Todoist migration done for user %d", u.ID) + //// 1. Get all labels and put them in a map for easier retrieval later on //labels := make(map[int64]*label) // -- 2.47.2 From 8ca58130acc77b26be433d70faaa420bad0e6587 Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 20 May 2020 23:05:34 +0200 Subject: [PATCH 07/16] Cleanup --- pkg/modules/migration/todoist/todoist.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index abc47e594..e59c94073 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -282,21 +282,5 @@ func (m *Migration) Migrate(u *user.User) (err error) { log.Debugf("[Todoist Migration] Done inserting data for user %d", u.ID) log.Debugf("[Todoist Migration] Todoist migration done for user %d", u.ID) - //// 1. Get all labels and put them in a map for easier retrieval later on - //labels := make(map[int64]*label) - // - - // 2. Get all projects - // 3. Get all lists - // 4. Get all tasks - - // Put all root tasks in a seperate list? - - // 5. Get all notes for all tasks and set them as description - // -> Notes contain file attachments, these need to also be added to the task - // 6. Get all reminders - // 7. Get all due dates? - // -> Reccurring also happens here - return nil } -- 2.47.2 From cc47442637d167ccfacc36ec0b3bada1946a3ea2 Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 20 May 2020 23:09:10 +0200 Subject: [PATCH 08/16] Add method and test structures to convert todoist to vikunja --- pkg/modules/migration/todoist/todoist.go | 4 ++ pkg/modules/migration/todoist/todoist_test.go | 44 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 pkg/modules/migration/todoist/todoist_test.go diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index e59c94073..869808b6c 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -204,6 +204,10 @@ func doPost(url string, form url.Values) (resp *http.Response, err error) { return hc.Do(req) } +func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.NamespaceWithLists, err error) { + +} + func getAccessTokenFromAuthToken(authToken string) (accessToken string, err error) { form := url.Values{ diff --git a/pkg/modules/migration/todoist/todoist_test.go b/pkg/modules/migration/todoist/todoist_test.go new file mode 100644 index 000000000..9f0235280 --- /dev/null +++ b/pkg/modules/migration/todoist/todoist_test.go @@ -0,0 +1,44 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2020 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package todoist + +import ( + "code.vikunja.io/api/pkg/models" + "github.com/stretchr/testify/assert" + "gopkg.in/d4l3k/messagediff.v1" + "testing" +) + +func TestConvertTodoistToVikunja(t *testing.T) { + testSync := &sync{ + Projects: nil, + Items: nil, + Labels: nil, + Notes: nil, + ProjectNotes: nil, + Reminders: nil, + } + + expectedHierachie := []*models.NamespaceWithLists{} + + hierachie, err := convertTodoistToVikunja(testSync) + assert.NoError(t, err) + assert.NotNil(t, hierachie) + if diff, equal := messagediff.PrettyDiff(hierachie, expectedHierachie); !equal { + t.Errorf("ListUser.ReadAll() = %v, want %v, diff: %v", hierachie, expectedHierachie, diff) + } +} -- 2.47.2 From 72bd28798faff8cd9fab182618c1b81ab85e55ef Mon Sep 17 00:00:00 2001 From: kolaente Date: Fri, 22 May 2020 18:11:37 +0200 Subject: [PATCH 09/16] Fix build --- pkg/modules/migration/todoist/todoist.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index 869808b6c..2d7678ca0 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -205,7 +205,7 @@ func doPost(url string, form url.Values) (resp *http.Response, err error) { } func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.NamespaceWithLists, err error) { - + return } func getAccessTokenFromAuthToken(authToken string) (accessToken string, err error) { -- 2.47.2 From 46ed64e02f25b35f19be8640b7ebf69e1ee898ff Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 23 May 2020 00:03:11 +0200 Subject: [PATCH 10/16] Started adding fixtures for testing --- pkg/modules/migration/todoist/todoist.go | 52 +-- pkg/modules/migration/todoist/todoist_test.go | 312 +++++++++++++++++- .../migration/wunderlist/wunderlist_test.go | 2 +- 3 files changed, 334 insertions(+), 32 deletions(-) diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index 2d7678ca0..6fe73ad37 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -75,28 +75,31 @@ type dueDate struct { } type item struct { - DayOrder int `json:"day_order"` - AssignedByUID int `json:"assigned_by_uid"` - Labels []int `json:"labels"` - SyncID int `json:"sync_id"` - SectionID int `json:"section_id"` - InHistory int `json:"in_history"` - ChildOrder int `json:"child_order"` - DateAdded time.Time `json:"date_added"` - ID int64 `json:"id"` - Content string `json:"content"` - Checked int `json:"checked"` - AddedByUID int `json:"added_by_uid"` - UserID int `json:"user_id"` - HasMoreNotes bool `json:"has_more_notes"` - Due *dueDate `json:"due"` - Priority int `json:"priority"` - ParentID int `json:"parent_id"` - IsDeleted int `json:"is_deleted"` - ResponsibleUID int `json:"responsible_uid"` - ProjectID int64 `json:"project_id"` - DateCompleted time.Time `json:"date_completed"` - Collapsed int `json:"collapsed"` + ID int `json:"id"` + LegacyID int `json:"legacy_id"` + UserID int `json:"user_id"` + ProjectID int `json:"project_id"` + LegacyProjectID int `json:"legacy_project_id"` + Content string `json:"content"` + Priority int `json:"priority"` + Due *dueDate `json:"due"` + ParentID int `json:"parent_id"` + LegacyParentID int `json:"legacy_parent_id"` + ChildOrder int `json:"child_order"` + SectionID int `json:"section_id"` + DayOrder int `json:"day_order"` + Collapsed int `json:"collapsed"` + Children interface{} `json:"children"` + Labels []int `json:"labels"` + AddedByUID int `json:"added_by_uid"` + AssignedByUID int `json:"assigned_by_uid"` + ResponsibleUID int `json:"responsible_uid"` + Checked int `json:"checked"` + InHistory int `json:"in_history"` + IsDeleted int `json:"is_deleted"` + DateAdded time.Time `json:"date_added"` + HasMoreNotes bool `json:"has_more_notes"` + DateCompleted time.Time `json:"date_completed"` } type fileAttachment struct { @@ -205,6 +208,11 @@ func doPost(url string, form url.Values) (resp *http.Response, err error) { } func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.NamespaceWithLists, err error) { + + // Priority 1 is low and 4 (max) is highest + + // If parenId is not 0, create a task relation + return } diff --git a/pkg/modules/migration/todoist/todoist_test.go b/pkg/modules/migration/todoist/todoist_test.go index 9f0235280..5a882af20 100644 --- a/pkg/modules/migration/todoist/todoist_test.go +++ b/pkg/modules/migration/todoist/todoist_test.go @@ -18,27 +18,321 @@ package todoist import ( "code.vikunja.io/api/pkg/models" + "code.vikunja.io/api/pkg/timeutil" "github.com/stretchr/testify/assert" "gopkg.in/d4l3k/messagediff.v1" + "strconv" "testing" + "time" ) func TestConvertTodoistToVikunja(t *testing.T) { - testSync := &sync{ - Projects: nil, - Items: nil, - Labels: nil, - Notes: nil, - ProjectNotes: nil, - Reminders: nil, + time1, err := time.Parse(time.RFC3339Nano, "2014-09-26T08:25:05Z") + assert.NoError(t, err) + time3, err := time.Parse(time.RFC3339Nano, "2014-10-21T08:25:05Z") + assert.NoError(t, err) + dueTime, err := time.Parse(time.RFC3339Nano, "2020-05-31T00:00:00Z") + assert.NoError(t, err) + nilTime, err := time.Parse(time.RFC3339Nano, "1970-01-01T00:00:00Z") + assert.NoError(t, err) + + makeTestItem := func(id, projectId int, hasDueDate, hasLabels, done bool) *item { + item := &item{ + ID: id, + UserID: 1855589, + ProjectID: projectId, + Content: "Task" + strconv.Itoa(id), + Priority: 1, + ParentID: 0, + ChildOrder: 1, + DateAdded: time1, + DateCompleted: nilTime, + } + + if done { + item.Checked = 1 + item.DateCompleted = time3 + } + + if hasLabels { + item.Labels = []int{} + } + + if hasDueDate { + item.Due = &dueDate{ + Date: "2020-05-31", + Timezone: nil, + IsRecurring: false, + } + } + + return item } - expectedHierachie := []*models.NamespaceWithLists{} + testSync := &sync{ + Projects: []*project{ + { + ID: 396936926, + Name: "Project1", + Color: 30, + ChildOrder: 1, + Collapsed: 0, + Shared: false, + IsDeleted: 0, + IsArchived: 0, + IsFavorite: 0, + }, + { + ID: 396936927, + Name: "Project2", + Color: 37, + ChildOrder: 1, + Collapsed: 0, + Shared: false, + IsDeleted: 0, + IsArchived: 0, + IsFavorite: 0, + }, + { + ID: 396936928, + Name: "Project3 - Archived", + Color: 37, + ChildOrder: 1, + Collapsed: 0, + Shared: false, + IsDeleted: 0, + IsArchived: 1, + IsFavorite: 0, + }, + }, + Items: []*item{ + makeTestItem(400000000, 396936926, false, false, false), + makeTestItem(400000001, 396936926, false, false, false), + makeTestItem(400000002, 396936926, false, false, false), + makeTestItem(400000003, 396936926, true, true, true), + makeTestItem(400000004, 396936926, false, true, false), + makeTestItem(400000005, 396936926, true, false, true), + makeTestItem(400000006, 396936926, true, false, true), + makeTestItem(400000007, 396936927, true, false, false), + makeTestItem(400000008, 396936927, true, false, false), + makeTestItem(400000009, 396936927, false, false, false), + makeTestItem(400000010, 396936927, false, false, true), + makeTestItem(400000101, 396936927, false, false, false), + makeTestItem(400000102, 396936927, true, true, false), + makeTestItem(400000103, 396936927, false, true, false), + makeTestItem(400000104, 396936927, false, true, false), + makeTestItem(400000105, 396936927, true, true, false), + makeTestItem(400000106, 396936926, true, true, true), + makeTestItem(400000107, 396936926, false, false, true), + makeTestItem(400000108, 396936926, false, false, true), + makeTestItem(400000109, 396936926, false, false, true), + makeTestItem(400000110, 396936928, false, false, true), + { + ID: 400000110, + UserID: 1855589, + ProjectID: 396936926, + Content: "Task with parent", + Priority: 1, + ParentID: 400000108, + ChildOrder: 1, + Checked: 0, + DateAdded: time1, + }, + }, + Labels: []*label{ + { + ID: 80000, + Name: "Label1", + Color: 30, + }, + { + ID: 80001, + Name: "Label2", + Color: 31, + }, + { + ID: 80002, + Name: "Label3", + Color: 32, + }, + { + ID: 80003, + Name: "Label4", + Color: 33, + }, + { + ID: 80004, + Name: "Label5", + Color: 34, + }, + { + ID: 80005, + Name: "Label6", + Color: 35, + }, + }, + Notes: []*note{ + { + ID: 101476, + PostedUID: 1855589, + ItemID: 400000000, + Content: "Lorem Ipsum dolor sit amet", + Posted: time1, + }, + { + ID: 101477, + PostedUID: 1855589, + ItemID: 400000001, + Content: "Lorem Ipsum dolor sit amet", + Posted: time1, + }, + { + ID: 101478, + PostedUID: 1855589, + ItemID: 400000003, + Content: "Lorem Ipsum dolor sit amet", + Posted: time1, + }, + { + ID: 101479, + PostedUID: 1855589, + ItemID: 400000010, + Content: "Lorem Ipsum dolor sit amet", + Posted: time1, + }, + { + ID: 101480, + PostedUID: 1855589, + ItemID: 400000101, + Content: "Lorem Ipsum dolor sit amet", + FileAttachment: &fileAttachment{ + FileName: "file.md", + FileType: "text/plain", + FileSize: 12345, + FileURL: "https://vikunja.io/testimage.jpg", // Using an image which we are hosting, so it'll still be up + UploadState: "completed", + }, + Posted: time1, + }, + }, + ProjectNotes: []*projectNote{ + { + ID: 102000, + Content: "Lorem Ipsum dolor sit amet", + ProjectID: 396936926, + Posted: time3, + PostedUID: 1855589, + }, + { + ID: 102001, + Content: "Lorem Ipsum dolor sit amet 2", + ProjectID: 396936926, + Posted: time3, + PostedUID: 1855589, + }, + { + ID: 102002, + Content: "Lorem Ipsum dolor sit amet 3", + ProjectID: 396936926, + Posted: time3, + PostedUID: 1855589, + }, + { + ID: 102003, + Content: "Lorem Ipsum dolor sit amet 4", + ProjectID: 396936927, + Posted: time3, + PostedUID: 1855589, + }, + { + ID: 102004, + Content: "Lorem Ipsum dolor sit amet 5", + ProjectID: 396936927, + Posted: time3, + PostedUID: 1855589, + }, + }, + Reminders: []*reminder{ + { + ID: 103000, + ItemID: 400000000, + Due: &dueDate{ + Date: "2020-06-15", + IsRecurring: false, + }, + MmOffset: 180, + }, + { + ID: 103001, + ItemID: 400000001, + Due: &dueDate{ + Date: "2020-06-15", + IsRecurring: false, + }, + }, + { + ID: 103002, + ItemID: 400000002, + Due: &dueDate{ + Date: "2020-07-15", + IsRecurring: true, + }, + }, + { + ID: 103003, + ItemID: 400000003, + Due: &dueDate{ + Date: "2020-06-15", + IsRecurring: false, + }, + }, + { + ID: 103004, + ItemID: 400000005, + Due: &dueDate{ + Date: "2020-06-15", + IsRecurring: false, + }, + }, + { + ID: 103006, + ItemID: 400000009, + Due: &dueDate{ + Date: "2020-06-15", + IsRecurring: false, + }, + }, + }, + } + + expectedHierachie := []*models.NamespaceWithLists{ + { + Namespace: models.Namespace{ + Title: "Migrated from todoist", + }, + Lists: []*models.List{ + { + Title: "Project1", + HexColor: todoistColors[30], + Tasks: []*models.Task{ + { + Title: "Task400000000", + Done: false, + DueDate: timeutil.FromTime(dueTime), + Priority: 1, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(time3), + }, + }, + }, + }, + }, + } hierachie, err := convertTodoistToVikunja(testSync) assert.NoError(t, err) assert.NotNil(t, hierachie) if diff, equal := messagediff.PrettyDiff(hierachie, expectedHierachie); !equal { - t.Errorf("ListUser.ReadAll() = %v, want %v, diff: %v", hierachie, expectedHierachie, diff) + t.Errorf("converted todoist data = %v, want %v, diff: %v", hierachie, expectedHierachie, diff) } } diff --git a/pkg/modules/migration/wunderlist/wunderlist_test.go b/pkg/modules/migration/wunderlist/wunderlist_test.go index 7b3fc6d00..11c1d8c2f 100644 --- a/pkg/modules/migration/wunderlist/wunderlist_test.go +++ b/pkg/modules/migration/wunderlist/wunderlist_test.go @@ -348,6 +348,6 @@ func TestWunderlistParsing(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, hierachie) if diff, equal := messagediff.PrettyDiff(hierachie, expectedHierachie); !equal { - t.Errorf("ListUser.ReadAll() = %v, want %v, diff: %v", hierachie, expectedHierachie, diff) + t.Errorf("converted wunderlist data = %v, want %v, diff: %v", hierachie, expectedHierachie, diff) } } -- 2.47.2 From efd08d666479bdf7df7572d53fb00f48199c4b8a Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 23 May 2020 17:41:42 +0200 Subject: [PATCH 11/16] Finish test fixtures --- pkg/modules/migration/todoist/todoist_test.go | 308 ++++++++++++++++-- 1 file changed, 276 insertions(+), 32 deletions(-) diff --git a/pkg/modules/migration/todoist/todoist_test.go b/pkg/modules/migration/todoist/todoist_test.go index 5a882af20..76ef6276d 100644 --- a/pkg/modules/migration/todoist/todoist_test.go +++ b/pkg/modules/migration/todoist/todoist_test.go @@ -17,10 +17,13 @@ package todoist import ( + "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/files" "code.vikunja.io/api/pkg/models" "code.vikunja.io/api/pkg/timeutil" "github.com/stretchr/testify/assert" "gopkg.in/d4l3k/messagediff.v1" + "io/ioutil" "strconv" "testing" "time" @@ -35,6 +38,8 @@ func TestConvertTodoistToVikunja(t *testing.T) { assert.NoError(t, err) nilTime, err := time.Parse(time.RFC3339Nano, "1970-01-01T00:00:00Z") assert.NoError(t, err) + exampleFile, err := ioutil.ReadFile(config.ServiceRootpath.GetString() + "/pkg/modules/migration/wunderlist/testimage.jpg") + assert.NoError(t, err) makeTestItem := func(id, projectId int, hasDueDate, hasLabels, done bool) *item { item := &item{ @@ -55,7 +60,12 @@ func TestConvertTodoistToVikunja(t *testing.T) { } if hasLabels { - item.Labels = []int{} + item.Labels = []int{ + 80000, + 80001, + 80002, + 80003, + } } if hasDueDate { @@ -113,6 +123,22 @@ func TestConvertTodoistToVikunja(t *testing.T) { makeTestItem(400000004, 396936926, false, true, false), makeTestItem(400000005, 396936926, true, false, true), makeTestItem(400000006, 396936926, true, false, true), + { + ID: 400000110, + UserID: 1855589, + ProjectID: 396936926, + Content: "Task with parent", + Priority: 1, + ParentID: 400000006, + ChildOrder: 1, + Checked: 0, + DateAdded: time1, + }, + makeTestItem(400000106, 396936926, true, true, true), + makeTestItem(400000107, 396936926, false, false, true), + makeTestItem(400000108, 396936926, false, false, true), + makeTestItem(400000109, 396936926, false, false, true), + makeTestItem(400000007, 396936927, true, false, false), makeTestItem(400000008, 396936927, true, false, false), makeTestItem(400000009, 396936927, false, false, false), @@ -122,22 +148,8 @@ func TestConvertTodoistToVikunja(t *testing.T) { makeTestItem(400000103, 396936927, false, true, false), makeTestItem(400000104, 396936927, false, true, false), makeTestItem(400000105, 396936927, true, true, false), - makeTestItem(400000106, 396936926, true, true, true), - makeTestItem(400000107, 396936926, false, false, true), - makeTestItem(400000108, 396936926, false, false, true), - makeTestItem(400000109, 396936926, false, false, true), + makeTestItem(400000110, 396936928, false, false, true), - { - ID: 400000110, - UserID: 1855589, - ProjectID: 396936926, - Content: "Task with parent", - Priority: 1, - ParentID: 400000108, - ChildOrder: 1, - Checked: 0, - DateAdded: time1, - }, }, Labels: []*label{ { @@ -160,16 +172,6 @@ func TestConvertTodoistToVikunja(t *testing.T) { Name: "Label4", Color: 33, }, - { - ID: 80004, - Name: "Label5", - Color: 34, - }, - { - ID: 80005, - Name: "Label6", - Color: 35, - }, }, Notes: []*note{ { @@ -264,9 +266,9 @@ func TestConvertTodoistToVikunja(t *testing.T) { }, { ID: 103001, - ItemID: 400000001, + ItemID: 400000000, Due: &dueDate{ - Date: "2020-06-15", + Date: "2020-06-16", IsRecurring: false, }, }, @@ -305,6 +307,25 @@ func TestConvertTodoistToVikunja(t *testing.T) { }, } + vikunjaLabels := []*models.Label{ + { + Title: "Label1", + HexColor: todoistColors[30], + }, + { + Title: "Label2", + HexColor: todoistColors[31], + }, + { + Title: "Label3", + HexColor: todoistColors[32], + }, + { + Title: "Label4", + HexColor: todoistColors[33], + }, + } + expectedHierachie := []*models.NamespaceWithLists{ { Namespace: models.Namespace{ @@ -312,16 +333,239 @@ func TestConvertTodoistToVikunja(t *testing.T) { }, Lists: []*models.List{ { - Title: "Project1", - HexColor: todoistColors[30], + Title: "Project1", + Description: "Lorem Ipsum dolor sit amet\nLorem Ipsum dolor sit amet 2\nLorem Ipsum dolor sit amet 3", + HexColor: todoistColors[30], Tasks: []*models.Task{ { - Title: "Task400000000", + Title: "Task400000000", + Description: "Lorem Ipsum dolor sit amet", + Done: false, + Priority: 1, + Created: timeutil.FromTime(time1), + Reminders: []timeutil.TimeStamp{ + timeutil.FromTime(time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC)), + timeutil.FromTime(time.Date(2020, time.June, 16, 0, 0, 0, 0, time.UTC)), + }, + }, + { + Title: "Task400000001", + Description: "Lorem Ipsum dolor sit amet", + Done: false, + Priority: 1, + Created: timeutil.FromTime(time1), + }, + { + Title: "Task400000002", + Done: false, + Priority: 1, + Created: timeutil.FromTime(time1), + Reminders: []timeutil.TimeStamp{ + timeutil.FromTime(time.Date(2020, time.July, 15, 0, 0, 0, 0, time.UTC)), + }, + }, + { + Title: "Task400000003", + Description: "Lorem Ipsum dolor sit amet", + Done: true, + DueDate: timeutil.FromTime(dueTime), + Priority: 1, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(time3), + Labels: vikunjaLabels, + Reminders: []timeutil.TimeStamp{ + timeutil.FromTime(time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC)), + }, + }, + { + Title: "Task400000004", + Done: false, + Priority: 1, + Created: timeutil.FromTime(time1), + Labels: vikunjaLabels, + Reminders: []timeutil.TimeStamp{ + timeutil.FromTime(time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC)), + }, + }, + { + Title: "Task400000005", + Done: true, + DueDate: timeutil.FromTime(dueTime), + Priority: 1, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(nilTime), + }, + { + Title: "Task400000006", + Done: true, + DueDate: timeutil.FromTime(dueTime), + Priority: 1, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(nilTime), + RelatedTasks: map[models.RelationKind][]*models.Task{ + models.RelationKindSubtask: { + { + Title: "Task with parent", + Done: false, + DueDate: timeutil.FromTime(dueTime), + Priority: 1, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(nilTime), + }, + }, + }, + }, + { + Title: "Task with parent", Done: false, DueDate: timeutil.FromTime(dueTime), Priority: 1, Created: timeutil.FromTime(time1), - DoneAt: timeutil.FromTime(time3), + DoneAt: timeutil.FromTime(nilTime), + RelatedTasks: map[models.RelationKind][]*models.Task{ + models.RelationKindParenttask: { + { + Title: "Task400000006", + Done: true, + DueDate: timeutil.FromTime(dueTime), + Priority: 1, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(nilTime), + }, + }, + }, + }, + { + Title: "Task400000106", + Done: true, + DueDate: timeutil.FromTime(dueTime), + Priority: 1, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(nilTime), + Labels: vikunjaLabels, + }, + { + Title: "Task400000107", + Done: true, + Priority: 1, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(nilTime), + }, + { + Title: "Task400000108", + Done: true, + Priority: 1, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(nilTime), + }, + { + Title: "Task400000109", + Done: true, + Priority: 1, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(nilTime), + }, + }, + }, + { + Title: "Project2", + Description: "Lorem Ipsum dolor sit amet 4\nLorem Ipsum dolor sit amet 5", + HexColor: todoistColors[37], + Tasks: []*models.Task{ + { + Title: "Task400000007", + Done: false, + DueDate: timeutil.FromTime(dueTime), + Priority: 1, + Created: timeutil.FromTime(time1), + }, + { + Title: "Task400000008", + Done: false, + DueDate: timeutil.FromTime(dueTime), + Priority: 1, + Created: timeutil.FromTime(time1), + }, + { + Title: "Task400000009", + Done: false, + Priority: 1, + Created: timeutil.FromTime(time1), + Reminders: []timeutil.TimeStamp{ + timeutil.FromTime(time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC)), + }, + }, + { + Title: "Task400000010", + Description: "Lorem Ipsum dolor sit amet", + Done: true, + Priority: 1, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(nilTime), + }, + { + Title: "Task400000101", + Description: "Lorem Ipsum dolor sit amet", + Done: false, + Priority: 1, + Created: timeutil.FromTime(time1), + Attachments: []*models.TaskAttachment{ + { + File: &files.File{ + Name: "file.md", + Mime: "text/plain", + Size: 12345, + Created: time1, + CreatedUnix: timeutil.FromTime(time1), + FileContent: exampleFile, + }, + Created: timeutil.FromTime(time1), + }, + }, + }, + { + Title: "Task400000102", + Done: false, + DueDate: timeutil.FromTime(dueTime), + Priority: 1, + Created: timeutil.FromTime(time1), + Labels: vikunjaLabels, + }, + { + Title: "Task400000103", + Done: false, + Priority: 1, + Created: timeutil.FromTime(time1), + Labels: vikunjaLabels, + }, + { + Title: "Task400000104", + Done: false, + Priority: 1, + Created: timeutil.FromTime(time1), + Labels: vikunjaLabels, + }, + { + Title: "Task400000105", + Done: false, + DueDate: timeutil.FromTime(dueTime), + Priority: 1, + Created: timeutil.FromTime(time1), + Labels: vikunjaLabels, + }, + }, + }, + { + Title: "Project3 - Archived", + HexColor: todoistColors[37], + IsArchived: true, + Tasks: []*models.Task{ + { + Title: "Task400000110", + Done: true, + Priority: 1, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(nilTime), }, }, }, -- 2.47.2 From c51d6654a5449ff66c08a6315ce739d208264f43 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 23 May 2020 17:59:03 +0200 Subject: [PATCH 12/16] Add creating labels when migrating --- .../migration/create_from_structure.go | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pkg/modules/migration/create_from_structure.go b/pkg/modules/migration/create_from_structure.go index 903ee08a2..22a61049d 100644 --- a/pkg/modules/migration/create_from_structure.go +++ b/pkg/modules/migration/create_from_structure.go @@ -30,6 +30,8 @@ func InsertFromStructure(str []*models.NamespaceWithLists, user *user.User) (err log.Debugf("[creating structure] Creating %d namespaces", len(str)) + labels := make(map[string]*models.Label) + // Create all namespaces for _, n := range str { err = n.Create(user) @@ -118,6 +120,31 @@ func InsertFromStructure(str []*models.NamespaceWithLists, user *user.User) (err log.Debugf("[creating structure] Created new attachment %d", a.ID) } } + + // Create all labels + for _, label := range t.Labels { + // Check if we already have a label with that name + color combination and use it + // If not, create one and save it for later + var l *models.Label + l, exists := labels[label.Title+label.HexColor] + if !exists { + err = label.Create(user) + if err != nil { + return err + } + labels[label.Title+label.HexColor] = label + l = label + } + + lt := &models.LabelTask{ + LabelID: l.ID, + TaskID: t.ID, + } + err = lt.Create(user) + if err != nil { + return err + } + } } } } -- 2.47.2 From 0cce5ca856b5c088c3f7f35d2e008644863a2da9 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 23 May 2020 19:03:46 +0200 Subject: [PATCH 13/16] Add logging to creating labels and debug logs --- .../migration/create_from_structure.go | 11 ++++++---- .../migration/create_from_structure_test.go | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/pkg/modules/migration/create_from_structure.go b/pkg/modules/migration/create_from_structure.go index 22a61049d..571e4bfcb 100644 --- a/pkg/modules/migration/create_from_structure.go +++ b/pkg/modules/migration/create_from_structure.go @@ -125,25 +125,28 @@ func InsertFromStructure(str []*models.NamespaceWithLists, user *user.User) (err for _, label := range t.Labels { // Check if we already have a label with that name + color combination and use it // If not, create one and save it for later - var l *models.Label - l, exists := labels[label.Title+label.HexColor] + var lb *models.Label + var exists bool + lb, exists = labels[label.Title+label.HexColor] if !exists { err = label.Create(user) if err != nil { return err } + log.Debugf("[creating structure] Created new label %d", label.ID) labels[label.Title+label.HexColor] = label - l = label + lb = label } lt := &models.LabelTask{ - LabelID: l.ID, + LabelID: lb.ID, TaskID: t.ID, } err = lt.Create(user) if err != nil { return err } + log.Debugf("[creating structure] Associated task %d with label %d", t.ID, lb.ID) } } } diff --git a/pkg/modules/migration/create_from_structure_test.go b/pkg/modules/migration/create_from_structure_test.go index 2b235da3f..75fea4fbe 100644 --- a/pkg/modules/migration/create_from_structure_test.go +++ b/pkg/modules/migration/create_from_structure_test.go @@ -69,6 +69,28 @@ func TestInsertFromStructure(t *testing.T) { }, }, }, + { + Title: "Task with labels", + Labels: []*models.Label{ + { + Title: "Label1", + HexColor: "ff00ff", + }, + { + Title: "Label2", + HexColor: "ff00ff", + }, + }, + }, + { + Title: "Task with same label", + Labels: []*models.Label{ + { + Title: "Label1", + HexColor: "ff00ff", + }, + }, + }, }, }, }, -- 2.47.2 From c5258989d65aab10502aae94ad8d2d5fa82bb01c Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 23 May 2020 19:16:52 +0200 Subject: [PATCH 14/16] Fix test init --- pkg/modules/migration/todoist/todoist_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/modules/migration/todoist/todoist_test.go b/pkg/modules/migration/todoist/todoist_test.go index 76ef6276d..24dede1c3 100644 --- a/pkg/modules/migration/todoist/todoist_test.go +++ b/pkg/modules/migration/todoist/todoist_test.go @@ -30,6 +30,9 @@ import ( ) func TestConvertTodoistToVikunja(t *testing.T) { + + config.InitConfig() + time1, err := time.Parse(time.RFC3339Nano, "2014-09-26T08:25:05Z") assert.NoError(t, err) time3, err := time.Parse(time.RFC3339Nano, "2014-10-21T08:25:05Z") -- 2.47.2 From f80163062e3a9ca14f117ea901b2ed678b5b51c8 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 23 May 2020 22:13:50 +0200 Subject: [PATCH 15/16] Add parsing logic + fix fixtures --- pkg/modules/migration/todoist/todoist.go | 156 +++++++++++++- pkg/modules/migration/todoist/todoist_test.go | 193 +++++++----------- 2 files changed, 231 insertions(+), 118 deletions(-) diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index 6fe73ad37..13828a3b3 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -19,9 +19,11 @@ package todoist import ( "bytes" "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/files" "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/models" "code.vikunja.io/api/pkg/modules/migration" + "code.vikunja.io/api/pkg/timeutil" "code.vikunja.io/api/pkg/user" "code.vikunja.io/api/pkg/utils" "encoding/json" @@ -132,7 +134,7 @@ type projectNote struct { IsDeleted int `json:"is_deleted"` Posted time.Time `json:"posted"` PostedUID int `json:"posted_uid"` - ProjectID int64 `json:"project_id"` + ProjectID int `json:"project_id"` UidsToNotify []int `json:"uids_to_notify"` } @@ -209,11 +211,157 @@ func doPost(url string, form url.Values) (resp *http.Response, err error) { func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.NamespaceWithLists, err error) { - // Priority 1 is low and 4 (max) is highest + newNamespace := &models.NamespaceWithLists{ + Namespace: models.Namespace{ + Title: "Migrated from todoist", + }, + } - // If parenId is not 0, create a task relation + // A map for all vikunja lists with the project id they're coming from as key + lists := make(map[int]*models.List, len(sync.Projects)) - return + // A map for all vikunja tasks with the todoist task id as key to find them easily and add more data + tasks := make(map[int]*models.Task, len(sync.Items)) + + // A map for all vikunja labels with the todoist id as key to find them easier + labels := make(map[int]*models.Label, len(sync.Labels)) + + for _, p := range sync.Projects { + list := &models.List{ + Title: p.Name, + HexColor: todoistColors[p.Color], + IsArchived: p.IsArchived == 1, + } + + lists[p.ID] = list + + newNamespace.Lists = append(newNamespace.Lists, list) + } + + for _, label := range sync.Labels { + labels[label.ID] = &models.Label{ + Title: label.Name, + HexColor: todoistColors[label.Color], + } + } + + for _, i := range sync.Items { + task := &models.Task{ + Title: i.Content, + Created: timeutil.FromTime(i.DateAdded), + Done: i.Checked == 1, + } + + // Only try to parse the task done at date if the task is actually done + // Sometimes weired things happen if we try to parse nil dates. + if task.Done { + task.DoneAt = timeutil.FromTime(i.DateCompleted) + } + + // Todoist priorities only range from 1 (lowest) and max 4 (highest), so we need to make slight adjustments + if i.Priority > 1 { + task.Priority = int64(i.Priority) + } + + // Put the due date together + if i.Due != nil { + dueDate, err := time.Parse("2006-01-02", i.Due.Date) + if err != nil { + return nil, err + } + task.DueDate = timeutil.FromTime(dueDate) + } + + // Put all labels together from earlier + for _, lID := range i.Labels { + task.Labels = append(task.Labels, labels[lID]) + } + + tasks[i.ID] = task + + lists[i.ProjectID].Tasks = append(lists[i.ProjectID].Tasks, task) + } + + // If the parenId of a task is not 0, create a task relation + // We're looping again here to make sure we have seem all tasks before and have them in our map + for _, i := range sync.Items { + if i.ParentID == 0 { + continue + } + + // Prevent all those nil errors + if tasks[i.ParentID].RelatedTasks == nil { + tasks[i.ParentID].RelatedTasks = make(models.RelatedTaskMap) + } + + tasks[i.ParentID].RelatedTasks[models.RelationKindSubtask] = append(tasks[i.ParentID].RelatedTasks[models.RelationKindSubtask], tasks[i.ID]) + } + + // Task Notes -> Task Descriptions + for _, n := range sync.Notes { + if tasks[n.ItemID].Description != "" { + tasks[n.ItemID].Description += "\n" + } + tasks[n.ItemID].Description += n.Content + + if n.FileAttachment == nil { + continue + } + + // Download the attachment and put it in the file + resp, err := http.Get(n.FileAttachment.FileURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + buf := &bytes.Buffer{} + _, err = buf.ReadFrom(resp.Body) + if err != nil { + return nil, err + } + + tasks[n.ItemID].Attachments = append(tasks[n.ItemID].Attachments, &models.TaskAttachment{ + File: &files.File{ + Name: n.FileAttachment.FileName, + Mime: n.FileAttachment.FileType, + Size: uint64(n.FileAttachment.FileSize), + Created: n.Posted, + CreatedUnix: timeutil.FromTime(n.Posted), + // We directly pass the file contents here to have a way to link the attachment to the file later. + // Because we don't have an ID for our task at this point of the migration, we cannot just throw all + // attachments in a slice and do the work of downloading and properly storing them later. + FileContent: buf.Bytes(), + }, + Created: timeutil.FromTime(n.Posted), + }) + } + + // Project Notes -> List Descriptions + for _, pn := range sync.ProjectNotes { + if lists[pn.ProjectID].Description != "" { + lists[pn.ProjectID].Description += "\n" + } + + lists[pn.ProjectID].Description += pn.Content + } + + // Reminders -> vikunja reminders + for _, r := range sync.Reminders { + if r.Due == nil { + continue + } + + date, err := time.Parse("2006-01-02", r.Due.Date) + if err != nil { + return nil, err + } + + tasks[r.ItemID].Reminders = append(tasks[r.ItemID].Reminders, timeutil.FromTime(date)) + } + + return []*models.NamespaceWithLists{ + newNamespace, + }, err } func getAccessTokenFromAuthToken(authToken string) (accessToken string, err error) { diff --git a/pkg/modules/migration/todoist/todoist_test.go b/pkg/modules/migration/todoist/todoist_test.go index 24dede1c3..19625a232 100644 --- a/pkg/modules/migration/todoist/todoist_test.go +++ b/pkg/modules/migration/todoist/todoist_test.go @@ -131,7 +131,7 @@ func TestConvertTodoistToVikunja(t *testing.T) { UserID: 1855589, ProjectID: 396936926, Content: "Task with parent", - Priority: 1, + Priority: 2, ParentID: 400000006, ChildOrder: 1, Checked: 0, @@ -152,7 +152,7 @@ func TestConvertTodoistToVikunja(t *testing.T) { makeTestItem(400000104, 396936927, false, true, false), makeTestItem(400000105, 396936927, true, true, false), - makeTestItem(400000110, 396936928, false, false, true), + makeTestItem(400000111, 396936928, false, false, true), }, Labels: []*label{ { @@ -344,7 +344,6 @@ func TestConvertTodoistToVikunja(t *testing.T) { Title: "Task400000000", Description: "Lorem Ipsum dolor sit amet", Done: false, - Priority: 1, Created: timeutil.FromTime(time1), Reminders: []timeutil.TimeStamp{ timeutil.FromTime(time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC)), @@ -355,14 +354,12 @@ func TestConvertTodoistToVikunja(t *testing.T) { Title: "Task400000001", Description: "Lorem Ipsum dolor sit amet", Done: false, - Priority: 1, Created: timeutil.FromTime(time1), }, { - Title: "Task400000002", - Done: false, - Priority: 1, - Created: timeutil.FromTime(time1), + Title: "Task400000002", + Done: false, + Created: timeutil.FromTime(time1), Reminders: []timeutil.TimeStamp{ timeutil.FromTime(time.Date(2020, time.July, 15, 0, 0, 0, 0, time.UTC)), }, @@ -372,7 +369,6 @@ func TestConvertTodoistToVikunja(t *testing.T) { Description: "Lorem Ipsum dolor sit amet", Done: true, DueDate: timeutil.FromTime(dueTime), - Priority: 1, Created: timeutil.FromTime(time1), DoneAt: timeutil.FromTime(time3), Labels: vikunjaLabels, @@ -381,37 +377,33 @@ func TestConvertTodoistToVikunja(t *testing.T) { }, }, { - Title: "Task400000004", - Done: false, - Priority: 1, - Created: timeutil.FromTime(time1), - Labels: vikunjaLabels, + Title: "Task400000004", + Done: false, + Created: timeutil.FromTime(time1), + Labels: vikunjaLabels, + }, + { + Title: "Task400000005", + Done: true, + DueDate: timeutil.FromTime(dueTime), + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(time3), Reminders: []timeutil.TimeStamp{ timeutil.FromTime(time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC)), }, }, { - Title: "Task400000005", - Done: true, - DueDate: timeutil.FromTime(dueTime), - Priority: 1, - Created: timeutil.FromTime(time1), - DoneAt: timeutil.FromTime(nilTime), - }, - { - Title: "Task400000006", - Done: true, - DueDate: timeutil.FromTime(dueTime), - Priority: 1, - Created: timeutil.FromTime(time1), - DoneAt: timeutil.FromTime(nilTime), + Title: "Task400000006", + Done: true, + DueDate: timeutil.FromTime(dueTime), + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(time3), RelatedTasks: map[models.RelationKind][]*models.Task{ models.RelationKindSubtask: { { Title: "Task with parent", Done: false, - DueDate: timeutil.FromTime(dueTime), - Priority: 1, + Priority: 2, Created: timeutil.FromTime(time1), DoneAt: timeutil.FromTime(nilTime), }, @@ -421,52 +413,35 @@ func TestConvertTodoistToVikunja(t *testing.T) { { Title: "Task with parent", Done: false, - DueDate: timeutil.FromTime(dueTime), - Priority: 1, - Created: timeutil.FromTime(time1), - DoneAt: timeutil.FromTime(nilTime), - RelatedTasks: map[models.RelationKind][]*models.Task{ - models.RelationKindParenttask: { - { - Title: "Task400000006", - Done: true, - DueDate: timeutil.FromTime(dueTime), - Priority: 1, - Created: timeutil.FromTime(time1), - DoneAt: timeutil.FromTime(nilTime), - }, - }, - }, - }, - { - Title: "Task400000106", - Done: true, - DueDate: timeutil.FromTime(dueTime), - Priority: 1, - Created: timeutil.FromTime(time1), - DoneAt: timeutil.FromTime(nilTime), - Labels: vikunjaLabels, - }, - { - Title: "Task400000107", - Done: true, - Priority: 1, + Priority: 2, Created: timeutil.FromTime(time1), DoneAt: timeutil.FromTime(nilTime), }, { - Title: "Task400000108", - Done: true, - Priority: 1, - Created: timeutil.FromTime(time1), - DoneAt: timeutil.FromTime(nilTime), + Title: "Task400000106", + Done: true, + DueDate: timeutil.FromTime(dueTime), + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(time3), + Labels: vikunjaLabels, }, { - Title: "Task400000109", - Done: true, - Priority: 1, - Created: timeutil.FromTime(time1), - DoneAt: timeutil.FromTime(nilTime), + Title: "Task400000107", + Done: true, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(time3), + }, + { + Title: "Task400000108", + Done: true, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(time3), + }, + { + Title: "Task400000109", + Done: true, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(time3), }, }, }, @@ -476,24 +451,21 @@ func TestConvertTodoistToVikunja(t *testing.T) { HexColor: todoistColors[37], Tasks: []*models.Task{ { - Title: "Task400000007", - Done: false, - DueDate: timeutil.FromTime(dueTime), - Priority: 1, - Created: timeutil.FromTime(time1), + Title: "Task400000007", + Done: false, + DueDate: timeutil.FromTime(dueTime), + Created: timeutil.FromTime(time1), }, { - Title: "Task400000008", - Done: false, - DueDate: timeutil.FromTime(dueTime), - Priority: 1, - Created: timeutil.FromTime(time1), + Title: "Task400000008", + Done: false, + DueDate: timeutil.FromTime(dueTime), + Created: timeutil.FromTime(time1), }, { - Title: "Task400000009", - Done: false, - Priority: 1, - Created: timeutil.FromTime(time1), + Title: "Task400000009", + Done: false, + Created: timeutil.FromTime(time1), Reminders: []timeutil.TimeStamp{ timeutil.FromTime(time.Date(2020, time.June, 15, 0, 0, 0, 0, time.UTC)), }, @@ -502,15 +474,13 @@ func TestConvertTodoistToVikunja(t *testing.T) { Title: "Task400000010", Description: "Lorem Ipsum dolor sit amet", Done: true, - Priority: 1, Created: timeutil.FromTime(time1), - DoneAt: timeutil.FromTime(nilTime), + DoneAt: timeutil.FromTime(time3), }, { Title: "Task400000101", Description: "Lorem Ipsum dolor sit amet", Done: false, - Priority: 1, Created: timeutil.FromTime(time1), Attachments: []*models.TaskAttachment{ { @@ -527,34 +497,30 @@ func TestConvertTodoistToVikunja(t *testing.T) { }, }, { - Title: "Task400000102", - Done: false, - DueDate: timeutil.FromTime(dueTime), - Priority: 1, - Created: timeutil.FromTime(time1), - Labels: vikunjaLabels, + Title: "Task400000102", + Done: false, + DueDate: timeutil.FromTime(dueTime), + Created: timeutil.FromTime(time1), + Labels: vikunjaLabels, }, { - Title: "Task400000103", - Done: false, - Priority: 1, - Created: timeutil.FromTime(time1), - Labels: vikunjaLabels, + Title: "Task400000103", + Done: false, + Created: timeutil.FromTime(time1), + Labels: vikunjaLabels, }, { - Title: "Task400000104", - Done: false, - Priority: 1, - Created: timeutil.FromTime(time1), - Labels: vikunjaLabels, + Title: "Task400000104", + Done: false, + Created: timeutil.FromTime(time1), + Labels: vikunjaLabels, }, { - Title: "Task400000105", - Done: false, - DueDate: timeutil.FromTime(dueTime), - Priority: 1, - Created: timeutil.FromTime(time1), - Labels: vikunjaLabels, + Title: "Task400000105", + Done: false, + DueDate: timeutil.FromTime(dueTime), + Created: timeutil.FromTime(time1), + Labels: vikunjaLabels, }, }, }, @@ -564,11 +530,10 @@ func TestConvertTodoistToVikunja(t *testing.T) { IsArchived: true, Tasks: []*models.Task{ { - Title: "Task400000110", - Done: true, - Priority: 1, - Created: timeutil.FromTime(time1), - DoneAt: timeutil.FromTime(nilTime), + Title: "Task400000111", + Done: true, + Created: timeutil.FromTime(time1), + DoneAt: timeutil.FromTime(time3), }, }, }, -- 2.47.2 From 866ea3992549fe95e6eba4956f84baa03834bad2 Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 23 May 2020 22:19:53 +0200 Subject: [PATCH 16/16] Add swagger docs + fix lint --- pkg/modules/migration/todoist/todoist.go | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index 13828a3b3..9d2dc859d 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -34,6 +34,7 @@ import ( "time" ) +// Migration is the todoist migration struct type Migration struct { Code string `json:"code"` } @@ -187,10 +188,28 @@ func init() { } } +// Name is used to get the name of the todoist migration - we're using the docs here to annotate the status route. +// @Summary Get migration status +// @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again. +// @tags migration +// @Produce json +// @Security JWTKeyAuth +// @Success 200 {object} migration.Status "The migration status" +// @Failure 500 {object} models.Message "Internal server error" +// @Router /migration/todoist/status [get] func (m *Migration) Name() string { return "todoist" } +// AuthURL returns the url users need to authenticate against +// @Summary Get the auth url from todoist +// @Description Returns the auth url where the user needs to get its auth code. This code can then be used to migrate everything from todoist to Vikunja. +// @tags migration +// @Produce json +// @Security JWTKeyAuth +// @Success 200 {object} handler.AuthURL "The auth url." +// @Failure 500 {object} models.Message "Internal server error" +// @Router /migration/todoist/auth [get] func (m *Migration) AuthURL() string { return "https://todoist.com/oauth/authorize" + "?client_id=" + config.MigrationTodoistClientID.GetString() + @@ -388,6 +407,17 @@ func getAccessTokenFromAuthToken(authToken string) (accessToken string, err erro return token.AccessToken, err } +// Migrate gets all tasks from todoist for a user and puts them into vikunja +// @Summary Migrate all lists, tasks etc. from todoist +// @Description Migrates all projects, tasks, notes, reminders, subtasks and files from todoist to vikunja. +// @tags migration +// @Accept json +// @Produce json +// @Security JWTKeyAuth +// @Param migrationCode body todoist.Migration true "The auth code previously obtained from the auth url. See the docs for /migration/todoist/auth." +// @Success 200 {object} models.Message "A message telling you everything was migrated successfully." +// @Failure 500 {object} models.Message "Internal server error" +// @Router /migration/todoist/migrate [post] func (m *Migration) Migrate(u *user.User) (err error) { log.Debugf("[Todoist Migration] Starting migration for user %d", u.ID) -- 2.47.2