From f5b5204776da55a70dd5ac1020d9d4ba4e579fbe Mon Sep 17 00:00:00 2001 From: konrad Date: Fri, 18 Dec 2020 11:12:05 +0000 Subject: [PATCH] Add Microsoft Todo migration (#737) Add more logs Fix lint Generate docs Add swagger docs Add microsoft todo parsing logic go mod tidy Add basic test structure Add more fields to tasks rename microsoft todo package Add getting microsoft todo data Add structs and helper methods Add microsoft todo config and routes Co-authored-by: kolaente Reviewed-on: https://kolaente.dev/vikunja/api/pulls/737 Co-Authored-By: konrad Co-Committed-By: konrad --- config.yml.sample | 15 + docs/content/doc/setup/config.md | 4 + go.sum | 3 +- pkg/config/config.go | 31 +- pkg/modules/migration/helpers.go | 14 + .../microsoft-todo/microsoft_todo.go | 407 ++++++++++++++++++ .../microsoft-todo/microsoft_todo_test.go | 169 ++++++++ pkg/modules/migration/todoist/todoist.go | 18 +- pkg/routes/api/v1/info.go | 6 + pkg/routes/routes.go | 12 + pkg/swagger/docs.go | 115 +++++ pkg/swagger/swagger.json | 115 +++++ pkg/swagger/swagger.yaml | 71 +++ 13 files changed, 951 insertions(+), 29 deletions(-) create mode 100644 pkg/modules/migration/microsoft-todo/microsoft_todo.go create mode 100644 pkg/modules/migration/microsoft-todo/microsoft_todo_test.go diff --git a/config.yml.sample b/config.yml.sample index 72426cdeb..e8a26824c 100644 --- a/config.yml.sample +++ b/config.yml.sample @@ -197,6 +197,21 @@ migration: # with the code obtained from the trello api. # Note that the vikunja frontend expects this to end on /migrate/trello. redirecturl: /migrate/trello + microsofttodo: + # Wheter to enable the microsoft todo migrator or not + enable: false + # The client id, required for making requests to the microsoft graph api + # See https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app#register-an-application + # for information about how to register your vikuinja instance. + clientid: + # The client secret, also required for making requests to the microsoft graph api + clientsecret: + # The url where clients are redirected after they authorized Vikunja to access their microsoft todo tasks. + # This needs to match the url you entered when registering your Vikunja instance at microsoft. + # This is usually the frontend url where the frontend then makes a request to /migration/microsoft-todo/migrate + # with the code obtained from the microsoft graph api. + # Note that the vikunja frontend expects this to be /migrate/microsoft-todo + redirecturl: /migrate/microsoft-todo avatar: # When using gravatar, this is the duration in seconds until a cached gravatar user avatar expires diff --git a/docs/content/doc/setup/config.md b/docs/content/doc/setup/config.md index 370420cfa..45a180f83 100644 --- a/docs/content/doc/setup/config.md +++ b/docs/content/doc/setup/config.md @@ -516,6 +516,10 @@ Default: `` Default: `` +### microsofttodo + +Default: `` + --- ## avatar diff --git a/go.sum b/go.sum index c437166d3..0710c0bba 100644 --- a/go.sum +++ b/go.sum @@ -692,6 +692,7 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw= @@ -843,8 +844,6 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 h1:sYNJzB4J8toYPQTM6pAkcmBRgw9SnQKP9oXCHfgy604= -golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 h1:3wPMTskHO3+O6jqTEXyFcsnuxMQOqYSaHsDxcbUXpqA= diff --git a/pkg/config/config.go b/pkg/config/config.go index 53debaaad..039d4d737 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -110,17 +110,21 @@ const ( FilesBasePath Key = `files.basepath` FilesMaxSize Key = `files.maxsize` - MigrationWunderlistEnable Key = `migration.wunderlist.enable` - MigrationWunderlistClientID Key = `migration.wunderlist.clientid` - MigrationWunderlistClientSecret Key = `migration.wunderlist.clientsecret` - MigrationWunderlistRedirectURL Key = `migration.wunderlist.redirecturl` - MigrationTodoistEnable Key = `migration.todoist.enable` - MigrationTodoistClientID Key = `migration.todoist.clientid` - MigrationTodoistClientSecret Key = `migration.todoist.clientsecret` - MigrationTodoistRedirectURL Key = `migration.todoist.redirecturl` - MigrationTrelloEnable Key = `migration.trello.enable` - MigrationTrelloKey Key = `migration.trello.key` - MigrationTrelloRedirectURL Key = `migration.trello.redirecturl` + MigrationWunderlistEnable Key = `migration.wunderlist.enable` + MigrationWunderlistClientID Key = `migration.wunderlist.clientid` + MigrationWunderlistClientSecret Key = `migration.wunderlist.clientsecret` + MigrationWunderlistRedirectURL Key = `migration.wunderlist.redirecturl` + MigrationTodoistEnable Key = `migration.todoist.enable` + MigrationTodoistClientID Key = `migration.todoist.clientid` + MigrationTodoistClientSecret Key = `migration.todoist.clientsecret` + MigrationTodoistRedirectURL Key = `migration.todoist.redirecturl` + MigrationTrelloEnable Key = `migration.trello.enable` + MigrationTrelloKey Key = `migration.trello.key` + MigrationTrelloRedirectURL Key = `migration.trello.redirecturl` + MigrationMicrosoftTodoEnable Key = `migration.microsofttodo.enable` + MigrationMicrosoftTodoClientID Key = `migration.microsofttodo.clientid` + MigrationMicrosoftTodoClientSecret Key = `migration.microsofttodo.clientsecret` + MigrationMicrosoftTodoRedirectURL Key = `migration.microsofttodo.redirecturl` CorsEnable Key = `cors.enable` CorsOrigins Key = `cors.origins` @@ -292,6 +296,7 @@ func InitDefaultConfig() { MigrationWunderlistEnable.setDefault(false) MigrationTodoistEnable.setDefault(false) MigrationTrelloEnable.setDefault(false) + MigrationMicrosoftTodoEnable.setDefault(false) // Avatar AvatarGravaterExpiration.setDefault(3600) // List Backgrounds @@ -349,6 +354,10 @@ func InitConfig() { MigrationTrelloRedirectURL.Set(ServiceFrontendurl.GetString() + "migrate/trello") } + if MigrationMicrosoftTodoRedirectURL.GetString() == "" { + MigrationMicrosoftTodoRedirectURL.Set(ServiceFrontendurl.GetString() + "migrate/microsoft-todo") + } + log.Printf("Using config file: %s", viper.ConfigFileUsed()) } diff --git a/pkg/modules/migration/helpers.go b/pkg/modules/migration/helpers.go index 0a89fff54..d5b6dbac5 100644 --- a/pkg/modules/migration/helpers.go +++ b/pkg/modules/migration/helpers.go @@ -20,6 +20,8 @@ import ( "bytes" "context" "net/http" + "net/url" + "strings" ) // DownloadFile downloads a file and returns its contents @@ -38,3 +40,15 @@ func DownloadFile(url string) (buf *bytes.Buffer, err error) { _, err = buf.ReadFrom(resp.Body) return } + +// DoPost makes a form encoded post request +func DoPost(url string, form url.Values) (resp *http.Response, err error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, url, strings.NewReader(form.Encode())) + if err != nil { + return + } + + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + hc := http.Client{} + return hc.Do(req) +} diff --git a/pkg/modules/migration/microsoft-todo/microsoft_todo.go b/pkg/modules/migration/microsoft-todo/microsoft_todo.go new file mode 100644 index 000000000..06dd9d4c7 --- /dev/null +++ b/pkg/modules/migration/microsoft-todo/microsoft_todo.go @@ -0,0 +1,407 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2020 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package microsofttodo + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/models" + "code.vikunja.io/api/pkg/modules/migration" + "code.vikunja.io/api/pkg/user" +) + +const apiScopes = `tasks.read tasks.read.shared` + +type Migration struct { + Code string `json:"code"` +} + +type apiTokenResponse struct { + TokenType string `json:"token_type"` + Scope string `json:"scope"` + ExpiresIn int `json:"expires_in"` + ExtExpiresIn int `json:"ext_expires_in"` + AccessToken string `json:"access_token"` +} + +type task struct { + OdataEtag string `json:"@odata.etag"` + Importance string `json:"importance"` + IsReminderOn bool `json:"isReminderOn"` + Status string `json:"status"` + Title string `json:"title"` + CreatedDateTime time.Time `json:"createdDateTime"` + LastModifiedDateTime time.Time `json:"lastModifiedDateTime"` + ID string `json:"id"` + Body *body `json:"body"` + DueDateTime *dateTimeTimeZone `json:"dueDateTime"` + Recurrence *recurrence `json:"recurrence"` + ReminderDateTime *dateTimeTimeZone `json:"reminderDateTime"` + CompletedDateTime *dateTimeTimeZone `json:"completedDateTime"` +} +type dateTimeTimeZone struct { + DateTime string `json:"dateTime"` + TimeZone string `json:"timeZone"` +} +type body struct { + Content string `json:"content"` + ContentType string `json:"contentType"` +} +type pattern struct { + Type string `json:"type"` + Interval int64 `json:"interval"` + Month int64 `json:"month"` + DayOfMonth int64 `json:"dayOfMonth"` + DaysOfWeek []string `json:"daysOfWeek"` + FirstDayOfWeek string `json:"firstDayOfWeek"` + Index string `json:"index"` +} +type taskRange struct { + Type string `json:"type"` + StartDate string `json:"startDate"` + EndDate string `json:"endDate"` + RecurrenceTimeZone string `json:"recurrenceTimeZone"` + NumberOfOccurrences int `json:"numberOfOccurrences"` +} +type recurrence struct { + Pattern *pattern `json:"pattern"` + Range *taskRange `json:"range"` +} + +type tasksResponse struct { + OdataContext string `json:"@odata.context"` + Value []*task `json:"value"` +} + +type list struct { + ID string `json:"id"` + OdataEtag string `json:"@odata.etag"` + DisplayName string `json:"displayName"` + IsOwner bool `json:"isOwner"` + IsShared bool `json:"isShared"` + WellknownListName string `json:"wellknownListName"` + Tasks []*task `json:"-"` // This field does not exist in the api, we're just using it to return a structure with everything at once +} + +type listsResponse struct { + OdataContext string `json:"@odata.context"` + Value []*list `json:"value"` +} + +func (dtt *dateTimeTimeZone) toTime() (t time.Time, err error) { + loc, err := time.LoadLocation(dtt.TimeZone) + if err != nil { + return t, err + } + + return time.ParseInLocation(time.RFC3339Nano, dtt.DateTime+"Z", loc) +} + +// AuthURL returns the url users need to authenticate against +// @Summary Get the auth url from Microsoft Todo +// @Description Returns the auth url where the user needs to get its auth code. This code can then be used to migrate everything from Microsoft Todo to Vikunja. +// @tags migration +// @Produce json +// @Security JWTKeyAuth +// @Success 200 {object} handler.AuthURL "The auth url." +// @Failure 500 {object} models.Message "Internal server error" +// @Router /migration/microsoft-todo/auth [get] +func (m *Migration) AuthURL() string { + return "https://login.microsoftonline.com/common/oauth2/v2.0/authorize" + + "?client_id=" + config.MigrationMicrosoftTodoClientID.GetString() + + "&response_type=code" + + "&redirect_uri=" + config.MigrationMicrosoftTodoRedirectURL.GetString() + + "&response_mode=query" + + "&scope=" + apiScopes +} + +// Name is used to get the name of the Microsoft Todo migration - we're using the docs here to annotate the status route. +// @Summary Get migration status +// @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again. +// @tags migration +// @Produce json +// @Security JWTKeyAuth +// @Success 200 {object} migration.Status "The migration status" +// @Failure 500 {object} models.Message "Internal server error" +// @Router /migration/microsoft-todo/status [get] +func (m *Migration) Name() string { + return "microsoft-todo" +} + +func getMicrosoftGraphAuthToken(code string) (accessToken string, err error) { + + form := url.Values{ + "client_id": []string{config.MigrationMicrosoftTodoClientID.GetString()}, + "client_secret": []string{config.MigrationMicrosoftTodoClientSecret.GetString()}, + "scope": []string{apiScopes}, + "code": []string{code}, + "redirect_uri": []string{config.MigrationMicrosoftTodoRedirectURL.GetString()}, + "grant_type": []string{"authorization_code"}, + } + resp, err := migration.DoPost("https://login.microsoftonline.com/common/oauth2/v2.0/token", form) + if err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode > 399 { + buf := &bytes.Buffer{} + _, _ = buf.ReadFrom(resp.Body) + return "", fmt.Errorf("got http status %d while trying to get token, error was %s", resp.StatusCode, buf.String()) + } + + token := &apiTokenResponse{} + err = json.NewDecoder(resp.Body).Decode(token) + return token.AccessToken, err +} + +func makeAuthenticatedGetRequest(token, urlPart string, v interface{}) error { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "https://graph.microsoft.com/v1.0/me/todo/"+urlPart, nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + buf := &bytes.Buffer{} + _, err = buf.ReadFrom(resp.Body) + if err != nil { + return err + } + + if resp.StatusCode > 399 { + return fmt.Errorf("Microsoft Graph API Error: Status Code: %d, Response was: %s", resp.StatusCode, buf.String()) + } + + // If the response is an empty json array, we need to exit here, otherwise this breaks the json parser since it + // expects a null for an empty slice + str := buf.String() + if str == "[]" { + return nil + } + + return json.Unmarshal(buf.Bytes(), v) +} + +func getMicrosoftTodoData(token string) (microsoftTodoData []*list, err error) { + + microsoftTodoData = []*list{} + + lists := &listsResponse{} + err = makeAuthenticatedGetRequest(token, "lists", lists) + if err != nil { + log.Errorf("[Microsoft Todo Migration] Could not get lists: %s", err) + return + } + + log.Debugf("[Microsoft Todo Migration] Got %d lists", len(lists.Value)) + + for _, list := range lists.Value { + tasksResponse := &tasksResponse{} + err = makeAuthenticatedGetRequest(token, "lists/"+list.ID+"/tasks", tasksResponse) + if err != nil { + log.Errorf("[Microsoft Todo Migration] Could not get tasks for list %s: %s", list.ID, err) + return + } + + log.Debugf("[Microsoft Todo Migration] Got %d tasks for list %s", len(tasksResponse.Value), list.ID) + + list.Tasks = tasksResponse.Value + + microsoftTodoData = append(microsoftTodoData, list) + } + + log.Debugf("[Microsoft Todo Migration] Got all tasks for %d lists", len(lists.Value)) + + return +} + +func convertMicrosoftTodoData(todoData []*list) (vikunjsStructure []*models.NamespaceWithLists, err error) { + + // One namespace with all lists + vikunjsStructure = []*models.NamespaceWithLists{ + { + Namespace: models.Namespace{ + Title: "Migrated from Microsoft Todo", + }, + Lists: []*models.List{}, + }, + } + + log.Debugf("[Microsoft Todo Migration] Converting %d lists", len(todoData)) + + for _, l := range todoData { + + log.Debugf("[Microsoft Todo Migration] Converting list %s", l.ID) + + // Lists only with title + list := &models.List{ + Title: l.DisplayName, + } + + log.Debugf("[Microsoft Todo Migration] Converting %d tasks", len(l.Tasks)) + + for _, t := range l.Tasks { + + log.Debugf("[Microsoft Todo Migration] Converting task %s", t.ID) + + task := &models.Task{ + Title: t.Title, + Done: t.Status == "completed", + } + + // Done Status + if task.Done { + log.Debugf("[Microsoft Todo Migration] Converting done at for task %s", t.ID) + task.DoneAt, err = t.CompletedDateTime.toTime() + if err != nil { + return + } + } + + // Description + if t.Body != nil && t.Body.ContentType == "text" { + task.Description = t.Body.Content + } + + // Priority + switch t.Importance { + case "low": + task.Priority = 1 + case "normal": + task.Priority = 2 + case "high": + task.Priority = 3 + default: + task.Priority = 0 + } + + // Reminders + if t.ReminderDateTime != nil { + log.Debugf("[Microsoft Todo Migration] Converting reminder for task %s", t.ID) + reminder, err := t.ReminderDateTime.toTime() + if err != nil { + return nil, err + } + + task.Reminders = []time.Time{reminder} + } + + // Due Date + if t.DueDateTime != nil { + log.Debugf("[Microsoft Todo Migration] Converting due date for task %s", t.ID) + dueDate, err := t.DueDateTime.toTime() + if err != nil { + return nil, err + } + + task.DueDate = dueDate + } + + // Repeating + if t.Recurrence != nil && t.Recurrence.Pattern != nil { + log.Debugf("[Microsoft Todo Migration] Converting recurring pattern for task %s", t.ID) + switch t.Recurrence.Pattern.Type { + case "daily": + task.RepeatAfter = t.Recurrence.Pattern.Interval * 60 * 60 * 24 + case "weekly": + task.RepeatAfter = t.Recurrence.Pattern.Interval * 60 * 60 * 24 * 7 + case "monthly": + task.RepeatAfter = t.Recurrence.Pattern.Interval * 60 * 60 * 24 * 30 + case "yearly": + task.RepeatAfter = t.Recurrence.Pattern.Interval * 60 * 60 * 24 * 365 + } + } + + list.Tasks = append(list.Tasks, task) + log.Debugf("[Microsoft Todo Migration] Done converted %d tasks", len(l.Tasks)) + } + + vikunjsStructure[0].Lists = append(vikunjsStructure[0].Lists, list) + log.Debugf("[Microsoft Todo Migration] Done converting list %s", l.ID) + } + + return +} + +// Migrate gets all tasks from Microsoft Todo for a user and puts them into vikunja +// @Summary Migrate all lists, tasks etc. from Microsoft Todo +// @Description Migrates all tasklinsts, tasks, notes and reminders from Microsoft Todo to Vikunja. +// @tags migration +// @Accept json +// @Produce json +// @Security JWTKeyAuth +// @Param migrationCode body microsofttodo.Migration true "The auth token previously obtained from the auth url. See the docs for /migration/microsoft-todo/auth." +// @Success 200 {object} models.Message "A message telling you everything was migrated successfully." +// @Failure 500 {object} models.Message "Internal server error" +// @Router /migration/microsoft-todo/migrate [post] +func (m *Migration) Migrate(user *user.User) (err error) { + + log.Debugf("[Microsoft Todo Migration] Start Microsoft Todo migration for user %d", user.ID) + log.Debugf("[Microsoft Todo Migration] Getting Microsoft Graph api token") + + token, err := getMicrosoftGraphAuthToken(m.Code) + if err != nil { + log.Debugf("[Microsoft Todo Migration] Error getting auth token: %s", err) + return + } + + log.Debugf("[Microsoft Todo Migration] Got Microsoft Graph api token") + log.Debugf("[Microsoft Todo Migration] Retrieving Microsoft Todo data") + + todoData, err := getMicrosoftTodoData(token) + if err != nil { + log.Debugf("[Microsoft Todo Migration] Error getting Microsoft Todo data: %s", err) + return + } + + log.Debugf("[Microsoft Todo Migration] Got Microsoft Todo data") + log.Debugf("[Microsoft Todo Migration] Start converting Microsoft Todo data") + + vikunjaStructure, err := convertMicrosoftTodoData(todoData) + if err != nil { + log.Debugf("[Microsoft Todo Migration] Error converting Microsoft Todo data: %s", err) + return + } + + log.Debugf("[Microsoft Todo Migration] Done converting Microsoft Todo data") + log.Debugf("[Microsoft Todo Migration] Creating new structure") + + err = migration.InsertFromStructure(vikunjaStructure, user) + if err != nil { + log.Debugf("[Microsoft Todo Migration] Error while creating new structure: %s", err) + return + } + + log.Debugf("[Microsoft Todo Migration] Created new structure") + log.Debugf("[Microsoft Todo Migration] Microsoft Todo migration done for user %d", user.ID) + + return +} diff --git a/pkg/modules/migration/microsoft-todo/microsoft_todo_test.go b/pkg/modules/migration/microsoft-todo/microsoft_todo_test.go new file mode 100644 index 000000000..f6ffed2dd --- /dev/null +++ b/pkg/modules/migration/microsoft-todo/microsoft_todo_test.go @@ -0,0 +1,169 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2020 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package microsofttodo + +import ( + "testing" + "time" + + "code.vikunja.io/api/pkg/models" + "github.com/d4l3k/messagediff" + "github.com/stretchr/testify/assert" +) + +func TestConverting(t *testing.T) { + + testtime := &dateTimeTimeZone{ + DateTime: "2020-12-18T03:00:00.4770000", + TimeZone: "UTC", + } + + testtimeTime, err := time.Parse(time.RFC3339Nano, "2020-12-18T03:00:00.4770000Z") + assert.NoError(t, err) + + microsoftTodoData := []*list{ + { + DisplayName: "List 1", + Tasks: []*task{ + { + Title: "Task 1", + Status: "notStarted", + Body: &body{ + Content: "This is a description", + ContentType: "text", + }, + }, + { + Title: "Task 2", + Status: "completed", + CompletedDateTime: testtime, + }, + { + Title: "Task 3", + Status: "notStarted", + Importance: "low", + }, + { + Title: "Task 4", + Status: "notStarted", + Importance: "high", + }, + { + Title: "Task 5", + Status: "notStarted", + IsReminderOn: true, + ReminderDateTime: testtime, + }, + { + Title: "Task 6", + Status: "notStarted", + DueDateTime: testtime, + }, + { + Title: "Task 7", + Status: "notStarted", + DueDateTime: testtime, + Recurrence: &recurrence{ + Pattern: &pattern{ + // Every week + Type: "weekly", + Interval: 1, + }, + }, + }, + }, + }, + { + DisplayName: "List 2", + Tasks: []*task{ + { + Title: "Task 1", + Status: "notStarted", + }, + { + Title: "Task 2", + Status: "notStarted", + }, + }, + }, + } + + expectedHierachie := []*models.NamespaceWithLists{ + { + Namespace: models.Namespace{ + Title: "Migrated from Microsoft Todo", + }, + Lists: []*models.List{ + { + Title: "List 1", + Tasks: []*models.Task{ + { + Title: "Task 1", + Description: "This is a description", + }, + { + Title: "Task 2", + Done: true, + DoneAt: testtimeTime, + }, + { + Title: "Task 3", + Priority: 1, + }, + { + Title: "Task 4", + Priority: 3, + }, + { + Title: "Task 5", + Reminders: []time.Time{ + testtimeTime, + }, + }, + { + Title: "Task 6", + DueDate: testtimeTime, + }, + { + Title: "Task 7", + DueDate: testtimeTime, + RepeatAfter: 60 * 60 * 24 * 7, // The amount of seconds in a week + }, + }, + }, + { + Title: "List 2", + Tasks: []*models.Task{ + { + Title: "Task 1", + }, + { + Title: "Task 2", + }, + }, + }, + }, + }, + } + + hierachie, err := convertMicrosoftTodoData(microsoftTodoData) + assert.NoError(t, err) + assert.NotNil(t, hierachie) + if diff, equal := messagediff.PrettyDiff(hierachie, expectedHierachie); !equal { + t.Errorf("converted microsoft todo data = %v, want %v, diff: %v", hierachie, expectedHierachie, diff) + } +} diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index b15459c41..ee4ebe7bd 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -18,13 +18,10 @@ package todoist import ( "bytes" - "context" "encoding/json" "fmt" - "net/http" "net/url" "sort" - "strings" "time" "code.vikunja.io/api/pkg/config" @@ -229,17 +226,6 @@ func (m *Migration) AuthURL() string { "&state=" + utils.MakeRandomString(32) } -func doPost(url string, form url.Values) (resp *http.Response, err error) { - req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, url, strings.NewReader(form.Encode())) - if err != nil { - return - } - - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - hc := http.Client{} - return hc.Do(req) -} - func convertTodoistToVikunja(sync *sync) (fullVikunjaHierachie []*models.NamespaceWithLists, err error) { newNamespace := &models.NamespaceWithLists{ @@ -451,7 +437,7 @@ func getAccessTokenFromAuthToken(authToken string) (accessToken string, err erro "code": []string{authToken}, "redirect_uri": []string{config.MigrationTodoistRedirectURL.GetString()}, } - resp, err := doPost("https://todoist.com/oauth/access_token", form) + resp, err := migration.DoPost("https://todoist.com/oauth/access_token", form) if err != nil { return } @@ -503,7 +489,7 @@ func (m *Migration) Migrate(u *user.User) (err error) { "sync_token": []string{"*"}, "resource_types": []string{"[\"all\"]"}, } - resp, err := doPost("https://api.todoist.com/sync/v8/sync", form) + resp, err := migration.DoPost("https://api.todoist.com/sync/v8/sync", form) if err != nil { return } diff --git a/pkg/routes/api/v1/info.go b/pkg/routes/api/v1/info.go index a9833a8c9..06ceea8dc 100644 --- a/pkg/routes/api/v1/info.go +++ b/pkg/routes/api/v1/info.go @@ -19,6 +19,8 @@ package v1 import ( "net/http" + microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo" + "code.vikunja.io/api/pkg/modules/migration/trello" "code.vikunja.io/api/pkg/log" @@ -121,6 +123,10 @@ func Info(c echo.Context) error { m := &trello.Migration{} info.AvailableMigrators = append(info.AvailableMigrators, m.Name()) } + if config.MigrationMicrosoftTodoEnable.GetBool() { + m := µsofttodo.Migration{} + info.AvailableMigrators = append(info.AvailableMigrators, m.Name()) + } if config.BackgroundsEnabled.GetBool() { if config.BackgroundsUploadEnabled.GetBool() { diff --git a/pkg/routes/routes.go b/pkg/routes/routes.go index a8ca3ec76..9d69931e0 100644 --- a/pkg/routes/routes.go +++ b/pkg/routes/routes.go @@ -50,6 +50,8 @@ import ( "strings" "time" + microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo" + "code.vikunja.io/api/pkg/modules/migration/trello" "code.vikunja.io/api/pkg/config" @@ -543,6 +545,16 @@ func registerAPIRoutes(a *echo.Group) { trelloMigrationHandler.RegisterRoutes(m) } + // Microsoft Todo + if config.MigrationMicrosoftTodoEnable.GetBool() { + microsoftTodoMigrationHandler := &migrationHandler.MigrationWeb{ + MigrationStruct: func() migration.Migrator { + return µsofttodo.Migration{} + }, + } + microsoftTodoMigrationHandler.RegisterRoutes(m) + } + // List Backgrounds if config.BackgroundsEnabled.GetBool() { a.GET("/lists/:list/background", backgroundHandler.GetListBackground) diff --git a/pkg/swagger/docs.go b/pkg/swagger/docs.go index c28f8c9af..2625060d2 100644 --- a/pkg/swagger/docs.go +++ b/pkg/swagger/docs.go @@ -2502,6 +2502,113 @@ var doc = `{ } } }, + "/migration/microsoft-todo/auth": { + "get": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Returns the auth url where the user needs to get its auth code. This code can then be used to migrate everything from Microsoft Todo to Vikunja.", + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Get the auth url from Microsoft Todo", + "responses": { + "200": { + "description": "The auth url.", + "schema": { + "$ref": "#/definitions/handler.AuthURL" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, + "/migration/microsoft-todo/migrate": { + "post": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Migrates all tasklinsts, tasks, notes and reminders from Microsoft Todo to Vikunja.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Migrate all lists, tasks etc. from Microsoft Todo", + "parameters": [ + { + "description": "The auth token previously obtained from the auth url. See the docs for /migration/microsoft-todo/auth.", + "name": "migrationCode", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/microsofttodo.Migration" + } + } + ], + "responses": { + "200": { + "description": "A message telling you everything was migrated successfully.", + "schema": { + "$ref": "#/definitions/models.Message" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, + "/migration/microsoft-todo/status": { + "get": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.", + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Get migration status", + "responses": { + "200": { + "description": "The migration status", + "schema": { + "$ref": "#/definitions/migration.Status" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, "/migration/todoist/auth": { "get": { "security": [ @@ -6657,6 +6764,14 @@ var doc = `{ } } }, + "microsofttodo.Migration": { + "type": "object", + "properties": { + "code": { + "type": "string" + } + } + }, "migration.Status": { "type": "object", "properties": { diff --git a/pkg/swagger/swagger.json b/pkg/swagger/swagger.json index 49bd76d01..d307e6daf 100644 --- a/pkg/swagger/swagger.json +++ b/pkg/swagger/swagger.json @@ -2485,6 +2485,113 @@ } } }, + "/migration/microsoft-todo/auth": { + "get": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Returns the auth url where the user needs to get its auth code. This code can then be used to migrate everything from Microsoft Todo to Vikunja.", + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Get the auth url from Microsoft Todo", + "responses": { + "200": { + "description": "The auth url.", + "schema": { + "$ref": "#/definitions/handler.AuthURL" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, + "/migration/microsoft-todo/migrate": { + "post": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Migrates all tasklinsts, tasks, notes and reminders from Microsoft Todo to Vikunja.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Migrate all lists, tasks etc. from Microsoft Todo", + "parameters": [ + { + "description": "The auth token previously obtained from the auth url. See the docs for /migration/microsoft-todo/auth.", + "name": "migrationCode", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/microsofttodo.Migration" + } + } + ], + "responses": { + "200": { + "description": "A message telling you everything was migrated successfully.", + "schema": { + "$ref": "#/definitions/models.Message" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, + "/migration/microsoft-todo/status": { + "get": { + "security": [ + { + "JWTKeyAuth": [] + } + ], + "description": "Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.", + "produces": [ + "application/json" + ], + "tags": [ + "migration" + ], + "summary": "Get migration status", + "responses": { + "200": { + "description": "The migration status", + "schema": { + "$ref": "#/definitions/migration.Status" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/models.Message" + } + } + } + } + }, "/migration/todoist/auth": { "get": { "security": [ @@ -6640,6 +6747,14 @@ } } }, + "microsofttodo.Migration": { + "type": "object", + "properties": { + "code": { + "type": "string" + } + } + }, "migration.Status": { "type": "object", "properties": { diff --git a/pkg/swagger/swagger.yaml b/pkg/swagger/swagger.yaml index 860a8b373..a834f1318 100644 --- a/pkg/swagger/swagger.yaml +++ b/pkg/swagger/swagger.yaml @@ -35,6 +35,11 @@ definitions: url: type: string type: object + microsofttodo.Migration: + properties: + code: + type: string + type: object migration.Status: properties: id: @@ -2671,6 +2676,72 @@ paths: summary: Login tags: - user + /migration/microsoft-todo/auth: + get: + description: Returns the auth url where the user needs to get its auth code. This code can then be used to migrate everything from Microsoft Todo to Vikunja. + produces: + - application/json + responses: + "200": + description: The auth url. + schema: + $ref: '#/definitions/handler.AuthURL' + "500": + description: Internal server error + schema: + $ref: '#/definitions/models.Message' + security: + - JWTKeyAuth: [] + summary: Get the auth url from Microsoft Todo + tags: + - migration + /migration/microsoft-todo/migrate: + post: + consumes: + - application/json + description: Migrates all tasklinsts, tasks, notes and reminders from Microsoft Todo to Vikunja. + parameters: + - description: The auth token previously obtained from the auth url. See the docs for /migration/microsoft-todo/auth. + in: body + name: migrationCode + required: true + schema: + $ref: '#/definitions/microsofttodo.Migration' + produces: + - application/json + responses: + "200": + description: A message telling you everything was migrated successfully. + schema: + $ref: '#/definitions/models.Message' + "500": + description: Internal server error + schema: + $ref: '#/definitions/models.Message' + security: + - JWTKeyAuth: [] + summary: Migrate all lists, tasks etc. from Microsoft Todo + tags: + - migration + /migration/microsoft-todo/status: + get: + description: Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again. + produces: + - application/json + responses: + "200": + description: The migration status + schema: + $ref: '#/definitions/migration.Status' + "500": + description: Internal server error + schema: + $ref: '#/definitions/models.Message' + security: + - JWTKeyAuth: [] + summary: Get migration status + tags: + - migration /migration/todoist/auth: get: description: Returns the auth url where the user needs to get its auth code. This code can then be used to migrate everything from todoist to Vikunja.