From feacbbff74d14d2c48e457acf6f1f9a6e3a84196 Mon Sep 17 00:00:00 2001 From: Erwan Martin Date: Wed, 27 Sep 2023 16:17:52 +0000 Subject: [PATCH 1/5] fix(caldav): do not update dates of tasks when repositioning them (#1605) When a task is updated, the position of the tasks of the whole project/bucket are updated. This leads to column "updated" of model Task to be updated quite often. However, that column is used for the ETag field of CALDAV. Thus, changing a task marks all the other tasks as updated, which prevents clients from synchronizing their edited tasks. Co-authored-by: Erwan Martin Reviewed-on: https://kolaente.dev/vikunja/api/pulls/1605 Co-authored-by: Erwan Martin Co-committed-by: Erwan Martin --- pkg/models/tasks.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go index 9b3598784..61242b969 100644 --- a/pkg/models/tasks.go +++ b/pkg/models/tasks.go @@ -1081,8 +1081,13 @@ func recalculateTaskKanbanPositions(s *xorm.Session, bucketID int64) (err error) currentPosition := maxPosition / float64(len(allTasks)) * (float64(i + 1)) + // Here we use "NoAutoTime() to prevent the ORM from updating column "updated" automatically. + // Otherwise, this signals to CalDAV clients that the task has changed, which is not the case. + // Consequence: when synchronizing a list of tasks, the first one immediately changes the date of all the + // following ones from the same batch, which are then unable to be updated. _, err = s.Cols("kanban_position"). Where("id = ?", task.ID). + NoAutoTime(). Update(&Task{KanbanPosition: currentPosition}) if err != nil { return @@ -1109,8 +1114,13 @@ func recalculateTaskPositions(s *xorm.Session, projectID int64) (err error) { currentPosition := maxPosition / float64(len(allTasks)) * (float64(i + 1)) + // Here we use "NoAutoTime() to prevent the ORM from updating column "updated" automatically. + // Otherwise, this signals to CalDAV clients that the task has changed, which is not the case. + // Consequence: when synchronizing a list of tasks, the first one immediately changes the date of all the + // following ones from the same batch, which are then unable to be updated. _, err = s.Cols("position"). Where("id = ?", task.ID). + NoAutoTime(). Update(&Task{Position: currentPosition}) if err != nil { return From 70d1903dcac67e33bdfdf54d0ba561af76dbf927 Mon Sep 17 00:00:00 2001 From: kolaente Date: Fri, 29 Sep 2023 12:30:53 +0200 Subject: [PATCH 2/5] docs: add typesense setup --- docs/content/doc/setup/typesense.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docs/content/doc/setup/typesense.md diff --git a/docs/content/doc/setup/typesense.md b/docs/content/doc/setup/typesense.md new file mode 100644 index 000000000..a1848b628 --- /dev/null +++ b/docs/content/doc/setup/typesense.md @@ -0,0 +1,23 @@ +--- +title: "Typesense" +date: 2023-09-29T12:23:55+02:00 +draft: false +menu: + sidebar: + parent: "setup" +--- + +# Use Typesense for enhanced search capabilities + +Vikunja supports using [Typesense](https://typesense.org/) for a better search experience. +Typesense allows fast fulltext search including fuzzy matching support. +It may return different results than what you'd get with a database-only search, but generally, the results are more relevant to what you're looking for. + +This document explains how to set up and use Typesense with Vikunja. + +## Setup + +1. First, install Typesense on your system. Refer to [their documentation](https://typesense.org/docs/guide/install-typesense.html) for specific instructions. +2. Once Typesense is available on your system and reachable by Vikunja, add the relevant configuration keys to your Vikunja config. [Check out the docs article about this]({{< ref "config.md#typesense">}}). +3. Index all tasks currently in Vikunja. To do that, run the `vikunja index` command with the api binary. This may take a while, depending on the size of your instance. +4. Restart the api. From now on, all task changes will be automatically indexed in Typesense. From 8f4ee3a089b403f7c0a7d384aea83946763cf52f Mon Sep 17 00:00:00 2001 From: kolaente Date: Fri, 29 Sep 2023 16:35:59 +0200 Subject: [PATCH 3/5] fix(typesense): make sure searching works when no task has a comment at index time --- pkg/models/typesense.go | 81 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/pkg/models/typesense.go b/pkg/models/typesense.go index a8bf1fa90..3c8ca6ec4 100644 --- a/pkg/models/typesense.go +++ b/pkg/models/typesense.go @@ -226,6 +226,11 @@ func ReindexAllTasks() (err error) { return fmt.Errorf("could not get all tasks: %s", err.Error()) } + err = indexDummyTask() + if err != nil { + return fmt.Errorf("could not index dummy task: %w", err) + } + err = reindexTasks(s, tasks) if err != nil { return fmt.Errorf("could not reindex all tasks: %s", err.Error()) @@ -292,6 +297,82 @@ func reindexTasks(s *xorm.Session, tasks map[int64]*Task) (err error) { return nil } +func indexDummyTask() (err error) { + // The initial sync should contain one dummy task with all related fields populated so that typesense + // creates the indexes properly. A little hacky, but gets the job done. + dummyTask := &typesenseTask{ + ID: "-100", + Title: "Dummytask", + Created: time.Now().Unix(), + Updated: time.Now().Unix(), + Reminders: []*TaskReminder{ + { + ID: -10, + TaskID: -100, + Reminder: time.Now(), + RelativePeriod: 10, + RelativeTo: ReminderRelationDueDate, + Created: time.Now(), + }, + }, + Assignees: []*user.User{ + { + ID: -100, + Username: "dummy", + Name: "dummy", + Email: "dummy@vikunja", + Created: time.Now(), + Updated: time.Now(), + }, + }, + Labels: []*Label{ + { + ID: -110, + Title: "dummylabel", + Description: "Lorem Ipsum Dummy", + HexColor: "000000", + Created: time.Now(), + Updated: time.Now(), + }, + }, + Attachments: []*TaskAttachment{ + { + ID: -120, + TaskID: -100, + Created: time.Now(), + }, + }, + Comments: []*TaskComment{ + { + ID: -220, + Comment: "Lorem Ipsum Dummy", + Created: time.Now(), + Updated: time.Now(), + Author: &user.User{ + ID: -100, + Username: "dummy", + Name: "dummy", + Email: "dummy@vikunja", + Created: time.Now(), + Updated: time.Now(), + }, + }, + }, + } + + _, err = typesenseClient.Collection("tasks"). + Documents(). + Create(dummyTask) + if err != nil { + return + } + + _, err = typesenseClient.Collection("tasks"). + Document(dummyTask.ID). + Delete() + return +} + type typesenseTask struct { ID string `json:"id"` Title string `json:"title"` From 98102e59f265fc356f74b316aea0283edce9cc39 Mon Sep 17 00:00:00 2001 From: kolaente Date: Fri, 29 Sep 2023 21:15:28 +0200 Subject: [PATCH 4/5] feat(typesense): add new tasks to typesense directly when they are created --- pkg/models/listeners.go | 33 ++++++++++++++++++++++++++++ pkg/models/typesense.go | 48 +++++++++++++++++++++++++++++------------ 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/pkg/models/listeners.go b/pkg/models/listeners.go index eee4a81d8..fcc23179e 100644 --- a/pkg/models/listeners.go +++ b/pkg/models/listeners.go @@ -63,6 +63,7 @@ func RegisterListeners() { events.RegisterListener((&TaskRelationDeletedEvent{}).Name(), &HandleTaskUpdateLastUpdated{}) if config.TypesenseEnabled.GetBool() { events.RegisterListener((&TaskDeletedEvent{}).Name(), &RemoveTaskFromTypesense{}) + events.RegisterListener((&TaskCreatedEvent{}).Name(), &AddTaskToTypesense{}) } } @@ -506,6 +507,38 @@ func (s *RemoveTaskFromTypesense) Handle(msg *message.Message) (err error) { return err } +// AddTaskToTypesense represents a listener +type AddTaskToTypesense struct { +} + +// Name defines the name for the AddTaskToTypesense listener +func (l *AddTaskToTypesense) Name() string { + return "add.task.to.typesense" +} + +// Handle is executed when the event AddTaskToTypesense listens on is fired +func (l *AddTaskToTypesense) Handle(msg *message.Message) (err error) { + event := &TaskCreatedEvent{} + err = json.Unmarshal(msg.Payload, event) + if err != nil { + return err + } + + log.Debugf("New task %d created, adding to typesense…", event.Task.ID) + + s := db.NewSession() + defer s.Close() + ttask, err := getTypesenseTaskForTask(s, event.Task, nil) + if err != nil { + return err + } + + _, err = typesenseClient.Collection("tasks"). + Documents(). + Create(ttask) + return +} + /////// // Project Event Listeners diff --git a/pkg/models/typesense.go b/pkg/models/typesense.go index 3c8ca6ec4..f7a4eea64 100644 --- a/pkg/models/typesense.go +++ b/pkg/models/typesense.go @@ -247,6 +247,36 @@ func ReindexAllTasks() (err error) { return } +func getTypesenseTaskForTask(s *xorm.Session, task *Task, projectsCache map[int64]*Project) (ttask *typesenseTask, err error) { + ttask = convertTaskToTypesenseTask(task) + + var p *Project + if projectsCache == nil { + p, err = GetProjectSimpleByID(s, task.ProjectID) + if err != nil { + return nil, fmt.Errorf("could not fetch project %d: %s", task.ProjectID, err.Error()) + } + } else { + var has bool + p, has = projectsCache[task.ProjectID] + if !has { + p, err = GetProjectSimpleByID(s, task.ProjectID) + if err != nil { + return nil, fmt.Errorf("could not fetch project %d: %s", task.ProjectID, err.Error()) + } + projectsCache[task.ProjectID] = p + } + } + + comment := &TaskComment{TaskID: task.ID} + ttask.Comments, _, _, err = comment.ReadAll(s, &user.User{ID: p.OwnerID}, "", -1, -1) + if err != nil { + return nil, fmt.Errorf("could not fetch comments for task %d: %s", task.ID, err.Error()) + } + + return +} + func reindexTasks(s *xorm.Session, tasks map[int64]*Task) (err error) { if len(tasks) == 0 { @@ -263,24 +293,13 @@ func reindexTasks(s *xorm.Session, tasks map[int64]*Task) (err error) { typesenseTasks := []interface{}{} for _, task := range tasks { - searchTask := convertTaskToTypesenseTask(task) - p, has := projects[task.ProjectID] - if !has { - p, err = GetProjectSimpleByID(s, task.ProjectID) - if err != nil { - return fmt.Errorf("could not fetch project %d: %s", task.ProjectID, err.Error()) - } - projects[task.ProjectID] = p - } - - comment := &TaskComment{TaskID: task.ID} - searchTask.Comments, _, _, err = comment.ReadAll(s, &user.User{ID: p.OwnerID}, "", -1, -1) + ttask, err := getTypesenseTaskForTask(s, task, projects) if err != nil { - return fmt.Errorf("could not fetch comments for task %d: %s", task.ID, err.Error()) + return err } - typesenseTasks = append(typesenseTasks, searchTask) + typesenseTasks = append(typesenseTasks, ttask) } _, err = typesenseClient.Collection("tasks"). @@ -487,6 +506,7 @@ func SyncUpdatedTasksIntoTypesense() (err error) { err = s. Where("updated >= ?", lastSync.SyncStartedAt). + And("updated != created"). // new tasks are already indexed via the event handler Find(tasks) if err != nil { _ = s.Rollback() From c217233e08804aa92b10cd8f10e6da4506b5aea3 Mon Sep 17 00:00:00 2001 From: kolaente Date: Fri, 29 Sep 2023 21:26:12 +0200 Subject: [PATCH 5/5] fix(typesense): getting all data from typesense --- pkg/models/task_search.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index eab515532..3085319ca 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -374,11 +374,14 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, Q: opts.search, QueryBy: "title, identifier, description, comments.comment", Page: pointer.Int(opts.page), - PerPage: pointer.Int(opts.perPage), ExhaustiveSearch: pointer.True(), FilterBy: pointer.String(strings.Join(filterBy, " && ")), } + if opts.perPage > 0 { + params.PerPage = pointer.Int(opts.perPage) + } + if sortby != "" { params.SortBy = pointer.String(sortby) }