Merge branch 'main' into main

docs/content/doc/setup/typesense.md

@@ -0,0 +1,23 @@
+---
+title: "Typesense"
+date: 2023-09-29T12:23:55+02:00
+draft: false
+menu:
+  sidebar:
+    parent: "setup"
+---
+
+# Use Typesense for enhanced search capabilities
+
+Vikunja supports using [Typesense](https://typesense.org/) for a better search experience.
+Typesense allows fast fulltext search including fuzzy matching support. 
+It may return different results than what you'd get with a database-only search, but generally, the results are more relevant to what you're looking for.
+
+This document explains how to set up and use Typesense with Vikunja.
+
+## Setup
+
+1. First, install Typesense on your system. Refer to [their documentation](https://typesense.org/docs/guide/install-typesense.html) for specific instructions.
+2. Once Typesense is available on your system and reachable by Vikunja, add the relevant configuration keys to your Vikunja config. [Check out the docs article about this]({{< ref "config.md#typesense">}}).
+3. Index all tasks currently in Vikunja. To do that, run the `vikunja index` command with the api binary. This may take a while, depending on the size of your instance.
+4. Restart the api. From now on, all task changes will be automatically indexed in Typesense.

pkg/models/listeners.go

@@ -63,6 +63,7 @@ func RegisterListeners() {
 	events.RegisterListener((&TaskRelationDeletedEvent{}).Name(), &HandleTaskUpdateLastUpdated{})
 	if config.TypesenseEnabled.GetBool() {
 		events.RegisterListener((&TaskDeletedEvent{}).Name(), &RemoveTaskFromTypesense{})
+		events.RegisterListener((&TaskCreatedEvent{}).Name(), &AddTaskToTypesense{})
 	}
 }
 
@@ -506,6 +507,38 @@ func (s *RemoveTaskFromTypesense) Handle(msg *message.Message) (err error) {
 	return err
 }
 
+// AddTaskToTypesense  represents a listener
+type AddTaskToTypesense struct {
+}
+
+// Name defines the name for the AddTaskToTypesense listener
+func (l *AddTaskToTypesense) Name() string {
+	return "add.task.to.typesense"
+}
+
+// Handle is executed when the event AddTaskToTypesense listens on is fired
+func (l *AddTaskToTypesense) Handle(msg *message.Message) (err error) {
+	event := &TaskCreatedEvent{}
+	err = json.Unmarshal(msg.Payload, event)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("New task %d created, adding to typesense…", event.Task.ID)
+
+	s := db.NewSession()
+	defer s.Close()
+	ttask, err := getTypesenseTaskForTask(s, event.Task, nil)
+	if err != nil {
+		return err
+	}
+
+	_, err = typesenseClient.Collection("tasks").
+		Documents().
+		Create(ttask)
+	return
+}
+
 ///////
 // Project Event Listeners
 

pkg/models/task_search.go

@@ -374,11 +374,14 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task,
 		Q:                opts.search,
 		QueryBy:          "title, identifier, description, comments.comment",
 		Page:             pointer.Int(opts.page),
-		PerPage:          pointer.Int(opts.perPage),
 		ExhaustiveSearch: pointer.True(),
 		FilterBy:         pointer.String(strings.Join(filterBy, " && ")),
 	}
 
+	if opts.perPage > 0 {
+		params.PerPage = pointer.Int(opts.perPage)
+	}
+
 	if sortby != "" {
 		params.SortBy = pointer.String(sortby)
 	}

pkg/models/tasks.go

@@ -1081,8 +1081,13 @@ func recalculateTaskKanbanPositions(s *xorm.Session, bucketID int64) (err error)
 
 		currentPosition := maxPosition / float64(len(allTasks)) * (float64(i + 1))
 
+		// Here we use "NoAutoTime() to prevent the ORM from updating column "updated" automatically.
+		// Otherwise, this signals to CalDAV clients that the task has changed, which is not the case.
+		// Consequence: when synchronizing a list of tasks, the first one immediately changes the date of all the
+		// following ones from the same batch, which are then unable to be updated.
 		_, err = s.Cols("kanban_position").
 			Where("id = ?", task.ID).
+			NoAutoTime().
 			Update(&Task{KanbanPosition: currentPosition})
 		if err != nil {
 			return
@@ -1109,8 +1114,13 @@ func recalculateTaskPositions(s *xorm.Session, projectID int64) (err error) {
 
 		currentPosition := maxPosition / float64(len(allTasks)) * (float64(i + 1))
 
+		// Here we use "NoAutoTime() to prevent the ORM from updating column "updated" automatically.
+		// Otherwise, this signals to CalDAV clients that the task has changed, which is not the case.
+		// Consequence: when synchronizing a list of tasks, the first one immediately changes the date of all the
+		// following ones from the same batch, which are then unable to be updated.
 		_, err = s.Cols("position").
 			Where("id = ?", task.ID).
+			NoAutoTime().
 			Update(&Task{Position: currentPosition})
 		if err != nil {
 			return

pkg/models/typesense.go

@@ -226,6 +226,11 @@ func ReindexAllTasks() (err error) {
 		return fmt.Errorf("could not get all tasks: %s", err.Error())
 	}
 
+	err = indexDummyTask()
+	if err != nil {
+		return fmt.Errorf("could not index dummy task: %w", err)
+	}
+
 	err = reindexTasks(s, tasks)
 	if err != nil {
 		return fmt.Errorf("could not reindex all tasks: %s", err.Error())
@@ -242,6 +247,36 @@ func ReindexAllTasks() (err error) {
 	return
 }
 
+func getTypesenseTaskForTask(s *xorm.Session, task *Task, projectsCache map[int64]*Project) (ttask *typesenseTask, err error) {
+	ttask = convertTaskToTypesenseTask(task)
+
+	var p *Project
+	if projectsCache == nil {
+		p, err = GetProjectSimpleByID(s, task.ProjectID)
+		if err != nil {
+			return nil, fmt.Errorf("could not fetch project %d: %s", task.ProjectID, err.Error())
+		}
+	} else {
+		var has bool
+		p, has = projectsCache[task.ProjectID]
+		if !has {
+			p, err = GetProjectSimpleByID(s, task.ProjectID)
+			if err != nil {
+				return nil, fmt.Errorf("could not fetch project %d: %s", task.ProjectID, err.Error())
+			}
+			projectsCache[task.ProjectID] = p
+		}
+	}
+
+	comment := &TaskComment{TaskID: task.ID}
+	ttask.Comments, _, _, err = comment.ReadAll(s, &user.User{ID: p.OwnerID}, "", -1, -1)
+	if err != nil {
+		return nil, fmt.Errorf("could not fetch comments for task %d: %s", task.ID, err.Error())
+	}
+
+	return
+}
+
 func reindexTasks(s *xorm.Session, tasks map[int64]*Task) (err error) {
 
 	if len(tasks) == 0 {
@@ -258,24 +293,13 @@ func reindexTasks(s *xorm.Session, tasks map[int64]*Task) (err error) {
 
 	typesenseTasks := []interface{}{}
 	for _, task := range tasks {
-		searchTask := convertTaskToTypesenseTask(task)
 
-		p, has := projects[task.ProjectID]
-		if !has {
-			p, err = GetProjectSimpleByID(s, task.ProjectID)
-			if err != nil {
-				return fmt.Errorf("could not fetch project %d: %s", task.ProjectID, err.Error())
-			}
-			projects[task.ProjectID] = p
-		}
-
-		comment := &TaskComment{TaskID: task.ID}
-		searchTask.Comments, _, _, err = comment.ReadAll(s, &user.User{ID: p.OwnerID}, "", -1, -1)
+		ttask, err := getTypesenseTaskForTask(s, task, projects)
 		if err != nil {
-			return fmt.Errorf("could not fetch comments for task %d: %s", task.ID, err.Error())
+			return err
 		}
 
-		typesenseTasks = append(typesenseTasks, searchTask)
+		typesenseTasks = append(typesenseTasks, ttask)
 	}
 
 	_, err = typesenseClient.Collection("tasks").
@@ -292,6 +316,82 @@ func reindexTasks(s *xorm.Session, tasks map[int64]*Task) (err error) {
 	return nil
 }
 
+func indexDummyTask() (err error) {
+	// The initial sync should contain one dummy task with all related fields populated so that typesense
+	// creates the indexes properly. A little hacky, but gets the job done.
+	dummyTask := &typesenseTask{
+		ID:      "-100",
+		Title:   "Dummytask",
+		Created: time.Now().Unix(),
+		Updated: time.Now().Unix(),
+		Reminders: []*TaskReminder{
+			{
+				ID:             -10,
+				TaskID:         -100,
+				Reminder:       time.Now(),
+				RelativePeriod: 10,
+				RelativeTo:     ReminderRelationDueDate,
+				Created:        time.Now(),
+			},
+		},
+		Assignees: []*user.User{
+			{
+				ID:       -100,
+				Username: "dummy",
+				Name:     "dummy",
+				Email:    "dummy@vikunja",
+				Created:  time.Now(),
+				Updated:  time.Now(),
+			},
+		},
+		Labels: []*Label{
+			{
+				ID:          -110,
+				Title:       "dummylabel",
+				Description: "Lorem Ipsum Dummy",
+				HexColor:    "000000",
+				Created:     time.Now(),
+				Updated:     time.Now(),
+			},
+		},
+		Attachments: []*TaskAttachment{
+			{
+				ID:      -120,
+				TaskID:  -100,
+				Created: time.Now(),
+			},
+		},
+		Comments: []*TaskComment{
+			{
+				ID:      -220,
+				Comment: "Lorem Ipsum Dummy",
+				Created: time.Now(),
+				Updated: time.Now(),
+				Author: &user.User{
+					ID:       -100,
+					Username: "dummy",
+					Name:     "dummy",
+					Email:    "dummy@vikunja",
+					Created:  time.Now(),
+					Updated:  time.Now(),
+				},
+			},
+		},
+	}
+
+	_, err = typesenseClient.Collection("tasks").
+		Documents().
+		Create(dummyTask)
+	if err != nil {
+		return
+	}
+
+	_, err = typesenseClient.Collection("tasks").
+		Document(dummyTask.ID).
+		Delete()
+	return
+}
+
 type typesenseTask struct {
 	ID                     string      `json:"id"`
 	Title                  string      `json:"title"`
@@ -406,6 +506,7 @@ func SyncUpdatedTasksIntoTypesense() (err error) {
 
 	err = s.
 		Where("updated >= ?", lastSync.SyncStartedAt).
+		And("updated != created"). // new tasks are already indexed via the event handler
 		Find(tasks)
 	if err != nil {
 		_ = s.Rollback()