feat(tasks): make sorting and filtering work with Typesense

pkg/models/task_search.go

@@ -18,6 +18,7 @@ package models
 
 import (
 	"code.vikunja.io/api/pkg/db"
+	"code.vikunja.io/api/pkg/log"
 	"code.vikunja.io/web"
 	"github.com/typesense/typesense-go/typesense/api"
 	"github.com/typesense/typesense-go/typesense/api/pointer"
@@ -39,15 +40,14 @@ type dbTaskSearcher struct {
 	hasFavoritesProject bool
 }
 
-func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) {
+func getOrderByDBStatement(opts *taskSearchOptions) (orderby string, err error) {
 	// Since xorm does not use placeholders for order by, it is possible to expose this with sql injection if we're directly
 	// passing user input to the db.
 	// As a workaround to prevent this, we check for valid column names here prior to passing it to the db.
-	var orderby string
 	for i, param := range opts.sortby {
 		// Validate the params
 		if err := param.validate(); err != nil {
-			return nil, totalCount, err
+			return "", err
 		}
 
 		// Mysql sorts columns with null values before ones without null value.
@@ -70,6 +70,16 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo
 		}
 	}
 
+	return
+}
+
+func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) {
+
+	orderby, err := getOrderByDBStatement(opts)
+	if err != nil {
+		return nil, 0, err
+	}
+
 	// Some filters need a special treatment since they are in a separate table
 	reminderFilters := []builder.Cond{}
 	assigneeFilters := []builder.Cond{}
@@ -242,10 +252,91 @@ type typesenseTaskSearcher struct {
 }
 
 func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) {
+
+	var sortbyFields []string
+	for i, param := range opts.sortby {
+		// Validate the params
+		if err := param.validate(); err != nil {
+			return nil, totalCount, err
+		}
+
+		// Typesense does not allow sorting by ID, so we sort by created timestamp instead
+		if param.sortBy == "id" {
+			param.sortBy = "created"
+		}
+
+		sortbyFields = append(sortbyFields, param.sortBy+"(missing_values:last):"+param.orderBy.String())
+
+		if i == 2 {
+			// Typesense supports up to 3 sorting parameters
+			// https://typesense.org/docs/0.25.0/api/search.html#ranking-and-sorting-parameters
+			break
+		}
+	}
+
+	sortby := strings.Join(sortbyFields, ",")
+
 	projectIDStrings := []string{}
 	for _, id := range opts.projectIDs {
 		projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10))
 	}
+	filterBy := []string{
+		"project_id: [" + strings.Join(projectIDStrings, ", ") + "]",
+	}
+
+	for _, f := range opts.filters {
+
+		filter := f.field
+
+		switch f.comparator {
+		case taskFilterComparatorEquals:
+			filter += ":="
+		case taskFilterComparatorNotEquals:
+			filter += ":!="
+		case taskFilterComparatorGreater:
+			filter += ":>"
+		case taskFilterComparatorGreateEquals:
+			filter += ":>="
+		case taskFilterComparatorLess:
+			filter += ":<"
+		case taskFilterComparatorLessEquals:
+			filter += ":<="
+		case taskFilterComparatorLike:
+			filter += ":"
+		//case taskFilterComparatorIn:
+		//filter += "["
+		case taskFilterComparatorInvalid:
+		// Nothing to do
+		default:
+			filter += ":="
+		}
+
+		switch f.value.(type) {
+		case string:
+			filter += f.value.(string)
+		case int:
+			filter += strconv.Itoa(f.value.(int))
+		case int64:
+			filter += strconv.FormatInt(f.value.(int64), 10)
+		case bool:
+			if f.value.(bool) {
+				filter += "true"
+			} else {
+				filter += "false"
+			}
+		default:
+			log.Errorf("Unknown search type %s=%v", f.field, f.value)
+		}
+
+		filterBy = append(filterBy, filter)
+	}
+
+	////////////////
+	// Actual search
+
+	if opts.search == "" {
+		opts.search = "*"
+	}
 
 	params := &api.SearchCollectionParams{
 		Q:                opts.search,
@@ -253,7 +344,11 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task,
 		Page:             pointer.Int(opts.page),
 		PerPage:          pointer.Int(opts.perPage),
 		ExhaustiveSearch: pointer.True(),
-		FilterBy:         pointer.String("project_id: [" + strings.Join(projectIDStrings, ", ") + "]"),
+		FilterBy:         pointer.String(strings.Join(filterBy, " && ")),
+	}
+
+	if sortby != "" {
+		params.SortBy = pointer.String(sortby)
 	}
 
 	result, err := typesenseClient.Collection("tasks").
@@ -275,6 +370,14 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task,
 
 	tasks = []*Task{}
 
-	err = t.s.In("id", taskIDs).Find(&tasks)
+	orderby, err := getOrderByDBStatement(opts)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	err = t.s.
+		In("id", taskIDs).
+		OrderBy(orderby).
+		Find(&tasks)
 	return tasks, int64(*result.Found), err
 }

pkg/models/tasks.go

@@ -309,7 +309,7 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op
 	}
 
 	tasks, totalItems, err = searcher.Search(opts)
-	return tasks, len(tasks), totalItems, nil
+	return tasks, len(tasks), totalItems, err
 }
 
 func getTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskSearchOptions) (tasks []*Task, resultCount int, totalItems int64, err error) {

pkg/models/typesense.go

@@ -58,14 +58,17 @@ func CreateTypesenseCollections() error {
 			{
 				Name: "id",
 				Type: "string",
+				Sort: pointer.True(),
 			},
 			{
 				Name: "title",
 				Type: "string",
+				Sort: pointer.True(),
 			},
 			{
 				Name: "description",
 				Type: "string",
+				Sort: pointer.True(),
 			},
 			{
 				Name: "done",
@@ -110,6 +113,7 @@ func CreateTypesenseCollections() error {
 			{
 				Name: "hex_color",
 				Type: "string",
+				Sort: pointer.True(),
 			},
 			{
 				Name: "percent_done",
@@ -118,6 +122,7 @@ func CreateTypesenseCollections() error {
 			{
 				Name: "identifier",
 				Type: "string",
+				Sort: pointer.True(),
 			},
 			{
 				Name: "index",
@@ -126,6 +131,7 @@ func CreateTypesenseCollections() error {
 			{
 				Name: "uid",
 				Type: "string",
+				Sort: pointer.True(),
 			},
 			{
 				Name: "cover_image_attachment_id",
@@ -244,7 +250,6 @@ func reindexTasks(s *xorm.Session, tasks map[int64]*Task) (err error) {
 		}
 
 		typesenseTasks = append(typesenseTasks, searchTask)
-
 	}
 
 	_, err = typesenseClient.Collection("tasks").
@@ -265,14 +270,14 @@ type typesenseTask struct {
 	Title                  string      `json:"title"`
 	Description            string      `json:"description"`
 	Done                   bool        `json:"done"`
-	DoneAt                 int64       `json:"done_at"`
+	DoneAt                 *int64      `json:"done_at"`
-	DueDate                int64       `json:"due_date"`
+	DueDate                *int64      `json:"due_date"`
 	ProjectID              int64       `json:"project_id"`
 	RepeatAfter            int64       `json:"repeat_after"`
 	RepeatMode             int         `json:"repeat_mode"`
 	Priority               int64       `json:"priority"`
-	StartDate              int64       `json:"start_date"`
+	StartDate              *int64      `json:"start_date"`
-	EndDate                int64       `json:"end_date"`
+	EndDate                *int64      `json:"end_date"`
 	HexColor               string      `json:"hex_color"`
 	PercentDone            float64     `json:"percent_done"`
 	Identifier             string      `json:"identifier"`
@@ -299,14 +304,14 @@ func convertTaskToTypesenseTask(task *Task) *typesenseTask {
 		Title:                  task.Title,
 		Description:            task.Description,
 		Done:                   task.Done,
-		DoneAt:                 task.DoneAt.UTC().Unix(),
+		DoneAt:                 pointer.Int64(task.DoneAt.UTC().Unix()),
-		DueDate:                task.DueDate.UTC().Unix(),
+		DueDate:                pointer.Int64(task.DueDate.UTC().Unix()),
 		ProjectID:              task.ProjectID,
 		RepeatAfter:            task.RepeatAfter,
 		RepeatMode:             int(task.RepeatMode),
 		Priority:               task.Priority,
-		StartDate:              task.StartDate.UTC().Unix(),
+		StartDate:              pointer.Int64(task.StartDate.UTC().Unix()),
-		EndDate:                task.EndDate.UTC().Unix(),
+		EndDate:                pointer.Int64(task.EndDate.UTC().Unix()),
 		HexColor:               task.HexColor,
 		PercentDone:            task.PercentDone,
 		Identifier:             task.Identifier,
@@ -327,16 +332,16 @@ func convertTaskToTypesenseTask(task *Task) *typesenseTask {
 	}
 
 	if task.DoneAt.IsZero() {
-		tt.DoneAt = 0
+		tt.DoneAt = nil
 	}
 	if task.DueDate.IsZero() {
-		tt.DueDate = 0
+		tt.DueDate = nil
 	}
 	if task.StartDate.IsZero() {
-		tt.StartDate = 0
+		tt.StartDate = nil
 	}
 	if task.EndDate.IsZero() {
-		tt.EndDate = 0
+		tt.EndDate = nil
 	}
 
 	return tt