From 2ca193e63b8d2fb2d0ecfcdbbb405e8ba99a63bd Mon Sep 17 00:00:00 2001 From: kolaente Date: Mon, 28 Aug 2023 19:10:18 +0200 Subject: [PATCH] feat(tasks): make sorting and filtering work with Typesense --- pkg/models/task_search.go | 113 ++++++++++++++++++++++++++++++++++++-- pkg/models/tasks.go | 2 +- pkg/models/typesense.go | 31 ++++++----- 3 files changed, 127 insertions(+), 19 deletions(-) diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index acb1c9cb5d8..3d1ea9e51d0 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -18,6 +18,7 @@ package models import ( "code.vikunja.io/api/pkg/db" + "code.vikunja.io/api/pkg/log" "code.vikunja.io/web" "github.com/typesense/typesense-go/typesense/api" "github.com/typesense/typesense-go/typesense/api/pointer" @@ -39,15 +40,14 @@ type dbTaskSearcher struct { hasFavoritesProject bool } -func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { +func getOrderByDBStatement(opts *taskSearchOptions) (orderby string, err error) { // Since xorm does not use placeholders for order by, it is possible to expose this with sql injection if we're directly // passing user input to the db. // As a workaround to prevent this, we check for valid column names here prior to passing it to the db. - var orderby string for i, param := range opts.sortby { // Validate the params if err := param.validate(); err != nil { - return nil, totalCount, err + return "", err } // Mysql sorts columns with null values before ones without null value. @@ -70,6 +70,16 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo } } + return +} + +func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { + + orderby, err := getOrderByDBStatement(opts) + if err != nil { + return nil, 0, err + } + // Some filters need a special treatment since they are in a separate table reminderFilters := []builder.Cond{} assigneeFilters := []builder.Cond{} @@ -242,10 +252,91 @@ type typesenseTaskSearcher struct { } func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { + + var sortbyFields []string + for i, param := range opts.sortby { + // Validate the params + if err := param.validate(); err != nil { + return nil, totalCount, err + } + + // Typesense does not allow sorting by ID, so we sort by created timestamp instead + if param.sortBy == "id" { + param.sortBy = "created" + } + + sortbyFields = append(sortbyFields, param.sortBy+"(missing_values:last):"+param.orderBy.String()) + + if i == 2 { + // Typesense supports up to 3 sorting parameters + // https://typesense.org/docs/0.25.0/api/search.html#ranking-and-sorting-parameters + break + } + } + + sortby := strings.Join(sortbyFields, ",") + projectIDStrings := []string{} for _, id := range opts.projectIDs { projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10)) } + filterBy := []string{ + "project_id: [" + strings.Join(projectIDStrings, ", ") + "]", + } + + for _, f := range opts.filters { + + filter := f.field + + switch f.comparator { + case taskFilterComparatorEquals: + filter += ":=" + case taskFilterComparatorNotEquals: + filter += ":!=" + case taskFilterComparatorGreater: + filter += ":>" + case taskFilterComparatorGreateEquals: + filter += ":>=" + case taskFilterComparatorLess: + filter += ":<" + case taskFilterComparatorLessEquals: + filter += ":<=" + case taskFilterComparatorLike: + filter += ":" + //case taskFilterComparatorIn: + //filter += "[" + case taskFilterComparatorInvalid: + // Nothing to do + default: + filter += ":=" + } + + switch f.value.(type) { + case string: + filter += f.value.(string) + case int: + filter += strconv.Itoa(f.value.(int)) + case int64: + filter += strconv.FormatInt(f.value.(int64), 10) + case bool: + if f.value.(bool) { + filter += "true" + } else { + filter += "false" + } + default: + log.Errorf("Unknown search type %s=%v", f.field, f.value) + } + + filterBy = append(filterBy, filter) + } + + //////////////// + // Actual search + + if opts.search == "" { + opts.search = "*" + } params := &api.SearchCollectionParams{ Q: opts.search, @@ -253,7 +344,11 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, Page: pointer.Int(opts.page), PerPage: pointer.Int(opts.perPage), ExhaustiveSearch: pointer.True(), - FilterBy: pointer.String("project_id: [" + strings.Join(projectIDStrings, ", ") + "]"), + FilterBy: pointer.String(strings.Join(filterBy, " && ")), + } + + if sortby != "" { + params.SortBy = pointer.String(sortby) } result, err := typesenseClient.Collection("tasks"). @@ -275,6 +370,14 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, tasks = []*Task{} - err = t.s.In("id", taskIDs).Find(&tasks) + orderby, err := getOrderByDBStatement(opts) + if err != nil { + return nil, 0, err + } + + err = t.s. + In("id", taskIDs). + OrderBy(orderby). + Find(&tasks) return tasks, int64(*result.Found), err } diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go index c4a3d91ac26..0061c53d38c 100644 --- a/pkg/models/tasks.go +++ b/pkg/models/tasks.go @@ -309,7 +309,7 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op } tasks, totalItems, err = searcher.Search(opts) - return tasks, len(tasks), totalItems, nil + return tasks, len(tasks), totalItems, err } func getTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskSearchOptions) (tasks []*Task, resultCount int, totalItems int64, err error) { diff --git a/pkg/models/typesense.go b/pkg/models/typesense.go index af72072fe24..37e1b41c2b4 100644 --- a/pkg/models/typesense.go +++ b/pkg/models/typesense.go @@ -58,14 +58,17 @@ func CreateTypesenseCollections() error { { Name: "id", Type: "string", + Sort: pointer.True(), }, { Name: "title", Type: "string", + Sort: pointer.True(), }, { Name: "description", Type: "string", + Sort: pointer.True(), }, { Name: "done", @@ -110,6 +113,7 @@ func CreateTypesenseCollections() error { { Name: "hex_color", Type: "string", + Sort: pointer.True(), }, { Name: "percent_done", @@ -118,6 +122,7 @@ func CreateTypesenseCollections() error { { Name: "identifier", Type: "string", + Sort: pointer.True(), }, { Name: "index", @@ -126,6 +131,7 @@ func CreateTypesenseCollections() error { { Name: "uid", Type: "string", + Sort: pointer.True(), }, { Name: "cover_image_attachment_id", @@ -244,7 +250,6 @@ func reindexTasks(s *xorm.Session, tasks map[int64]*Task) (err error) { } typesenseTasks = append(typesenseTasks, searchTask) - } _, err = typesenseClient.Collection("tasks"). @@ -265,14 +270,14 @@ type typesenseTask struct { Title string `json:"title"` Description string `json:"description"` Done bool `json:"done"` - DoneAt int64 `json:"done_at"` - DueDate int64 `json:"due_date"` + DoneAt *int64 `json:"done_at"` + DueDate *int64 `json:"due_date"` ProjectID int64 `json:"project_id"` RepeatAfter int64 `json:"repeat_after"` RepeatMode int `json:"repeat_mode"` Priority int64 `json:"priority"` - StartDate int64 `json:"start_date"` - EndDate int64 `json:"end_date"` + StartDate *int64 `json:"start_date"` + EndDate *int64 `json:"end_date"` HexColor string `json:"hex_color"` PercentDone float64 `json:"percent_done"` Identifier string `json:"identifier"` @@ -299,14 +304,14 @@ func convertTaskToTypesenseTask(task *Task) *typesenseTask { Title: task.Title, Description: task.Description, Done: task.Done, - DoneAt: task.DoneAt.UTC().Unix(), - DueDate: task.DueDate.UTC().Unix(), + DoneAt: pointer.Int64(task.DoneAt.UTC().Unix()), + DueDate: pointer.Int64(task.DueDate.UTC().Unix()), ProjectID: task.ProjectID, RepeatAfter: task.RepeatAfter, RepeatMode: int(task.RepeatMode), Priority: task.Priority, - StartDate: task.StartDate.UTC().Unix(), - EndDate: task.EndDate.UTC().Unix(), + StartDate: pointer.Int64(task.StartDate.UTC().Unix()), + EndDate: pointer.Int64(task.EndDate.UTC().Unix()), HexColor: task.HexColor, PercentDone: task.PercentDone, Identifier: task.Identifier, @@ -327,16 +332,16 @@ func convertTaskToTypesenseTask(task *Task) *typesenseTask { } if task.DoneAt.IsZero() { - tt.DoneAt = 0 + tt.DoneAt = nil } if task.DueDate.IsZero() { - tt.DueDate = 0 + tt.DueDate = nil } if task.StartDate.IsZero() { - tt.StartDate = 0 + tt.StartDate = nil } if task.EndDate.IsZero() { - tt.EndDate = 0 + tt.EndDate = nil } return tt