yverry
/
api
forked from vikunja/vikunja
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(tasks): make sorting and filtering work with Typesense

This commit is contained in:
kolaente 2023-08-28 19:10:18 +02:00
parent 09cfe41e4f
commit 2ca193e63b
Signed by untrusted user: konrad
GPG Key ID: F40E70337AB24C9B
3 changed files with 127 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
pkg/models/task_search.go
View File

@ -18,6 +18,7 @@ package models
import (
"code.vikunja.io/api/pkg/db"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/web"
"github.com/typesense/typesense-go/typesense/api"
"github.com/typesense/typesense-go/typesense/api/pointer"
@ -39,15 +40,14 @@ type dbTaskSearcher struct {
hasFavoritesProject bool
}
func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) {
func getOrderByDBStatement(opts *taskSearchOptions) (orderby string, err error) {
// Since xorm does not use placeholders for order by, it is possible to expose this with sql injection if we're directly
// passing user input to the db.
// As a workaround to prevent this, we check for valid column names here prior to passing it to the db.
var orderby string
for i, param := range opts.sortby {
// Validate the params
if err := param.validate(); err != nil {
return nil, totalCount, err
return "", err
}
// Mysql sorts columns with null values before ones without null value.
@ -70,6 +70,16 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo
}
}
return
}
func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) {
orderby, err := getOrderByDBStatement(opts)
if err != nil {
return nil, 0, err
}
// Some filters need a special treatment since they are in a separate table
reminderFilters := []builder.Cond{}
assigneeFilters := []builder.Cond{}
@ -242,10 +252,91 @@ type typesenseTaskSearcher struct {
}
func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) {
var sortbyFields []string
for i, param := range opts.sortby {
// Validate the params
if err := param.validate(); err != nil {
return nil, totalCount, err
}
// Typesense does not allow sorting by ID, so we sort by created timestamp instead
if param.sortBy == "id" {
param.sortBy = "created"
}
sortbyFields = append(sortbyFields, param.sortBy+"(missing_values:last):"+param.orderBy.String())
if i == 2 {
// Typesense supports up to 3 sorting parameters
// https://typesense.org/docs/0.25.0/api/search.html#ranking-and-sorting-parameters
break
}
}
sortby := strings.Join(sortbyFields, ",")
projectIDStrings := []string{}
for _, id := range opts.projectIDs {
projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10))
}
filterBy := []string{
"project_id: [" + strings.Join(projectIDStrings, ", ") + "]",
}
for _, f := range opts.filters {
filter := f.field
switch f.comparator {
case taskFilterComparatorEquals:
filter += ":="
case taskFilterComparatorNotEquals:
filter += ":!="
case taskFilterComparatorGreater:
filter += ":>"
case taskFilterComparatorGreateEquals:
filter += ":>="
case taskFilterComparatorLess:
filter += ":<"
case taskFilterComparatorLessEquals:
filter += ":<="
case taskFilterComparatorLike:
filter += ":"
//case taskFilterComparatorIn:
//filter += "["
case taskFilterComparatorInvalid:
// Nothing to do
default:
filter += ":="
}
switch f.value.(type) {
case string:
filter += f.value.(string)
case int:
filter += strconv.Itoa(f.value.(int))
case int64:
filter += strconv.FormatInt(f.value.(int64), 10)
case bool:
if f.value.(bool) {
filter += "true"
} else {
filter += "false"
}
default:
log.Errorf("Unknown search type %s=%v", f.field, f.value)
}
filterBy = append(filterBy, filter)
}
////////////////
// Actual search
if opts.search == "" {
opts.search = "*"
}
params := &api.SearchCollectionParams{
Q: opts.search,
@ -253,7 +344,11 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task,
Page: pointer.Int(opts.page),
PerPage: pointer.Int(opts.perPage),
ExhaustiveSearch: pointer.True(),
FilterBy: pointer.String("project_id: [" + strings.Join(projectIDStrings, ", ") + "]"),
FilterBy: pointer.String(strings.Join(filterBy, " && ")),
}
if sortby != "" {
params.SortBy = pointer.String(sortby)
}
result, err := typesenseClient.Collection("tasks").
@ -275,6 +370,14 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task,
tasks = []*Task{}
err = t.s.In("id", taskIDs).Find(&tasks)
orderby, err := getOrderByDBStatement(opts)
if err != nil {
return nil, 0, err
}
err = t.s.
In("id", taskIDs).
OrderBy(orderby).
Find(&tasks)
return tasks, int64(*result.Found), err
}

2
pkg/models/tasks.go
View File

@ -309,7 +309,7 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op
}
tasks, totalItems, err = searcher.Search(opts)
return tasks, len(tasks), totalItems, nil
return tasks, len(tasks), totalItems, err
}
func getTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskSearchOptions) (tasks []*Task, resultCount int, totalItems int64, err error) {

31
pkg/models/typesense.go
View File

@ -58,14 +58,17 @@ func CreateTypesenseCollections() error {
{
Name: "id",
Type: "string",
Sort: pointer.True(),
},
{
Name: "title",
Type: "string",
Sort: pointer.True(),
},
{
Name: "description",
Type: "string",
Sort: pointer.True(),
},
{
Name: "done",
@ -110,6 +113,7 @@ func CreateTypesenseCollections() error {
{
Name: "hex_color",
Type: "string",
Sort: pointer.True(),
},
{
Name: "percent_done",
@ -118,6 +122,7 @@ func CreateTypesenseCollections() error {
{
Name: "identifier",
Type: "string",
Sort: pointer.True(),
},
{
Name: "index",
@ -126,6 +131,7 @@ func CreateTypesenseCollections() error {
{
Name: "uid",
Type: "string",
Sort: pointer.True(),
},
{
Name: "cover_image_attachment_id",
@ -244,7 +250,6 @@ func reindexTasks(s *xorm.Session, tasks map[int64]*Task) (err error) {
}
typesenseTasks = append(typesenseTasks, searchTask)
}
_, err = typesenseClient.Collection("tasks").
@ -265,14 +270,14 @@ type typesenseTask struct {
Title string `json:"title"`
Description string `json:"description"`
Done bool `json:"done"`
DoneAt int64 `json:"done_at"`
DueDate int64 `json:"due_date"`
DoneAt *int64 `json:"done_at"`
DueDate *int64 `json:"due_date"`
ProjectID int64 `json:"project_id"`
RepeatAfter int64 `json:"repeat_after"`
RepeatMode int `json:"repeat_mode"`
Priority int64 `json:"priority"`
StartDate int64 `json:"start_date"`
EndDate int64 `json:"end_date"`
StartDate *int64 `json:"start_date"`
EndDate *int64 `json:"end_date"`
HexColor string `json:"hex_color"`
PercentDone float64 `json:"percent_done"`
Identifier string `json:"identifier"`
@ -299,14 +304,14 @@ func convertTaskToTypesenseTask(task *Task) *typesenseTask {
Title: task.Title,
Description: task.Description,
Done: task.Done,
DoneAt: task.DoneAt.UTC().Unix(),
DueDate: task.DueDate.UTC().Unix(),
DoneAt: pointer.Int64(task.DoneAt.UTC().Unix()),
DueDate: pointer.Int64(task.DueDate.UTC().Unix()),
ProjectID: task.ProjectID,
RepeatAfter: task.RepeatAfter,
RepeatMode: int(task.RepeatMode),
Priority: task.Priority,
StartDate: task.StartDate.UTC().Unix(),
EndDate: task.EndDate.UTC().Unix(),
StartDate: pointer.Int64(task.StartDate.UTC().Unix()),
EndDate: pointer.Int64(task.EndDate.UTC().Unix()),
HexColor: task.HexColor,
PercentDone: task.PercentDone,
Identifier: task.Identifier,
@ -327,16 +332,16 @@ func convertTaskToTypesenseTask(task *Task) *typesenseTask {
}
if task.DoneAt.IsZero() {
tt.DoneAt = 0
tt.DoneAt = nil
}
if task.DueDate.IsZero() {
tt.DueDate = 0
tt.DueDate = nil
}
if task.StartDate.IsZero() {
tt.StartDate = 0
tt.StartDate = nil
}
if task.EndDate.IsZero() {
tt.EndDate = 0
tt.EndDate = nil
}
return tt