From 010b4ce783750641d5614baa4e79e046e6f3c842 Mon Sep 17 00:00:00 2001 From: kolaente Date: Mon, 28 Aug 2023 12:14:50 +0200 Subject: [PATCH] feat(tasks): add searching via typesense --- pkg/cmd/index.go | 5 + pkg/models/export.go | 2 +- pkg/models/project.go | 2 +- pkg/models/project_duplicate.go | 2 +- pkg/models/task_collection.go | 4 +- pkg/models/task_search.go | 280 ++++++++++++++++++++++++++++++++ pkg/models/tasks.go | 212 ++---------------------- pkg/models/typesense.go | 2 +- 8 files changed, 308 insertions(+), 201 deletions(-) create mode 100644 pkg/models/task_search.go diff --git a/pkg/cmd/index.go b/pkg/cmd/index.go index 68aaf9945f3..36eb7492db7 100644 --- a/pkg/cmd/index.go +++ b/pkg/cmd/index.go @@ -40,6 +40,8 @@ var indexCmd = &cobra.Command{ return } + log.Infof("Indexing… This may take a while.") + err := models.CreateTypesenseCollections() if err != nil { log.Critical(err.Error()) @@ -48,6 +50,9 @@ var indexCmd = &cobra.Command{ err = models.ReindexAllTasks() if err != nil { log.Critical(err.Error()) + return } + + log.Infof("Done!") }, } diff --git a/pkg/models/export.go b/pkg/models/export.go index 28f45b55acb..30b33bc3d9f 100644 --- a/pkg/models/export.go +++ b/pkg/models/export.go @@ -153,7 +153,7 @@ func exportProjectsAndTasks(s *xorm.Session, u *user.User, wr *zip.Writer) (task projectIDs = append(projectIDs, p.ID) } - tasks, _, _, err := getTasksForProjects(s, rawProjects, u, &taskOptions{ + tasks, _, _, err := getTasksForProjects(s, rawProjects, u, &taskSearchOptions{ page: 0, perPage: -1, }) diff --git a/pkg/models/project.go b/pkg/models/project.go index 8105445e231..d14adb33ba0 100644 --- a/pkg/models/project.go +++ b/pkg/models/project.go @@ -969,7 +969,7 @@ func (p *Project) Delete(s *xorm.Session, a web.Auth) (err error) { // Delete all tasks on that project // Using the loop to make sure all related entities to all tasks are properly deleted as well. - tasks, _, _, err := getRawTasksForProjects(s, []*Project{p}, a, &taskOptions{}) + tasks, _, _, err := getRawTasksForProjects(s, []*Project{p}, a, &taskSearchOptions{}) if err != nil { return } diff --git a/pkg/models/project_duplicate.go b/pkg/models/project_duplicate.go index 6aab4707ca1..2efcf60369a 100644 --- a/pkg/models/project_duplicate.go +++ b/pkg/models/project_duplicate.go @@ -209,7 +209,7 @@ func (pd *ProjectDuplicate) Create(s *xorm.Session, doer web.Auth) (err error) { func duplicateTasks(s *xorm.Session, doer web.Auth, ld *ProjectDuplicate, bucketMap map[int64]int64) (err error) { // Get all tasks + all task details - tasks, _, _, err := getTasksForProjects(s, []*Project{{ID: ld.ProjectID}}, doer, &taskOptions{}) + tasks, _, _, err := getTasksForProjects(s, []*Project{{ID: ld.ProjectID}}, doer, &taskSearchOptions{}) if err != nil { return err } diff --git a/pkg/models/task_collection.go b/pkg/models/task_collection.go index de4b228b68f..c1a9397131f 100644 --- a/pkg/models/task_collection.go +++ b/pkg/models/task_collection.go @@ -80,7 +80,7 @@ func validateTaskField(fieldName string) error { return ErrInvalidTaskField{TaskField: fieldName} } -func getTaskFilterOptsFromCollection(tf *TaskCollection) (opts *taskOptions, err error) { +func getTaskFilterOptsFromCollection(tf *TaskCollection) (opts *taskSearchOptions, err error) { if len(tf.SortByArr) > 0 { tf.SortBy = append(tf.SortBy, tf.SortByArr...) } @@ -108,7 +108,7 @@ func getTaskFilterOptsFromCollection(tf *TaskCollection) (opts *taskOptions, err sort = append(sort, param) } - opts = &taskOptions{ + opts = &taskSearchOptions{ sortby: sort, filterConcat: taskFilterConcatinator(tf.FilterConcat), filterIncludeNulls: tf.FilterIncludeNulls, diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go new file mode 100644 index 00000000000..acb1c9cb5d8 --- /dev/null +++ b/pkg/models/task_search.go @@ -0,0 +1,280 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2023 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package models + +import ( + "code.vikunja.io/api/pkg/db" + "code.vikunja.io/web" + "github.com/typesense/typesense-go/typesense/api" + "github.com/typesense/typesense-go/typesense/api/pointer" + "strconv" + "strings" + + "xorm.io/builder" + "xorm.io/xorm" + "xorm.io/xorm/schemas" +) + +type taskSearcher interface { + Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) +} + +type dbTaskSearcher struct { + s *xorm.Session + a web.Auth + hasFavoritesProject bool +} + +func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { + // Since xorm does not use placeholders for order by, it is possible to expose this with sql injection if we're directly + // passing user input to the db. + // As a workaround to prevent this, we check for valid column names here prior to passing it to the db. + var orderby string + for i, param := range opts.sortby { + // Validate the params + if err := param.validate(); err != nil { + return nil, totalCount, err + } + + // Mysql sorts columns with null values before ones without null value. + // Because it does not have support for NULLS FIRST or NULLS LAST we work around this by + // first sorting for null (or not null) values and then the order we actually want to. + if db.Type() == schemas.MYSQL { + orderby += "`" + param.sortBy + "` IS NULL, " + } + + orderby += "`" + param.sortBy + "` " + param.orderBy.String() + + // Postgres and sqlite allow us to control how columns with null values are sorted. + // To make that consistent with the sort order we have and other dbms, we're adding a separate clause here. + if db.Type() == schemas.POSTGRES || db.Type() == schemas.SQLITE { + orderby += " NULLS LAST" + } + + if (i + 1) < len(opts.sortby) { + orderby += ", " + } + } + + // Some filters need a special treatment since they are in a separate table + reminderFilters := []builder.Cond{} + assigneeFilters := []builder.Cond{} + labelFilters := []builder.Cond{} + projectFilters := []builder.Cond{} + + var filters = make([]builder.Cond, 0, len(opts.filters)) + // To still find tasks with nil values, we exclude 0s when comparing with >/< values. + for _, f := range opts.filters { + if f.field == "reminders" { + f.field = "reminder" // This is the name in the db + filter, err := getFilterCond(f, opts.filterIncludeNulls) + if err != nil { + return nil, totalCount, err + } + reminderFilters = append(reminderFilters, filter) + continue + } + + if f.field == "assignees" { + if f.comparator == taskFilterComparatorLike { + return nil, totalCount, err + } + f.field = "username" + filter, err := getFilterCond(f, opts.filterIncludeNulls) + if err != nil { + return nil, totalCount, err + } + assigneeFilters = append(assigneeFilters, filter) + continue + } + + if f.field == "labels" || f.field == "label_id" { + f.field = "label_id" + filter, err := getFilterCond(f, opts.filterIncludeNulls) + if err != nil { + return nil, totalCount, err + } + labelFilters = append(labelFilters, filter) + continue + } + + if f.field == "parent_project" || f.field == "parent_project_id" { + f.field = "parent_project_id" + filter, err := getFilterCond(f, opts.filterIncludeNulls) + if err != nil { + return nil, totalCount, err + } + projectFilters = append(projectFilters, filter) + continue + } + + filter, err := getFilterCond(f, opts.filterIncludeNulls) + if err != nil { + return nil, totalCount, err + } + filters = append(filters, filter) + } + + // Then return all tasks for that projects + var where builder.Cond + + if opts.search != "" { + where = + builder.Or( + db.ILIKE("title", opts.search), + db.ILIKE("description", opts.search), + ) + + searchIndex := getTaskIndexFromSearchString(opts.search) + if searchIndex > 0 { + where = builder.Or(where, builder.Eq{"`index`": searchIndex}) + } + } + + var projectIDCond builder.Cond + var favoritesCond builder.Cond + if len(opts.projectIDs) > 0 { + projectIDCond = builder.In("project_id", opts.projectIDs) + } + + if d.hasFavoritesProject { + // All favorite tasks for that user + favCond := builder. + Select("entity_id"). + From("favorites"). + Where( + builder.And( + builder.Eq{"user_id": d.a.GetID()}, + builder.Eq{"kind": FavoriteKindTask}, + )) + + favoritesCond = builder.In("id", favCond) + } + + if len(reminderFilters) > 0 { + filters = append(filters, getFilterCondForSeparateTable("task_reminders", opts.filterConcat, reminderFilters)) + } + + if len(assigneeFilters) > 0 { + assigneeFilter := []builder.Cond{ + builder.In("user_id", + builder.Select("id"). + From("users"). + Where(builder.Or(assigneeFilters...)), + )} + filters = append(filters, getFilterCondForSeparateTable("task_assignees", opts.filterConcat, assigneeFilter)) + } + + if len(labelFilters) > 0 { + filters = append(filters, getFilterCondForSeparateTable("label_tasks", opts.filterConcat, labelFilters)) + } + + if len(projectFilters) > 0 { + var filtercond builder.Cond + if opts.filterConcat == filterConcatOr { + filtercond = builder.Or(projectFilters...) + } + if opts.filterConcat == filterConcatAnd { + filtercond = builder.And(projectFilters...) + } + + cond := builder.In( + "project_id", + builder. + Select("id"). + From("projects"). + Where(filtercond), + ) + filters = append(filters, cond) + } + + var filterCond builder.Cond + if len(filters) > 0 { + if opts.filterConcat == filterConcatOr { + filterCond = builder.Or(filters...) + } + if opts.filterConcat == filterConcatAnd { + filterCond = builder.And(filters...) + } + } + + limit, start := getLimitFromPageIndex(opts.page, opts.perPage) + cond := builder.And(builder.Or(projectIDCond, favoritesCond), where, filterCond) + + query := d.s.Where(cond) + if limit > 0 { + query = query.Limit(limit, start) + } + + tasks = []*Task{} + err = query.OrderBy(orderby).Find(&tasks) + if err != nil { + return nil, totalCount, err + } + + queryCount := d.s.Where(cond) + totalCount, err = queryCount. + Count(&Task{}) + if err != nil { + return nil, totalCount, err + + } + + return +} + +type typesenseTaskSearcher struct { + s *xorm.Session +} + +func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { + projectIDStrings := []string{} + for _, id := range opts.projectIDs { + projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10)) + } + + params := &api.SearchCollectionParams{ + Q: opts.search, + QueryBy: "title, description, comments.comment", + Page: pointer.Int(opts.page), + PerPage: pointer.Int(opts.perPage), + ExhaustiveSearch: pointer.True(), + FilterBy: pointer.String("project_id: [" + strings.Join(projectIDStrings, ", ") + "]"), + } + + result, err := typesenseClient.Collection("tasks"). + Documents(). + Search(params) + if err != nil { + return + } + + taskIDs := []int64{} + for _, h := range *result.Hits { + hit := *h.Document + taskID, err := strconv.ParseInt(hit["id"].(string), 10, 64) + if err != nil { + return nil, 0, err + } + taskIDs = append(taskIDs, taskID) + } + + tasks = []*Task{} + + err = t.s.In("id", taskIDs).Find(&tasks) + return tasks, int64(*result.Found), err +} diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go index 7dd7be5168e..c4a3d91ac26 100644 --- a/pkg/models/tasks.go +++ b/pkg/models/tasks.go @@ -25,7 +25,6 @@ import ( "time" "code.vikunja.io/api/pkg/config" - "code.vikunja.io/api/pkg/db" "code.vikunja.io/api/pkg/events" "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/user" @@ -36,7 +35,6 @@ import ( "github.com/jinzhu/copier" "xorm.io/builder" "xorm.io/xorm" - "xorm.io/xorm/schemas" ) type TaskRepeatMode int @@ -167,7 +165,7 @@ const ( filterConcatOr = "or" ) -type taskOptions struct { +type taskSearchOptions struct { search string page int perPage int @@ -175,6 +173,7 @@ type taskOptions struct { filters []*taskFilter filterConcat taskFilterConcatinator filterIncludeNulls bool + projectIDs []int64 } // ReadAll is a dummy function to still have that endpoint documented @@ -266,7 +265,7 @@ func getTaskIndexFromSearchString(s string) (index int64) { } //nolint:gocyclo -func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskOptions) (tasks []*Task, resultCount int, totalItems int64, err error) { +func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskSearchOptions) (tasks []*Task, resultCount int, totalItems int64, err error) { // If the user does not have any projects, don't try to get any tasks if len(projects) == 0 { @@ -279,14 +278,14 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op } // Get all project IDs and get the tasks - var projectIDs []int64 + opts.projectIDs = []int64{} var hasFavoritesProject bool - for _, l := range projects { - if l.ID == FavoritesPseudoProject.ID { + for _, p := range projects { + if p.ID == FavoritesPseudoProject.ID { hasFavoritesProject = true continue } - projectIDs = append(projectIDs, l.ID) + opts.projectIDs = append(opts.projectIDs, p.ID) } // Add the id parameter as the last parameter to sortby by default, but only if it is not already passed as the last parameter. @@ -298,199 +297,22 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op }) } - // Since xorm does not use placeholders for order by, it is possible to expose this with sql injection if we're directly - // passing user input to the db. - // As a workaround to prevent this, we check for valid column names here prior to passing it to the db. - var orderby string - for i, param := range opts.sortby { - // Validate the params - if err := param.validate(); err != nil { - return nil, 0, 0, err - } - - // Mysql sorts columns with null values before ones without null value. - // Because it does not have support for NULLS FIRST or NULLS LAST we work around this by - // first sorting for null (or not null) values and then the order we actually want to. - if db.Type() == schemas.MYSQL { - orderby += "`" + param.sortBy + "` IS NULL, " - } - - orderby += "`" + param.sortBy + "` " + param.orderBy.String() - - // Postgres and sqlite allow us to control how columns with null values are sorted. - // To make that consistent with the sort order we have and other dbms, we're adding a separate clause here. - if db.Type() == schemas.POSTGRES || db.Type() == schemas.SQLITE { - orderby += " NULLS LAST" - } - - if (i + 1) < len(opts.sortby) { - orderby += ", " + var searcher taskSearcher = &dbTaskSearcher{ + s: s, + a: a, + hasFavoritesProject: hasFavoritesProject, + } + if config.TypesenseEnabled.GetBool() { + searcher = &typesenseTaskSearcher{ + s: s, } } - // Some filters need a special treatment since they are in a separate table - reminderFilters := []builder.Cond{} - assigneeFilters := []builder.Cond{} - labelFilters := []builder.Cond{} - projectFilters := []builder.Cond{} - - var filters = make([]builder.Cond, 0, len(opts.filters)) - // To still find tasks with nil values, we exclude 0s when comparing with >/< values. - for _, f := range opts.filters { - if f.field == "reminders" { - f.field = "reminder" // This is the name in the db - filter, err := getFilterCond(f, opts.filterIncludeNulls) - if err != nil { - return nil, 0, 0, err - } - reminderFilters = append(reminderFilters, filter) - continue - } - - if f.field == "assignees" { - if f.comparator == taskFilterComparatorLike { - return nil, 0, 0, ErrInvalidTaskFilterValue{Field: f.field, Value: f.value} - } - f.field = "username" - filter, err := getFilterCond(f, opts.filterIncludeNulls) - if err != nil { - return nil, 0, 0, err - } - assigneeFilters = append(assigneeFilters, filter) - continue - } - - if f.field == "labels" || f.field == "label_id" { - f.field = "label_id" - filter, err := getFilterCond(f, opts.filterIncludeNulls) - if err != nil { - return nil, 0, 0, err - } - labelFilters = append(labelFilters, filter) - continue - } - - if f.field == "parent_project" || f.field == "parent_project_id" { - f.field = "parent_project_id" - filter, err := getFilterCond(f, opts.filterIncludeNulls) - if err != nil { - return nil, 0, 0, err - } - projectFilters = append(projectFilters, filter) - continue - } - - filter, err := getFilterCond(f, opts.filterIncludeNulls) - if err != nil { - return nil, 0, 0, err - } - filters = append(filters, filter) - } - - // Then return all tasks for that projects - var where builder.Cond - - if opts.search != "" { - where = db.ILIKE("title", opts.search) - - searchIndex := getTaskIndexFromSearchString(opts.search) - if searchIndex > 0 { - where = builder.Or(where, builder.Eq{"`index`": searchIndex}) - } - } - - var projectIDCond builder.Cond - var favoritesCond builder.Cond - if len(projectIDs) > 0 { - projectIDCond = builder.In("project_id", projectIDs) - } - - if hasFavoritesProject { - // All favorite tasks for that user - favCond := builder. - Select("entity_id"). - From("favorites"). - Where( - builder.And( - builder.Eq{"user_id": a.GetID()}, - builder.Eq{"kind": FavoriteKindTask}, - )) - - favoritesCond = builder.In("id", favCond) - } - - if len(reminderFilters) > 0 { - filters = append(filters, getFilterCondForSeparateTable("task_reminders", opts.filterConcat, reminderFilters)) - } - - if len(assigneeFilters) > 0 { - assigneeFilter := []builder.Cond{ - builder.In("user_id", - builder.Select("id"). - From("users"). - Where(builder.Or(assigneeFilters...)), - )} - filters = append(filters, getFilterCondForSeparateTable("task_assignees", opts.filterConcat, assigneeFilter)) - } - - if len(labelFilters) > 0 { - filters = append(filters, getFilterCondForSeparateTable("label_tasks", opts.filterConcat, labelFilters)) - } - - if len(projectFilters) > 0 { - var filtercond builder.Cond - if opts.filterConcat == filterConcatOr { - filtercond = builder.Or(projectFilters...) - } - if opts.filterConcat == filterConcatAnd { - filtercond = builder.And(projectFilters...) - } - - cond := builder.In( - "project_id", - builder. - Select("id"). - From("projects"). - Where(filtercond), - ) - filters = append(filters, cond) - } - - var filterCond builder.Cond - if len(filters) > 0 { - if opts.filterConcat == filterConcatOr { - filterCond = builder.Or(filters...) - } - if opts.filterConcat == filterConcatAnd { - filterCond = builder.And(filters...) - } - } - - limit, start := getLimitFromPageIndex(opts.page, opts.perPage) - cond := builder.And(builder.Or(projectIDCond, favoritesCond), where, filterCond) - - query := s.Where(cond) - if limit > 0 { - query = query.Limit(limit, start) - } - - tasks = []*Task{} - err = query.OrderBy(orderby).Find(&tasks) - if err != nil { - return nil, 0, 0, err - } - - queryCount := s.Where(cond) - totalItems, err = queryCount. - Count(&Task{}) - if err != nil { - return nil, 0, 0, err - } - + tasks, totalItems, err = searcher.Search(opts) return tasks, len(tasks), totalItems, nil } -func getTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskOptions) (tasks []*Task, resultCount int, totalItems int64, err error) { +func getTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskSearchOptions) (tasks []*Task, resultCount int, totalItems int64, err error) { tasks, resultCount, totalItems, err = getRawTasksForProjects(s, projects, a, opts) if err != nil { diff --git a/pkg/models/typesense.go b/pkg/models/typesense.go index 61a5020dbe8..04d8742eb30 100644 --- a/pkg/models/typesense.go +++ b/pkg/models/typesense.go @@ -248,7 +248,7 @@ type typesenseTask struct { Reminders interface{} `json:"reminders"` Assignees interface{} `json:"assignees"` Labels interface{} `json:"labels"` - //RelatedTasks interface{} `json:"related_tasks"` + //RelatedTasks interface{} `json:"related_tasks"` // TODO Attachments interface{} `json:"attachments"` Comments interface{} `json:"comments"` }