feat: use flexsearch for all local searches (#997)

Co-authored-by: kolaente <k@knt.li>
Reviewed-on: vikunja/frontend#997
Reviewed-by: dpschen <dpschen@noreply.kolaente.de>
Co-authored-by: konrad <k@knt.li>
Co-committed-by: konrad <k@knt.li>

package.json

@@ -30,6 +30,7 @@
     "dompurify": "2.3.3",
     "easymde": "2.15.0",
     "flatpickr": "4.6.9",
+    "flexsearch": "^0.7.21",
     "highlight.js": "11.3.1",
     "is-touch-device": "1.0.1",
     "lodash.clonedeep": "4.5.0",
@@ -54,6 +55,7 @@
     "@fortawesome/free-regular-svg-icons": "5.15.4",
     "@fortawesome/free-solid-svg-icons": "5.15.4",
     "@fortawesome/vue-fontawesome": "3.0.0-5",
+    "@types/flexsearch": "^0.7.2",
     "@types/jest": "27.0.2",
     "@typescript-eslint/eslint-plugin": "5.3.1",
     "@typescript-eslint/parser": "5.3.1",
@@ -81,8 +83,8 @@
     "typescript": "4.4.4",
     "vite": "2.6.14",
     "vite-plugin-pwa": "0.11.5",
-    "vue-tsc": "0.29.4",
     "vite-svg-loader": "3.1.0",
+    "vue-tsc": "0.29.4",
     "wait-on": "6.0.0",
     "workbox-cli": "6.3.0"
   },

src/components/namespace/namespace-search.vue

@@ -25,15 +25,7 @@ export default {
 	},
 	computed: {
 		namespaces() {
-			if (this.query === '') {
-				return []
-			}
-			
-			return this.$store.state.namespaces.namespaces.filter(n => {
-				return !n.isArchived && 
-					n.id > 0 &&
-					n.title.toLowerCase().includes(this.query.toLowerCase())
-			})
+			return this.$store.getters['namespaces/searchNamespace'](this.query)
 		},
 	},
 	methods: {

src/components/quick-actions/quick-actions.vue

@@ -110,40 +110,32 @@ export default {
 		results() {
 			let lists = []
 			if (this.searchMode === SEARCH_MODE_ALL || this.searchMode === SEARCH_MODE_LISTS) {
-				const ncache = {}
-
-				const history = getHistory()
-				// Puts recently visited lists at the top
-				const allLists = [...new Set([
-					...history.map(l => {
-						return this.$store.getters['lists/getListById'](l.id)
-					}),
-					...Object.values(this.$store.state.lists)])]
-
 				const {list} = this.parsedQuery
 
 				if (list === null) {
 					lists = []
 				} else {
+					const ncache = {}
+					const history = getHistory()
+					// Puts recently visited lists at the top
+					const allLists = [...new Set([
+						...history.map(l => {
+							return this.$store.getters['lists/getListById'](l.id)
+						}),
+						...this.$store.getters['lists/searchList'](list),
+					])]
+
 					lists = allLists.filter(l => {
 						if (typeof l === 'undefined' || l === null) {
 							return false
 						}
 
-						if (l.isArchived) {
-							return false
-						}
-
 						if (typeof ncache[l.namespaceId] === 'undefined') {
 							ncache[l.namespaceId] = this.$store.getters['namespaces/getNamespaceById'](l.namespaceId)
 						}
 
-						if (ncache[l.namespaceId].isArchived) {
-							return false
-						}
-
-						return l.title.toLowerCase().includes(list.toLowerCase())
-					}) ?? []
+						return !ncache[l.namespaceId].isArchived
+					})
 				}
 			}
 

src/components/tasks/partials/listSearch.vue

@@ -1,7 +1,6 @@
 <template>
 	<multiselect
 		class="control is-expanded"
-		:loading="listSerivce.loading"
 		:placeholder="$t('list.search')"
 		@search="findLists"
 		:search-results="foundLists"
@@ -18,7 +17,6 @@
 </template>
 
 <script>
-import ListService from '../../../services/list'
 import ListModel from '../../../models/list'
 import Multiselect from '@/components/input/multiselect.vue'
 
@@ -26,7 +24,6 @@ export default {
 	name: 'listSearch',
 	data() {
 		return {
-			listSerivce: new ListService(),
 			list: new ListModel(),
 			foundLists: [],
 		}
@@ -50,17 +47,8 @@ export default {
 		},
 	},
 	methods: {
-		async findLists(query) {
-			if (query === '') {
-				this.clearAll()
-				return
-			}
-
-			this.foundLists = await this.listSerivce.getAll({}, {s: query})
-		},
-
-		clearAll() {
-			this.foundLists = []
+		findLists(query) {
+			this.foundLists = this.$store.getters['lists/searchList'](query)
 		},
 
 		select(list) {
@@ -82,6 +70,6 @@ export default {
 
 <style lang="scss" scoped>
 .list-namespace-title {
-  color: $grey-500;
+	color: $grey-500;
 }
 </style>

src/helpers/labels.test.ts

@@ -1,20 +1,25 @@
 import {filterLabelsByQuery} from './labels'
+import {createNewIndexer} from '../indexes'
+
+const {add} = createNewIndexer('labels', ['title', 'description'])
 
 describe('filter labels', () => {
 	const state = {
-		labels: [
-			{id: 1, title: 'label1'},
-			{id: 2, title: 'label2'},
-			{id: 3, title: 'label3'},
-			{id: 4, title: 'label4'},
-			{id: 5, title: 'label5'},
-			{id: 6, title: 'label6'},
-			{id: 7, title: 'label7'},
-			{id: 8, title: 'label8'},
-			{id: 9, title: 'label9'},
-		],
+		labels: {
+			1: {id: 1, title: 'label1'},
+			2: {id: 2, title: 'label2'},
+			3: {id: 3, title: 'label3'},
+			4: {id: 4, title: 'label4'},
+			5: {id: 5, title: 'label5'},
+			6: {id: 6, title: 'label6'},
+			7: {id: 7, title: 'label7'},
+			8: {id: 8, title: 'label8'},
+			9: {id: 9, title: 'label9'},
+		},
 	}
 
+	Object.values(state.labels).forEach(add)
+
 	it('should return an empty array for an empty query', () => {
 		const labels = filterLabelsByQuery(state, [], '')
 
@@ -31,7 +36,7 @@ describe('filter labels', () => {
 			id: number,
 			title: string,
 		}
-		
+
 		const labelsToHide: label[] = [{id: 1, title: 'label1'}]
 		const labels = filterLabelsByQuery(state, labelsToHide, 'label1')
 

src/helpers/labels.ts

@@ -1,10 +1,16 @@
-interface label {
+import {createNewIndexer} from '../indexes'
+
+const {search} = createNewIndexer('labels', ['title', 'description'])
+
+export interface label {
 	id: number,
 	title: string,
 }
 
 interface labelState {
-	labels: label[],
+	labels: {
+		[k: number]: label,
+	},
 }
 
 /**
@@ -15,17 +21,12 @@ interface labelState {
  * @returns {Array}
  */
 export function filterLabelsByQuery(state: labelState, labelsToHide: label[], query: string) {
-	if (query === '') {
-		return []
-	}
+	const labelIdsToHide: number[] = labelsToHide.map(({id}) => id)
 
-	const labelQuery = query.toLowerCase()
-	const labelIds = labelsToHide.map(({id}) => id)
-	return Object
-		.values(state.labels)
-		.filter(({id, title}) => {
-			return !labelIds.includes(id) && title.toLowerCase().includes(labelQuery)
-		})
+	return search(query)
+			?.filter(value => !labelIdsToHide.includes(value))
+			.map(id => state.labels[id])
+		|| []
 }
 
 

src/indexes/index.ts

@@ -0,0 +1,52 @@
+import {Document, SimpleDocumentSearchResultSetUnit} from 'flexsearch'
+
+export interface withId {
+	id: number,
+}
+
+const indexes: { [k: string]: Document<withId> } = {}
+
+export const createNewIndexer = (name: string, fieldsToIndex: string[]) => {
+	if (typeof indexes[name] === 'undefined') {
+		indexes[name] = new Document<withId>({
+			tokenize: 'full',
+			document: {
+				id: 'id',
+				index: fieldsToIndex,
+			},
+		})
+	}
+
+	const index = indexes[name]
+
+	function add(item: withId) {
+		return index.add(item.id, item)
+	}
+
+	function remove(item: withId) {
+		return index.remove(item.id)
+	}
+
+	function update(item: withId) {
+		return index.update(item.id, item)
+	}
+
+	function search(query: string | null): number[] | null {
+		if (query === '' || query === null) {
+			return null
+		}
+
+		// @ts-ignore
+		return index.search(query)
+				?.flatMap(r => r.result)
+				.filter((value, index, self) => self.indexOf(value) === index)
+			|| null
+	}
+
+	return {
+		add,
+		remove,
+		update,
+		search,
+	}
+}

src/store/modules/labels.js

@@ -3,6 +3,9 @@ import {setLoading} from '@/store/helper'
 import {success} from '@/message'
 import {i18n} from '@/i18n'
 import {getLabelsByIds, filterLabelsByQuery} from '@/helpers/labels'
+import {createNewIndexer} from '@/indexes'
+
+const {add, remove, update} = createNewIndexer('labels', ['title', 'description'])
 
 async function getAllLabels(page = 1) {
 	const labelService = new LabelService()
@@ -26,13 +29,16 @@ export default {
 		setLabels(state, labels) {
 			labels.forEach(l => {
 				state.labels[l.id] = l
+				add(l)
 			})
 		},
 		setLabel(state, label) {
 			state.labels[label.id] = label
+			update(label)
 		},
 		removeLabelById(state, label) {
 			delete state.labels[label.id]
+			remove(label)
 		},
 		setLoaded(state, loaded) {
 			state.loaded = loaded

src/store/modules/lists.js

@@ -1,6 +1,9 @@
 import ListService from '@/services/list'
 import {setLoading} from '@/store/helper'
 import {removeListFromHistory} from '@/modules/listHistory.ts'
+import {createNewIndexer} from '@/indexes'
+
+const {add, remove, search, update} = createNewIndexer('lists', ['title', 'description'])
 
 const FavoriteListsNamespace = -2
 
@@ -11,14 +14,17 @@ export default {
 	mutations: {
 		setList(state, list) {
 			state[list.id] = list
+			update(list)
 		},
 		setLists(state, lists) {
 			lists.forEach(l => {
 				state[l.id] = l
+				add(l)
 			})
 		},
 		removeListById(state, list) {
 			delete state[list.id]
+			remove(list)
 		},
 	},
 	getters: {
@@ -34,6 +40,13 @@ export default {
 			})
 			return typeof list === 'undefined' ? null : list
 		},
+		searchList: state => (query, includeArchived = false) => {
+			return search(query)
+					?.filter(value => value > 0)
+					.map(id => state[id])
+					.filter(list => list.isArchived === includeArchived)
+				|| []
+		},
 	},
 	actions: {
 		toggleListFavorite(ctx, list) {
@@ -66,7 +79,7 @@ export default {
 				await listService.update(list)
 				ctx.commit('setList', list)
 				ctx.commit('namespaces/setListInNamespaceById', list, {root: true})
-				
+
 				// the returned list from listService.update is the same!
 				// in order to not validate vuex mutations we have to create a new copy
 				const newList = {
@@ -81,7 +94,7 @@ export default {
 				ctx.dispatch('namespaces/loadNamespacesIfFavoritesDontExist', null, {root: true})
 				ctx.dispatch('namespaces/removeFavoritesNamespaceIfEmpty', null, {root: true})
 				return newList
-			} catch(e) {
+			} catch (e) {
 				// Reset the list state to the initial one to avoid confusion for the user
 				ctx.commit('setList', {
 					...list,
@@ -97,13 +110,13 @@ export default {
 			const cancel = setLoading(ctx, 'lists')
 			const listService = new ListService()
 
-			try {	
+			try {
 				const response = await listService.delete(list)
 				ctx.commit('removeListById', list)
 				ctx.commit('namespaces/removeListFromNamespaceById', list, {root: true})
 				removeListFromHistory({id: list.id})
 				return response
-			} finally{
+			} finally {
 				cancel()
 			}
 		},

src/store/modules/namespaces.js

@@ -1,5 +1,8 @@
 import NamespaceService from '../../services/namespace'
 import {setLoading} from '@/store/helper'
+import {createNewIndexer} from '@/indexes'
+
+const {add, remove, search, update} = createNewIndexer('namespaces', ['title', 'description'])
 
 export default {
 	namespaced: true,
@@ -9,6 +12,9 @@ export default {
 	mutations: {
 		namespaces(state, namespaces) {
 			state.namespaces = namespaces
+			namespaces.forEach(n => {
+				add(n)
+			})
 		},
 		setNamespaceById(state, namespace) {
 			const namespaceIndex = state.namespaces.findIndex(n => n.id === namespace.id)
@@ -22,8 +28,9 @@ export default {
 			if (!namespace.lists || namespace.lists.length === 0) {
 				namespace.lists = state.namespaces[namespaceIndex].lists
 			}
-			
+
 			state.namespaces[namespaceIndex] = namespace
+			update(namespace)
 		},
 		setListInNamespaceById(state, list) {
 			for (const n in state.namespaces) {
@@ -43,11 +50,13 @@ export default {
 		},
 		addNamespace(state, namespace) {
 			state.namespaces.push(namespace)
+			add(namespace)
 		},
 		removeNamespaceById(state, namespaceId) {
 			for (const n in state.namespaces) {
 				if (state.namespaces[n].id === namespaceId) {
 					state.namespaces.splice(n, 1)
+					remove(state.namespaces[n])
 					return
 				}
 			}
@@ -78,11 +87,11 @@ export default {
 	getters: {
 		getListAndNamespaceById: state => (listId, ignorePseudoNamespaces = false) => {
 			for (const n in state.namespaces) {
-				
-				if(ignorePseudoNamespaces && state.namespaces[n].id < 0) {
+
+				if (ignorePseudoNamespaces && state.namespaces[n].id < 0) {
 					continue
 				}
-				
+
 				for (const l in state.namespaces[n].lists) {
 					if (state.namespaces[n].lists[l].id === listId) {
 						return {
@@ -97,6 +106,13 @@ export default {
 		getNamespaceById: state => namespaceId => {
 			return state.namespaces.find(({id}) => id == namespaceId) || null
 		},
+		searchNamespace: (state, getters) => query => {
+			return search(query)
+					?.filter(value => value > 0)
+					.map(getters.getNamespaceById)
+					.filter(n => n !== null)
+				|| []
+		},
 	},
 	actions: {
 		async loadNamespaces(ctx) {
@@ -107,12 +123,12 @@ export default {
 				// We always load all namespaces and filter them on the frontend
 				const namespaces = await namespaceService.getAll({}, {is_archived: true})
 				ctx.commit('namespaces', namespaces)
-				
+
 				// Put all lists in the list state
 				const lists = namespaces.flatMap(({lists}) => lists)
-				
+
 				ctx.commit('lists/setLists', lists, {root: true})
-				
+
 				return namespaces
 			} finally {
 				cancel()

yarn.lock

@@ -3168,6 +3168,11 @@
   resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f"
   integrity sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==
 
+"@types/flexsearch@^0.7.2":
+  version "0.7.2"
+  resolved "https://registry.yarnpkg.com/@types/flexsearch/-/flexsearch-0.7.2.tgz#05d982d292e70fcb9fc59347a4a4f98c4ecd9e56"
+  integrity sha512-Nq0CSpOCyUhaF7tAXSvMtoyBMPGlhNyF+uElhIrrgSiXDmX/bnn9jUX7Us3l81Hzowb9rcgNISke0Nj+3xhd3g==
+
 "@types/glob@^7.1.1":
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb"
@@ -7121,6 +7126,11 @@ flatten@^1.0.2:
   resolved "https://registry.yarnpkg.com/flatten/-/flatten-1.0.3.tgz#c1283ac9f27b368abc1e36d1ff7b04501a30356b"
   integrity sha512-dVsPA/UwQ8+2uoFe5GHtiBMu48dWLTdsuEd7CKGlZlD78r1TTWBvDuFaFGKCo/ZfEr95Uk56vZoX86OsHkUeIg==
 
+flexsearch@^0.7.21:
+  version "0.7.21"
+  resolved "https://registry.yarnpkg.com/flexsearch/-/flexsearch-0.7.21.tgz#0f5ede3f2aae67ddc351efbe3b24b69d29e9d48b"
+  integrity sha512-W7cHV7Hrwjid6lWmy0IhsWDFQboWSng25U3VVywpHOTJnnAZNPScog67G+cVpeX9f7yDD21ih0WDrMMT+JoaYg==
+
 flush-write-stream@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-2.0.0.tgz#6f58e776154f5eefacff92a6e5a681c88ac50f7c"