From dee46d527ace3028c76a1e588718a897db6002de Mon Sep 17 00:00:00 2001 From: kolaente Date: Mon, 28 Aug 2023 11:11:30 +0200 Subject: [PATCH 01/11] feat(tasks): add typesense indexing --- go.mod | 46 ++++-- go.sum | 81 +++++++++++ pkg/cmd/index.go | 53 +++++++ pkg/config/config.go | 7 + pkg/initialize/init.go | 3 + pkg/models/typesense.go | 303 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 479 insertions(+), 14 deletions(-) create mode 100644 pkg/cmd/index.go create mode 100644 pkg/models/typesense.go diff --git a/go.mod b/go.mod index eaa4b4ebdd0..db5b9b4d48b 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/gocarina/gocsv v0.0.0-20230616125104-99d496ca653d github.com/golang-jwt/jwt/v5 v5.0.0 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/hashicorp/go-version v1.6.0 github.com/iancoleman/strcase v0.3.0 github.com/jinzhu/copier v0.3.5 @@ -64,12 +64,12 @@ require ( github.com/ulule/limiter/v3 v3.11.2 github.com/wneessen/go-mail v0.4.0 github.com/yuin/goldmark v1.5.4 - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.12.0 golang.org/x/image v0.11.0 golang.org/x/oauth2 v0.10.0 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.10.0 - golang.org/x/term v0.10.0 + golang.org/x/sys v0.11.0 + golang.org/x/term v0.11.0 gopkg.in/d4l3k/messagediff.v1 v1.2.1 gopkg.in/yaml.v3 v3.0.1 src.techknowlogick.com/xgo v1.7.1-0.20230711181658-617d3b65dd40 @@ -85,25 +85,35 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/andybalholm/brotli v1.0.5 // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/beevik/etree v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect + github.com/bytedance/sonic v1.10.0 // indirect github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect + github.com/chenzhuoyu/iasm v0.9.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/deepmap/oapi-codegen v1.13.4 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-chi/chi/v5 v5.0.8 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gin-gonic/gin v1.9.1 // indirect + github.com/go-chi/chi/v5 v5.0.10 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.6.1 // indirect github.com/go-jose/go-jose/v3 v3.0.0 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/spec v0.20.4 // indirect - github.com/go-openapi/swag v0.19.15 // indirect - github.com/goccy/go-json v0.10.0 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.15.1 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -113,14 +123,16 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/laurent22/ical-go v0.1.1-0.20181107184520-7e5d6ade8eef // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/lithammer/shortuuid/v3 v3.0.7 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -129,32 +141,38 @@ require ( github.com/onsi/ginkgo v1.16.4 // indirect github.com/onsi/gomega v1.16.0 // indirect github.com/paulmach/orb v0.9.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pelletier/go-toml/v2 v2.0.9 // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect + github.com/sony/gobreaker v0.5.0 // indirect github.com/spf13/cast v1.5.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/typesense/typesense-go v0.8.0 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect github.com/urfave/cli/v2 v2.3.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 // indirect go.opentelemetry.io/otel v1.15.0 // indirect go.opentelemetry.io/otel/trace v1.15.0 // indirect - golang.org/x/mod v0.9.0 // indirect - golang.org/x/net v0.12.0 // indirect + golang.org/x/arch v0.4.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/tools v0.11.1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 704e64945e5..5ca2573c414 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,7 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/ThreeDotsLabs/watermill v1.2.0 h1:TU3TML1dnQ/ifK09F2+4JQk2EKhmhXe7Qv7eb5ZpTS8= @@ -76,6 +77,8 @@ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -98,10 +101,15 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= +github.com/bytedance/sonic v1.10.0 h1:qtNZduETEIWJVIyDl01BeNxur2rW9OwTQ/yBqFRkKEk= +github.com/bytedance/sonic v1.10.0/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY= github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -112,6 +120,12 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= +github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo= +github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -142,6 +156,8 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deepmap/oapi-codegen v1.13.4 h1:lRRQ8JAXaz5/4oidKFyk3fFZFQsbv0BzRtvDKDnvIfM= +github.com/deepmap/oapi-codegen v1.13.4/go.mod h1:/h5nFQbTAMz4S/WtBz8sBfamlGByYKDr21O2uoNgCYI= github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= @@ -183,8 +199,14 @@ github.com/getsentry/sentry-go v0.23.0 h1:dn+QRCeJv4pPt9OjVXiMcGIBIefaTJPw/h0bZW github.com/getsentry/sentry-go v0.23.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= +github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= @@ -205,6 +227,8 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= +github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= @@ -212,6 +236,14 @@ github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.15.1 h1:BSe8uhN+xQ4r5guV/ywQI4gO59C2raYcGffYWZEjZzM= +github.com/go-playground/validator/v10 v10.15.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -227,6 +259,8 @@ github.com/goccy/go-json v0.8.1/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGF github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -311,6 +345,8 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -438,6 +474,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6 h1:SwcnSwBR7X/5EHJQlXBockkJVIMRVt5yKaesBPMtyZQ= github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6/go.mod h1:WrYiIuiXUMIvTDAQw97C+9l0CnBmCcvosPjN3XDqS/o= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -448,6 +485,12 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kolaente/caldav-go v3.0.1-0.20190610114120-2a4eb8b5dcc9+incompatible h1:q7DbyV+sFjEoTuuUdRDNl2nlyfztkZgxVVCV7JhzIkY= github.com/kolaente/caldav-go v3.0.1-0.20190610114120-2a4eb8b5dcc9+incompatible/go.mod h1:y1UhTNI4g0hVymJrI6yJ5/ohy09hNBeU8iJEZjgdDOw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -475,6 +518,8 @@ github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8 github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/laurent22/ical-go v0.1.1-0.20181107184520-7e5d6ade8eef h1:RZnRnSID1skF35j/15KJ6hKZkdIC/teQClJK5wP5LU4= github.com/laurent22/ical-go v0.1.1-0.20181107184520-7e5d6ade8eef/go.mod h1:4LATl0uhhtytR6p9n1AlktDyIz4u2iUnWEdI3L/hXiw= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -523,6 +568,8 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= @@ -596,6 +643,8 @@ github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKf github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -642,11 +691,15 @@ github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -672,6 +725,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= +github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= @@ -686,6 +741,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -702,6 +758,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -715,6 +773,12 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tkuchiki/go-timezone v0.2.2 h1:MdHR65KwgVTwWFQrota4SKzc4L5EfuH5SdZZGtk/P2Q= github.com/tkuchiki/go-timezone v0.2.2/go.mod h1:oFweWxYl35C/s7HMVZXiA19Jr9Y0qJHMaG/J2TES4LY= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/typesense/typesense-go v0.8.0 h1:jb0pk8LuizYaNgPdoC7lLK16HsYijshHtp2SJe4wVKs= +github.com/typesense/typesense-go v0.8.0/go.mod h1:4mq4FYHzU7csU/KHaZoyG2bCSKl7GrCeyAr2YhXT1/0= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulule/limiter/v3 v3.11.2 h1:P4yOrxoEMJbOTfRJR2OzjL90oflzYPPmWg+dvwN2tHA= github.com/ulule/limiter/v3 v3.11.2/go.mod h1:QG5GnFOCV+k7lrL5Y8kgEeeflPH3+Cviqlqa8SVSQxI= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -775,6 +839,9 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.4.0 h1:A8WCeEWhLwPBKNbFi5Wv5UTCBx5zzubnXDlMOFAzFMc= +golang.org/x/arch v0.4.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -801,6 +868,8 @@ golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -845,6 +914,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -895,6 +966,8 @@ golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -995,6 +1068,8 @@ golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1003,6 +1078,8 @@ golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1089,6 +1166,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.11.1 h1:ojD5zOW8+7dOGzdnNgersm8aPfcDjhMp12UfG93NIMc= +golang.org/x/tools v0.11.1/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1349,7 +1428,9 @@ modernc.org/tcl v1.8.13/go.mod h1:V+q/Ef0IJaNUSECieLU4o+8IScapxnMyFV6i/7uQlAY= modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk= modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.2.19/go.mod h1:+ZpP0pc4zz97eukOzW3xagV/lS82IpPN9NGG5pNF9vY= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/pkg/cmd/index.go b/pkg/cmd/index.go new file mode 100644 index 00000000000..68aaf9945f3 --- /dev/null +++ b/pkg/cmd/index.go @@ -0,0 +1,53 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2023 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package cmd + +import ( + "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/initialize" + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/models" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(indexCmd) +} + +var indexCmd = &cobra.Command{ + Use: "index", + Short: "Reindex all of Vikunja's data into Typesense. This will remove any existing index.", + PreRun: func(cmd *cobra.Command, args []string) { + initialize.FullInit() + }, + Run: func(cmd *cobra.Command, args []string) { + if !config.TypesenseEnabled.GetBool() { + log.Error("Typesense not enabled") + return + } + + err := models.CreateTypesenseCollections() + if err != nil { + log.Critical(err.Error()) + return + } + err = models.ReindexAllTasks() + if err != nil { + log.Critical(err.Error()) + } + }, +} diff --git a/pkg/config/config.go b/pkg/config/config.go index eae242ad20e..f0f3fb0f8de 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -87,6 +87,10 @@ const ( DatabaseSslRootCert Key = `database.sslrootcert` DatabaseTLS Key = `database.tls` + TypesenseEnabled Key = `typesense.enabled` + TypesenseURL Key = `typesense.url` + TypesenseAPIKey Key = `typesense.apikey` + MailerEnabled Key = `mailer.enabled` MailerHost Key = `mailer.host` MailerPort Key = `mailer.port` @@ -317,6 +321,9 @@ func InitDefaultConfig() { DatabaseSslRootCert.setDefault("") DatabaseTLS.setDefault("false") + // Typesense + TypesenseEnabled.setDefault(false) + // Mailer MailerEnabled.setDefault(false) MailerHost.setDefault("") diff --git a/pkg/initialize/init.go b/pkg/initialize/init.go index e2205590976..aa1cd69306b 100644 --- a/pkg/initialize/init.go +++ b/pkg/initialize/init.go @@ -74,6 +74,9 @@ func FullInit() { // Set Engine InitEngines() + // Init Typesense + models.InitTypesense() + // Start the mail daemon mail.StartMailDaemon() diff --git a/pkg/models/typesense.go b/pkg/models/typesense.go new file mode 100644 index 00000000000..61a5020dbe8 --- /dev/null +++ b/pkg/models/typesense.go @@ -0,0 +1,303 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2023 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package models + +import ( + "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/db" + "code.vikunja.io/api/pkg/user" + "fmt" + + "github.com/typesense/typesense-go/typesense" + "github.com/typesense/typesense-go/typesense/api" + "github.com/typesense/typesense-go/typesense/api/pointer" +) + +var typesenseClient *typesense.Client + +func InitTypesense() { + if !config.TypesenseEnabled.GetBool() { + return + } + + typesenseClient = typesense.NewClient( + typesense.WithServer(config.TypesenseURL.GetString()), + typesense.WithAPIKey(config.TypesenseAPIKey.GetString())) +} + +func CreateTypesenseCollections() error { + taskSchema := &api.CollectionSchema{ + Name: "tasks", + EnableNestedFields: pointer.True(), + Fields: []api.Field{ + { + Name: "id", + Type: "string", + }, + { + Name: "title", + Type: "string", + }, + { + Name: "description", + Type: "string", + }, + { + Name: "done", + Type: "bool", + }, + { + Name: "done_at", + Type: "int64", // unix timestamp + Optional: pointer.True(), + }, + { + Name: "due_date", + Type: "int64", // unix timestamp + Optional: pointer.True(), + }, + { + Name: "project_id", + Type: "int64", + }, + { + Name: "repeat_after", + Type: "int64", + }, + { + Name: "repeat_mode", + Type: "int32", + }, + { + Name: "priority", + Type: "int64", + }, + { + Name: "start_date", + Type: "int64", // unix timestamp + Optional: pointer.True(), + }, + { + Name: "end_date", + Type: "int64", // unix timestamp + Optional: pointer.True(), + }, + { + Name: "hex_color", + Type: "string", + }, + { + Name: "percent_done", + Type: "float", + }, + { + Name: "identifier", + Type: "string", + }, + { + Name: "index", + Type: "int64", + }, + { + Name: "uid", + Type: "string", + }, + { + Name: "cover_image_attachment_id", + Type: "int64", + }, + { + Name: "created", + Type: "int64", // unix timestamp + }, + { + Name: "updated", + Type: "int64", // unix timestamp + }, + { + Name: "bucket_id", + Type: "int64", + }, + { + Name: "position", + Type: "float", + }, + { + Name: "kanban_position", + Type: "float", + }, + { + Name: "created_by_id", + Type: "int64", + }, + { + Name: "reminders", + Type: "object[]", // TODO + Optional: pointer.True(), + }, + { + Name: "assignees", + Type: "object[]", // TODO + Optional: pointer.True(), + }, + { + Name: "labels", + Type: "object[]", // TODO + Optional: pointer.True(), + }, + { + Name: "related_tasks", + Type: "object[]", // TODO + Optional: pointer.True(), + }, + { + Name: "attachments", + Type: "object[]", // TODO + Optional: pointer.True(), + }, + { + Name: "comments", + Type: "object[]", // TODO + Optional: pointer.True(), + }, + }, + } + + // delete any collection which might exist + _, _ = typesenseClient.Collection("tasks").Delete() + + _, err := typesenseClient.Collections().Create(taskSchema) + return err +} + +func ReindexAllTasks() (err error) { + tasks := make(map[int64]*Task) + + s := db.NewSession() + defer s.Close() + + err = s.Find(tasks) + if err != nil { + return err + } + + err = addMoreInfoToTasks(s, tasks, &user.User{ID: 1}) + if err != nil { + return err + } + + for _, task := range tasks { + searchTask := convertTaskToTypesenseTask(task) + + comment := &TaskComment{TaskID: task.ID} + searchTask.Comments, _, _, err = comment.ReadAll(s, task.CreatedBy, "", -1, -1) + if err != nil { + return err + } + + _, err = typesenseClient.Collection("tasks"). + Documents(). + Create(searchTask) + if err != nil { + return err + } + } + + return nil +} + +type typesenseTask struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Done bool `json:"done"` + DoneAt int64 `json:"done_at"` + DueDate int64 `json:"due_date"` + ProjectID int64 `json:"project_id"` + RepeatAfter int64 `json:"repeat_after"` + RepeatMode int `json:"repeat_mode"` + Priority int64 `json:"priority"` + StartDate int64 `json:"start_date"` + EndDate int64 `json:"end_date"` + HexColor string `json:"hex_color"` + PercentDone float64 `json:"percent_done"` + Identifier string `json:"identifier"` + Index int64 `json:"index"` + UID string `json:"uid"` + CoverImageAttachmentID int64 `json:"cover_image_attachment_id"` + Created int64 `json:"created"` + Updated int64 `json:"updated"` + BucketID int64 `json:"bucket_id"` + Position float64 `json:"position"` + KanbanPosition float64 `json:"kanban_position"` + CreatedByID int64 `json:"created_by_id"` + Reminders interface{} `json:"reminders"` + Assignees interface{} `json:"assignees"` + Labels interface{} `json:"labels"` + //RelatedTasks interface{} `json:"related_tasks"` + Attachments interface{} `json:"attachments"` + Comments interface{} `json:"comments"` +} + +func convertTaskToTypesenseTask(task *Task) *typesenseTask { + tt := &typesenseTask{ + ID: fmt.Sprintf("%d", task.ID), + Title: task.Title, + Description: task.Description, + Done: task.Done, + DoneAt: task.DoneAt.UTC().Unix(), + DueDate: task.DueDate.UTC().Unix(), + ProjectID: task.ProjectID, + RepeatAfter: task.RepeatAfter, + RepeatMode: int(task.RepeatMode), + Priority: task.Priority, + StartDate: task.StartDate.UTC().Unix(), + EndDate: task.EndDate.UTC().Unix(), + HexColor: task.HexColor, + PercentDone: task.PercentDone, + Identifier: task.Identifier, + Index: task.Index, + UID: task.UID, + CoverImageAttachmentID: task.CoverImageAttachmentID, + Created: task.Created.UTC().Unix(), + Updated: task.Updated.UTC().Unix(), + BucketID: task.BucketID, + Position: task.Position, + KanbanPosition: task.KanbanPosition, + CreatedByID: task.CreatedByID, + Reminders: task.Reminders, + Assignees: task.Assignees, + Labels: task.Labels, + //RelatedTasks: task.RelatedTasks, + Attachments: task.Attachments, + } + + if task.DoneAt.IsZero() { + tt.DoneAt = 0 + } + if task.DueDate.IsZero() { + tt.DueDate = 0 + } + if task.StartDate.IsZero() { + tt.StartDate = 0 + } + if task.EndDate.IsZero() { + tt.EndDate = 0 + } + + return tt +} From 010b4ce783750641d5614baa4e79e046e6f3c842 Mon Sep 17 00:00:00 2001 From: kolaente Date: Mon, 28 Aug 2023 12:14:50 +0200 Subject: [PATCH 02/11] feat(tasks): add searching via typesense --- pkg/cmd/index.go | 5 + pkg/models/export.go | 2 +- pkg/models/project.go | 2 +- pkg/models/project_duplicate.go | 2 +- pkg/models/task_collection.go | 4 +- pkg/models/task_search.go | 280 ++++++++++++++++++++++++++++++++ pkg/models/tasks.go | 212 ++---------------------- pkg/models/typesense.go | 2 +- 8 files changed, 308 insertions(+), 201 deletions(-) create mode 100644 pkg/models/task_search.go diff --git a/pkg/cmd/index.go b/pkg/cmd/index.go index 68aaf9945f3..36eb7492db7 100644 --- a/pkg/cmd/index.go +++ b/pkg/cmd/index.go @@ -40,6 +40,8 @@ var indexCmd = &cobra.Command{ return } + log.Infof("Indexing… This may take a while.") + err := models.CreateTypesenseCollections() if err != nil { log.Critical(err.Error()) @@ -48,6 +50,9 @@ var indexCmd = &cobra.Command{ err = models.ReindexAllTasks() if err != nil { log.Critical(err.Error()) + return } + + log.Infof("Done!") }, } diff --git a/pkg/models/export.go b/pkg/models/export.go index 28f45b55acb..30b33bc3d9f 100644 --- a/pkg/models/export.go +++ b/pkg/models/export.go @@ -153,7 +153,7 @@ func exportProjectsAndTasks(s *xorm.Session, u *user.User, wr *zip.Writer) (task projectIDs = append(projectIDs, p.ID) } - tasks, _, _, err := getTasksForProjects(s, rawProjects, u, &taskOptions{ + tasks, _, _, err := getTasksForProjects(s, rawProjects, u, &taskSearchOptions{ page: 0, perPage: -1, }) diff --git a/pkg/models/project.go b/pkg/models/project.go index 8105445e231..d14adb33ba0 100644 --- a/pkg/models/project.go +++ b/pkg/models/project.go @@ -969,7 +969,7 @@ func (p *Project) Delete(s *xorm.Session, a web.Auth) (err error) { // Delete all tasks on that project // Using the loop to make sure all related entities to all tasks are properly deleted as well. - tasks, _, _, err := getRawTasksForProjects(s, []*Project{p}, a, &taskOptions{}) + tasks, _, _, err := getRawTasksForProjects(s, []*Project{p}, a, &taskSearchOptions{}) if err != nil { return } diff --git a/pkg/models/project_duplicate.go b/pkg/models/project_duplicate.go index 6aab4707ca1..2efcf60369a 100644 --- a/pkg/models/project_duplicate.go +++ b/pkg/models/project_duplicate.go @@ -209,7 +209,7 @@ func (pd *ProjectDuplicate) Create(s *xorm.Session, doer web.Auth) (err error) { func duplicateTasks(s *xorm.Session, doer web.Auth, ld *ProjectDuplicate, bucketMap map[int64]int64) (err error) { // Get all tasks + all task details - tasks, _, _, err := getTasksForProjects(s, []*Project{{ID: ld.ProjectID}}, doer, &taskOptions{}) + tasks, _, _, err := getTasksForProjects(s, []*Project{{ID: ld.ProjectID}}, doer, &taskSearchOptions{}) if err != nil { return err } diff --git a/pkg/models/task_collection.go b/pkg/models/task_collection.go index de4b228b68f..c1a9397131f 100644 --- a/pkg/models/task_collection.go +++ b/pkg/models/task_collection.go @@ -80,7 +80,7 @@ func validateTaskField(fieldName string) error { return ErrInvalidTaskField{TaskField: fieldName} } -func getTaskFilterOptsFromCollection(tf *TaskCollection) (opts *taskOptions, err error) { +func getTaskFilterOptsFromCollection(tf *TaskCollection) (opts *taskSearchOptions, err error) { if len(tf.SortByArr) > 0 { tf.SortBy = append(tf.SortBy, tf.SortByArr...) } @@ -108,7 +108,7 @@ func getTaskFilterOptsFromCollection(tf *TaskCollection) (opts *taskOptions, err sort = append(sort, param) } - opts = &taskOptions{ + opts = &taskSearchOptions{ sortby: sort, filterConcat: taskFilterConcatinator(tf.FilterConcat), filterIncludeNulls: tf.FilterIncludeNulls, diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go new file mode 100644 index 00000000000..acb1c9cb5d8 --- /dev/null +++ b/pkg/models/task_search.go @@ -0,0 +1,280 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2023 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package models + +import ( + "code.vikunja.io/api/pkg/db" + "code.vikunja.io/web" + "github.com/typesense/typesense-go/typesense/api" + "github.com/typesense/typesense-go/typesense/api/pointer" + "strconv" + "strings" + + "xorm.io/builder" + "xorm.io/xorm" + "xorm.io/xorm/schemas" +) + +type taskSearcher interface { + Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) +} + +type dbTaskSearcher struct { + s *xorm.Session + a web.Auth + hasFavoritesProject bool +} + +func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { + // Since xorm does not use placeholders for order by, it is possible to expose this with sql injection if we're directly + // passing user input to the db. + // As a workaround to prevent this, we check for valid column names here prior to passing it to the db. + var orderby string + for i, param := range opts.sortby { + // Validate the params + if err := param.validate(); err != nil { + return nil, totalCount, err + } + + // Mysql sorts columns with null values before ones without null value. + // Because it does not have support for NULLS FIRST or NULLS LAST we work around this by + // first sorting for null (or not null) values and then the order we actually want to. + if db.Type() == schemas.MYSQL { + orderby += "`" + param.sortBy + "` IS NULL, " + } + + orderby += "`" + param.sortBy + "` " + param.orderBy.String() + + // Postgres and sqlite allow us to control how columns with null values are sorted. + // To make that consistent with the sort order we have and other dbms, we're adding a separate clause here. + if db.Type() == schemas.POSTGRES || db.Type() == schemas.SQLITE { + orderby += " NULLS LAST" + } + + if (i + 1) < len(opts.sortby) { + orderby += ", " + } + } + + // Some filters need a special treatment since they are in a separate table + reminderFilters := []builder.Cond{} + assigneeFilters := []builder.Cond{} + labelFilters := []builder.Cond{} + projectFilters := []builder.Cond{} + + var filters = make([]builder.Cond, 0, len(opts.filters)) + // To still find tasks with nil values, we exclude 0s when comparing with >/< values. + for _, f := range opts.filters { + if f.field == "reminders" { + f.field = "reminder" // This is the name in the db + filter, err := getFilterCond(f, opts.filterIncludeNulls) + if err != nil { + return nil, totalCount, err + } + reminderFilters = append(reminderFilters, filter) + continue + } + + if f.field == "assignees" { + if f.comparator == taskFilterComparatorLike { + return nil, totalCount, err + } + f.field = "username" + filter, err := getFilterCond(f, opts.filterIncludeNulls) + if err != nil { + return nil, totalCount, err + } + assigneeFilters = append(assigneeFilters, filter) + continue + } + + if f.field == "labels" || f.field == "label_id" { + f.field = "label_id" + filter, err := getFilterCond(f, opts.filterIncludeNulls) + if err != nil { + return nil, totalCount, err + } + labelFilters = append(labelFilters, filter) + continue + } + + if f.field == "parent_project" || f.field == "parent_project_id" { + f.field = "parent_project_id" + filter, err := getFilterCond(f, opts.filterIncludeNulls) + if err != nil { + return nil, totalCount, err + } + projectFilters = append(projectFilters, filter) + continue + } + + filter, err := getFilterCond(f, opts.filterIncludeNulls) + if err != nil { + return nil, totalCount, err + } + filters = append(filters, filter) + } + + // Then return all tasks for that projects + var where builder.Cond + + if opts.search != "" { + where = + builder.Or( + db.ILIKE("title", opts.search), + db.ILIKE("description", opts.search), + ) + + searchIndex := getTaskIndexFromSearchString(opts.search) + if searchIndex > 0 { + where = builder.Or(where, builder.Eq{"`index`": searchIndex}) + } + } + + var projectIDCond builder.Cond + var favoritesCond builder.Cond + if len(opts.projectIDs) > 0 { + projectIDCond = builder.In("project_id", opts.projectIDs) + } + + if d.hasFavoritesProject { + // All favorite tasks for that user + favCond := builder. + Select("entity_id"). + From("favorites"). + Where( + builder.And( + builder.Eq{"user_id": d.a.GetID()}, + builder.Eq{"kind": FavoriteKindTask}, + )) + + favoritesCond = builder.In("id", favCond) + } + + if len(reminderFilters) > 0 { + filters = append(filters, getFilterCondForSeparateTable("task_reminders", opts.filterConcat, reminderFilters)) + } + + if len(assigneeFilters) > 0 { + assigneeFilter := []builder.Cond{ + builder.In("user_id", + builder.Select("id"). + From("users"). + Where(builder.Or(assigneeFilters...)), + )} + filters = append(filters, getFilterCondForSeparateTable("task_assignees", opts.filterConcat, assigneeFilter)) + } + + if len(labelFilters) > 0 { + filters = append(filters, getFilterCondForSeparateTable("label_tasks", opts.filterConcat, labelFilters)) + } + + if len(projectFilters) > 0 { + var filtercond builder.Cond + if opts.filterConcat == filterConcatOr { + filtercond = builder.Or(projectFilters...) + } + if opts.filterConcat == filterConcatAnd { + filtercond = builder.And(projectFilters...) + } + + cond := builder.In( + "project_id", + builder. + Select("id"). + From("projects"). + Where(filtercond), + ) + filters = append(filters, cond) + } + + var filterCond builder.Cond + if len(filters) > 0 { + if opts.filterConcat == filterConcatOr { + filterCond = builder.Or(filters...) + } + if opts.filterConcat == filterConcatAnd { + filterCond = builder.And(filters...) + } + } + + limit, start := getLimitFromPageIndex(opts.page, opts.perPage) + cond := builder.And(builder.Or(projectIDCond, favoritesCond), where, filterCond) + + query := d.s.Where(cond) + if limit > 0 { + query = query.Limit(limit, start) + } + + tasks = []*Task{} + err = query.OrderBy(orderby).Find(&tasks) + if err != nil { + return nil, totalCount, err + } + + queryCount := d.s.Where(cond) + totalCount, err = queryCount. + Count(&Task{}) + if err != nil { + return nil, totalCount, err + + } + + return +} + +type typesenseTaskSearcher struct { + s *xorm.Session +} + +func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { + projectIDStrings := []string{} + for _, id := range opts.projectIDs { + projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10)) + } + + params := &api.SearchCollectionParams{ + Q: opts.search, + QueryBy: "title, description, comments.comment", + Page: pointer.Int(opts.page), + PerPage: pointer.Int(opts.perPage), + ExhaustiveSearch: pointer.True(), + FilterBy: pointer.String("project_id: [" + strings.Join(projectIDStrings, ", ") + "]"), + } + + result, err := typesenseClient.Collection("tasks"). + Documents(). + Search(params) + if err != nil { + return + } + + taskIDs := []int64{} + for _, h := range *result.Hits { + hit := *h.Document + taskID, err := strconv.ParseInt(hit["id"].(string), 10, 64) + if err != nil { + return nil, 0, err + } + taskIDs = append(taskIDs, taskID) + } + + tasks = []*Task{} + + err = t.s.In("id", taskIDs).Find(&tasks) + return tasks, int64(*result.Found), err +} diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go index 7dd7be5168e..c4a3d91ac26 100644 --- a/pkg/models/tasks.go +++ b/pkg/models/tasks.go @@ -25,7 +25,6 @@ import ( "time" "code.vikunja.io/api/pkg/config" - "code.vikunja.io/api/pkg/db" "code.vikunja.io/api/pkg/events" "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/user" @@ -36,7 +35,6 @@ import ( "github.com/jinzhu/copier" "xorm.io/builder" "xorm.io/xorm" - "xorm.io/xorm/schemas" ) type TaskRepeatMode int @@ -167,7 +165,7 @@ const ( filterConcatOr = "or" ) -type taskOptions struct { +type taskSearchOptions struct { search string page int perPage int @@ -175,6 +173,7 @@ type taskOptions struct { filters []*taskFilter filterConcat taskFilterConcatinator filterIncludeNulls bool + projectIDs []int64 } // ReadAll is a dummy function to still have that endpoint documented @@ -266,7 +265,7 @@ func getTaskIndexFromSearchString(s string) (index int64) { } //nolint:gocyclo -func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskOptions) (tasks []*Task, resultCount int, totalItems int64, err error) { +func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskSearchOptions) (tasks []*Task, resultCount int, totalItems int64, err error) { // If the user does not have any projects, don't try to get any tasks if len(projects) == 0 { @@ -279,14 +278,14 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op } // Get all project IDs and get the tasks - var projectIDs []int64 + opts.projectIDs = []int64{} var hasFavoritesProject bool - for _, l := range projects { - if l.ID == FavoritesPseudoProject.ID { + for _, p := range projects { + if p.ID == FavoritesPseudoProject.ID { hasFavoritesProject = true continue } - projectIDs = append(projectIDs, l.ID) + opts.projectIDs = append(opts.projectIDs, p.ID) } // Add the id parameter as the last parameter to sortby by default, but only if it is not already passed as the last parameter. @@ -298,199 +297,22 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op }) } - // Since xorm does not use placeholders for order by, it is possible to expose this with sql injection if we're directly - // passing user input to the db. - // As a workaround to prevent this, we check for valid column names here prior to passing it to the db. - var orderby string - for i, param := range opts.sortby { - // Validate the params - if err := param.validate(); err != nil { - return nil, 0, 0, err - } - - // Mysql sorts columns with null values before ones without null value. - // Because it does not have support for NULLS FIRST or NULLS LAST we work around this by - // first sorting for null (or not null) values and then the order we actually want to. - if db.Type() == schemas.MYSQL { - orderby += "`" + param.sortBy + "` IS NULL, " - } - - orderby += "`" + param.sortBy + "` " + param.orderBy.String() - - // Postgres and sqlite allow us to control how columns with null values are sorted. - // To make that consistent with the sort order we have and other dbms, we're adding a separate clause here. - if db.Type() == schemas.POSTGRES || db.Type() == schemas.SQLITE { - orderby += " NULLS LAST" - } - - if (i + 1) < len(opts.sortby) { - orderby += ", " + var searcher taskSearcher = &dbTaskSearcher{ + s: s, + a: a, + hasFavoritesProject: hasFavoritesProject, + } + if config.TypesenseEnabled.GetBool() { + searcher = &typesenseTaskSearcher{ + s: s, } } - // Some filters need a special treatment since they are in a separate table - reminderFilters := []builder.Cond{} - assigneeFilters := []builder.Cond{} - labelFilters := []builder.Cond{} - projectFilters := []builder.Cond{} - - var filters = make([]builder.Cond, 0, len(opts.filters)) - // To still find tasks with nil values, we exclude 0s when comparing with >/< values. - for _, f := range opts.filters { - if f.field == "reminders" { - f.field = "reminder" // This is the name in the db - filter, err := getFilterCond(f, opts.filterIncludeNulls) - if err != nil { - return nil, 0, 0, err - } - reminderFilters = append(reminderFilters, filter) - continue - } - - if f.field == "assignees" { - if f.comparator == taskFilterComparatorLike { - return nil, 0, 0, ErrInvalidTaskFilterValue{Field: f.field, Value: f.value} - } - f.field = "username" - filter, err := getFilterCond(f, opts.filterIncludeNulls) - if err != nil { - return nil, 0, 0, err - } - assigneeFilters = append(assigneeFilters, filter) - continue - } - - if f.field == "labels" || f.field == "label_id" { - f.field = "label_id" - filter, err := getFilterCond(f, opts.filterIncludeNulls) - if err != nil { - return nil, 0, 0, err - } - labelFilters = append(labelFilters, filter) - continue - } - - if f.field == "parent_project" || f.field == "parent_project_id" { - f.field = "parent_project_id" - filter, err := getFilterCond(f, opts.filterIncludeNulls) - if err != nil { - return nil, 0, 0, err - } - projectFilters = append(projectFilters, filter) - continue - } - - filter, err := getFilterCond(f, opts.filterIncludeNulls) - if err != nil { - return nil, 0, 0, err - } - filters = append(filters, filter) - } - - // Then return all tasks for that projects - var where builder.Cond - - if opts.search != "" { - where = db.ILIKE("title", opts.search) - - searchIndex := getTaskIndexFromSearchString(opts.search) - if searchIndex > 0 { - where = builder.Or(where, builder.Eq{"`index`": searchIndex}) - } - } - - var projectIDCond builder.Cond - var favoritesCond builder.Cond - if len(projectIDs) > 0 { - projectIDCond = builder.In("project_id", projectIDs) - } - - if hasFavoritesProject { - // All favorite tasks for that user - favCond := builder. - Select("entity_id"). - From("favorites"). - Where( - builder.And( - builder.Eq{"user_id": a.GetID()}, - builder.Eq{"kind": FavoriteKindTask}, - )) - - favoritesCond = builder.In("id", favCond) - } - - if len(reminderFilters) > 0 { - filters = append(filters, getFilterCondForSeparateTable("task_reminders", opts.filterConcat, reminderFilters)) - } - - if len(assigneeFilters) > 0 { - assigneeFilter := []builder.Cond{ - builder.In("user_id", - builder.Select("id"). - From("users"). - Where(builder.Or(assigneeFilters...)), - )} - filters = append(filters, getFilterCondForSeparateTable("task_assignees", opts.filterConcat, assigneeFilter)) - } - - if len(labelFilters) > 0 { - filters = append(filters, getFilterCondForSeparateTable("label_tasks", opts.filterConcat, labelFilters)) - } - - if len(projectFilters) > 0 { - var filtercond builder.Cond - if opts.filterConcat == filterConcatOr { - filtercond = builder.Or(projectFilters...) - } - if opts.filterConcat == filterConcatAnd { - filtercond = builder.And(projectFilters...) - } - - cond := builder.In( - "project_id", - builder. - Select("id"). - From("projects"). - Where(filtercond), - ) - filters = append(filters, cond) - } - - var filterCond builder.Cond - if len(filters) > 0 { - if opts.filterConcat == filterConcatOr { - filterCond = builder.Or(filters...) - } - if opts.filterConcat == filterConcatAnd { - filterCond = builder.And(filters...) - } - } - - limit, start := getLimitFromPageIndex(opts.page, opts.perPage) - cond := builder.And(builder.Or(projectIDCond, favoritesCond), where, filterCond) - - query := s.Where(cond) - if limit > 0 { - query = query.Limit(limit, start) - } - - tasks = []*Task{} - err = query.OrderBy(orderby).Find(&tasks) - if err != nil { - return nil, 0, 0, err - } - - queryCount := s.Where(cond) - totalItems, err = queryCount. - Count(&Task{}) - if err != nil { - return nil, 0, 0, err - } - + tasks, totalItems, err = searcher.Search(opts) return tasks, len(tasks), totalItems, nil } -func getTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskOptions) (tasks []*Task, resultCount int, totalItems int64, err error) { +func getTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskSearchOptions) (tasks []*Task, resultCount int, totalItems int64, err error) { tasks, resultCount, totalItems, err = getRawTasksForProjects(s, projects, a, opts) if err != nil { diff --git a/pkg/models/typesense.go b/pkg/models/typesense.go index 61a5020dbe8..04d8742eb30 100644 --- a/pkg/models/typesense.go +++ b/pkg/models/typesense.go @@ -248,7 +248,7 @@ type typesenseTask struct { Reminders interface{} `json:"reminders"` Assignees interface{} `json:"assignees"` Labels interface{} `json:"labels"` - //RelatedTasks interface{} `json:"related_tasks"` + //RelatedTasks interface{} `json:"related_tasks"` // TODO Attachments interface{} `json:"attachments"` Comments interface{} `json:"comments"` } From 1a82d6da44b14b988a9046ed2d8d0eb014275220 Mon Sep 17 00:00:00 2001 From: kolaente Date: Mon, 28 Aug 2023 13:26:40 +0200 Subject: [PATCH 03/11] feat(tasks): add periodic resync of updated tasks to Typesense --- pkg/initialize/init.go | 1 + pkg/migration/20230828125443.go | 46 +++++++++++ pkg/models/typesense.go | 135 ++++++++++++++++++++++++++++++-- 3 files changed, 175 insertions(+), 7 deletions(-) create mode 100644 pkg/migration/20230828125443.go diff --git a/pkg/initialize/init.go b/pkg/initialize/init.go index aa1cd69306b..37211c0970f 100644 --- a/pkg/initialize/init.go +++ b/pkg/initialize/init.go @@ -89,6 +89,7 @@ func FullInit() { models.RegisterUserDeletionCron() models.RegisterOldExportCleanupCron() openid.CleanupSavedOpenIDProviders() + models.RegisterPeriodicTypesenseResyncCron() // Start processing events go func() { diff --git a/pkg/migration/20230828125443.go b/pkg/migration/20230828125443.go new file mode 100644 index 00000000000..eed1c9edcd2 --- /dev/null +++ b/pkg/migration/20230828125443.go @@ -0,0 +1,46 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public Licensee as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public Licensee for more details. +// +// You should have received a copy of the GNU Affero General Public Licensee +// along with this program. If not, see . + +package migration + +import ( + "src.techknowlogick.com/xormigrate" + "time" + "xorm.io/xorm" +) + +type typesenseSync20230828125443 struct { + Collection string `xorm:"not null"` + SyncStartedAt time.Time `xorm:"not null"` + SyncFinishedAt time.Time `xorm:"null"` +} + +func (typesenseSync20230828125443) TableName() string { + return "typesense_sync" +} + +func init() { + migrations = append(migrations, &xormigrate.Migration{ + ID: "20230828125443", + Description: "", + Migrate: func(tx *xorm.Engine) error { + return tx.Sync2(typesenseSync20230828125443{}) + }, + Rollback: func(tx *xorm.Engine) error { + return nil + }, + }) +} diff --git a/pkg/models/typesense.go b/pkg/models/typesense.go index 04d8742eb30..af72072fe24 100644 --- a/pkg/models/typesense.go +++ b/pkg/models/typesense.go @@ -17,16 +17,27 @@ package models import ( + "code.vikunja.io/api/pkg/cron" + "code.vikunja.io/api/pkg/log" + "fmt" + "time" + "xorm.io/xorm" + "code.vikunja.io/api/pkg/config" "code.vikunja.io/api/pkg/db" "code.vikunja.io/api/pkg/user" - "fmt" "github.com/typesense/typesense-go/typesense" "github.com/typesense/typesense-go/typesense/api" "github.com/typesense/typesense-go/typesense/api/pointer" ) +type TypesenseSync struct { + Collection string `xorm:"not null"` + SyncStartedAt time.Time `xorm:"not null"` + SyncFinishedAt time.Time `xorm:"null"` +} + var typesenseClient *typesense.Client func InitTypesense() { @@ -190,16 +201,39 @@ func ReindexAllTasks() (err error) { s := db.NewSession() defer s.Close() + currentSync := &TypesenseSync{ + Collection: "tasks", + SyncStartedAt: time.Now(), + } + _, err = s.Insert(currentSync) + if err != nil { + return err + } + err = s.Find(tasks) if err != nil { return err } + err = reindexTasks(s, tasks) + if err != nil { + return err + } + + currentSync.SyncFinishedAt = time.Now() + _, err = s.Where("collection = ?", "tasks"). + Cols("sync_finished_at"). + Update(currentSync) + return +} + +func reindexTasks(s *xorm.Session, tasks map[int64]*Task) (err error) { err = addMoreInfoToTasks(s, tasks, &user.User{ID: 1}) if err != nil { return err } + typesenseTasks := []interface{}{} for _, task := range tasks { searchTask := convertTaskToTypesenseTask(task) @@ -209,12 +243,18 @@ func ReindexAllTasks() (err error) { return err } - _, err = typesenseClient.Collection("tasks"). - Documents(). - Create(searchTask) - if err != nil { - return err - } + typesenseTasks = append(typesenseTasks, searchTask) + + } + + _, err = typesenseClient.Collection("tasks"). + Documents(). + Import(typesenseTasks, &api.ImportDocumentsParams{ + Action: pointer.String("upsert"), + BatchSize: pointer.Int(100), + }) + if err != nil { + return err } return nil @@ -301,3 +341,84 @@ func convertTaskToTypesenseTask(task *Task) *typesenseTask { return tt } + +func SyncUpdatedTasksIntoTypesense() (err error) { + tasks := make(map[int64]*Task) + + s := db.NewSession() + _ = s.Begin() + defer s.Close() + + lastSync := &TypesenseSync{} + has, err := s.Where("collection = ?", "tasks"). + Get(lastSync) + if err != nil { + _ = s.Rollback() + return err + } + + if !has { + log.Errorf("[Typesense Sync] No typesense sync stats yet, please run a full index via the CLI first") + _ = s.Rollback() + return + } + + currentSync := &TypesenseSync{SyncStartedAt: time.Now()} + _, err = s.Where("collection = ?", "tasks"). + Cols("sync_started_at", "sync_finished_at"). + Update(currentSync) + if err != nil { + _ = s.Rollback() + return + } + + err = s. + Where("updated >= ?", lastSync.SyncStartedAt). + Find(tasks) + if err != nil { + _ = s.Rollback() + return + } + + if len(tasks) > 0 { + log.Debugf("[Typesense Sync] Updating %d tasks", len(tasks)) + + err = reindexTasks(s, tasks) + if err != nil { + _ = s.Rollback() + return + } + } + + if len(tasks) == 0 { + log.Debugf("[Typesense Sync] No tasks changed since the last sync, not syncing") + } + + currentSync.SyncFinishedAt = time.Now() + _, err = s.Where("collection = ?", "tasks"). + Cols("sync_finished_at"). + Update(currentSync) + if err != nil { + _ = s.Rollback() + return err + } + + return s.Commit() +} + +func RegisterPeriodicTypesenseResyncCron() { + if !config.TypesenseEnabled.GetBool() { + log.Debugf("[Typesense Sync] Typesense is disabled, not setting up sync cron") + return + } + + err := cron.Schedule("* * * * *", func() { + err := SyncUpdatedTasksIntoTypesense() + if err != nil { + log.Fatalf("[Typesense Sync] Could not sync updated tasks into typesense: %s", err) + } + }) + if err != nil { + log.Fatalf("[Typesense Sync] Could not register typesense resync cron: %s", err) + } +} From 09cfe41e4fed27e1f3829c6b95f5c53bb0f548e4 Mon Sep 17 00:00:00 2001 From: kolaente Date: Mon, 28 Aug 2023 13:34:47 +0200 Subject: [PATCH 04/11] feat(tasks): remove deleted tasks from Typesense --- pkg/models/listeners.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/pkg/models/listeners.go b/pkg/models/listeners.go index 738485ea78d..4522ab92260 100644 --- a/pkg/models/listeners.go +++ b/pkg/models/listeners.go @@ -17,7 +17,9 @@ package models import ( + "code.vikunja.io/api/pkg/config" "encoding/json" + "strconv" "code.vikunja.io/api/pkg/db" "code.vikunja.io/api/pkg/events" @@ -58,6 +60,9 @@ func RegisterListeners() { events.RegisterListener((&TaskAttachmentDeletedEvent{}).Name(), &HandleTaskUpdateLastUpdated{}) events.RegisterListener((&TaskRelationCreatedEvent{}).Name(), &HandleTaskUpdateLastUpdated{}) events.RegisterListener((&TaskRelationDeletedEvent{}).Name(), &HandleTaskUpdateLastUpdated{}) + if config.TypesenseEnabled.GetBool() { + events.RegisterListener((&TaskDeletedEvent{}).Name(), &RemoveTaskFromTypesense{}) + } } ////// @@ -474,6 +479,32 @@ func (s *HandleTaskUpdateLastUpdated) Handle(msg *message.Message) (err error) { return updateTaskLastUpdated(sess, &Task{ID: taskIDInt}) } +// RemoveTaskFromTypesense represents a listener +type RemoveTaskFromTypesense struct { +} + +// Name defines the name for the RemoveTaskFromTypesense listener +func (s *RemoveTaskFromTypesense) Name() string { + return "remove.task.from.typesense" +} + +// Handle is executed when the event RemoveTaskFromTypesense listens on is fired +func (s *RemoveTaskFromTypesense) Handle(msg *message.Message) (err error) { + event := &TaskDeletedEvent{} + err = json.Unmarshal(msg.Payload, event) + if err != nil { + return err + } + + log.Debugf("[Typesense Sync] Removing task %d from Typesense", event.Task.ID) + + _, err = typesenseClient. + Collection("tasks"). + Document(strconv.FormatInt(event.Task.ID, 10)). + Delete() + return err +} + /////// // Project Event Listeners From 2ca193e63b8d2fb2d0ecfcdbbb405e8ba99a63bd Mon Sep 17 00:00:00 2001 From: kolaente Date: Mon, 28 Aug 2023 19:10:18 +0200 Subject: [PATCH 05/11] feat(tasks): make sorting and filtering work with Typesense --- pkg/models/task_search.go | 113 ++++++++++++++++++++++++++++++++++++-- pkg/models/tasks.go | 2 +- pkg/models/typesense.go | 31 ++++++----- 3 files changed, 127 insertions(+), 19 deletions(-) diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index acb1c9cb5d8..3d1ea9e51d0 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -18,6 +18,7 @@ package models import ( "code.vikunja.io/api/pkg/db" + "code.vikunja.io/api/pkg/log" "code.vikunja.io/web" "github.com/typesense/typesense-go/typesense/api" "github.com/typesense/typesense-go/typesense/api/pointer" @@ -39,15 +40,14 @@ type dbTaskSearcher struct { hasFavoritesProject bool } -func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { +func getOrderByDBStatement(opts *taskSearchOptions) (orderby string, err error) { // Since xorm does not use placeholders for order by, it is possible to expose this with sql injection if we're directly // passing user input to the db. // As a workaround to prevent this, we check for valid column names here prior to passing it to the db. - var orderby string for i, param := range opts.sortby { // Validate the params if err := param.validate(); err != nil { - return nil, totalCount, err + return "", err } // Mysql sorts columns with null values before ones without null value. @@ -70,6 +70,16 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo } } + return +} + +func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { + + orderby, err := getOrderByDBStatement(opts) + if err != nil { + return nil, 0, err + } + // Some filters need a special treatment since they are in a separate table reminderFilters := []builder.Cond{} assigneeFilters := []builder.Cond{} @@ -242,10 +252,91 @@ type typesenseTaskSearcher struct { } func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { + + var sortbyFields []string + for i, param := range opts.sortby { + // Validate the params + if err := param.validate(); err != nil { + return nil, totalCount, err + } + + // Typesense does not allow sorting by ID, so we sort by created timestamp instead + if param.sortBy == "id" { + param.sortBy = "created" + } + + sortbyFields = append(sortbyFields, param.sortBy+"(missing_values:last):"+param.orderBy.String()) + + if i == 2 { + // Typesense supports up to 3 sorting parameters + // https://typesense.org/docs/0.25.0/api/search.html#ranking-and-sorting-parameters + break + } + } + + sortby := strings.Join(sortbyFields, ",") + projectIDStrings := []string{} for _, id := range opts.projectIDs { projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10)) } + filterBy := []string{ + "project_id: [" + strings.Join(projectIDStrings, ", ") + "]", + } + + for _, f := range opts.filters { + + filter := f.field + + switch f.comparator { + case taskFilterComparatorEquals: + filter += ":=" + case taskFilterComparatorNotEquals: + filter += ":!=" + case taskFilterComparatorGreater: + filter += ":>" + case taskFilterComparatorGreateEquals: + filter += ":>=" + case taskFilterComparatorLess: + filter += ":<" + case taskFilterComparatorLessEquals: + filter += ":<=" + case taskFilterComparatorLike: + filter += ":" + //case taskFilterComparatorIn: + //filter += "[" + case taskFilterComparatorInvalid: + // Nothing to do + default: + filter += ":=" + } + + switch f.value.(type) { + case string: + filter += f.value.(string) + case int: + filter += strconv.Itoa(f.value.(int)) + case int64: + filter += strconv.FormatInt(f.value.(int64), 10) + case bool: + if f.value.(bool) { + filter += "true" + } else { + filter += "false" + } + default: + log.Errorf("Unknown search type %s=%v", f.field, f.value) + } + + filterBy = append(filterBy, filter) + } + + //////////////// + // Actual search + + if opts.search == "" { + opts.search = "*" + } params := &api.SearchCollectionParams{ Q: opts.search, @@ -253,7 +344,11 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, Page: pointer.Int(opts.page), PerPage: pointer.Int(opts.perPage), ExhaustiveSearch: pointer.True(), - FilterBy: pointer.String("project_id: [" + strings.Join(projectIDStrings, ", ") + "]"), + FilterBy: pointer.String(strings.Join(filterBy, " && ")), + } + + if sortby != "" { + params.SortBy = pointer.String(sortby) } result, err := typesenseClient.Collection("tasks"). @@ -275,6 +370,14 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, tasks = []*Task{} - err = t.s.In("id", taskIDs).Find(&tasks) + orderby, err := getOrderByDBStatement(opts) + if err != nil { + return nil, 0, err + } + + err = t.s. + In("id", taskIDs). + OrderBy(orderby). + Find(&tasks) return tasks, int64(*result.Found), err } diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go index c4a3d91ac26..0061c53d38c 100644 --- a/pkg/models/tasks.go +++ b/pkg/models/tasks.go @@ -309,7 +309,7 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op } tasks, totalItems, err = searcher.Search(opts) - return tasks, len(tasks), totalItems, nil + return tasks, len(tasks), totalItems, err } func getTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskSearchOptions) (tasks []*Task, resultCount int, totalItems int64, err error) { diff --git a/pkg/models/typesense.go b/pkg/models/typesense.go index af72072fe24..37e1b41c2b4 100644 --- a/pkg/models/typesense.go +++ b/pkg/models/typesense.go @@ -58,14 +58,17 @@ func CreateTypesenseCollections() error { { Name: "id", Type: "string", + Sort: pointer.True(), }, { Name: "title", Type: "string", + Sort: pointer.True(), }, { Name: "description", Type: "string", + Sort: pointer.True(), }, { Name: "done", @@ -110,6 +113,7 @@ func CreateTypesenseCollections() error { { Name: "hex_color", Type: "string", + Sort: pointer.True(), }, { Name: "percent_done", @@ -118,6 +122,7 @@ func CreateTypesenseCollections() error { { Name: "identifier", Type: "string", + Sort: pointer.True(), }, { Name: "index", @@ -126,6 +131,7 @@ func CreateTypesenseCollections() error { { Name: "uid", Type: "string", + Sort: pointer.True(), }, { Name: "cover_image_attachment_id", @@ -244,7 +250,6 @@ func reindexTasks(s *xorm.Session, tasks map[int64]*Task) (err error) { } typesenseTasks = append(typesenseTasks, searchTask) - } _, err = typesenseClient.Collection("tasks"). @@ -265,14 +270,14 @@ type typesenseTask struct { Title string `json:"title"` Description string `json:"description"` Done bool `json:"done"` - DoneAt int64 `json:"done_at"` - DueDate int64 `json:"due_date"` + DoneAt *int64 `json:"done_at"` + DueDate *int64 `json:"due_date"` ProjectID int64 `json:"project_id"` RepeatAfter int64 `json:"repeat_after"` RepeatMode int `json:"repeat_mode"` Priority int64 `json:"priority"` - StartDate int64 `json:"start_date"` - EndDate int64 `json:"end_date"` + StartDate *int64 `json:"start_date"` + EndDate *int64 `json:"end_date"` HexColor string `json:"hex_color"` PercentDone float64 `json:"percent_done"` Identifier string `json:"identifier"` @@ -299,14 +304,14 @@ func convertTaskToTypesenseTask(task *Task) *typesenseTask { Title: task.Title, Description: task.Description, Done: task.Done, - DoneAt: task.DoneAt.UTC().Unix(), - DueDate: task.DueDate.UTC().Unix(), + DoneAt: pointer.Int64(task.DoneAt.UTC().Unix()), + DueDate: pointer.Int64(task.DueDate.UTC().Unix()), ProjectID: task.ProjectID, RepeatAfter: task.RepeatAfter, RepeatMode: int(task.RepeatMode), Priority: task.Priority, - StartDate: task.StartDate.UTC().Unix(), - EndDate: task.EndDate.UTC().Unix(), + StartDate: pointer.Int64(task.StartDate.UTC().Unix()), + EndDate: pointer.Int64(task.EndDate.UTC().Unix()), HexColor: task.HexColor, PercentDone: task.PercentDone, Identifier: task.Identifier, @@ -327,16 +332,16 @@ func convertTaskToTypesenseTask(task *Task) *typesenseTask { } if task.DoneAt.IsZero() { - tt.DoneAt = 0 + tt.DoneAt = nil } if task.DueDate.IsZero() { - tt.DueDate = 0 + tt.DueDate = nil } if task.StartDate.IsZero() { - tt.StartDate = 0 + tt.StartDate = nil } if task.EndDate.IsZero() { - tt.EndDate = 0 + tt.EndDate = nil } return tt From 748651447adad960650e1204b9b5f3df26426afb Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 29 Aug 2023 09:31:36 +0200 Subject: [PATCH 06/11] feat(tasks): find tasks by their identifier when searching with Typesense --- pkg/models/task_search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index 3d1ea9e51d0..0ef5a9fa264 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -340,7 +340,7 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, params := &api.SearchCollectionParams{ Q: opts.search, - QueryBy: "title, description, comments.comment", + QueryBy: "title, identifier, description, comments.comment", Page: pointer.Int(opts.page), PerPage: pointer.Int(opts.perPage), ExhaustiveSearch: pointer.True(), From d0e3062beeecc7a63a038fa55f7488290b3711db Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 29 Aug 2023 10:52:25 +0200 Subject: [PATCH 07/11] feat(tasks): allow filtering for reminders, assignees and labels with Typesense --- pkg/models/task_search.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index 0ef5a9fa264..07674a7f04f 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -286,6 +286,21 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, for _, f := range opts.filters { + if f.field == "reminders" { + f.field = "reminders.reminder" + continue + } + + if f.field == "assignees" { + f.field = "assignees.username" + continue + } + + if f.field == "labels" || f.field == "label_id" { + f.field = "labels.id" + continue + } + filter := f.field switch f.comparator { From 4f2796ac580a473aeab732553eb0b849865a5a9f Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 29 Aug 2023 11:14:36 +0200 Subject: [PATCH 08/11] fix(filters): make "in" filter comparator work with Typesense --- pkg/models/task_search.go | 55 +++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index 07674a7f04f..b33dc1dacde 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -251,6 +251,35 @@ type typesenseTaskSearcher struct { s *xorm.Session } +func convertFilterValues(value interface{}) string { + if _, is := value.([]interface{}); is { + filter := []string{} + for _, v := range value.([]interface{}) { + filter = append(filter, convertFilterValues(v)) + } + + return strings.Join(filter, ",") + } + + switch value.(type) { + case string: + return value.(string) + case int: + return strconv.Itoa(value.(int)) + case int64: + return strconv.FormatInt(value.(int64), 10) + case bool: + if value.(bool) { + return "true" + } + + return "false" + } + + log.Errorf("Unknown search type for value %v", value) + return "" +} + func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { var sortbyFields []string @@ -288,17 +317,14 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, if f.field == "reminders" { f.field = "reminders.reminder" - continue } if f.field == "assignees" { f.field = "assignees.username" - continue } if f.field == "labels" || f.field == "label_id" { f.field = "labels.id" - continue } filter := f.field @@ -318,29 +344,18 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, filter += ":<=" case taskFilterComparatorLike: filter += ":" - //case taskFilterComparatorIn: - //filter += "[" + case taskFilterComparatorIn: + filter += ":[" case taskFilterComparatorInvalid: // Nothing to do default: filter += ":=" } - switch f.value.(type) { - case string: - filter += f.value.(string) - case int: - filter += strconv.Itoa(f.value.(int)) - case int64: - filter += strconv.FormatInt(f.value.(int64), 10) - case bool: - if f.value.(bool) { - filter += "true" - } else { - filter += "false" - } - default: - log.Errorf("Unknown search type %s=%v", f.field, f.value) + filter += convertFilterValues(f.value) + + if f.comparator == taskFilterComparatorIn { + filter += "]" } filterBy = append(filterBy, filter) From c1ccbe818665ff0d4368c5c010d0c92cee44be96 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 29 Aug 2023 11:23:52 +0200 Subject: [PATCH 09/11] feat(docs): update sample config and docs about Typesense config --- config.yml.sample | 12 +++++++++ docs/content/doc/setup/config.md | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/config.yml.sample b/config.yml.sample index 159780fc8e0..fa1820bdf04 100644 --- a/config.yml.sample +++ b/config.yml.sample @@ -91,6 +91,18 @@ database: # Enable SSL/TLS for mysql connections. Options: false, true, skip-verify, preferred tls: false +typesense: + # Whether to enable the Typesense integration. If true, all tasks will be synced to the configured Typesense + # instance and all search and filtering will run through Typesense instead of only through the database. + # Typesense allows fast fulltext search including fuzzy matching support. It may return different results than + # what you'd get with a database-only search. + enabled: false + # The url to the Typesense instance you want to use. Can be hosted locally or in Typesense Cloud as long + # as Vikunja is able to reach it. + url: '' + # The Typesense API key you want to use. + apikey: '' + redis: # Whether to enable redis or not enabled: false diff --git a/docs/content/doc/setup/config.md b/docs/content/doc/setup/config.md index a1d5f227ffb..8cc0d147cea 100644 --- a/docs/content/doc/setup/config.md +++ b/docs/content/doc/setup/config.md @@ -494,6 +494,49 @@ Full path: `database.tls` Environment path: `VIKUNJA_DATABASE_TLS` +--- + +## typesense + + + +### enabled + +Whether to enable the Typesense integration. If true, all tasks will be synced to the configured Typesense +instance and all search and filtering will run through Typesense instead of only through the database. +Typesense allows fast fulltext search including fuzzy matching support. It may return different results than +what you'd get with a database-only search. + +Default: `false` + +Full path: `typesense.enabled` + +Environment path: `VIKUNJA_TYPESENSE_ENABLED` + + +### url + +The url to the Typesense instance you want to use. Can be hosted locally or in Typesense Cloud as long +as Vikunja is able to reach it. + +Default: `` + +Full path: `typesense.url` + +Environment path: `VIKUNJA_TYPESENSE_URL` + + +### apikey + +The Typesense API key you want to use. + +Default: `` + +Full path: `typesense.apikey` + +Environment path: `VIKUNJA_TYPESENSE_APIKEY` + + --- ## redis From 29317b980e68b7e10b127e7e93afff1dd56ace3e Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 29 Aug 2023 11:32:45 +0200 Subject: [PATCH 10/11] fix: lint --- .golangci.yml | 6 ++++++ pkg/cmd/index.go | 2 +- pkg/migration/20230828125443.go | 3 ++- pkg/models/listeners.go | 3 ++- pkg/models/task_search.go | 18 ++++++++++-------- pkg/models/typesense.go | 7 ++++--- 6 files changed, 25 insertions(+), 14 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index e3e51dd5e65..e0b55c189c2 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -97,3 +97,9 @@ issues: - musttag - path: pkg/models/task_collection.go text: 'append result not assigned to the same slice' + - text: 'string `label_id` has 3 occurrences, make it a constant' + linters: + - goconst + - text: 'string `labels` has 3 occurrences, make it a constant' + linters: + - goconst diff --git a/pkg/cmd/index.go b/pkg/cmd/index.go index 36eb7492db7..299ec315f18 100644 --- a/pkg/cmd/index.go +++ b/pkg/cmd/index.go @@ -1,5 +1,5 @@ // Vikunja is a to-do list application to facilitate your life. -// Copyright 2018-2023 Vikunja and contributors. All rights reserved. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public Licensee as published by diff --git a/pkg/migration/20230828125443.go b/pkg/migration/20230828125443.go index eed1c9edcd2..8958bafcb9f 100644 --- a/pkg/migration/20230828125443.go +++ b/pkg/migration/20230828125443.go @@ -17,8 +17,9 @@ package migration import ( - "src.techknowlogick.com/xormigrate" "time" + + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) diff --git a/pkg/models/listeners.go b/pkg/models/listeners.go index 4522ab92260..7f571e76899 100644 --- a/pkg/models/listeners.go +++ b/pkg/models/listeners.go @@ -17,10 +17,11 @@ package models import ( - "code.vikunja.io/api/pkg/config" "encoding/json" "strconv" + "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/db" "code.vikunja.io/api/pkg/events" "code.vikunja.io/api/pkg/log" diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index b33dc1dacde..1ac8762eed3 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -1,5 +1,5 @@ // Vikunja is a to-do list application to facilitate your life. -// Copyright 2018-2023 Vikunja and contributors. All rights reserved. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public Licensee as published by @@ -17,13 +17,14 @@ package models import ( + "strconv" + "strings" + "code.vikunja.io/api/pkg/db" "code.vikunja.io/api/pkg/log" "code.vikunja.io/web" "github.com/typesense/typesense-go/typesense/api" "github.com/typesense/typesense-go/typesense/api/pointer" - "strconv" - "strings" "xorm.io/builder" "xorm.io/xorm" @@ -73,6 +74,7 @@ func getOrderByDBStatement(opts *taskSearchOptions) (orderby string, err error) return } +//nolint:gocyclo func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { orderby, err := getOrderByDBStatement(opts) @@ -261,15 +263,15 @@ func convertFilterValues(value interface{}) string { return strings.Join(filter, ",") } - switch value.(type) { + switch v := value.(type) { case string: - return value.(string) + return v case int: - return strconv.Itoa(value.(int)) + return strconv.Itoa(v) case int64: - return strconv.FormatInt(value.(int64), 10) + return strconv.FormatInt(v, 10) case bool: - if value.(bool) { + if v { return "true" } diff --git a/pkg/models/typesense.go b/pkg/models/typesense.go index 37e1b41c2b4..43e4b409cc1 100644 --- a/pkg/models/typesense.go +++ b/pkg/models/typesense.go @@ -1,5 +1,5 @@ // Vikunja is a to-do list application to facilitate your life. -// Copyright 2018-2023 Vikunja and contributors. All rights reserved. +// Copyright 2018-2021 Vikunja and contributors. All rights reserved. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public Licensee as published by @@ -17,10 +17,11 @@ package models import ( - "code.vikunja.io/api/pkg/cron" - "code.vikunja.io/api/pkg/log" "fmt" "time" + + "code.vikunja.io/api/pkg/cron" + "code.vikunja.io/api/pkg/log" "xorm.io/xorm" "code.vikunja.io/api/pkg/config" From 02184663e551b62d50a7ab5fdce3bb5a01931a81 Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 29 Aug 2023 11:40:53 +0200 Subject: [PATCH 11/11] fix(filter): assignee search by partial username test --- pkg/models/task_collection_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/models/task_collection_test.go b/pkg/models/task_collection_test.go index 5da5631c7d1..822666f47c4 100644 --- a/pkg/models/task_collection_test.go +++ b/pkg/models/task_collection_test.go @@ -1076,7 +1076,7 @@ func TestTaskCollection_ReadAll(t *testing.T) { }, args: defaultArgs, want: []*Task{}, - wantErr: true, + wantErr: false, }, { name: "filter assignees in by id",