From d56a611be70bf7dd4e1510f4dafbb91200221805 Mon Sep 17 00:00:00 2001 From: konrad Date: Sat, 10 Oct 2020 16:53:59 +0000 Subject: [PATCH] Key-Value Storages (#674) Fix lint Add docs for keyvalue config options Use keyvalue store to cache unsplash photo results Cleanup Use keyvalue store for upload avatar Use keyvalue store for initials avatar Fix initializing metrics Use keyvalue for metrics Add IncryBy and DecrBy methods to increase or decrease a value Fix lint Return custom error if a key does not exist Init keyvalue storage Follow the keyvalue storage setting for things like cache and other Add docs Add configuration of the storage backend Add redis keyvalue storage implementation Add doc comments Add methods to use storage through the package itself Add memory implementation for keyvalue store Co-authored-by: kolaente Reviewed-on: https://kolaente.dev/vikunja/api/pulls/674 Co-Authored-By: konrad Co-Committed-By: konrad --- config.yml.sample | 18 +++- docs/content/doc/setup/config.md | 26 +++-- pkg/config/config.go | 12 +++ pkg/db/db.go | 2 +- pkg/initialize/init.go | 4 + pkg/metrics/active_users.go | 28 +----- pkg/metrics/metrics.go | 25 +++-- pkg/modules/avatar/initials/initials.go | 75 ++++++++------ pkg/modules/avatar/upload/upload.go | 54 ++++++----- pkg/modules/background/unsplash/unsplash.go | 35 ++++--- pkg/modules/keyvalue/error/error.go | 52 ++++++++++ pkg/modules/keyvalue/keyvalue.go | 72 ++++++++++++++ pkg/modules/keyvalue/memory/memory.go | 102 ++++++++++++++++++++ pkg/modules/keyvalue/redis/redis.go | 78 +++++++++++++++ pkg/red/redis.go | 4 + pkg/routes/metrics.go | 4 - 16 files changed, 469 insertions(+), 122 deletions(-) create mode 100644 pkg/modules/keyvalue/error/error.go create mode 100644 pkg/modules/keyvalue/keyvalue.go create mode 100644 pkg/modules/keyvalue/memory/memory.go create mode 100644 pkg/modules/keyvalue/redis/redis.go diff --git a/config.yml.sample b/config.yml.sample index e86b344b866..6f69f81bef0 100644 --- a/config.yml.sample +++ b/config.yml.sample @@ -61,8 +61,10 @@ database: cache: # If cache is enabled or not enabled: false - # Cache type. Possible values are memory or redis, you'll need to enable redis below when using redis - type: memory + # Cache type. Possible values are "keyvalue", "memory" or "redis". + # When choosing "keyvalue" this setting follows the one configured in the "keyvalue" section. + # When choosing "redis" you will need to configure the redis connection seperately. + type: keyvalue # When using memory this defines the maximum size an element can take maxelementsize: 1000 @@ -136,8 +138,10 @@ ratelimit: period: 60 # The max number of requests a user is allowed to do in the configured time period limit: 100 - # The store where the limit counter for each user is stored. Possible values are "memory" or "redis" - store: memory + # The store where the limit counter for each user is stored. + # Possible values are "keyvalue", "memory" or "redis". + # When choosing "keyvalue" this setting follows the one configured in the "keyvalue" section. + store: keyvalue files: # The path where files are stored @@ -205,3 +209,9 @@ backgrounds: legal: imprinturl: privacyurl: + +# Key Value Storage settings +# The Key Value Storage is used for different kinds of things like metrics and a few cache systems. +keyvalue: + # The type of the storage backend. Can be either "memory" or "redis". If "redis" is chosen it needs to be configured seperately. + type: "memory" diff --git a/docs/content/doc/setup/config.md b/docs/content/doc/setup/config.md index 283e8c82b4d..5a645cc1166 100644 --- a/docs/content/doc/setup/config.md +++ b/docs/content/doc/setup/config.md @@ -104,8 +104,10 @@ database: cache: # If cache is enabled or not enabled: false - # Cache type. Possible values are memory or redis, you'll need to enable redis below when using redis - type: memory + # Cache type. Possible values are "keyvalue", "memory" or "redis". + # When choosing "keyvalue" this setting follows the one configured in the "keyvalue" section. + # When choosing "redis" you will need to configure the redis connection seperately. + type: keyvalue # When using memory this defines the maximum size an element can take maxelementsize: 1000 @@ -169,7 +171,7 @@ log: http: "stdout" # Echo has its own logging which usually is unnessecary, which is why it is disabled by default. Possible values are stdout, stderr, file or off to disable standard logging. echo: "off" - + ratelimit: # whether or not to enable the rate limit enabled: false @@ -179,8 +181,10 @@ ratelimit: period: 60 # The max number of requests a user is allowed to do in the configured time period limit: 100 - # The store where the limit counter for each user is stored. Possible values are "memory" or "redis" - store: memory + # The store where the limit counter for each user is stored. + # Possible values are "keyvalue", "memory" or "redis". + # When choosing "keyvalue" this setting follows the one configured in the "keyvalue" section. + store: keyvalue files: # The path where files are stored @@ -234,7 +238,7 @@ backgrounds: unsplash: # Whether to enable setting backgrounds from unsplash as list backgrounds enabled: false - # You need to create an application for your installation at https://unsplash.com/oauth/applications/new + # You need to create an application for your installation at https://unsplash.com/oauth/applications/new # and set the access token below. accesstoken: # The unsplash application id is only used for pingback and required as per their api guidelines. @@ -246,6 +250,12 @@ backgrounds: # Legal urls # Will be shown in the frontend if configured here legal: - imprinturl: - privacyurl: + imprinturl: + privacyurl: + +# Key Value Storage settings +# The Key Value Storage is used for different kinds of things like metrics and a few cache systems. +keyvalue: + # The type of the storage backend. Can be either "memory" or "redis". If "redis" is chosen it needs to be configured seperately. + type: "memory" {{< /highlight >}} diff --git a/pkg/config/config.go b/pkg/config/config.go index aa861f073c9..f17f7b26def 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -124,6 +124,8 @@ const ( BackgroundsUnsplashEnabled Key = `backgrounds.providers.unsplash.enabled` BackgroundsUnsplashAccessToken Key = `backgrounds.providers.unsplash.accesstoken` BackgroundsUnsplashApplicationID Key = `backgrounds.providers.unsplash.applicationid` + + KeyvalueType Key = `keyvalue.type` ) // GetString returns a string config value @@ -277,6 +279,8 @@ func InitDefaultConfig() { BackgroundsEnabled.setDefault(true) BackgroundsUploadEnabled.setDefault(true) BackgroundsUnsplashEnabled.setDefault(false) + // Key Value + KeyvalueType.setDefault("memory") } // InitConfig initializes the config, sets defaults etc. @@ -310,6 +314,14 @@ func InitConfig() { return } + if CacheType.GetString() == "keyvalue" { + CacheType.Set(KeyvalueType.GetString()) + } + + if RateLimitStore.GetString() == "keyvalue" { + RateLimitStore.Set(KeyvalueType.GetString()) + } + log.Printf("Using config file: %s", viper.ConfigFileUsed()) } diff --git a/pkg/db/db.go b/pkg/db/db.go index 73251251f06..5224232a6f7 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -90,7 +90,7 @@ func CreateDBEngine() (engine *xorm.Engine, err error) { cacher := caches.NewLRUCacher(caches.NewMemoryStore(), config.CacheMaxElementSize.GetInt()) engine.SetDefaultCacher(cacher) case "redis": - cacher := xrc.NewRedisCacher(config.RedisEnabled.GetString(), config.RedisPassword.GetString(), xrc.DEFAULT_EXPIRATION, engine.Logger()) + cacher := xrc.NewRedisCacher(config.RedisHost.GetString(), config.RedisPassword.GetString(), xrc.DEFAULT_EXPIRATION, engine.Logger()) engine.SetDefaultCacher(cacher) default: log.Info("Did not find a valid cache type. Caching disabled. Please refer to the docs for poosible cache types.") diff --git a/pkg/initialize/init.go b/pkg/initialize/init.go index dc9f89e822f..e08a9fd98bf 100644 --- a/pkg/initialize/init.go +++ b/pkg/initialize/init.go @@ -23,6 +23,7 @@ import ( "code.vikunja.io/api/pkg/mail" "code.vikunja.io/api/pkg/migration" "code.vikunja.io/api/pkg/models" + "code.vikunja.io/api/pkg/modules/keyvalue" migrator "code.vikunja.io/api/pkg/modules/migration" "code.vikunja.io/api/pkg/red" "code.vikunja.io/api/pkg/user" @@ -36,6 +37,9 @@ func LightInit() { // Init redis red.InitRedis() + // Init keyvalue store + keyvalue.InitStorage() + // Set logger log.InitLogger() } diff --git a/pkg/metrics/active_users.go b/pkg/metrics/active_users.go index af3c72821d7..328be356df8 100644 --- a/pkg/metrics/active_users.go +++ b/pkg/metrics/active_users.go @@ -17,10 +17,9 @@ package metrics import ( - "bytes" "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/modules/keyvalue" "code.vikunja.io/web" - "encoding/gob" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "sync" @@ -91,36 +90,19 @@ func SetUserActive(a web.Auth) (err error) { // getActiveUsers returns the active users from redis func getActiveUsers() (users activeUsersMap, err error) { - - activeUsersR, err := r.Get(ActiveUsersKey).Bytes() - if err != nil { - if err.Error() == "redis: nil" { - return users, nil - } - return - } - - var b bytes.Buffer - _, err = b.Write(activeUsersR) + u, err := keyvalue.Get(ActiveUsersKey) if err != nil { return nil, err } - d := gob.NewDecoder(&b) - if err := d.Decode(&users); err != nil { - return nil, err - } + + users = u.(activeUsersMap) return } // PushActiveUsers pushed the content of the activeUsers map to redis func PushActiveUsers() (err error) { - var b bytes.Buffer - e := gob.NewEncoder(&b) activeUsers.mutex.Lock() defer activeUsers.mutex.Unlock() - if err := e.Encode(activeUsers.users); err != nil { - return err - } - return r.Set(ActiveUsersKey, b.Bytes(), 0).Err() + return keyvalue.Put(ActiveUsersKey, activeUsers.users) } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 715543ef871..0d1a953d964 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -19,14 +19,12 @@ package metrics import ( "code.vikunja.io/api/pkg/config" "code.vikunja.io/api/pkg/log" - "code.vikunja.io/api/pkg/red" - "github.com/go-redis/redis/v7" + "code.vikunja.io/api/pkg/modules/keyvalue" + e "code.vikunja.io/api/pkg/modules/keyvalue/error" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) -var r *redis.Client - const ( // ListCountKey is the name of the key in which we save the list count ListCountKey = `listcount` @@ -46,8 +44,6 @@ const ( // InitMetrics Initializes the metrics func InitMetrics() { - r = red.GetRedis() - // init active users, sometimes we'll have garbage from previous runs in redis instead if err := PushActiveUsers(); err != nil { log.Fatalf("Could not set initial count for active users, error was %s", err) @@ -101,18 +97,21 @@ func InitMetrics() { // GetCount returns the current count from redis func GetCount(key string) (count int64, err error) { - count, err = r.Get(key).Int64() - if err != nil && err.Error() != "redis: nil" { - return + cnt, err := keyvalue.Get(key) + if err != nil { + if e.IsErrValueNotFoundForKey(err) { + return 0, nil + } + return 0, err } - err = nil + count = cnt.(int64) return } // SetCount sets the list count to a given value func SetCount(count int64, key string) error { - return r.Set(key, count, 0).Err() + return keyvalue.Put(key, count) } // UpdateCount updates a count with a given amount @@ -121,13 +120,13 @@ func UpdateCount(update int64, key string) { return } if update > 0 { - err := r.IncrBy(key, update).Err() + err := keyvalue.IncrBy(key, update) if err != nil { log.Error(err.Error()) } } if update < 0 { - err := r.DecrBy(key, update).Err() + err := keyvalue.DecrBy(key, update) if err != nil { log.Error(err.Error()) } diff --git a/pkg/modules/avatar/initials/initials.go b/pkg/modules/avatar/initials/initials.go index 727c8b0dd52..0defdc66aaa 100644 --- a/pkg/modules/avatar/initials/initials.go +++ b/pkg/modules/avatar/initials/initials.go @@ -17,14 +17,12 @@ package initials import ( + "bytes" "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/modules/keyvalue" + e "code.vikunja.io/api/pkg/modules/keyvalue/error" "code.vikunja.io/api/pkg/user" "github.com/disintegration/imaging" - "strconv" - "strings" - "sync" - - "bytes" "github.com/golang/freetype/truetype" "golang.org/x/image/font" "golang.org/x/image/font/gofont/goregular" @@ -33,6 +31,8 @@ import ( "image/color" "image/draw" "image/png" + "strconv" + "strings" ) // Provider represents the provider implementation of the initials provider @@ -51,19 +51,8 @@ var ( {121, 134, 203, 255}, {241, 185, 29, 255}, } - - // Contain the created avatars with a size of defaultSize - cache = map[int64]*image.RGBA64{} - cacheLock = sync.Mutex{} - cacheResized = map[string][]byte{} - cacheResizedLock = sync.Mutex{} ) -func init() { - cache = make(map[int64]*image.RGBA64) - cacheResized = make(map[string][]byte) -} - const ( dpi = 72 defaultSize = 1024 @@ -124,10 +113,26 @@ func drawImage(text rune, bg *color.RGBA) (img *image.RGBA64, err error) { return img, err } +func getCacheKey(prefix string, keys ...int64) string { + result := "avatar_initials_" + prefix + for i, key := range keys { + result += strconv.Itoa(int(key)) + if i < len(keys) { + result += "_" + } + } + return result +} + func getAvatarForUser(u *user.User) (fullSizeAvatar *image.RGBA64, err error) { - var cached bool - fullSizeAvatar, cached = cache[u.ID] - if !cached { + cacheKey := getCacheKey("full", u.ID) + + a, err := keyvalue.Get(cacheKey) + if err != nil && !e.IsErrValueNotFoundForKey(err) { + return nil, err + } + + if err != nil && e.IsErrValueNotFoundForKey(err) { log.Debugf("Initials avatar for user %d not cached, creating...", u.ID) firstRune := []rune(strings.ToUpper(u.Username))[0] bg := avatarBgColors[int(u.ID)%len(avatarBgColors)] // Random color based on the user id @@ -136,21 +141,27 @@ func getAvatarForUser(u *user.User) (fullSizeAvatar *image.RGBA64, err error) { if err != nil { return nil, err } - cacheLock.Lock() - cache[u.ID] = fullSizeAvatar - cacheLock.Unlock() + err = keyvalue.Put(cacheKey, fullSizeAvatar) + if err != nil { + return nil, err + } + } else { + fullSizeAvatar = a.(*image.RGBA64) } - return fullSizeAvatar, err + return fullSizeAvatar, nil } // GetAvatar returns an initials avatar for a user func (p *Provider) GetAvatar(u *user.User, size int64) (avatar []byte, mimeType string, err error) { + cacheKey := getCacheKey("resized", u.ID, size) - var cached bool - cacheKey := strconv.Itoa(int(u.ID)) + "_" + strconv.Itoa(int(size)) - avatar, cached = cacheResized[cacheKey] - if !cached { + a, err := keyvalue.Get(cacheKey) + if err != nil && !e.IsErrValueNotFoundForKey(err) { + return nil, "", err + } + + if err != nil && e.IsErrValueNotFoundForKey(err) { log.Debugf("Initials avatar for user %d and size %d not cached, creating...", u.ID, size) fullAvatar, err := getAvatarForUser(u) if err != nil { @@ -164,12 +175,14 @@ func (p *Provider) GetAvatar(u *user.User, size int64) (avatar []byte, mimeType return nil, "", err } avatar = buf.Bytes() - cacheResizedLock.Lock() - cacheResized[cacheKey] = avatar - cacheResizedLock.Unlock() + err = keyvalue.Put(cacheKey, avatar) + if err != nil { + return nil, "", err + } } else { + avatar = a.([]byte) log.Debugf("Serving initials avatar for user %d and size %d from cache", u.ID, size) } - return avatar, "image/png", err + return avatar, "image/png", nil } diff --git a/pkg/modules/avatar/upload/upload.go b/pkg/modules/avatar/upload/upload.go index b2dd5deca68..357bc287d77 100644 --- a/pkg/modules/avatar/upload/upload.go +++ b/pkg/modules/avatar/upload/upload.go @@ -20,25 +20,16 @@ import ( "bytes" "code.vikunja.io/api/pkg/files" "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/modules/keyvalue" + e "code.vikunja.io/api/pkg/modules/keyvalue/error" "code.vikunja.io/api/pkg/user" "github.com/disintegration/imaging" "image" "image/png" "io/ioutil" - "sync" + "strconv" ) -var ( - // This is a map with a map so we're able to clear all cached avatar (in all sizes) for one user at once - // The first map has as key the user id, the second one has the size as key - resizedCache = map[int64]map[int64][]byte{} - resizedCacheLock = sync.Mutex{} -) - -func init() { - resizedCache = make(map[int64]map[int64][]byte) -} - // Provider represents the upload avatar provider type Provider struct { } @@ -46,19 +37,32 @@ type Provider struct { // GetAvatar returns an uploaded user avatar func (p *Provider) GetAvatar(u *user.User, size int64) (avatar []byte, mimeType string, err error) { - a, cached := resizedCache[u.ID] - if cached { + cacheKey := "avatar_upload_" + strconv.Itoa(int(u.ID)) + + ai, err := keyvalue.Get(cacheKey) + if err != nil && !e.IsErrValueNotFoundForKey(err) { + return nil, "", err + } + + var cached map[int64][]byte + + if ai != nil { + cached = ai.(map[int64][]byte) + } + + if err != nil && e.IsErrValueNotFoundForKey(err) { + // Nothing ever cached for this user so we need to create the size map to avoid panics + cached = make(map[int64][]byte) + } else { + a := ai.(map[int64][]byte) if a != nil && a[size] != nil { log.Debugf("Serving uploaded avatar for user %d and size %d from cache.", u.ID, size) return a[size], "", nil } // This means we have a map for the user, but nothing in it. if a == nil { - resizedCache[u.ID] = make(map[int64][]byte) + cached = make(map[int64][]byte) } - } else { - // Nothing ever cached for this user so we need to create the size map to avoid panics - resizedCache[u.ID] = make(map[int64][]byte) } log.Debugf("Uploaded avatar for user %d and size %d not cached, resizing and caching.", u.ID, size) @@ -84,15 +88,17 @@ func (p *Provider) GetAvatar(u *user.User, size int64) (avatar []byte, mimeType } avatar, err = ioutil.ReadAll(buf) - resizedCacheLock.Lock() - resizedCache[u.ID][size] = avatar - resizedCacheLock.Unlock() + if err != nil { + return nil, "", err + } + cached[size] = avatar + err = keyvalue.Put(cacheKey, cached) return avatar, f.Mime, err } // InvalidateCache invalidates the avatar cache for a user func InvalidateCache(u *user.User) { - resizedCacheLock.Lock() - delete(resizedCache, u.ID) - resizedCacheLock.Unlock() + if err := keyvalue.Del("avatar_upload_" + strconv.Itoa(int(u.ID))); err != nil { + log.Errorf("Could not invalidate upload avatar cache for user %d, error was %s", u.ID, err) + } } diff --git a/pkg/modules/background/unsplash/unsplash.go b/pkg/modules/background/unsplash/unsplash.go index f2c6db42e2b..cacd404b26b 100644 --- a/pkg/modules/background/unsplash/unsplash.go +++ b/pkg/modules/background/unsplash/unsplash.go @@ -23,6 +23,8 @@ import ( "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/models" "code.vikunja.io/api/pkg/modules/background" + "code.vikunja.io/api/pkg/modules/keyvalue" + e "code.vikunja.io/api/pkg/modules/keyvalue/error" "code.vikunja.io/web" "encoding/json" "net/http" @@ -32,7 +34,10 @@ import ( "time" ) -const unsplashAPIURL = `https://api.unsplash.com/` +const ( + unsplashAPIURL = `https://api.unsplash.com/` + cachePrefix = `unsplash_photo_` +) // Provider represents an unsplash image provider type Provider struct { @@ -72,10 +77,6 @@ type Photo struct { } `json:"links"` } -// Very simple caching method - pretty much only used to retain information when saving an image -// FIXME: Should use a proper cache -var photos map[string]*Photo - // We're caching the initial collection to save a few api requests as this is retrieved every time a // user opens the settings page. type initialCollection struct { @@ -87,10 +88,6 @@ type initialCollection struct { var emptySearchResult *initialCollection -func init() { - photos = make(map[string]*Photo) -} - func doGet(url string, result ...interface{}) (err error) { req, err := http.NewRequest("GET", unsplashAPIURL+url, nil) if err != nil { @@ -120,15 +117,21 @@ func getImageID(fullURL string) string { // Gets an unsplash photo either from cache or directly from the unsplash api func getUnsplashPhotoInfoByID(photoID string) (photo *Photo, err error) { - var exists bool - photo, exists = photos[photoID] - if !exists { + + p, err := keyvalue.Get(cachePrefix + photoID) + if err != nil && !e.IsErrValueNotFoundForKey(err) { + return nil, err + } + + if err != nil && e.IsErrValueNotFoundForKey(err) { log.Debugf("Image information for unsplash photo %s not cached, requesting from unsplash...", photoID) photo = &Photo{} err = doGet("photos/"+photoID, photo) if err != nil { return } + } else { + photo = p.(*Photo) } return } @@ -180,7 +183,9 @@ func (p *Provider) Search(search string, page int64) (result []*background.Image AuthorName: p.User.Name, }, }) - photos[p.ID] = p + if err := keyvalue.Put(cachePrefix+p.ID, p); err != nil { + return nil, err + } } // Put the collection in cache @@ -213,7 +218,9 @@ func (p *Provider) Search(search string, page int64) (result []*background.Image AuthorName: p.User.Name, }, }) - photos[p.ID] = p + if err := keyvalue.Put(cachePrefix+p.ID, p); err != nil { + return nil, err + } } return diff --git a/pkg/modules/keyvalue/error/error.go b/pkg/modules/keyvalue/error/error.go new file mode 100644 index 00000000000..fbaac3ab632 --- /dev/null +++ b/pkg/modules/keyvalue/error/error.go @@ -0,0 +1,52 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2020 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package error + +import "fmt" + +// ErrValueNotFoundForKey represents an error where a key could not be found +type ErrValueNotFoundForKey struct { + Key string +} + +// Error is the error implementation +func (e *ErrValueNotFoundForKey) Error() string { + return fmt.Sprintf("could not find value for key %s", e.Key) +} + +// IsErrValueNotFoundForKey checks if an error is ErrValueNotFoundForKey +func IsErrValueNotFoundForKey(err error) bool { + _, is := err.(*ErrValueNotFoundForKey) + return is +} + +// ErrValueHasWrongType represents an error where a value saved at key has the wrong value +type ErrValueHasWrongType struct { + Key string + ExpectedValue string +} + +// Error is the error implementation +func (e *ErrValueHasWrongType) Error() string { + return fmt.Sprintf("value at key %s has the wrong value, expexted was %s", e.Key, e.ExpectedValue) +} + +// IsErrValueHasWrongType checks if an error is ErrValueHasWrongType +func IsErrValueHasWrongType(err error) bool { + _, is := err.(*ErrValueHasWrongType) + return is +} diff --git a/pkg/modules/keyvalue/keyvalue.go b/pkg/modules/keyvalue/keyvalue.go new file mode 100644 index 00000000000..df798640d8d --- /dev/null +++ b/pkg/modules/keyvalue/keyvalue.go @@ -0,0 +1,72 @@ +// Copyright 2020 Vikunja and contriubtors. All rights reserved. +// +// This file is part of Vikunja. +// +// Vikunja is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Vikunja is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Vikunja. If not, see . + +package keyvalue + +import ( + "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/modules/keyvalue/memory" + "code.vikunja.io/api/pkg/modules/keyvalue/redis" +) + +// Storage defines an interface for saving key-value pairs +type Storage interface { + Put(key string, value interface{}) (err error) + Get(key string) (value interface{}, err error) + Del(key string) (err error) + IncrBy(key string, update int64) (err error) + DecrBy(key string, update int64) (err error) +} + +var store Storage + +// InitStorage initializes the configured storage backend +func InitStorage() { + switch config.KeyvalueType.GetString() { + case "redis": + store = redis.NewStorage() + case "memory": + fallthrough + default: + store = memory.NewStorage() + } +} + +// Put puts a value in the storage backend +func Put(key string, value interface{}) error { + return store.Put(key, value) +} + +// Get returns a value from a storage backend +func Get(key string) (value interface{}, err error) { + return store.Get(key) +} + +// Del removes a save value from a storage backend +func Del(key string) (err error) { + return store.Del(key) +} + +// IncrBy increases a value at key by the amount in update +func IncrBy(key string, update int64) (err error) { + return store.IncrBy(key, update) +} + +// DecrBy increases a value at key by the amount in update +func DecrBy(key string, update int64) (err error) { + return store.DecrBy(key, update) +} diff --git a/pkg/modules/keyvalue/memory/memory.go b/pkg/modules/keyvalue/memory/memory.go new file mode 100644 index 00000000000..043573f7d29 --- /dev/null +++ b/pkg/modules/keyvalue/memory/memory.go @@ -0,0 +1,102 @@ +// Copyright 2020 Vikunja and contriubtors. All rights reserved. +// +// This file is part of Vikunja. +// +// Vikunja is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Vikunja is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Vikunja. If not, see . + +package memory + +import ( + e "code.vikunja.io/api/pkg/modules/keyvalue/error" + "sync" +) + +// Storage is the memory implementation of a storage backend +type Storage struct { + store map[string]interface{} + mutex sync.Mutex +} + +// NewStorage creates a new memory storage +func NewStorage() *Storage { + s := &Storage{} + s.store = make(map[string]interface{}) + return s +} + +// Put puts a value into the memory storage +func (s *Storage) Put(key string, value interface{}) (err error) { + s.mutex.Lock() + defer s.mutex.Unlock() + s.store[key] = value + return nil +} + +// Get retrieves a saved value from memory storage +func (s *Storage) Get(key string) (value interface{}, err error) { + s.mutex.Lock() + defer s.mutex.Unlock() + + var exists bool + value, exists = s.store[key] + if !exists { + return nil, &e.ErrValueNotFoundForKey{Key: key} + } + + return +} + +// Del removes a saved value from a memory storage +func (s *Storage) Del(key string) (err error) { + s.mutex.Lock() + defer s.mutex.Unlock() + delete(s.store, key) + return nil +} + +// IncrBy increases the value saved at key by the amount provided through update +// It assumes the value saved for the key either does not exist or has a type of int64 +func (s *Storage) IncrBy(key string, update int64) (err error) { + s.mutex.Lock() + defer s.mutex.Unlock() + + v, err := s.Get(key) + if err != nil && !e.IsErrValueNotFoundForKey(err) { + return err + } + val, is := v.(int64) + if !is { + return &e.ErrValueHasWrongType{Key: key, ExpectedValue: "int64"} + } + s.store[key] = val + update + return nil +} + +// DecrBy decreases the value saved at key by the amount provided through update +// It assumes the value saved for the key either does not exist or has a type of int64 +func (s *Storage) DecrBy(key string, update int64) (err error) { + s.mutex.Lock() + defer s.mutex.Unlock() + + v, err := s.Get(key) + if err != nil && !e.IsErrValueNotFoundForKey(err) { + return err + } + val, is := v.(int64) + if !is { + return &e.ErrValueHasWrongType{Key: key, ExpectedValue: "int64"} + } + s.store[key] = val - update + return nil +} diff --git a/pkg/modules/keyvalue/redis/redis.go b/pkg/modules/keyvalue/redis/redis.go new file mode 100644 index 00000000000..1422ceb1530 --- /dev/null +++ b/pkg/modules/keyvalue/redis/redis.go @@ -0,0 +1,78 @@ +// Copyright 2020 Vikunja and contriubtors. All rights reserved. +// +// This file is part of Vikunja. +// +// Vikunja is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Vikunja is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Vikunja. If not, see . + +package redis + +import ( + e "code.vikunja.io/api/pkg/modules/keyvalue/error" + "code.vikunja.io/api/pkg/red" + "encoding/json" + "github.com/go-redis/redis/v7" +) + +// Storage is a redis implementation of a keyvalue storage +type Storage struct { + client *redis.Client +} + +// NewStorage creates a new redis key value storage +func NewStorage() *Storage { + red.InitRedis() + + return &Storage{ + client: red.GetRedis(), + } +} + +// Put puts a value into redis +func (s *Storage) Put(key string, value interface{}) (err error) { + v, err := json.Marshal(value) + if err != nil { + return err + } + + return s.client.Set(key, v, 0).Err() +} + +// Get retrieves a saved value from redis +func (s *Storage) Get(key string) (value interface{}, err error) { + b, err := s.client.Get(key).Bytes() + if err != nil { + if err == redis.Nil { + return nil, &e.ErrValueNotFoundForKey{Key: key} + } + return nil, err + } + + err = json.Unmarshal(b, value) + return +} + +// Del removed a value from redis +func (s *Storage) Del(key string) (err error) { + return s.client.Del(key).Err() +} + +// IncrBy increases the value saved at key by the amount provided through update +func (s *Storage) IncrBy(key string, update int64) (err error) { + return s.client.IncrBy(key, update).Err() +} + +// DecrBy decreases the value saved at key by the amount provided through update +func (s *Storage) DecrBy(key string, update int64) (err error) { + return s.client.DecrBy(key, update).Err() +} diff --git a/pkg/red/redis.go b/pkg/red/redis.go index 016fb587310..4ae7e9a8462 100644 --- a/pkg/red/redis.go +++ b/pkg/red/redis.go @@ -26,6 +26,10 @@ var r *redis.Client // InitRedis initializes a redis connection func InitRedis() { + if r != nil { + return + } + if !config.RedisEnabled.GetBool() { return } diff --git a/pkg/routes/metrics.go b/pkg/routes/metrics.go index a0a9031c797..61722f0699f 100644 --- a/pkg/routes/metrics.go +++ b/pkg/routes/metrics.go @@ -33,10 +33,6 @@ func setupMetrics(a *echo.Group) { return } - if !config.RedisEnabled.GetBool() { - log.Fatal("You have to enable redis in order to use metrics") - } - metrics.InitMetrics() type countable struct {