Key-Value Storages (#674)
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
Fix lint Add docs for keyvalue config options Use keyvalue store to cache unsplash photo results Cleanup Use keyvalue store for upload avatar Use keyvalue store for initials avatar Fix initializing metrics Use keyvalue for metrics Add IncryBy and DecrBy methods to increase or decrease a value Fix lint Return custom error if a key does not exist Init keyvalue storage Follow the keyvalue storage setting for things like cache and other Add docs Add configuration of the storage backend Add redis keyvalue storage implementation Add doc comments Add methods to use storage through the package itself Add memory implementation for keyvalue store Co-authored-by: kolaente <k@knt.li> Reviewed-on: vikunja/api#674 Co-Authored-By: konrad <konrad@kola-entertainments.de> Co-Committed-By: konrad <konrad@kola-entertainments.de>
This commit is contained in:
parent
bf5d8af3f6
commit
d56a611be7
@ -61,8 +61,10 @@ database:
|
||||
cache:
|
||||
# If cache is enabled or not
|
||||
enabled: false
|
||||
# Cache type. Possible values are memory or redis, you'll need to enable redis below when using redis
|
||||
type: memory
|
||||
# Cache type. Possible values are "keyvalue", "memory" or "redis".
|
||||
# When choosing "keyvalue" this setting follows the one configured in the "keyvalue" section.
|
||||
# When choosing "redis" you will need to configure the redis connection seperately.
|
||||
type: keyvalue
|
||||
# When using memory this defines the maximum size an element can take
|
||||
maxelementsize: 1000
|
||||
|
||||
@ -136,8 +138,10 @@ ratelimit:
|
||||
period: 60
|
||||
# The max number of requests a user is allowed to do in the configured time period
|
||||
limit: 100
|
||||
# The store where the limit counter for each user is stored. Possible values are "memory" or "redis"
|
||||
store: memory
|
||||
# The store where the limit counter for each user is stored.
|
||||
# Possible values are "keyvalue", "memory" or "redis".
|
||||
# When choosing "keyvalue" this setting follows the one configured in the "keyvalue" section.
|
||||
store: keyvalue
|
||||
|
||||
files:
|
||||
# The path where files are stored
|
||||
@ -205,3 +209,9 @@ backgrounds:
|
||||
legal:
|
||||
imprinturl:
|
||||
privacyurl:
|
||||
|
||||
# Key Value Storage settings
|
||||
# The Key Value Storage is used for different kinds of things like metrics and a few cache systems.
|
||||
keyvalue:
|
||||
# The type of the storage backend. Can be either "memory" or "redis". If "redis" is chosen it needs to be configured seperately.
|
||||
type: "memory"
|
||||
|
@ -104,8 +104,10 @@ database:
|
||||
cache:
|
||||
# If cache is enabled or not
|
||||
enabled: false
|
||||
# Cache type. Possible values are memory or redis, you'll need to enable redis below when using redis
|
||||
type: memory
|
||||
# Cache type. Possible values are "keyvalue", "memory" or "redis".
|
||||
# When choosing "keyvalue" this setting follows the one configured in the "keyvalue" section.
|
||||
# When choosing "redis" you will need to configure the redis connection seperately.
|
||||
type: keyvalue
|
||||
# When using memory this defines the maximum size an element can take
|
||||
maxelementsize: 1000
|
||||
|
||||
@ -169,7 +171,7 @@ log:
|
||||
http: "stdout"
|
||||
# Echo has its own logging which usually is unnessecary, which is why it is disabled by default. Possible values are stdout, stderr, file or off to disable standard logging.
|
||||
echo: "off"
|
||||
|
||||
|
||||
ratelimit:
|
||||
# whether or not to enable the rate limit
|
||||
enabled: false
|
||||
@ -179,8 +181,10 @@ ratelimit:
|
||||
period: 60
|
||||
# The max number of requests a user is allowed to do in the configured time period
|
||||
limit: 100
|
||||
# The store where the limit counter for each user is stored. Possible values are "memory" or "redis"
|
||||
store: memory
|
||||
# The store where the limit counter for each user is stored.
|
||||
# Possible values are "keyvalue", "memory" or "redis".
|
||||
# When choosing "keyvalue" this setting follows the one configured in the "keyvalue" section.
|
||||
store: keyvalue
|
||||
|
||||
files:
|
||||
# The path where files are stored
|
||||
@ -234,7 +238,7 @@ backgrounds:
|
||||
unsplash:
|
||||
# Whether to enable setting backgrounds from unsplash as list backgrounds
|
||||
enabled: false
|
||||
# You need to create an application for your installation at https://unsplash.com/oauth/applications/new
|
||||
# You need to create an application for your installation at https://unsplash.com/oauth/applications/new
|
||||
# and set the access token below.
|
||||
accesstoken:
|
||||
# The unsplash application id is only used for pingback and required as per their api guidelines.
|
||||
@ -246,6 +250,12 @@ backgrounds:
|
||||
# Legal urls
|
||||
# Will be shown in the frontend if configured here
|
||||
legal:
|
||||
imprinturl:
|
||||
privacyurl:
|
||||
imprinturl:
|
||||
privacyurl:
|
||||
|
||||
# Key Value Storage settings
|
||||
# The Key Value Storage is used for different kinds of things like metrics and a few cache systems.
|
||||
keyvalue:
|
||||
# The type of the storage backend. Can be either "memory" or "redis". If "redis" is chosen it needs to be configured seperately.
|
||||
type: "memory"
|
||||
{{< /highlight >}}
|
||||
|
@ -124,6 +124,8 @@ const (
|
||||
BackgroundsUnsplashEnabled Key = `backgrounds.providers.unsplash.enabled`
|
||||
BackgroundsUnsplashAccessToken Key = `backgrounds.providers.unsplash.accesstoken`
|
||||
BackgroundsUnsplashApplicationID Key = `backgrounds.providers.unsplash.applicationid`
|
||||
|
||||
KeyvalueType Key = `keyvalue.type`
|
||||
)
|
||||
|
||||
// GetString returns a string config value
|
||||
@ -277,6 +279,8 @@ func InitDefaultConfig() {
|
||||
BackgroundsEnabled.setDefault(true)
|
||||
BackgroundsUploadEnabled.setDefault(true)
|
||||
BackgroundsUnsplashEnabled.setDefault(false)
|
||||
// Key Value
|
||||
KeyvalueType.setDefault("memory")
|
||||
}
|
||||
|
||||
// InitConfig initializes the config, sets defaults etc.
|
||||
@ -310,6 +314,14 @@ func InitConfig() {
|
||||
return
|
||||
}
|
||||
|
||||
if CacheType.GetString() == "keyvalue" {
|
||||
CacheType.Set(KeyvalueType.GetString())
|
||||
}
|
||||
|
||||
if RateLimitStore.GetString() == "keyvalue" {
|
||||
RateLimitStore.Set(KeyvalueType.GetString())
|
||||
}
|
||||
|
||||
log.Printf("Using config file: %s", viper.ConfigFileUsed())
|
||||
}
|
||||
|
||||
|
@ -90,7 +90,7 @@ func CreateDBEngine() (engine *xorm.Engine, err error) {
|
||||
cacher := caches.NewLRUCacher(caches.NewMemoryStore(), config.CacheMaxElementSize.GetInt())
|
||||
engine.SetDefaultCacher(cacher)
|
||||
case "redis":
|
||||
cacher := xrc.NewRedisCacher(config.RedisEnabled.GetString(), config.RedisPassword.GetString(), xrc.DEFAULT_EXPIRATION, engine.Logger())
|
||||
cacher := xrc.NewRedisCacher(config.RedisHost.GetString(), config.RedisPassword.GetString(), xrc.DEFAULT_EXPIRATION, engine.Logger())
|
||||
engine.SetDefaultCacher(cacher)
|
||||
default:
|
||||
log.Info("Did not find a valid cache type. Caching disabled. Please refer to the docs for poosible cache types.")
|
||||
|
@ -23,6 +23,7 @@ import (
|
||||
"code.vikunja.io/api/pkg/mail"
|
||||
"code.vikunja.io/api/pkg/migration"
|
||||
"code.vikunja.io/api/pkg/models"
|
||||
"code.vikunja.io/api/pkg/modules/keyvalue"
|
||||
migrator "code.vikunja.io/api/pkg/modules/migration"
|
||||
"code.vikunja.io/api/pkg/red"
|
||||
"code.vikunja.io/api/pkg/user"
|
||||
@ -36,6 +37,9 @@ func LightInit() {
|
||||
// Init redis
|
||||
red.InitRedis()
|
||||
|
||||
// Init keyvalue store
|
||||
keyvalue.InitStorage()
|
||||
|
||||
// Set logger
|
||||
log.InitLogger()
|
||||
}
|
||||
|
@ -17,10 +17,9 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
"code.vikunja.io/api/pkg/modules/keyvalue"
|
||||
"code.vikunja.io/web"
|
||||
"encoding/gob"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"sync"
|
||||
@ -91,36 +90,19 @@ func SetUserActive(a web.Auth) (err error) {
|
||||
|
||||
// getActiveUsers returns the active users from redis
|
||||
func getActiveUsers() (users activeUsersMap, err error) {
|
||||
|
||||
activeUsersR, err := r.Get(ActiveUsersKey).Bytes()
|
||||
if err != nil {
|
||||
if err.Error() == "redis: nil" {
|
||||
return users, nil
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
var b bytes.Buffer
|
||||
_, err = b.Write(activeUsersR)
|
||||
u, err := keyvalue.Get(ActiveUsersKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := gob.NewDecoder(&b)
|
||||
if err := d.Decode(&users); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
users = u.(activeUsersMap)
|
||||
return
|
||||
}
|
||||
|
||||
// PushActiveUsers pushed the content of the activeUsers map to redis
|
||||
func PushActiveUsers() (err error) {
|
||||
var b bytes.Buffer
|
||||
e := gob.NewEncoder(&b)
|
||||
activeUsers.mutex.Lock()
|
||||
defer activeUsers.mutex.Unlock()
|
||||
if err := e.Encode(activeUsers.users); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return r.Set(ActiveUsersKey, b.Bytes(), 0).Err()
|
||||
return keyvalue.Put(ActiveUsersKey, activeUsers.users)
|
||||
}
|
||||
|
@ -19,14 +19,12 @@ package metrics
|
||||
import (
|
||||
"code.vikunja.io/api/pkg/config"
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
"code.vikunja.io/api/pkg/red"
|
||||
"github.com/go-redis/redis/v7"
|
||||
"code.vikunja.io/api/pkg/modules/keyvalue"
|
||||
e "code.vikunja.io/api/pkg/modules/keyvalue/error"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var r *redis.Client
|
||||
|
||||
const (
|
||||
// ListCountKey is the name of the key in which we save the list count
|
||||
ListCountKey = `listcount`
|
||||
@ -46,8 +44,6 @@ const (
|
||||
|
||||
// InitMetrics Initializes the metrics
|
||||
func InitMetrics() {
|
||||
r = red.GetRedis()
|
||||
|
||||
// init active users, sometimes we'll have garbage from previous runs in redis instead
|
||||
if err := PushActiveUsers(); err != nil {
|
||||
log.Fatalf("Could not set initial count for active users, error was %s", err)
|
||||
@ -101,18 +97,21 @@ func InitMetrics() {
|
||||
|
||||
// GetCount returns the current count from redis
|
||||
func GetCount(key string) (count int64, err error) {
|
||||
count, err = r.Get(key).Int64()
|
||||
if err != nil && err.Error() != "redis: nil" {
|
||||
return
|
||||
cnt, err := keyvalue.Get(key)
|
||||
if err != nil {
|
||||
if e.IsErrValueNotFoundForKey(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
err = nil
|
||||
count = cnt.(int64)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// SetCount sets the list count to a given value
|
||||
func SetCount(count int64, key string) error {
|
||||
return r.Set(key, count, 0).Err()
|
||||
return keyvalue.Put(key, count)
|
||||
}
|
||||
|
||||
// UpdateCount updates a count with a given amount
|
||||
@ -121,13 +120,13 @@ func UpdateCount(update int64, key string) {
|
||||
return
|
||||
}
|
||||
if update > 0 {
|
||||
err := r.IncrBy(key, update).Err()
|
||||
err := keyvalue.IncrBy(key, update)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
}
|
||||
if update < 0 {
|
||||
err := r.DecrBy(key, update).Err()
|
||||
err := keyvalue.DecrBy(key, update)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
|
@ -17,14 +17,12 @@
|
||||
package initials
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
"code.vikunja.io/api/pkg/modules/keyvalue"
|
||||
e "code.vikunja.io/api/pkg/modules/keyvalue/error"
|
||||
"code.vikunja.io/api/pkg/user"
|
||||
"github.com/disintegration/imaging"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"bytes"
|
||||
"github.com/golang/freetype/truetype"
|
||||
"golang.org/x/image/font"
|
||||
"golang.org/x/image/font/gofont/goregular"
|
||||
@ -33,6 +31,8 @@ import (
|
||||
"image/color"
|
||||
"image/draw"
|
||||
"image/png"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Provider represents the provider implementation of the initials provider
|
||||
@ -51,19 +51,8 @@ var (
|
||||
{121, 134, 203, 255},
|
||||
{241, 185, 29, 255},
|
||||
}
|
||||
|
||||
// Contain the created avatars with a size of defaultSize
|
||||
cache = map[int64]*image.RGBA64{}
|
||||
cacheLock = sync.Mutex{}
|
||||
cacheResized = map[string][]byte{}
|
||||
cacheResizedLock = sync.Mutex{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
cache = make(map[int64]*image.RGBA64)
|
||||
cacheResized = make(map[string][]byte)
|
||||
}
|
||||
|
||||
const (
|
||||
dpi = 72
|
||||
defaultSize = 1024
|
||||
@ -124,10 +113,26 @@ func drawImage(text rune, bg *color.RGBA) (img *image.RGBA64, err error) {
|
||||
return img, err
|
||||
}
|
||||
|
||||
func getCacheKey(prefix string, keys ...int64) string {
|
||||
result := "avatar_initials_" + prefix
|
||||
for i, key := range keys {
|
||||
result += strconv.Itoa(int(key))
|
||||
if i < len(keys) {
|
||||
result += "_"
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func getAvatarForUser(u *user.User) (fullSizeAvatar *image.RGBA64, err error) {
|
||||
var cached bool
|
||||
fullSizeAvatar, cached = cache[u.ID]
|
||||
if !cached {
|
||||
cacheKey := getCacheKey("full", u.ID)
|
||||
|
||||
a, err := keyvalue.Get(cacheKey)
|
||||
if err != nil && !e.IsErrValueNotFoundForKey(err) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err != nil && e.IsErrValueNotFoundForKey(err) {
|
||||
log.Debugf("Initials avatar for user %d not cached, creating...", u.ID)
|
||||
firstRune := []rune(strings.ToUpper(u.Username))[0]
|
||||
bg := avatarBgColors[int(u.ID)%len(avatarBgColors)] // Random color based on the user id
|
||||
@ -136,21 +141,27 @@ func getAvatarForUser(u *user.User) (fullSizeAvatar *image.RGBA64, err error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cacheLock.Lock()
|
||||
cache[u.ID] = fullSizeAvatar
|
||||
cacheLock.Unlock()
|
||||
err = keyvalue.Put(cacheKey, fullSizeAvatar)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
fullSizeAvatar = a.(*image.RGBA64)
|
||||
}
|
||||
|
||||
return fullSizeAvatar, err
|
||||
return fullSizeAvatar, nil
|
||||
}
|
||||
|
||||
// GetAvatar returns an initials avatar for a user
|
||||
func (p *Provider) GetAvatar(u *user.User, size int64) (avatar []byte, mimeType string, err error) {
|
||||
cacheKey := getCacheKey("resized", u.ID, size)
|
||||
|
||||
var cached bool
|
||||
cacheKey := strconv.Itoa(int(u.ID)) + "_" + strconv.Itoa(int(size))
|
||||
avatar, cached = cacheResized[cacheKey]
|
||||
if !cached {
|
||||
a, err := keyvalue.Get(cacheKey)
|
||||
if err != nil && !e.IsErrValueNotFoundForKey(err) {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
if err != nil && e.IsErrValueNotFoundForKey(err) {
|
||||
log.Debugf("Initials avatar for user %d and size %d not cached, creating...", u.ID, size)
|
||||
fullAvatar, err := getAvatarForUser(u)
|
||||
if err != nil {
|
||||
@ -164,12 +175,14 @@ func (p *Provider) GetAvatar(u *user.User, size int64) (avatar []byte, mimeType
|
||||
return nil, "", err
|
||||
}
|
||||
avatar = buf.Bytes()
|
||||
cacheResizedLock.Lock()
|
||||
cacheResized[cacheKey] = avatar
|
||||
cacheResizedLock.Unlock()
|
||||
err = keyvalue.Put(cacheKey, avatar)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
} else {
|
||||
avatar = a.([]byte)
|
||||
log.Debugf("Serving initials avatar for user %d and size %d from cache", u.ID, size)
|
||||
}
|
||||
|
||||
return avatar, "image/png", err
|
||||
return avatar, "image/png", nil
|
||||
}
|
||||
|
@ -20,25 +20,16 @@ import (
|
||||
"bytes"
|
||||
"code.vikunja.io/api/pkg/files"
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
"code.vikunja.io/api/pkg/modules/keyvalue"
|
||||
e "code.vikunja.io/api/pkg/modules/keyvalue/error"
|
||||
"code.vikunja.io/api/pkg/user"
|
||||
"github.com/disintegration/imaging"
|
||||
"image"
|
||||
"image/png"
|
||||
"io/ioutil"
|
||||
"sync"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
var (
|
||||
// This is a map with a map so we're able to clear all cached avatar (in all sizes) for one user at once
|
||||
// The first map has as key the user id, the second one has the size as key
|
||||
resizedCache = map[int64]map[int64][]byte{}
|
||||
resizedCacheLock = sync.Mutex{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
resizedCache = make(map[int64]map[int64][]byte)
|
||||
}
|
||||
|
||||
// Provider represents the upload avatar provider
|
||||
type Provider struct {
|
||||
}
|
||||
@ -46,19 +37,32 @@ type Provider struct {
|
||||
// GetAvatar returns an uploaded user avatar
|
||||
func (p *Provider) GetAvatar(u *user.User, size int64) (avatar []byte, mimeType string, err error) {
|
||||
|
||||
a, cached := resizedCache[u.ID]
|
||||
if cached {
|
||||
cacheKey := "avatar_upload_" + strconv.Itoa(int(u.ID))
|
||||
|
||||
ai, err := keyvalue.Get(cacheKey)
|
||||
if err != nil && !e.IsErrValueNotFoundForKey(err) {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
var cached map[int64][]byte
|
||||
|
||||
if ai != nil {
|
||||
cached = ai.(map[int64][]byte)
|
||||
}
|
||||
|
||||
if err != nil && e.IsErrValueNotFoundForKey(err) {
|
||||
// Nothing ever cached for this user so we need to create the size map to avoid panics
|
||||
cached = make(map[int64][]byte)
|
||||
} else {
|
||||
a := ai.(map[int64][]byte)
|
||||
if a != nil && a[size] != nil {
|
||||
log.Debugf("Serving uploaded avatar for user %d and size %d from cache.", u.ID, size)
|
||||
return a[size], "", nil
|
||||
}
|
||||
// This means we have a map for the user, but nothing in it.
|
||||
if a == nil {
|
||||
resizedCache[u.ID] = make(map[int64][]byte)
|
||||
cached = make(map[int64][]byte)
|
||||
}
|
||||
} else {
|
||||
// Nothing ever cached for this user so we need to create the size map to avoid panics
|
||||
resizedCache[u.ID] = make(map[int64][]byte)
|
||||
}
|
||||
|
||||
log.Debugf("Uploaded avatar for user %d and size %d not cached, resizing and caching.", u.ID, size)
|
||||
@ -84,15 +88,17 @@ func (p *Provider) GetAvatar(u *user.User, size int64) (avatar []byte, mimeType
|
||||
}
|
||||
|
||||
avatar, err = ioutil.ReadAll(buf)
|
||||
resizedCacheLock.Lock()
|
||||
resizedCache[u.ID][size] = avatar
|
||||
resizedCacheLock.Unlock()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
cached[size] = avatar
|
||||
err = keyvalue.Put(cacheKey, cached)
|
||||
return avatar, f.Mime, err
|
||||
}
|
||||
|
||||
// InvalidateCache invalidates the avatar cache for a user
|
||||
func InvalidateCache(u *user.User) {
|
||||
resizedCacheLock.Lock()
|
||||
delete(resizedCache, u.ID)
|
||||
resizedCacheLock.Unlock()
|
||||
if err := keyvalue.Del("avatar_upload_" + strconv.Itoa(int(u.ID))); err != nil {
|
||||
log.Errorf("Could not invalidate upload avatar cache for user %d, error was %s", u.ID, err)
|
||||
}
|
||||
}
|
||||
|
@ -23,6 +23,8 @@ import (
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
"code.vikunja.io/api/pkg/models"
|
||||
"code.vikunja.io/api/pkg/modules/background"
|
||||
"code.vikunja.io/api/pkg/modules/keyvalue"
|
||||
e "code.vikunja.io/api/pkg/modules/keyvalue/error"
|
||||
"code.vikunja.io/web"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
@ -32,7 +34,10 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const unsplashAPIURL = `https://api.unsplash.com/`
|
||||
const (
|
||||
unsplashAPIURL = `https://api.unsplash.com/`
|
||||
cachePrefix = `unsplash_photo_`
|
||||
)
|
||||
|
||||
// Provider represents an unsplash image provider
|
||||
type Provider struct {
|
||||
@ -72,10 +77,6 @@ type Photo struct {
|
||||
} `json:"links"`
|
||||
}
|
||||
|
||||
// Very simple caching method - pretty much only used to retain information when saving an image
|
||||
// FIXME: Should use a proper cache
|
||||
var photos map[string]*Photo
|
||||
|
||||
// We're caching the initial collection to save a few api requests as this is retrieved every time a
|
||||
// user opens the settings page.
|
||||
type initialCollection struct {
|
||||
@ -87,10 +88,6 @@ type initialCollection struct {
|
||||
|
||||
var emptySearchResult *initialCollection
|
||||
|
||||
func init() {
|
||||
photos = make(map[string]*Photo)
|
||||
}
|
||||
|
||||
func doGet(url string, result ...interface{}) (err error) {
|
||||
req, err := http.NewRequest("GET", unsplashAPIURL+url, nil)
|
||||
if err != nil {
|
||||
@ -120,15 +117,21 @@ func getImageID(fullURL string) string {
|
||||
|
||||
// Gets an unsplash photo either from cache or directly from the unsplash api
|
||||
func getUnsplashPhotoInfoByID(photoID string) (photo *Photo, err error) {
|
||||
var exists bool
|
||||
photo, exists = photos[photoID]
|
||||
if !exists {
|
||||
|
||||
p, err := keyvalue.Get(cachePrefix + photoID)
|
||||
if err != nil && !e.IsErrValueNotFoundForKey(err) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err != nil && e.IsErrValueNotFoundForKey(err) {
|
||||
log.Debugf("Image information for unsplash photo %s not cached, requesting from unsplash...", photoID)
|
||||
photo = &Photo{}
|
||||
err = doGet("photos/"+photoID, photo)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
photo = p.(*Photo)
|
||||
}
|
||||
return
|
||||
}
|
||||
@ -180,7 +183,9 @@ func (p *Provider) Search(search string, page int64) (result []*background.Image
|
||||
AuthorName: p.User.Name,
|
||||
},
|
||||
})
|
||||
photos[p.ID] = p
|
||||
if err := keyvalue.Put(cachePrefix+p.ID, p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Put the collection in cache
|
||||
@ -213,7 +218,9 @@ func (p *Provider) Search(search string, page int64) (result []*background.Image
|
||||
AuthorName: p.User.Name,
|
||||
},
|
||||
})
|
||||
photos[p.ID] = p
|
||||
if err := keyvalue.Put(cachePrefix+p.ID, p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
|
52
pkg/modules/keyvalue/error/error.go
Normal file
52
pkg/modules/keyvalue/error/error.go
Normal file
@ -0,0 +1,52 @@
|
||||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package error
|
||||
|
||||
import "fmt"
|
||||
|
||||
// ErrValueNotFoundForKey represents an error where a key could not be found
|
||||
type ErrValueNotFoundForKey struct {
|
||||
Key string
|
||||
}
|
||||
|
||||
// Error is the error implementation
|
||||
func (e *ErrValueNotFoundForKey) Error() string {
|
||||
return fmt.Sprintf("could not find value for key %s", e.Key)
|
||||
}
|
||||
|
||||
// IsErrValueNotFoundForKey checks if an error is ErrValueNotFoundForKey
|
||||
func IsErrValueNotFoundForKey(err error) bool {
|
||||
_, is := err.(*ErrValueNotFoundForKey)
|
||||
return is
|
||||
}
|
||||
|
||||
// ErrValueHasWrongType represents an error where a value saved at key has the wrong value
|
||||
type ErrValueHasWrongType struct {
|
||||
Key string
|
||||
ExpectedValue string
|
||||
}
|
||||
|
||||
// Error is the error implementation
|
||||
func (e *ErrValueHasWrongType) Error() string {
|
||||
return fmt.Sprintf("value at key %s has the wrong value, expexted was %s", e.Key, e.ExpectedValue)
|
||||
}
|
||||
|
||||
// IsErrValueHasWrongType checks if an error is ErrValueHasWrongType
|
||||
func IsErrValueHasWrongType(err error) bool {
|
||||
_, is := err.(*ErrValueHasWrongType)
|
||||
return is
|
||||
}
|
72
pkg/modules/keyvalue/keyvalue.go
Normal file
72
pkg/modules/keyvalue/keyvalue.go
Normal file
@ -0,0 +1,72 @@
|
||||
// Copyright 2020 Vikunja and contriubtors. All rights reserved.
|
||||
//
|
||||
// This file is part of Vikunja.
|
||||
//
|
||||
// Vikunja is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// Vikunja is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Vikunja. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package keyvalue
|
||||
|
||||
import (
|
||||
"code.vikunja.io/api/pkg/config"
|
||||
"code.vikunja.io/api/pkg/modules/keyvalue/memory"
|
||||
"code.vikunja.io/api/pkg/modules/keyvalue/redis"
|
||||
)
|
||||
|
||||
// Storage defines an interface for saving key-value pairs
|
||||
type Storage interface {
|
||||
Put(key string, value interface{}) (err error)
|
||||
Get(key string) (value interface{}, err error)
|
||||
Del(key string) (err error)
|
||||
IncrBy(key string, update int64) (err error)
|
||||
DecrBy(key string, update int64) (err error)
|
||||
}
|
||||
|
||||
var store Storage
|
||||
|
||||
// InitStorage initializes the configured storage backend
|
||||
func InitStorage() {
|
||||
switch config.KeyvalueType.GetString() {
|
||||
case "redis":
|
||||
store = redis.NewStorage()
|
||||
case "memory":
|
||||
fallthrough
|
||||
default:
|
||||
store = memory.NewStorage()
|
||||
}
|
||||
}
|
||||
|
||||
// Put puts a value in the storage backend
|
||||
func Put(key string, value interface{}) error {
|
||||
return store.Put(key, value)
|
||||
}
|
||||
|
||||
// Get returns a value from a storage backend
|
||||
func Get(key string) (value interface{}, err error) {
|
||||
return store.Get(key)
|
||||
}
|
||||
|
||||
// Del removes a save value from a storage backend
|
||||
func Del(key string) (err error) {
|
||||
return store.Del(key)
|
||||
}
|
||||
|
||||
// IncrBy increases a value at key by the amount in update
|
||||
func IncrBy(key string, update int64) (err error) {
|
||||
return store.IncrBy(key, update)
|
||||
}
|
||||
|
||||
// DecrBy increases a value at key by the amount in update
|
||||
func DecrBy(key string, update int64) (err error) {
|
||||
return store.DecrBy(key, update)
|
||||
}
|
102
pkg/modules/keyvalue/memory/memory.go
Normal file
102
pkg/modules/keyvalue/memory/memory.go
Normal file
@ -0,0 +1,102 @@
|
||||
// Copyright 2020 Vikunja and contriubtors. All rights reserved.
|
||||
//
|
||||
// This file is part of Vikunja.
|
||||
//
|
||||
// Vikunja is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// Vikunja is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Vikunja. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package memory
|
||||
|
||||
import (
|
||||
e "code.vikunja.io/api/pkg/modules/keyvalue/error"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Storage is the memory implementation of a storage backend
|
||||
type Storage struct {
|
||||
store map[string]interface{}
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
// NewStorage creates a new memory storage
|
||||
func NewStorage() *Storage {
|
||||
s := &Storage{}
|
||||
s.store = make(map[string]interface{})
|
||||
return s
|
||||
}
|
||||
|
||||
// Put puts a value into the memory storage
|
||||
func (s *Storage) Put(key string, value interface{}) (err error) {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
s.store[key] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get retrieves a saved value from memory storage
|
||||
func (s *Storage) Get(key string) (value interface{}, err error) {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
|
||||
var exists bool
|
||||
value, exists = s.store[key]
|
||||
if !exists {
|
||||
return nil, &e.ErrValueNotFoundForKey{Key: key}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Del removes a saved value from a memory storage
|
||||
func (s *Storage) Del(key string) (err error) {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
delete(s.store, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// IncrBy increases the value saved at key by the amount provided through update
|
||||
// It assumes the value saved for the key either does not exist or has a type of int64
|
||||
func (s *Storage) IncrBy(key string, update int64) (err error) {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
|
||||
v, err := s.Get(key)
|
||||
if err != nil && !e.IsErrValueNotFoundForKey(err) {
|
||||
return err
|
||||
}
|
||||
val, is := v.(int64)
|
||||
if !is {
|
||||
return &e.ErrValueHasWrongType{Key: key, ExpectedValue: "int64"}
|
||||
}
|
||||
s.store[key] = val + update
|
||||
return nil
|
||||
}
|
||||
|
||||
// DecrBy decreases the value saved at key by the amount provided through update
|
||||
// It assumes the value saved for the key either does not exist or has a type of int64
|
||||
func (s *Storage) DecrBy(key string, update int64) (err error) {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
|
||||
v, err := s.Get(key)
|
||||
if err != nil && !e.IsErrValueNotFoundForKey(err) {
|
||||
return err
|
||||
}
|
||||
val, is := v.(int64)
|
||||
if !is {
|
||||
return &e.ErrValueHasWrongType{Key: key, ExpectedValue: "int64"}
|
||||
}
|
||||
s.store[key] = val - update
|
||||
return nil
|
||||
}
|
78
pkg/modules/keyvalue/redis/redis.go
Normal file
78
pkg/modules/keyvalue/redis/redis.go
Normal file
@ -0,0 +1,78 @@
|
||||
// Copyright 2020 Vikunja and contriubtors. All rights reserved.
|
||||
//
|
||||
// This file is part of Vikunja.
|
||||
//
|
||||
// Vikunja is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// Vikunja is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Vikunja. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package redis
|
||||
|
||||
import (
|
||||
e "code.vikunja.io/api/pkg/modules/keyvalue/error"
|
||||
"code.vikunja.io/api/pkg/red"
|
||||
"encoding/json"
|
||||
"github.com/go-redis/redis/v7"
|
||||
)
|
||||
|
||||
// Storage is a redis implementation of a keyvalue storage
|
||||
type Storage struct {
|
||||
client *redis.Client
|
||||
}
|
||||
|
||||
// NewStorage creates a new redis key value storage
|
||||
func NewStorage() *Storage {
|
||||
red.InitRedis()
|
||||
|
||||
return &Storage{
|
||||
client: red.GetRedis(),
|
||||
}
|
||||
}
|
||||
|
||||
// Put puts a value into redis
|
||||
func (s *Storage) Put(key string, value interface{}) (err error) {
|
||||
v, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.client.Set(key, v, 0).Err()
|
||||
}
|
||||
|
||||
// Get retrieves a saved value from redis
|
||||
func (s *Storage) Get(key string) (value interface{}, err error) {
|
||||
b, err := s.client.Get(key).Bytes()
|
||||
if err != nil {
|
||||
if err == redis.Nil {
|
||||
return nil, &e.ErrValueNotFoundForKey{Key: key}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(b, value)
|
||||
return
|
||||
}
|
||||
|
||||
// Del removed a value from redis
|
||||
func (s *Storage) Del(key string) (err error) {
|
||||
return s.client.Del(key).Err()
|
||||
}
|
||||
|
||||
// IncrBy increases the value saved at key by the amount provided through update
|
||||
func (s *Storage) IncrBy(key string, update int64) (err error) {
|
||||
return s.client.IncrBy(key, update).Err()
|
||||
}
|
||||
|
||||
// DecrBy decreases the value saved at key by the amount provided through update
|
||||
func (s *Storage) DecrBy(key string, update int64) (err error) {
|
||||
return s.client.DecrBy(key, update).Err()
|
||||
}
|
@ -26,6 +26,10 @@ var r *redis.Client
|
||||
|
||||
// InitRedis initializes a redis connection
|
||||
func InitRedis() {
|
||||
if r != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if !config.RedisEnabled.GetBool() {
|
||||
return
|
||||
}
|
||||
|
@ -33,10 +33,6 @@ func setupMetrics(a *echo.Group) {
|
||||
return
|
||||
}
|
||||
|
||||
if !config.RedisEnabled.GetBool() {
|
||||
log.Fatal("You have to enable redis in order to use metrics")
|
||||
}
|
||||
|
||||
metrics.InitMetrics()
|
||||
|
||||
type countable struct {
|
||||
|
Loading…
x
Reference in New Issue
Block a user