diff --git a/pkg/config/config.go b/pkg/config/config.go
index 98f4d41fa..9577b2d88 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -46,7 +46,6 @@ const (
 	ServiceEnableTaskAttachments Key = `service.enabletaskattachments`
 	ServiceTimeZone              Key = `service.timezone`
 	ServiceEnableTaskComments    Key = `service.enabletaskcomments`
-	ServiceGravatarExpiration    Key = `service.gravatarexpiration`
 
 	DatabaseType                  Key = `database.type`
 	DatabaseHost                  Key = `database.host`
@@ -103,6 +102,9 @@ const (
 	CorsEnable  Key = `cors.enable`
 	CorsOrigins Key = `cors.origins`
 	CorsMaxAge  Key = `cors.maxage`
+
+	AvatarProvider           Key = `avatar.provider`
+	AvatarGravaterExpiration Key = `avatar.gravatarexpiration`
 )
 
 // GetString returns a string config value
@@ -174,7 +176,6 @@ func InitDefaultConfig() {
 	ServiceEnableTaskAttachments.setDefault(true)
 	ServiceTimeZone.setDefault("GMT")
 	ServiceEnableTaskComments.setDefault(true)
-	ServiceGravatarExpiration.setDefault(3600)
 
 	// Database
 	DatabaseType.setDefault("sqlite")
@@ -230,6 +231,9 @@ func InitDefaultConfig() {
 	CorsMaxAge.setDefault(0)
 	// Migration
 	MigrationWunderlistEnable.setDefault(false)
+	// Avatar
+	AvatarProvider.setDefault("gravatar")
+	AvatarGravaterExpiration.setDefault(3600)
 }
 
 // InitConfig initializes the config, sets defaults etc.
diff --git a/pkg/modules/avatar/empty/empty.go b/pkg/modules/avatar/empty/empty.go
new file mode 100644
index 000000000..f698fdd9e
--- /dev/null
+++ b/pkg/modules/avatar/empty/empty.go
@@ -0,0 +1,43 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package empty
+
+import "code.vikunja.io/api/pkg/user"
+
+type Provider struct {
+}
+
+const defaultAvatar string = `<?xml version="1.0" encoding="UTF-8"?>
+<svg width="128" height="128" version="1.1" viewBox="0 0 33.867 33.867" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xlink="http://www.w3.org/1999/xlink">
+<metadata>
+<rdf:RDF>
+<cc:Work rdf:about="">
+<dc:format>image/svg+xml</dc:format>
+<dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
+<dc:title/>
+</cc:Work>
+</rdf:RDF>
+</metadata>
+<g transform="matrix(.1916 0 0 .1916 -163.17 -2538.8)" fill="#ccc" shape-rendering="auto">
+<circle cx="940.01" cy="13319" r="49.346" color="#000000" color-rendering="auto" image-rendering="auto" solid-color="#000000" style="isolation:auto;mix-blend-mode:normal"/>
+<path d="m940.01 13375a88.385 50.907 0 0 0-88.385 50.908 88.385 50.907 0 0 0 0.0948 1.542l176.59-0.596a88.385 50.907 0 0 0 0.071-0.812v-0.303a88.385 50.907 0 0 0-88.375-50.739z" color="#000000" color-rendering="auto" image-rendering="auto" solid-color="#000000" style="isolation:auto;mix-blend-mode:normal"/>
+</g>
+</svg>`
+
+func (p *Provider) GetAvatar(user *user.User, size int64) (avatar []byte, mimeType string, err error) {
+	return []byte(defaultAvatar), "image/svg+xml", nil
+}
diff --git a/pkg/modules/avatar/gravatar/gravatar.go b/pkg/modules/avatar/gravatar/gravatar.go
index 737e06ba9..075df0a03 100644
--- a/pkg/modules/avatar/gravatar/gravatar.go
+++ b/pkg/modules/avatar/gravatar/gravatar.go
@@ -53,7 +53,7 @@ func (g *Provider) GetAvatar(user *user.User, size int64) ([]byte, string, error
 		// elaped is alway < 0 so the next check would always succeed.
 		// To have it make sense, we flip that.
 		elapsed := time.Until(a.loadedAt) * -1
-		needsRefetch = elapsed > time.Duration(config.ServiceGravatarExpiration.GetInt64())*time.Second
+		needsRefetch = elapsed > time.Duration(config.AvatarGravaterExpiration.GetInt64())*time.Second
 		if needsRefetch {
 			log.Debugf("Refetching avatar for user %d after %v", user.ID, elapsed)
 		} else {
diff --git a/pkg/routes/api/v1/avatar.go b/pkg/routes/api/v1/avatar.go
index 429d440c3..f0b02f44a 100644
--- a/pkg/routes/api/v1/avatar.go
+++ b/pkg/routes/api/v1/avatar.go
@@ -17,7 +17,10 @@
 package v1
 
 import (
+	"code.vikunja.io/api/pkg/config"
 	"code.vikunja.io/api/pkg/log"
+	"code.vikunja.io/api/pkg/modules/avatar"
+	"code.vikunja.io/api/pkg/modules/avatar/empty"
 	"code.vikunja.io/api/pkg/modules/avatar/gravatar"
 	user2 "code.vikunja.io/api/pkg/user"
 	"code.vikunja.io/web/handler"
@@ -51,7 +54,13 @@ func GetAvatar(c echo.Context) error {
 	// Initialize the avatar provider
 	// For now, we only have one avatar provider, in the future there could be multiple which
 	// could be changed based on user settings etc.
-	avatarProvider := gravatar.Provider{}
+	var avatarProvider avatar.Provider
+	switch config.AvatarProvider.GetString() {
+	case "gravatar":
+		avatarProvider = &gravatar.Provider{}
+	default:
+		avatarProvider = &empty.Provider{}
+	}
 
 	size := c.QueryParam("size")
 	var sizeInt int64 = 250 // Default size of 250
@@ -64,11 +73,11 @@ func GetAvatar(c echo.Context) error {
 	}
 
 	// Get the avatar
-	avatar, mimeType, err := avatarProvider.GetAvatar(user, sizeInt)
+	a, mimeType, err := avatarProvider.GetAvatar(user, sizeInt)
 	if err != nil {
 		log.Errorf("Error getting avatar for user %d: %v", user.ID, err)
 		return handler.HandleHTTPError(err, c)
 	}
 
-	return c.Blob(http.StatusOK, mimeType, avatar)
+	return c.Blob(http.StatusOK, mimeType, a)
 }