diff --git a/docs/content/doc/usage/errors.md b/docs/content/doc/usage/errors.md
index e0c05faa0..b33b550a0 100644
--- a/docs/content/doc/usage/errors.md
+++ b/docs/content/doc/usage/errors.md
@@ -90,6 +90,7 @@ This document describes the different errors Vikunja can return.
 | 4019 | 400 | Invalid task filter value. |
 | 4020 | 400 | The provided attachment does not belong to that task. |
 | 4021 | 400 | This user is already assigned to that task. |
+| 4022 | 400 | The task has a relative reminder which does not specify relative to what. |
 
 ## Namespace
 
diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go
index 2e82c359b..7db8929bf 100644
--- a/pkg/models/tasks.go
+++ b/pkg/models/tasks.go
@@ -1513,24 +1513,21 @@ func (t *Task) overwriteRemindersWithReminderDates(reminderDates []time.Time) {
 func updateRelativeReminderDates(task *Task) (err error) {
 	for _, reminder := range task.Reminders {
 		relativeDuration := time.Duration(reminder.RelativePeriod) * time.Second
+		if reminder.RelativeTo != "" {
+			reminder.Reminder = time.Time{}
+		}
 		switch reminder.RelativeTo {
 		case ReminderRelationDueDate:
 			if !task.DueDate.IsZero() {
 				reminder.Reminder = task.DueDate.Add(relativeDuration)
-			} else {
-				reminder.Reminder = time.Time{}
 			}
 		case ReminderRelationStartDate:
 			if !task.StartDate.IsZero() {
 				reminder.Reminder = task.StartDate.Add(relativeDuration)
-			} else {
-				reminder.Reminder = time.Time{}
 			}
 		case ReminderRelationEndDate:
 			if !task.EndDate.IsZero() {
 				reminder.Reminder = task.EndDate.Add(relativeDuration)
-			} else {
-				reminder.Reminder = time.Time{}
 			}
 		default:
 			if reminder.RelativePeriod != 0 {