Restore command (#593)

Add waiting for changes to config file

Add max size for config files

Restore files

Restore database file

Expose migrate to

Move init stuff to seperate package

Add restoring config file

Co-authored-by: kolaente <k@knt.li>
Reviewed-on: vikunja/api#593

pkg/cmd/cmd.go

@@ -17,15 +17,6 @@
 package cmd
 
 import (
-	"code.vikunja.io/api/pkg/config"
-	"code.vikunja.io/api/pkg/files"
-	"code.vikunja.io/api/pkg/log"
-	"code.vikunja.io/api/pkg/mail"
-	"code.vikunja.io/api/pkg/migration"
-	"code.vikunja.io/api/pkg/models"
-	migrator "code.vikunja.io/api/pkg/modules/migration"
-	"code.vikunja.io/api/pkg/red"
-	"code.vikunja.io/api/pkg/user"
 	"fmt"
 	"github.com/spf13/cobra"
 	"os"
@@ -54,48 +45,3 @@ func Execute() {
 		os.Exit(1)
 	}
 }
-
-// Will only fullInit config, redis, logger but no db connection.
-func lightInit() {
-	// Init the config
-	config.InitConfig()
-
-	// Init redis
-	red.InitRedis()
-
-	// Set logger
-	log.InitLogger()
-}
-
-// Initializes all kinds of things in the right order
-func fullInit() {
-
-	lightInit()
-
-	// Run the migrations
-	migration.Migrate(nil)
-
-	// Set Engine
-	err := models.SetEngine()
-	if err != nil {
-		log.Fatal(err.Error())
-	}
-	err = user.InitDB()
-	if err != nil {
-		log.Fatal(err.Error())
-	}
-	err = files.SetEngine()
-	if err != nil {
-		log.Fatal(err.Error())
-	}
-	err = migrator.InitDB()
-	if err != nil {
-		log.Fatal(err.Error())
-	}
-
-	// Initialize the files handler
-	files.InitFileHandler()
-
-	// Start the mail daemon
-	mail.StartMailDaemon()
-}

pkg/cmd/dump.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"code.vikunja.io/api/pkg/initialize"
 	"code.vikunja.io/api/pkg/log"
 	"code.vikunja.io/api/pkg/modules/dump"
 	"github.com/spf13/cobra"
@@ -31,7 +32,7 @@ var dumpCmd = &cobra.Command{
 	Use:   "dump",
 	Short: "Dump all vikunja data into a zip file. Includes config, files and db.",
 	PreRun: func(cmd *cobra.Command, args []string) {
-		fullInit()
+		initialize.FullInit()
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		filename := "vikunja-dump_" + time.Now().Format("2006-01-02_15-03-05") + ".zip"

pkg/cmd/migrate.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"code.vikunja.io/api/pkg/initialize"
 	"code.vikunja.io/api/pkg/migration"
 	"github.com/spf13/cobra"
 )
@@ -36,7 +37,7 @@ var migrateCmd = &cobra.Command{
 	Use:   "migrate",
 	Short: "Run all database migrations which didn't already run.",
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
-		lightInit()
+		initialize.LightInit()
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		migration.Migrate(nil)

pkg/cmd/restore.go

@@ -0,0 +1,42 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"code.vikunja.io/api/pkg/initialize"
+	"code.vikunja.io/api/pkg/log"
+	"code.vikunja.io/api/pkg/modules/dump"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(restoreCmd)
+}
+
+var restoreCmd = &cobra.Command{
+	Use:   "restore [filename]",
+	Short: "Restores all vikunja data from a vikunja dump.",
+	Args:  cobra.ExactArgs(1),
+	PreRun: func(cmd *cobra.Command, args []string) {
+		initialize.FullInit()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := dump.Restore(args[0]); err != nil {
+			log.Critical(err.Error())
+		}
+	},
+}

pkg/cmd/testmail.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"code.vikunja.io/api/pkg/initialize"
 	"code.vikunja.io/api/pkg/log"
 	"code.vikunja.io/api/pkg/mail"
 	"github.com/spf13/cobra"
@@ -31,7 +32,7 @@ var testmailCmd = &cobra.Command{
 	Short: "Send a test mail using the configured smtp connection",
 	Args:  cobra.ExactArgs(1),
 	PreRun: func(cmd *cobra.Command, args []string) {
-		lightInit()
+		initialize.LightInit()
 
 		// Start the mail daemon
 		mail.StartMailDaemon()

pkg/cmd/web.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"code.vikunja.io/api/pkg/config"
+	"code.vikunja.io/api/pkg/initialize"
 	"code.vikunja.io/api/pkg/log"
 	"code.vikunja.io/api/pkg/routes"
 	"code.vikunja.io/api/pkg/swagger"
@@ -37,7 +38,7 @@ var webCmd = &cobra.Command{
 	Use:   "web",
 	Short: "Starts the rest api web server",
 	PreRun: func(cmd *cobra.Command, args []string) {
-		fullInit()
+		initialize.FullInit()
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 

pkg/db/db.go

@@ -168,3 +168,20 @@ func initSqliteEngine() (engine *xorm.Engine, err error) {
 
 	return xorm.NewEngine("sqlite3", path)
 }
+
+// WipeEverything wipes all tables and their data. Use with caution...
+func WipeEverything() error {
+
+	tables, err := x.DBMetas()
+	if err != nil {
+		return err
+	}
+
+	for _, t := range tables {
+		if err := x.DropTables(t.Name); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/db/dump.go

@@ -40,3 +40,15 @@ func Dump() (data map[string][]byte, err error) {
 
 	return
 }
+
+// Restore restores a table with all its entries
+func Restore(table string, contents []map[string]interface{}) (err error) {
+
+	for _, content := range contents {
+		if _, err := x.Table(table).Insert(content); err != nil {
+			return err
+		}
+	}
+
+	return
+}

pkg/files/files.go

@@ -96,7 +96,7 @@ func Create(f io.ReadCloser, realname string, realsize uint64, a web.Auth) (file
 	}
 
 	// Save the file to storage with its new ID as path
-	err = afs.WriteReader(file.getFileName(), f)
+	err = file.Save(f)
 	return
 }
 
@@ -113,3 +113,8 @@ func (f *File) Delete() (err error) {
 	err = afs.Remove(f.getFileName())
 	return
 }
+
+// Save saves a file to storage
+func (f *File) Save(fcontent io.ReadCloser) error {
+	return afs.WriteReader(f.getFileName(), fcontent)
+}

pkg/initialize/init.go

@@ -0,0 +1,79 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package initialize
+
+import (
+	"code.vikunja.io/api/pkg/config"
+	"code.vikunja.io/api/pkg/files"
+	"code.vikunja.io/api/pkg/log"
+	"code.vikunja.io/api/pkg/mail"
+	"code.vikunja.io/api/pkg/migration"
+	"code.vikunja.io/api/pkg/models"
+	migrator "code.vikunja.io/api/pkg/modules/migration"
+	"code.vikunja.io/api/pkg/red"
+	"code.vikunja.io/api/pkg/user"
+)
+
+// LightInit will only fullInit config, redis, logger but no db connection.
+func LightInit() {
+	// Init the config
+	config.InitConfig()
+
+	// Init redis
+	red.InitRedis()
+
+	// Set logger
+	log.InitLogger()
+}
+
+// InitEngines intializes all db connections
+func InitEngines() {
+	err := models.SetEngine()
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+	err = user.InitDB()
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+	err = files.SetEngine()
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+	err = migrator.InitDB()
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+}
+
+// FullInit initializes all kinds of things in the right order
+func FullInit() {
+
+	LightInit()
+
+	// Run the migrations
+	migration.Migrate(nil)
+
+	// Set Engine
+	InitEngines()
+
+	// Initialize the files handler
+	files.InitFileHandler()
+
+	// Start the mail daemon
+	mail.StartMailDaemon()
+}

pkg/migration/migration.go

@@ -104,6 +104,15 @@ func Rollback(migrationID string) {
 	log.Info("Rolled back successfully.")
 }
 
+// MigrateTo executes all migrations up to a certain point
+func MigrateTo(migrationID string, x *xorm.Engine) error {
+	m := initMigration(x)
+	if err := m.MigrateTo(migrationID); err != nil {
+		return err
+	}
+	return nil
+}
+
 // Deletes a column from a table. All arguments are strings, to let them be standalone and not depending on any struct.
 func dropTableColum(x *xorm.Engine, tableName, col string) error {
 

pkg/modules/dump/restore.go

@@ -0,0 +1,217 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package dump
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"code.vikunja.io/api/pkg/db"
+	"code.vikunja.io/api/pkg/files"
+	"code.vikunja.io/api/pkg/initialize"
+	"code.vikunja.io/api/pkg/log"
+	"code.vikunja.io/api/pkg/migration"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"src.techknowlogick.com/xormigrate"
+	"strconv"
+	"strings"
+)
+
+const maxConfigSize = 5 * 1024 * 1024 // 5 MB, should be largely enough
+
+// Restore takes a zip file name and restores it
+func Restore(filename string) error {
+
+	r, err := zip.OpenReader(filename)
+	if err != nil {
+		return fmt.Errorf("could not open zip file: %s", err)
+	}
+
+	log.Warning("Restoring a dump will wipe your current installation!")
+	log.Warning("To confirm, please type 'Yes, I understand' and confirm with enter:")
+	cr := bufio.NewReader(os.Stdin)
+	text, err := cr.ReadString('\n')
+	if err != nil {
+		return fmt.Errorf("could not read confirmation message: %s", err)
+	}
+	if text != "Yes, I understand\n" {
+		return fmt.Errorf("invalid confirmation message")
+	}
+
+	// Find the configFile, database and files files
+	var configFile *zip.File
+	dbfiles := make(map[string]*zip.File)
+	filesFiles := make(map[string]*zip.File)
+	for _, file := range r.File {
+		if strings.HasPrefix(file.Name, "config") {
+			configFile = file
+			continue
+		}
+		if strings.HasPrefix(file.Name, "database/") {
+			fname := strings.ReplaceAll(file.Name, "database/", "")
+			dbfiles[fname[:len(fname)-5]] = file
+			continue
+		}
+		if strings.HasPrefix(file.Name, "files/") {
+			filesFiles[strings.ReplaceAll(file.Name, "files/", "")] = file
+		}
+	}
+	if configFile == nil {
+		return fmt.Errorf("dump does not contain a config file")
+	}
+
+	///////
+	// Restore the config file
+	if configFile.UncompressedSize64 > maxConfigSize {
+		return fmt.Errorf("config file too large, is %d, max size is %d", configFile.UncompressedSize64, maxConfigSize)
+	}
+
+	outFile, err := os.OpenFile(configFile.Name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, configFile.Mode())
+	if err != nil {
+		return fmt.Errorf("could not open config file for writing: %s", err)
+	}
+
+	cfgr, err := configFile.Open()
+	if err != nil {
+		return err
+	}
+
+	// #nosec - We eliminated the potential decompression bomb by erroring out above if the file is larger than a threshold.
+	_, err = io.Copy(outFile, cfgr)
+	if err != nil {
+		return fmt.Errorf("could not create config file: %s", err)
+	}
+
+	_ = cfgr.Close()
+	_ = outFile.Close()
+
+	log.Infof("The config file has been restored to '%s'.", configFile.Name)
+	log.Infof("You can now make changes to it, hit enter when you're done.")
+	if _, err := bufio.NewReader(os.Stdin).ReadString('\n'); err != nil {
+		return fmt.Errorf("could not read from stdin: %s", err)
+	}
+	log.Info("Restoring...")
+
+	// Init the configFile again since the restored configuration is most likely different from the one before
+	initialize.LightInit()
+	initialize.InitEngines()
+	files.InitFileHandler()
+
+	///////
+	// Restore the db
+	// Start by wiping everything
+	if err := db.WipeEverything(); err != nil {
+		return fmt.Errorf("could not wipe database: %s", err)
+	}
+	log.Info("Wiped database.")
+
+	// Because we don't explicitly saved the table definitions, we take the last ran db migration from the dump
+	// and execute everything until that point.
+	migrations := dbfiles["migration"]
+	rc, err := migrations.Open()
+	if err != nil {
+		return fmt.Errorf("could not open migrations: %s", err)
+	}
+	defer rc.Close()
+
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(rc); err != nil {
+		return fmt.Errorf("could not read migrations: %s", err)
+	}
+
+	ms := []*xormigrate.Migration{}
+	if err := json.Unmarshal(buf.Bytes(), &ms); err != nil {
+		return fmt.Errorf("could not read migrations: %s", err)
+	}
+	sort.Slice(ms, func(i, j int) bool {
+		return ms[i].ID > ms[j].ID
+	})
+
+	lastMigration := ms[len(ms)-1]
+	if err := migration.MigrateTo(lastMigration.ID, nil); err != nil {
+		return fmt.Errorf("could not create db structure: %s", err)
+	}
+
+	// Restore all db data
+	for table, d := range dbfiles {
+		content, err := unmarshalFileToJSON(d)
+		if err != nil {
+			return fmt.Errorf("could not read table %s: %s", table, err)
+		}
+		if err := db.Restore(table, content); err != nil {
+			return fmt.Errorf("could not restore table data for table %s: %s", table, err)
+		}
+		log.Infof("Restored table %s", table)
+	}
+	log.Infof("Restored %d tables", len(dbfiles))
+
+	// Run migrations again to migrate a potentially outdated dump
+	migration.Migrate(nil)
+
+	///////
+	// Restore Files
+	for i, file := range filesFiles {
+		id, err := strconv.ParseInt(i, 10, 64)
+		if err != nil {
+			return fmt.Errorf("could not parse file id %s: %s", i, err)
+		}
+
+		f := &files.File{ID: id}
+
+		fc, err := file.Open()
+		if err != nil {
+			return fmt.Errorf("could not open file %s: %s", i, err)
+		}
+
+		if err := f.Save(fc); err != nil {
+			return fmt.Errorf("could not save file: %s", err)
+		}
+
+		_ = fc.Close()
+		log.Infof("Restored file %s", i)
+	}
+	log.Infof("Restored %d files.", len(filesFiles))
+
+	///////
+	// Done
+	log.Infof("Done restoring dump.")
+
+	return nil
+}
+
+func unmarshalFileToJSON(file *zip.File) (contents []map[string]interface{}, err error) {
+	rc, err := file.Open()
+	if err != nil {
+		return
+	}
+	defer rc.Close()
+
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(rc); err != nil {
+		return nil, err
+	}
+
+	contents = []map[string]interface{}{}
+	if err := json.Unmarshal(buf.Bytes(), &contents); err != nil {
+		return nil, err
+	}
+	return
+}