From e4f150bbe3b46118b7b92c4d97bc3e62de4aa87d Mon Sep 17 00:00:00 2001 From: konrad Date: Sun, 21 Jun 2020 15:30:48 +0000 Subject: [PATCH] Restore command (#593) Add waiting for changes to config file Add max size for config files Restore files Restore database file Expose migrate to Move init stuff to seperate package Add restoring config file Co-authored-by: kolaente Reviewed-on: https://kolaente.dev/vikunja/api/pulls/593 --- pkg/cmd/cmd.go | 54 --------- pkg/cmd/dump.go | 3 +- pkg/cmd/migrate.go | 3 +- pkg/cmd/restore.go | 42 +++++++ pkg/cmd/testmail.go | 3 +- pkg/cmd/web.go | 3 +- pkg/db/db.go | 17 +++ pkg/db/dump.go | 12 ++ pkg/files/files.go | 7 +- pkg/initialize/init.go | 79 +++++++++++++ pkg/migration/migration.go | 9 ++ pkg/modules/dump/restore.go | 217 ++++++++++++++++++++++++++++++++++++ 12 files changed, 390 insertions(+), 59 deletions(-) create mode 100644 pkg/cmd/restore.go create mode 100644 pkg/initialize/init.go create mode 100644 pkg/modules/dump/restore.go diff --git a/pkg/cmd/cmd.go b/pkg/cmd/cmd.go index 53a4f79043..0b742583de 100644 --- a/pkg/cmd/cmd.go +++ b/pkg/cmd/cmd.go @@ -17,15 +17,6 @@ package cmd import ( - "code.vikunja.io/api/pkg/config" - "code.vikunja.io/api/pkg/files" - "code.vikunja.io/api/pkg/log" - "code.vikunja.io/api/pkg/mail" - "code.vikunja.io/api/pkg/migration" - "code.vikunja.io/api/pkg/models" - migrator "code.vikunja.io/api/pkg/modules/migration" - "code.vikunja.io/api/pkg/red" - "code.vikunja.io/api/pkg/user" "fmt" "github.com/spf13/cobra" "os" @@ -54,48 +45,3 @@ func Execute() { os.Exit(1) } } - -// Will only fullInit config, redis, logger but no db connection. -func lightInit() { - // Init the config - config.InitConfig() - - // Init redis - red.InitRedis() - - // Set logger - log.InitLogger() -} - -// Initializes all kinds of things in the right order -func fullInit() { - - lightInit() - - // Run the migrations - migration.Migrate(nil) - - // Set Engine - err := models.SetEngine() - if err != nil { - log.Fatal(err.Error()) - } - err = user.InitDB() - if err != nil { - log.Fatal(err.Error()) - } - err = files.SetEngine() - if err != nil { - log.Fatal(err.Error()) - } - err = migrator.InitDB() - if err != nil { - log.Fatal(err.Error()) - } - - // Initialize the files handler - files.InitFileHandler() - - // Start the mail daemon - mail.StartMailDaemon() -} diff --git a/pkg/cmd/dump.go b/pkg/cmd/dump.go index 34f36b1626..235698bd51 100644 --- a/pkg/cmd/dump.go +++ b/pkg/cmd/dump.go @@ -17,6 +17,7 @@ package cmd import ( + "code.vikunja.io/api/pkg/initialize" "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/modules/dump" "github.com/spf13/cobra" @@ -31,7 +32,7 @@ var dumpCmd = &cobra.Command{ Use: "dump", Short: "Dump all vikunja data into a zip file. Includes config, files and db.", PreRun: func(cmd *cobra.Command, args []string) { - fullInit() + initialize.FullInit() }, Run: func(cmd *cobra.Command, args []string) { filename := "vikunja-dump_" + time.Now().Format("2006-01-02_15-03-05") + ".zip" diff --git a/pkg/cmd/migrate.go b/pkg/cmd/migrate.go index a53387c8af..016db19ff7 100644 --- a/pkg/cmd/migrate.go +++ b/pkg/cmd/migrate.go @@ -17,6 +17,7 @@ package cmd import ( + "code.vikunja.io/api/pkg/initialize" "code.vikunja.io/api/pkg/migration" "github.com/spf13/cobra" ) @@ -36,7 +37,7 @@ var migrateCmd = &cobra.Command{ Use: "migrate", Short: "Run all database migrations which didn't already run.", PersistentPreRun: func(cmd *cobra.Command, args []string) { - lightInit() + initialize.LightInit() }, Run: func(cmd *cobra.Command, args []string) { migration.Migrate(nil) diff --git a/pkg/cmd/restore.go b/pkg/cmd/restore.go new file mode 100644 index 0000000000..54719d44cd --- /dev/null +++ b/pkg/cmd/restore.go @@ -0,0 +1,42 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2020 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package cmd + +import ( + "code.vikunja.io/api/pkg/initialize" + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/modules/dump" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(restoreCmd) +} + +var restoreCmd = &cobra.Command{ + Use: "restore [filename]", + Short: "Restores all vikunja data from a vikunja dump.", + Args: cobra.ExactArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + initialize.FullInit() + }, + Run: func(cmd *cobra.Command, args []string) { + if err := dump.Restore(args[0]); err != nil { + log.Critical(err.Error()) + } + }, +} diff --git a/pkg/cmd/testmail.go b/pkg/cmd/testmail.go index 95b9cf5190..f02d4b3c8d 100644 --- a/pkg/cmd/testmail.go +++ b/pkg/cmd/testmail.go @@ -17,6 +17,7 @@ package cmd import ( + "code.vikunja.io/api/pkg/initialize" "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/mail" "github.com/spf13/cobra" @@ -31,7 +32,7 @@ var testmailCmd = &cobra.Command{ Short: "Send a test mail using the configured smtp connection", Args: cobra.ExactArgs(1), PreRun: func(cmd *cobra.Command, args []string) { - lightInit() + initialize.LightInit() // Start the mail daemon mail.StartMailDaemon() diff --git a/pkg/cmd/web.go b/pkg/cmd/web.go index c293510902..a727cebd8f 100644 --- a/pkg/cmd/web.go +++ b/pkg/cmd/web.go @@ -18,6 +18,7 @@ package cmd import ( "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/initialize" "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/routes" "code.vikunja.io/api/pkg/swagger" @@ -37,7 +38,7 @@ var webCmd = &cobra.Command{ Use: "web", Short: "Starts the rest api web server", PreRun: func(cmd *cobra.Command, args []string) { - fullInit() + initialize.FullInit() }, Run: func(cmd *cobra.Command, args []string) { diff --git a/pkg/db/db.go b/pkg/db/db.go index 713ace74aa..457d87fb75 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -168,3 +168,20 @@ func initSqliteEngine() (engine *xorm.Engine, err error) { return xorm.NewEngine("sqlite3", path) } + +// WipeEverything wipes all tables and their data. Use with caution... +func WipeEverything() error { + + tables, err := x.DBMetas() + if err != nil { + return err + } + + for _, t := range tables { + if err := x.DropTables(t.Name); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/db/dump.go b/pkg/db/dump.go index f608016469..70bd3f988b 100644 --- a/pkg/db/dump.go +++ b/pkg/db/dump.go @@ -40,3 +40,15 @@ func Dump() (data map[string][]byte, err error) { return } + +// Restore restores a table with all its entries +func Restore(table string, contents []map[string]interface{}) (err error) { + + for _, content := range contents { + if _, err := x.Table(table).Insert(content); err != nil { + return err + } + } + + return +} diff --git a/pkg/files/files.go b/pkg/files/files.go index 060f6f5ce7..94b5505a76 100644 --- a/pkg/files/files.go +++ b/pkg/files/files.go @@ -96,7 +96,7 @@ func Create(f io.ReadCloser, realname string, realsize uint64, a web.Auth) (file } // Save the file to storage with its new ID as path - err = afs.WriteReader(file.getFileName(), f) + err = file.Save(f) return } @@ -113,3 +113,8 @@ func (f *File) Delete() (err error) { err = afs.Remove(f.getFileName()) return } + +// Save saves a file to storage +func (f *File) Save(fcontent io.ReadCloser) error { + return afs.WriteReader(f.getFileName(), fcontent) +} diff --git a/pkg/initialize/init.go b/pkg/initialize/init.go new file mode 100644 index 0000000000..dc9f89e822 --- /dev/null +++ b/pkg/initialize/init.go @@ -0,0 +1,79 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2020 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package initialize + +import ( + "code.vikunja.io/api/pkg/config" + "code.vikunja.io/api/pkg/files" + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/mail" + "code.vikunja.io/api/pkg/migration" + "code.vikunja.io/api/pkg/models" + migrator "code.vikunja.io/api/pkg/modules/migration" + "code.vikunja.io/api/pkg/red" + "code.vikunja.io/api/pkg/user" +) + +// LightInit will only fullInit config, redis, logger but no db connection. +func LightInit() { + // Init the config + config.InitConfig() + + // Init redis + red.InitRedis() + + // Set logger + log.InitLogger() +} + +// InitEngines intializes all db connections +func InitEngines() { + err := models.SetEngine() + if err != nil { + log.Fatal(err.Error()) + } + err = user.InitDB() + if err != nil { + log.Fatal(err.Error()) + } + err = files.SetEngine() + if err != nil { + log.Fatal(err.Error()) + } + err = migrator.InitDB() + if err != nil { + log.Fatal(err.Error()) + } +} + +// FullInit initializes all kinds of things in the right order +func FullInit() { + + LightInit() + + // Run the migrations + migration.Migrate(nil) + + // Set Engine + InitEngines() + + // Initialize the files handler + files.InitFileHandler() + + // Start the mail daemon + mail.StartMailDaemon() +} diff --git a/pkg/migration/migration.go b/pkg/migration/migration.go index c112e7044b..c6970bf8cd 100644 --- a/pkg/migration/migration.go +++ b/pkg/migration/migration.go @@ -104,6 +104,15 @@ func Rollback(migrationID string) { log.Info("Rolled back successfully.") } +// MigrateTo executes all migrations up to a certain point +func MigrateTo(migrationID string, x *xorm.Engine) error { + m := initMigration(x) + if err := m.MigrateTo(migrationID); err != nil { + return err + } + return nil +} + // Deletes a column from a table. All arguments are strings, to let them be standalone and not depending on any struct. func dropTableColum(x *xorm.Engine, tableName, col string) error { diff --git a/pkg/modules/dump/restore.go b/pkg/modules/dump/restore.go new file mode 100644 index 0000000000..504458dbbd --- /dev/null +++ b/pkg/modules/dump/restore.go @@ -0,0 +1,217 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-2020 Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package dump + +import ( + "archive/zip" + "bufio" + "bytes" + "code.vikunja.io/api/pkg/db" + "code.vikunja.io/api/pkg/files" + "code.vikunja.io/api/pkg/initialize" + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/migration" + "encoding/json" + "fmt" + "io" + "os" + "sort" + "src.techknowlogick.com/xormigrate" + "strconv" + "strings" +) + +const maxConfigSize = 5 * 1024 * 1024 // 5 MB, should be largely enough + +// Restore takes a zip file name and restores it +func Restore(filename string) error { + + r, err := zip.OpenReader(filename) + if err != nil { + return fmt.Errorf("could not open zip file: %s", err) + } + + log.Warning("Restoring a dump will wipe your current installation!") + log.Warning("To confirm, please type 'Yes, I understand' and confirm with enter:") + cr := bufio.NewReader(os.Stdin) + text, err := cr.ReadString('\n') + if err != nil { + return fmt.Errorf("could not read confirmation message: %s", err) + } + if text != "Yes, I understand\n" { + return fmt.Errorf("invalid confirmation message") + } + + // Find the configFile, database and files files + var configFile *zip.File + dbfiles := make(map[string]*zip.File) + filesFiles := make(map[string]*zip.File) + for _, file := range r.File { + if strings.HasPrefix(file.Name, "config") { + configFile = file + continue + } + if strings.HasPrefix(file.Name, "database/") { + fname := strings.ReplaceAll(file.Name, "database/", "") + dbfiles[fname[:len(fname)-5]] = file + continue + } + if strings.HasPrefix(file.Name, "files/") { + filesFiles[strings.ReplaceAll(file.Name, "files/", "")] = file + } + } + if configFile == nil { + return fmt.Errorf("dump does not contain a config file") + } + + /////// + // Restore the config file + if configFile.UncompressedSize64 > maxConfigSize { + return fmt.Errorf("config file too large, is %d, max size is %d", configFile.UncompressedSize64, maxConfigSize) + } + + outFile, err := os.OpenFile(configFile.Name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, configFile.Mode()) + if err != nil { + return fmt.Errorf("could not open config file for writing: %s", err) + } + + cfgr, err := configFile.Open() + if err != nil { + return err + } + + // #nosec - We eliminated the potential decompression bomb by erroring out above if the file is larger than a threshold. + _, err = io.Copy(outFile, cfgr) + if err != nil { + return fmt.Errorf("could not create config file: %s", err) + } + + _ = cfgr.Close() + _ = outFile.Close() + + log.Infof("The config file has been restored to '%s'.", configFile.Name) + log.Infof("You can now make changes to it, hit enter when you're done.") + if _, err := bufio.NewReader(os.Stdin).ReadString('\n'); err != nil { + return fmt.Errorf("could not read from stdin: %s", err) + } + log.Info("Restoring...") + + // Init the configFile again since the restored configuration is most likely different from the one before + initialize.LightInit() + initialize.InitEngines() + files.InitFileHandler() + + /////// + // Restore the db + // Start by wiping everything + if err := db.WipeEverything(); err != nil { + return fmt.Errorf("could not wipe database: %s", err) + } + log.Info("Wiped database.") + + // Because we don't explicitly saved the table definitions, we take the last ran db migration from the dump + // and execute everything until that point. + migrations := dbfiles["migration"] + rc, err := migrations.Open() + if err != nil { + return fmt.Errorf("could not open migrations: %s", err) + } + defer rc.Close() + + var buf bytes.Buffer + if _, err := buf.ReadFrom(rc); err != nil { + return fmt.Errorf("could not read migrations: %s", err) + } + + ms := []*xormigrate.Migration{} + if err := json.Unmarshal(buf.Bytes(), &ms); err != nil { + return fmt.Errorf("could not read migrations: %s", err) + } + sort.Slice(ms, func(i, j int) bool { + return ms[i].ID > ms[j].ID + }) + + lastMigration := ms[len(ms)-1] + if err := migration.MigrateTo(lastMigration.ID, nil); err != nil { + return fmt.Errorf("could not create db structure: %s", err) + } + + // Restore all db data + for table, d := range dbfiles { + content, err := unmarshalFileToJSON(d) + if err != nil { + return fmt.Errorf("could not read table %s: %s", table, err) + } + if err := db.Restore(table, content); err != nil { + return fmt.Errorf("could not restore table data for table %s: %s", table, err) + } + log.Infof("Restored table %s", table) + } + log.Infof("Restored %d tables", len(dbfiles)) + + // Run migrations again to migrate a potentially outdated dump + migration.Migrate(nil) + + /////// + // Restore Files + for i, file := range filesFiles { + id, err := strconv.ParseInt(i, 10, 64) + if err != nil { + return fmt.Errorf("could not parse file id %s: %s", i, err) + } + + f := &files.File{ID: id} + + fc, err := file.Open() + if err != nil { + return fmt.Errorf("could not open file %s: %s", i, err) + } + + if err := f.Save(fc); err != nil { + return fmt.Errorf("could not save file: %s", err) + } + + _ = fc.Close() + log.Infof("Restored file %s", i) + } + log.Infof("Restored %d files.", len(filesFiles)) + + /////// + // Done + log.Infof("Done restoring dump.") + + return nil +} + +func unmarshalFileToJSON(file *zip.File) (contents []map[string]interface{}, err error) { + rc, err := file.Open() + if err != nil { + return + } + defer rc.Close() + + var buf bytes.Buffer + if _, err := buf.ReadFrom(rc); err != nil { + return nil, err + } + + contents = []map[string]interface{}{} + if err := json.Unmarshal(buf.Bytes(), &contents); err != nil { + return nil, err + } + return +}