vikunja
/
vikunja
6
43
Fork 71
Code Issues 150 Pull Requests 15 Packages Releases 32 Activity

Restore command #593

Merged
konrad merged 8 commits from feature/restore into master 2020-06-21 15:30:51 +00:00
Conversation 0 Commits 8 Files Changed 12 +390 -59
12 changed files with 390 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
pkg/cmd/cmd.go
View File

@ -17,15 +17,6 @@
package cmd
import (
"code.vikunja.io/api/pkg/config"
"code.vikunja.io/api/pkg/files"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/mail"
"code.vikunja.io/api/pkg/migration"
"code.vikunja.io/api/pkg/models"
migrator "code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/red"
"code.vikunja.io/api/pkg/user"
"fmt"
"github.com/spf13/cobra"
"os"
@ -54,48 +45,3 @@ func Execute() {
os.Exit(1)
}
}
// Will only fullInit config, redis, logger but no db connection.
func lightInit() {
// Init the config
config.InitConfig()
// Init redis
red.InitRedis()
// Set logger
log.InitLogger()
}
// Initializes all kinds of things in the right order
func fullInit() {
lightInit()
// Run the migrations
migration.Migrate(nil)
// Set Engine
err := models.SetEngine()
if err != nil {
log.Fatal(err.Error())
}
err = user.InitDB()
if err != nil {
log.Fatal(err.Error())
}
err = files.SetEngine()
if err != nil {
log.Fatal(err.Error())
}
err = migrator.InitDB()
if err != nil {
log.Fatal(err.Error())
}
// Initialize the files handler
files.InitFileHandler()
// Start the mail daemon
mail.StartMailDaemon()
}

3
pkg/cmd/dump.go
View File

@ -17,6 +17,7 @@
package cmd
import (
"code.vikunja.io/api/pkg/initialize"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/modules/dump"
"github.com/spf13/cobra"
@ -31,7 +32,7 @@ var dumpCmd = &cobra.Command{
Use: "dump",
Short: "Dump all vikunja data into a zip file. Includes config, files and db.",
PreRun: func(cmd *cobra.Command, args []string) {
fullInit()
initialize.FullInit()
},
Run: func(cmd *cobra.Command, args []string) {
filename := "vikunja-dump_" + time.Now().Format("2006-01-02_15-03-05") + ".zip"

3
pkg/cmd/migrate.go
View File

@ -17,6 +17,7 @@
package cmd
import (
"code.vikunja.io/api/pkg/initialize"
"code.vikunja.io/api/pkg/migration"
"github.com/spf13/cobra"
)
@ -36,7 +37,7 @@ var migrateCmd = &cobra.Command{
Use: "migrate",
Short: "Run all database migrations which didn't already run.",
PersistentPreRun: func(cmd *cobra.Command, args []string) {
lightInit()
initialize.LightInit()
},
Run: func(cmd *cobra.Command, args []string) {
migration.Migrate(nil)

42
pkg/cmd/restore.go Normal file
View File

@ -0,0 +1,42 @@
// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package cmd
import (
"code.vikunja.io/api/pkg/initialize"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/modules/dump"
"github.com/spf13/cobra"
)
func init() {
rootCmd.AddCommand(restoreCmd)
}
var restoreCmd = &cobra.Command{
Use: "restore [filename]",
Short: "Restores all vikunja data from a vikunja dump.",
Args: cobra.ExactArgs(1),
PreRun: func(cmd *cobra.Command, args []string) {
initialize.FullInit()
},
Run: func(cmd *cobra.Command, args []string) {
if err := dump.Restore(args[0]); err != nil {
log.Critical(err.Error())
}
},
}

3
pkg/cmd/testmail.go
View File

@ -17,6 +17,7 @@
package cmd
import (
"code.vikunja.io/api/pkg/initialize"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/mail"
"github.com/spf13/cobra"
@ -31,7 +32,7 @@ var testmailCmd = &cobra.Command{
Short: "Send a test mail using the configured smtp connection",
Args: cobra.ExactArgs(1),
PreRun: func(cmd *cobra.Command, args []string) {
lightInit()
initialize.LightInit()
// Start the mail daemon
mail.StartMailDaemon()

3
pkg/cmd/web.go
View File

@ -18,6 +18,7 @@ package cmd
import (
"code.vikunja.io/api/pkg/config"
"code.vikunja.io/api/pkg/initialize"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/routes"
"code.vikunja.io/api/pkg/swagger"
@ -37,7 +38,7 @@ var webCmd = &cobra.Command{
Use: "web",
Short: "Starts the rest api web server",
PreRun: func(cmd *cobra.Command, args []string) {
fullInit()
initialize.FullInit()
},
Run: func(cmd *cobra.Command, args []string) {

17
pkg/db/db.go
View File

@ -168,3 +168,20 @@ func initSqliteEngine() (engine *xorm.Engine, err error) {
return xorm.NewEngine("sqlite3", path)
}
// WipeEverything wipes all tables and their data. Use with caution...
func WipeEverything() error {
tables, err := x.DBMetas()
if err != nil {
return err
}
for _, t := range tables {
if err := x.DropTables(t.Name); err != nil {
return err
}
}
return nil
}

12
pkg/db/dump.go
View File

@ -40,3 +40,15 @@ func Dump() (data map[string][]byte, err error) {
return
}
// Restore restores a table with all its entries
func Restore(table string, contents []map[string]interface{}) (err error) {
for _, content := range contents {
if _, err := x.Table(table).Insert(content); err != nil {
return err
}
}
return
}

7
pkg/files/files.go
View File

@ -96,7 +96,7 @@ func Create(f io.ReadCloser, realname string, realsize uint64, a web.Auth) (file
}
// Save the file to storage with its new ID as path
err = afs.WriteReader(file.getFileName(), f)
err = file.Save(f)
return
}
@ -113,3 +113,8 @@ func (f *File) Delete() (err error) {
err = afs.Remove(f.getFileName())
return
}
// Save saves a file to storage
func (f *File) Save(fcontent io.ReadCloser) error {
return afs.WriteReader(f.getFileName(), fcontent)
}

79
pkg/initialize/init.go Normal file
View File

@ -0,0 +1,79 @@
// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package initialize
import (
"code.vikunja.io/api/pkg/config"
"code.vikunja.io/api/pkg/files"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/mail"
"code.vikunja.io/api/pkg/migration"
"code.vikunja.io/api/pkg/models"
migrator "code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/red"
"code.vikunja.io/api/pkg/user"
)
// LightInit will only fullInit config, redis, logger but no db connection.
func LightInit() {
// Init the config
config.InitConfig()
// Init redis
red.InitRedis()
// Set logger
log.InitLogger()
}
// InitEngines intializes all db connections
func InitEngines() {
err := models.SetEngine()
if err != nil {
log.Fatal(err.Error())
}
err = user.InitDB()
if err != nil {
log.Fatal(err.Error())
}
err = files.SetEngine()
if err != nil {
log.Fatal(err.Error())
}
err = migrator.InitDB()
if err != nil {
log.Fatal(err.Error())
}
}
// FullInit initializes all kinds of things in the right order
func FullInit() {
LightInit()
// Run the migrations
migration.Migrate(nil)
// Set Engine
InitEngines()
// Initialize the files handler
files.InitFileHandler()
// Start the mail daemon
mail.StartMailDaemon()
}

9
pkg/migration/migration.go
View File

@ -104,6 +104,15 @@ func Rollback(migrationID string) {
log.Info("Rolled back successfully.")
}
// MigrateTo executes all migrations up to a certain point
func MigrateTo(migrationID string, x *xorm.Engine) error {
m := initMigration(x)
if err := m.MigrateTo(migrationID); err != nil {
return err
}
return nil
}
// Deletes a column from a table. All arguments are strings, to let them be standalone and not depending on any struct.
func dropTableColum(x *xorm.Engine, tableName, col string) error {

217
pkg/modules/dump/restore.go Normal file
View File

@ -0,0 +1,217 @@
// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package dump
import (
"archive/zip"
"bufio"
"bytes"
"code.vikunja.io/api/pkg/db"
"code.vikunja.io/api/pkg/files"
"code.vikunja.io/api/pkg/initialize"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/migration"
"encoding/json"
"fmt"
"io"
"os"
"sort"
"src.techknowlogick.com/xormigrate"
"strconv"
"strings"
)
const maxConfigSize = 5 * 1024 * 1024 // 5 MB, should be largely enough
// Restore takes a zip file name and restores it
func Restore(filename string) error {
r, err := zip.OpenReader(filename)
if err != nil {
return fmt.Errorf("could not open zip file: %s", err)
}
log.Warning("Restoring a dump will wipe your current installation!")
log.Warning("To confirm, please type 'Yes, I understand' and confirm with enter:")
cr := bufio.NewReader(os.Stdin)
text, err := cr.ReadString('\n')
if err != nil {
return fmt.Errorf("could not read confirmation message: %s", err)
}
if text != "Yes, I understand\n" {
return fmt.Errorf("invalid confirmation message")
}
// Find the configFile, database and files files
var configFile *zip.File
dbfiles := make(map[string]*zip.File)
filesFiles := make(map[string]*zip.File)
for _, file := range r.File {
if strings.HasPrefix(file.Name, "config") {
configFile = file
continue
}
if strings.HasPrefix(file.Name, "database/") {
fname := strings.ReplaceAll(file.Name, "database/", "")
dbfiles[fname[:len(fname)-5]] = file
continue
}
if strings.HasPrefix(file.Name, "files/") {
filesFiles[strings.ReplaceAll(file.Name, "files/", "")] = file
}
}
if configFile == nil {
return fmt.Errorf("dump does not contain a config file")
}
///////
// Restore the config file
if configFile.UncompressedSize64 > maxConfigSize {
return fmt.Errorf("config file too large, is %d, max size is %d", configFile.UncompressedSize64, maxConfigSize)
}
outFile, err := os.OpenFile(configFile.Name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, configFile.Mode())
if err != nil {
return fmt.Errorf("could not open config file for writing: %s", err)
}
cfgr, err := configFile.Open()
if err != nil {
return err
}
// #nosec - We eliminated the potential decompression bomb by erroring out above if the file is larger than a threshold.
_, err = io.Copy(outFile, cfgr)
if err != nil {
return fmt.Errorf("could not create config file: %s", err)
}
_ = cfgr.Close()
_ = outFile.Close()
log.Infof("The config file has been restored to '%s'.", configFile.Name)
log.Infof("You can now make changes to it, hit enter when you're done.")
if _, err := bufio.NewReader(os.Stdin).ReadString('\n'); err != nil {
return fmt.Errorf("could not read from stdin: %s", err)
}
log.Info("Restoring...")
// Init the configFile again since the restored configuration is most likely different from the one before
initialize.LightInit()
initialize.InitEngines()
files.InitFileHandler()
///////
// Restore the db
// Start by wiping everything
if err := db.WipeEverything(); err != nil {
return fmt.Errorf("could not wipe database: %s", err)
}
log.Info("Wiped database.")
// Because we don't explicitly saved the table definitions, we take the last ran db migration from the dump
// and execute everything until that point.
migrations := dbfiles["migration"]
rc, err := migrations.Open()
if err != nil {
return fmt.Errorf("could not open migrations: %s", err)
}
defer rc.Close()
var buf bytes.Buffer
if _, err := buf.ReadFrom(rc); err != nil {
return fmt.Errorf("could not read migrations: %s", err)
}
ms := []*xormigrate.Migration{}
if err := json.Unmarshal(buf.Bytes(), &ms); err != nil {
return fmt.Errorf("could not read migrations: %s", err)
}
sort.Slice(ms, func(i, j int) bool {
return ms[i].ID > ms[j].ID
})
lastMigration := ms[len(ms)-1]
if err := migration.MigrateTo(lastMigration.ID, nil); err != nil {
return fmt.Errorf("could not create db structure: %s", err)
}
// Restore all db data
for table, d := range dbfiles {
content, err := unmarshalFileToJSON(d)
if err != nil {
return fmt.Errorf("could not read table %s: %s", table, err)
}
if err := db.Restore(table, content); err != nil {
return fmt.Errorf("could not restore table data for table %s: %s", table, err)
}
log.Infof("Restored table %s", table)
}
log.Infof("Restored %d tables", len(dbfiles))
// Run migrations again to migrate a potentially outdated dump
migration.Migrate(nil)
///////
// Restore Files
for i, file := range filesFiles {
id, err := strconv.ParseInt(i, 10, 64)
if err != nil {
return fmt.Errorf("could not parse file id %s: %s", i, err)
}
f := &files.File{ID: id}
fc, err := file.Open()
if err != nil {
return fmt.Errorf("could not open file %s: %s", i, err)
}
if err := f.Save(fc); err != nil {
return fmt.Errorf("could not save file: %s", err)
}
_ = fc.Close()
log.Infof("Restored file %s", i)
}
log.Infof("Restored %d files.", len(filesFiles))
///////
// Done
log.Infof("Done restoring dump.")
return nil
}
func unmarshalFileToJSON(file *zip.File) (contents []map[string]interface{}, err error) {
rc, err := file.Open()
if err != nil {
return
}
defer rc.Close()
var buf bytes.Buffer
if _, err := buf.ReadFrom(rc); err != nil {
return nil, err
}
contents = []map[string]interface{}{}
if err := json.Unmarshal(buf.Bytes(), &contents); err != nil {
return nil, err
}
return
}