Add dump command (#592)

Fix files location in dump

Fix gitignore

Add docs

Add dumps to gitignore

Move dump to seperate package

logging

Dump files

Dump version

Dump database

Dump config

Co-authored-by: kolaente <k@knt.li>
Reviewed-on: vikunja/api#592

.gitignore

@@ -20,3 +20,4 @@ docs/resources/
 pkg/static/templates_vfsdata.go
 files/
 !pkg/files/
+vikunja-dump*

docs/content/doc/usage/cli.md

@@ -81,4 +81,14 @@ Starts Vikunja's REST api server.
 Usage:
 {{< highlight bash >}}
 $ vikunja web    
-{{< /highlight >}}
+{{< /highlight >}}
+
+### `dump`
+
+Creates a zip file with all vikunja-related files.
+This includes config, version, all files and the full database.
+
+Usage:
+{{< highlight bash >}}
+$ vikunja dump
+{{< /highlight >}}

pkg/cmd/dump.go

@@ -0,0 +1,39 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"code.vikunja.io/api/pkg/modules/dump"
+	"github.com/spf13/cobra"
+	"time"
+)
+
+func init() {
+	rootCmd.AddCommand(dumpCmd)
+}
+
+var dumpCmd = &cobra.Command{
+	Use:   "dump",
+	Short: "Dump all vikunja data into a zip file. Includes config, files and db.",
+	PreRun: func(cmd *cobra.Command, args []string) {
+		fullInit()
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		filename := "vikunja-dump_" + time.Now().Format("2006-01-02_15-03-05") + ".zip"
+		dump.Dump(filename)
+	},
+}

pkg/db/dump.go

@@ -0,0 +1,42 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package db
+
+import "encoding/json"
+
+// Dump dumps all database tables
+func Dump() (data map[string][]byte, err error) {
+	tables, err := x.DBMetas()
+	if err != nil {
+		return
+	}
+
+	data = make(map[string][]byte, len(tables))
+	for _, table := range tables {
+		entries := []map[string]interface{}{}
+		err := x.Table(table.Name).Find(&entries)
+		if err != nil {
+			return nil, err
+		}
+		data[table.Name], err = json.Marshal(entries)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return
+}

pkg/files/dump.go

@@ -0,0 +1,43 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package files
+
+import "bytes"
+
+// Dump dumps all saved files
+// This only includes the raw files, no db entries.
+func Dump() (allFiles map[int64][]byte, err error) {
+	files := []*File{}
+	err = x.Find(&files)
+	if err != nil {
+		return
+	}
+
+	allFiles = make(map[int64][]byte, len(files))
+	for _, file := range files {
+		if err := file.LoadFileByID(); err != nil {
+			return nil, err
+		}
+		var buf bytes.Buffer
+		if _, err := buf.ReadFrom(file.File); err != nil {
+			return nil, err
+		}
+		allFiles[file.ID] = buf.Bytes()
+	}
+
+	return
+}

pkg/modules/dump/dump.go

@@ -0,0 +1,136 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-2020 Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package dump
+
+import (
+	"archive/zip"
+	"code.vikunja.io/api/pkg/db"
+	"code.vikunja.io/api/pkg/files"
+	"code.vikunja.io/api/pkg/log"
+	"code.vikunja.io/api/pkg/version"
+	"github.com/spf13/viper"
+	"io"
+	"os"
+	"strconv"
+)
+
+// Change to deflate to gain better compression
+// see http://golang.org/pkg/archive/zip/#pkg-constants
+const compressionUsed = zip.Deflate
+
+// Dump creates a zip file with all vikunja files at filename
+func Dump(filename string) {
+	dumpFile, err := os.Create(filename)
+	if err != nil {
+		log.Criticalf("Error opening dump file: %s", err)
+	}
+	defer dumpFile.Close()
+
+	dumpWriter := zip.NewWriter(dumpFile)
+	defer dumpWriter.Close()
+
+	// Config
+	log.Info("Start dumping config file...")
+	err = writeFileToZip(viper.ConfigFileUsed(), dumpWriter)
+	if err != nil {
+		log.Criticalf("Error saving config file: %s", err)
+	}
+	log.Info("Dumped config file")
+
+	// Version
+	log.Info("Start dumping version file...")
+	err = writeBytesToZip("VERSION", []byte(version.Version), dumpWriter)
+	if err != nil {
+		log.Criticalf("Error saving version: %s", err)
+	}
+	log.Info("Dumped version")
+
+	// Database
+	log.Info("Start dumping database...")
+	data, err := db.Dump()
+	if err != nil {
+		log.Criticalf("Error saving database data: %s", err)
+	}
+	for t, d := range data {
+		err = writeBytesToZip("database/"+t+".json", d, dumpWriter)
+		if err != nil {
+			log.Criticalf("Error writing database table %s: %s", t, err)
+		}
+	}
+	log.Info("Dumped database")
+
+	// Files
+	log.Info("Start dumping files...")
+	allFiles, err := files.Dump()
+	if err != nil {
+		log.Criticalf("Error saving file: %s", err)
+	}
+	for fid, fcontent := range allFiles {
+		err = writeBytesToZip("files/"+strconv.FormatInt(fid, 10), fcontent, dumpWriter)
+		if err != nil {
+			log.Criticalf("Error writing file %d: %s", fid, err)
+		}
+	}
+	log.Infof("Dumped files")
+
+	log.Info("Done creating dump")
+	log.Infof("Dump file saved at %s", filename)
+
+}
+
+func writeFileToZip(filename string, writer *zip.Writer) error {
+	// #nosec
+	fileToZip, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer fileToZip.Close()
+
+	// Get the file information
+	info, err := fileToZip.Stat()
+	if err != nil {
+		return err
+	}
+
+	header, err := zip.FileInfoHeader(info)
+	if err != nil {
+		return err
+	}
+
+	header.Name = info.Name()
+	header.Method = compressionUsed
+
+	w, err := writer.CreateHeader(header)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(w, fileToZip)
+	return err
+}
+
+func writeBytesToZip(filename string, data []byte, writer *zip.Writer) (err error) {
+	header := &zip.FileHeader{
+		Name:   filename,
+		Method: compressionUsed,
+	}
+	w, err := writer.CreateHeader(header)
+	if err != nil {
+		return err
+	}
+	_, err = w.Write(data)
+	return
+}