2021-09-04 19:26:31 +00:00
// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-2021 Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public Licensee as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public Licensee for more details.
//
// You should have received a copy of the GNU Affero General Public Licensee
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package vikunjafile
import (
"archive/zip"
"bytes"
"encoding/json"
"fmt"
"io"
"strconv"
"strings"
"code.vikunja.io/api/pkg/db"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/user"
2023-01-13 17:32:54 +00:00
"github.com/hashicorp/go-version"
2021-09-04 19:26:31 +00:00
)
const logPrefix = "[Vikunja File Import] "
type FileMigrator struct {
}
// Name is used to get the name of the vikunja-file migration - we're using the docs here to annotate the status route.
// @Summary Get migration status
// @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.
// @tags migration
// @Produce json
// @Security JWTKeyAuth
// @Success 200 {object} migration.Status "The migration status"
// @Failure 500 {object} models.Message "Internal server error"
// @Router /migration/vikunja-file/status [get]
func ( v * FileMigrator ) Name ( ) string {
return "vikunja-file"
}
// Migrate takes a vikunja file export, parses it and imports everything in it into Vikunja.
2022-11-13 16:07:01 +00:00
// @Summary Import all projects, tasks etc. from a Vikunja data export
2021-09-04 19:26:31 +00:00
// @Description Imports all projects, tasks, notes, reminders, subtasks and files from a Vikunjda data export into Vikunja.
// @tags migration
2023-04-03 00:17:09 +00:00
// @Accept x-www-form-urlencoded
2021-09-04 19:26:31 +00:00
// @Produce json
// @Security JWTKeyAuth
// @Param import formData string true "The Vikunja export zip file."
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
// @Failure 500 {object} models.Message "Internal server error"
// @Router /migration/vikunja-file/migrate [post]
func ( v * FileMigrator ) Migrate ( user * user . User , file io . ReaderAt , size int64 ) error {
r , err := zip . NewReader ( file , size )
if err != nil {
2022-03-27 14:55:37 +00:00
return fmt . Errorf ( "could not open import file: %w" , err )
2021-09-04 19:26:31 +00:00
}
log . Debugf ( logPrefix + "Importing a zip file containing %d files" , len ( r . File ) )
var dataFile * zip . File
var filterFile * zip . File
2023-01-13 17:32:54 +00:00
var versionFile * zip . File
2021-09-04 19:26:31 +00:00
storedFiles := make ( map [ int64 ] * zip . File )
for _ , f := range r . File {
if strings . HasPrefix ( f . Name , "files/" ) {
fname := strings . ReplaceAll ( f . Name , "files/" , "" )
id , err := strconv . ParseInt ( fname , 10 , 64 )
if err != nil {
2022-03-27 14:55:37 +00:00
return fmt . Errorf ( "could not convert file id: %w" , err )
2021-09-04 19:26:31 +00:00
}
storedFiles [ id ] = f
log . Debugf ( logPrefix + "Found a blob file" )
continue
}
if f . Name == "data.json" {
dataFile = f
log . Debugf ( logPrefix + "Found a data file" )
continue
}
if f . Name == "filters.json" {
filterFile = f
log . Debugf ( logPrefix + "Found a filter file" )
}
2023-01-13 17:32:54 +00:00
if f . Name == "VERSION" {
versionFile = f
log . Debugf ( logPrefix + "Found a version file" )
}
2021-09-04 19:26:31 +00:00
}
if dataFile == nil {
return fmt . Errorf ( "no data file provided" )
}
log . Debugf ( logPrefix + "" )
2023-01-13 17:32:54 +00:00
//////
// Check if we're able to import this dump
vf , err := versionFile . Open ( )
if err != nil {
return fmt . Errorf ( "could not open version file: %w" , err )
}
var bufVersion bytes . Buffer
if _ , err := bufVersion . ReadFrom ( vf ) ; err != nil {
return fmt . Errorf ( "could not read version file: %w" , err )
}
dumpedVersion , err := version . NewVersion ( bufVersion . String ( ) )
if err != nil {
return err
}
minVersion , err := version . NewVersion ( "0.20.1+61" )
if err != nil {
return err
}
if dumpedVersion . LessThan ( minVersion ) {
return fmt . Errorf ( "export was created with an older version, need at least %s but the export needs at least %s" , dumpedVersion , minVersion )
}
2021-09-04 19:26:31 +00:00
//////
// Import the bulk of Vikunja data
df , err := dataFile . Open ( )
if err != nil {
2022-03-27 14:55:37 +00:00
return fmt . Errorf ( "could not open data file: %w" , err )
2021-09-04 19:26:31 +00:00
}
defer df . Close ( )
var bufData bytes . Buffer
if _ , err := bufData . ReadFrom ( df ) ; err != nil {
2022-03-27 14:55:37 +00:00
return fmt . Errorf ( "could not read data file: %w" , err )
2021-09-04 19:26:31 +00:00
}
2023-01-13 17:32:54 +00:00
projects := [ ] * models . ProjectWithTasksAndBuckets { }
if err := json . Unmarshal ( bufData . Bytes ( ) , & projects ) ; err != nil {
2022-03-27 14:55:37 +00:00
return fmt . Errorf ( "could not read data: %w" , err )
2021-09-04 19:26:31 +00:00
}
2023-01-13 17:32:54 +00:00
for _ , p := range projects {
err = addDetailsToProjectAndChildren ( p , storedFiles )
if err != nil {
return err
2021-09-04 19:26:31 +00:00
}
}
2023-01-13 17:32:54 +00:00
err = migration . InsertFromStructure ( projects , user )
2021-09-04 19:26:31 +00:00
if err != nil {
2022-03-27 14:55:37 +00:00
return fmt . Errorf ( "could not insert data: %w" , err )
2021-09-04 19:26:31 +00:00
}
if filterFile == nil {
log . Debugf ( logPrefix + "No filter file found" )
return nil
}
///////
// Import filters
ff , err := filterFile . Open ( )
if err != nil {
2022-03-27 14:55:37 +00:00
return fmt . Errorf ( "could not open filters file: %w" , err )
2021-09-04 19:26:31 +00:00
}
defer ff . Close ( )
var bufFilter bytes . Buffer
if _ , err := bufFilter . ReadFrom ( ff ) ; err != nil {
2022-03-27 14:55:37 +00:00
return fmt . Errorf ( "could not read filters file: %w" , err )
2021-09-04 19:26:31 +00:00
}
filters := [ ] * models . SavedFilter { }
if err := json . Unmarshal ( bufFilter . Bytes ( ) , & filters ) ; err != nil {
2022-03-27 14:55:37 +00:00
return fmt . Errorf ( "could not read filter data: %w" , err )
2021-09-04 19:26:31 +00:00
}
log . Debugf ( logPrefix + "Importing %d saved filters" , len ( filters ) )
s := db . NewSession ( )
defer s . Close ( )
for _ , f := range filters {
f . ID = 0
err = f . Create ( s , user )
if err != nil {
_ = s . Rollback ( )
return err
}
}
return s . Commit ( )
}
2023-01-13 17:32:54 +00:00
func addDetailsToProjectAndChildren ( p * models . ProjectWithTasksAndBuckets , storedFiles map [ int64 ] * zip . File ) ( err error ) {
err = addDetailsToProject ( p , storedFiles )
if err != nil {
return err
}
for _ , cp := range p . ChildProjects {
err = addDetailsToProjectAndChildren ( cp , storedFiles )
if err != nil {
return
}
}
return
}
func addDetailsToProject ( l * models . ProjectWithTasksAndBuckets , storedFiles map [ int64 ] * zip . File ) ( err error ) {
if b , exists := storedFiles [ l . BackgroundFileID ] ; exists {
bf , err := b . Open ( )
if err != nil {
return fmt . Errorf ( "could not open project background file %d for reading: %w" , l . BackgroundFileID , err )
}
var buf bytes . Buffer
if _ , err := buf . ReadFrom ( bf ) ; err != nil {
return fmt . Errorf ( "could not read project background file %d: %w" , l . BackgroundFileID , err )
}
l . BackgroundInformation = & buf
}
for _ , t := range l . Tasks {
for _ , label := range t . Labels {
label . ID = 0
}
for _ , comment := range t . Comments {
comment . ID = 0
}
for _ , attachment := range t . Attachments {
attachmentFile , exists := storedFiles [ attachment . File . ID ]
if ! exists {
log . Debugf ( logPrefix + "Could not find attachment file %d for attachment %d" , attachment . File . ID , attachment . ID )
continue
}
af , err := attachmentFile . Open ( )
if err != nil {
return fmt . Errorf ( "could not open attachment %d for reading: %w" , attachment . ID , err )
}
var buf bytes . Buffer
if _ , err := buf . ReadFrom ( af ) ; err != nil {
return fmt . Errorf ( "could not read attachment %d: %w" , attachment . ID , err )
}
attachment . ID = 0
attachment . File . ID = 0
attachment . File . FileContent = buf . Bytes ( )
}
}
return
}