diff --git a/go.mod b/go.mod index ccbdf9113..3698955a5 100644 --- a/go.mod +++ b/go.mod @@ -111,6 +111,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/go.sum b/go.sum index be92024bf..fa218aac2 100644 --- a/go.sum +++ b/go.sum @@ -370,6 +370,8 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= diff --git a/pkg/models/export.go b/pkg/models/export.go index 54ff0abce..df208d066 100644 --- a/pkg/models/export.go +++ b/pkg/models/export.go @@ -124,7 +124,7 @@ func ExportUserData(s *xorm.Session, u *user.User) (err error) { func exportProjectsAndTasks(s *xorm.Session, u *user.User, wr *zip.Writer) (taskIDs []int64, err error) { // Get all projects - rawProjectsMap, _, _, err := getRawProjectsForUser( + rawProjects, _, _, err := getRawProjectsForUser( s, &projectOptions{ search: "", @@ -137,22 +137,23 @@ func exportProjectsAndTasks(s *xorm.Session, u *user.User, wr *zip.Writer) (task return taskIDs, err } - if len(rawProjectsMap) == 0 { + if len(rawProjects) == 0 { return } - projects := []*Project{} - projectsMap := make(map[int64]*ProjectWithTasksAndBuckets, len(rawProjectsMap)) + projects := []*ProjectWithTasksAndBuckets{} + projectsMap := make(map[int64]*ProjectWithTasksAndBuckets, len(rawProjects)) projectIDs := []int64{} - for _, p := range rawProjectsMap { - projects = append(projects, p) - projectsMap[p.ID] = &ProjectWithTasksAndBuckets{ + for _, p := range rawProjects { + pp := &ProjectWithTasksAndBuckets{ Project: *p, } + projects = append(projects, pp) + projectsMap[p.ID] = pp projectIDs = append(projectIDs, p.ID) } - tasks, _, _, err := getTasksForProjects(s, projects, u, &taskOptions{ + tasks, _, _, err := getTasksForProjects(s, rawProjects, u, &taskOptions{ page: 0, perPage: -1, }) diff --git a/pkg/modules/migration/vikunja-file/export.zip b/pkg/modules/migration/vikunja-file/export.zip index c22c81289..0e9442ecb 100644 Binary files a/pkg/modules/migration/vikunja-file/export.zip and b/pkg/modules/migration/vikunja-file/export.zip differ diff --git a/pkg/modules/migration/vikunja-file/export_pre_0.21.0.zip b/pkg/modules/migration/vikunja-file/export_pre_0.21.0.zip new file mode 100644 index 000000000..9289c3224 Binary files /dev/null and b/pkg/modules/migration/vikunja-file/export_pre_0.21.0.zip differ diff --git a/pkg/modules/migration/vikunja-file/vikunja.go b/pkg/modules/migration/vikunja-file/vikunja.go index 60266fb87..7ee90b6f1 100644 --- a/pkg/modules/migration/vikunja-file/vikunja.go +++ b/pkg/modules/migration/vikunja-file/vikunja.go @@ -30,6 +30,8 @@ import ( "code.vikunja.io/api/pkg/models" "code.vikunja.io/api/pkg/modules/migration" "code.vikunja.io/api/pkg/user" + + "github.com/hashicorp/go-version" ) const logPrefix = "[Vikunja File Import] " @@ -71,6 +73,7 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er var dataFile *zip.File var filterFile *zip.File + var versionFile *zip.File storedFiles := make(map[int64]*zip.File) for _, f := range r.File { if strings.HasPrefix(f.Name, "files/") { @@ -92,6 +95,10 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er filterFile = f log.Debugf(logPrefix + "Found a filter file") } + if f.Name == "VERSION" { + versionFile = f + log.Debugf(logPrefix + "Found a version file") + } } if dataFile == nil { @@ -100,6 +107,31 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er log.Debugf(logPrefix + "") + ////// + // Check if we're able to import this dump + vf, err := versionFile.Open() + if err != nil { + return fmt.Errorf("could not open version file: %w", err) + } + + var bufVersion bytes.Buffer + if _, err := bufVersion.ReadFrom(vf); err != nil { + return fmt.Errorf("could not read version file: %w", err) + } + + dumpedVersion, err := version.NewVersion(bufVersion.String()) + if err != nil { + return err + } + minVersion, err := version.NewVersion("0.20.1+61") + if err != nil { + return err + } + + if dumpedVersion.LessThan(minVersion) { + return fmt.Errorf("export was created with an older version, need at least %s but the export needs at least %s", dumpedVersion, minVersion) + } + ////// // Import the bulk of Vikunja data df, err := dataFile.Open() @@ -113,57 +145,19 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er return fmt.Errorf("could not read data file: %w", err) } - parent := []*models.ProjectWithTasksAndBuckets{} - if err := json.Unmarshal(bufData.Bytes(), &parent); err != nil { + projects := []*models.ProjectWithTasksAndBuckets{} + if err := json.Unmarshal(bufData.Bytes(), &projects); err != nil { return fmt.Errorf("could not read data: %w", err) } - for _, n := range parent { - for _, l := range n.ChildProjects { - if b, exists := storedFiles[l.BackgroundFileID]; exists { - bf, err := b.Open() - if err != nil { - return fmt.Errorf("could not open project background file %d for reading: %w", l.BackgroundFileID, err) - } - var buf bytes.Buffer - if _, err := buf.ReadFrom(bf); err != nil { - return fmt.Errorf("could not read project background file %d: %w", l.BackgroundFileID, err) - } - - l.BackgroundInformation = &buf - } - - for _, t := range l.Tasks { - for _, label := range t.Labels { - label.ID = 0 - } - for _, comment := range t.Comments { - comment.ID = 0 - } - for _, attachment := range t.Attachments { - attachmentFile, exists := storedFiles[attachment.File.ID] - if !exists { - log.Debugf(logPrefix+"Could not find attachment file %d for attachment %d", attachment.File.ID, attachment.ID) - continue - } - af, err := attachmentFile.Open() - if err != nil { - return fmt.Errorf("could not open attachment %d for reading: %w", attachment.ID, err) - } - var buf bytes.Buffer - if _, err := buf.ReadFrom(af); err != nil { - return fmt.Errorf("could not read attachment %d: %w", attachment.ID, err) - } - - attachment.ID = 0 - attachment.File.ID = 0 - attachment.File.FileContent = buf.Bytes() - } - } + for _, p := range projects { + err = addDetailsToProjectAndChildren(p, storedFiles) + if err != nil { + return err } } - err = migration.InsertFromStructure(parent, user) + err = migration.InsertFromStructure(projects, user) if err != nil { return fmt.Errorf("could not insert data: %w", err) } @@ -207,3 +201,64 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er return s.Commit() } + +func addDetailsToProjectAndChildren(p *models.ProjectWithTasksAndBuckets, storedFiles map[int64]*zip.File) (err error) { + err = addDetailsToProject(p, storedFiles) + if err != nil { + return err + } + + for _, cp := range p.ChildProjects { + err = addDetailsToProjectAndChildren(cp, storedFiles) + if err != nil { + return + } + } + + return +} + +func addDetailsToProject(l *models.ProjectWithTasksAndBuckets, storedFiles map[int64]*zip.File) (err error) { + if b, exists := storedFiles[l.BackgroundFileID]; exists { + bf, err := b.Open() + if err != nil { + return fmt.Errorf("could not open project background file %d for reading: %w", l.BackgroundFileID, err) + } + var buf bytes.Buffer + if _, err := buf.ReadFrom(bf); err != nil { + return fmt.Errorf("could not read project background file %d: %w", l.BackgroundFileID, err) + } + + l.BackgroundInformation = &buf + } + + for _, t := range l.Tasks { + for _, label := range t.Labels { + label.ID = 0 + } + for _, comment := range t.Comments { + comment.ID = 0 + } + for _, attachment := range t.Attachments { + attachmentFile, exists := storedFiles[attachment.File.ID] + if !exists { + log.Debugf(logPrefix+"Could not find attachment file %d for attachment %d", attachment.File.ID, attachment.ID) + continue + } + af, err := attachmentFile.Open() + if err != nil { + return fmt.Errorf("could not open attachment %d for reading: %w", attachment.ID, err) + } + var buf bytes.Buffer + if _, err := buf.ReadFrom(af); err != nil { + return fmt.Errorf("could not read attachment %d: %w", attachment.ID, err) + } + + attachment.ID = 0 + attachment.File.ID = 0 + attachment.File.FileContent = buf.Bytes() + } + } + + return +} diff --git a/pkg/modules/migration/vikunja-file/vikunja_test.go b/pkg/modules/migration/vikunja-file/vikunja_test.go index 5ce8bbf81..676a5ebc3 100644 --- a/pkg/modules/migration/vikunja-file/vikunja_test.go +++ b/pkg/modules/migration/vikunja-file/vikunja_test.go @@ -27,49 +27,71 @@ import ( ) func TestVikunjaFileMigrator_Migrate(t *testing.T) { - db.LoadAndAssertFixtures(t) + t.Run("migrate successfully", func(t *testing.T) { + db.LoadAndAssertFixtures(t) - m := &FileMigrator{} - u := &user.User{ID: 1} + m := &FileMigrator{} + u := &user.User{ID: 1} - f, err := os.Open(config.ServiceRootpath.GetString() + "/pkg/modules/migration/vikunja-file/export.zip") - if err != nil { - t.Fatalf("Could not open file: %s", err) - } - defer f.Close() - s, err := f.Stat() - if err != nil { - t.Fatalf("Could not stat file: %s", err) - } + f, err := os.Open(config.ServiceRootpath.GetString() + "/pkg/modules/migration/vikunja-file/export.zip") + if err != nil { + t.Fatalf("Could not open file: %s", err) + } + defer f.Close() + s, err := f.Stat() + if err != nil { + t.Fatalf("Could not stat file: %s", err) + } - err = m.Migrate(u, f, s.Size()) - assert.NoError(t, err) - db.AssertExists(t, "projects", map[string]interface{}{ - "title": "Test project", - "owner_id": u.ID, - }, false) - db.AssertExists(t, "projects", map[string]interface{}{ - "title": "A project with a background", - "owner_id": u.ID, - }, false) - db.AssertExists(t, "tasks", map[string]interface{}{ - "title": "Some other task", - "created_by_id": u.ID, - }, false) - db.AssertExists(t, "task_comments", map[string]interface{}{ - "comment": "This is a comment", - "author_id": u.ID, - }, false) - db.AssertExists(t, "files", map[string]interface{}{ - "name": "cristiano-mozzillo-v3d5uBB26yA-unsplash.jpg", - "created_by_id": u.ID, - }, false) - db.AssertExists(t, "labels", map[string]interface{}{ - "title": "test", - "created_by_id": u.ID, - }, false) - db.AssertExists(t, "buckets", map[string]interface{}{ - "title": "Test Bucket", - "created_by_id": u.ID, - }, false) + err = m.Migrate(u, f, s.Size()) + assert.NoError(t, err) + db.AssertExists(t, "projects", map[string]interface{}{ + "title": "test project", + "owner_id": u.ID, + }, false) + db.AssertExists(t, "projects", map[string]interface{}{ + "title": "Inbox", + "owner_id": u.ID, + }, false) + db.AssertExists(t, "tasks", map[string]interface{}{ + "title": "some other task", + "created_by_id": u.ID, + }, false) + db.AssertExists(t, "task_comments", map[string]interface{}{ + "comment": "This is a comment", + "author_id": u.ID, + }, false) + db.AssertExists(t, "files", map[string]interface{}{ + "name": "grant-whitty-546453-unsplash.jpg", + "created_by_id": u.ID, + }, false) + db.AssertExists(t, "labels", map[string]interface{}{ + "title": "test", + "created_by_id": u.ID, + }, false) + db.AssertExists(t, "buckets", map[string]interface{}{ + "title": "Test Bucket", + "created_by_id": u.ID, + }, false) + }) + t.Run("should not accept an old import", func(t *testing.T) { + db.LoadAndAssertFixtures(t) + + m := &FileMigrator{} + u := &user.User{ID: 1} + + f, err := os.Open(config.ServiceRootpath.GetString() + "/pkg/modules/migration/vikunja-file/export_pre_0.21.0.zip") + if err != nil { + t.Fatalf("Could not open file: %s", err) + } + defer f.Close() + s, err := f.Stat() + if err != nil { + t.Fatalf("Could not stat file: %s", err) + } + + err = m.Migrate(u, f, s.Size()) + assert.Error(t, err) + assert.ErrorContainsf(t, err, "export was created with an older version", "Invalid error message") + }) }