vikunja
/
vikunja
6
43
Fork 71
Code Issues 150 Pull Requests 15 Packages Releases 32 Activity

fix(migration): make file migration work with new structure

This commit is contained in:
kolaente 2023-01-13 18:32:54 +01:00
parent 2f81791735
commit 4b55e2ce03
Signed by: konrad
GPG Key ID: F40E70337AB24C9B
7 changed files with 176 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -111,6 +111,7 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect

2
go.sum
View File

@ -370,6 +370,8 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=

17
pkg/models/export.go
View File

@ -124,7 +124,7 @@ func ExportUserData(s *xorm.Session, u *user.User) (err error) {
func exportProjectsAndTasks(s *xorm.Session, u *user.User, wr *zip.Writer) (taskIDs []int64, err error) {
// Get all projects
rawProjectsMap, _, _, err := getRawProjectsForUser(
rawProjects, _, _, err := getRawProjectsForUser(
s,
&projectOptions{
search: "",
@ -137,22 +137,23 @@ func exportProjectsAndTasks(s *xorm.Session, u *user.User, wr *zip.Writer) (task
return taskIDs, err
}
if len(rawProjectsMap) == 0 {
if len(rawProjects) == 0 {
return
}
projects := []*Project{}
projectsMap := make(map[int64]*ProjectWithTasksAndBuckets, len(rawProjectsMap))
projects := []*ProjectWithTasksAndBuckets{}
projectsMap := make(map[int64]*ProjectWithTasksAndBuckets, len(rawProjects))
projectIDs := []int64{}
for _, p := range rawProjectsMap {
projects = append(projects, p)
projectsMap[p.ID] = &ProjectWithTasksAndBuckets{
for _, p := range rawProjects {
pp := &ProjectWithTasksAndBuckets{
Project: *p,
}
projects = append(projects, pp)
projectsMap[p.ID] = pp
projectIDs = append(projectIDs, p.ID)
}
tasks, _, _, err := getTasksForProjects(s, projects, u, &taskOptions{
tasks, _, _, err := getTasksForProjects(s, rawProjects, u, &taskOptions{
page: 0,
perPage: -1,
})

BIN
pkg/modules/migration/vikunja-file/export.zip
View File

Binary file not shown.

BIN
pkg/modules/migration/vikunja-file/export_pre_0.21.0.zip Normal file
View File

Binary file not shown.

145
pkg/modules/migration/vikunja-file/vikunja.go
View File

@ -30,6 +30,8 @@ import (
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/user"
"github.com/hashicorp/go-version"
)
const logPrefix = "[Vikunja File Import] "
@ -71,6 +73,7 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er
var dataFile *zip.File
var filterFile *zip.File
var versionFile *zip.File
storedFiles := make(map[int64]*zip.File)
for _, f := range r.File {
if strings.HasPrefix(f.Name, "files/") {
@ -92,6 +95,10 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er
filterFile = f
log.Debugf(logPrefix + "Found a filter file")
}
if f.Name == "VERSION" {
versionFile = f
log.Debugf(logPrefix + "Found a version file")
}
}
if dataFile == nil {
@ -100,6 +107,31 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er
log.Debugf(logPrefix + "")
//////
// Check if we're able to import this dump
vf, err := versionFile.Open()
if err != nil {
return fmt.Errorf("could not open version file: %w", err)
}
var bufVersion bytes.Buffer
if _, err := bufVersion.ReadFrom(vf); err != nil {
return fmt.Errorf("could not read version file: %w", err)
}
dumpedVersion, err := version.NewVersion(bufVersion.String())
if err != nil {
return err
}
minVersion, err := version.NewVersion("0.20.1+61")
if err != nil {
return err
}
if dumpedVersion.LessThan(minVersion) {
return fmt.Errorf("export was created with an older version, need at least %s but the export needs at least %s", dumpedVersion, minVersion)
}
//////
// Import the bulk of Vikunja data
df, err := dataFile.Open()
@ -113,57 +145,19 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er
return fmt.Errorf("could not read data file: %w", err)
}
parent := []*models.ProjectWithTasksAndBuckets{}
if err := json.Unmarshal(bufData.Bytes(), &parent); err != nil {
projects := []*models.ProjectWithTasksAndBuckets{}
if err := json.Unmarshal(bufData.Bytes(), &projects); err != nil {
return fmt.Errorf("could not read data: %w", err)
}
for _, n := range parent {
for _, l := range n.ChildProjects {
if b, exists := storedFiles[l.BackgroundFileID]; exists {
bf, err := b.Open()
if err != nil {
return fmt.Errorf("could not open project background file %d for reading: %w", l.BackgroundFileID, err)
}
var buf bytes.Buffer
if _, err := buf.ReadFrom(bf); err != nil {
return fmt.Errorf("could not read project background file %d: %w", l.BackgroundFileID, err)
}
l.BackgroundInformation = &buf
}
for _, t := range l.Tasks {
for _, label := range t.Labels {
label.ID = 0
}
for _, comment := range t.Comments {
comment.ID = 0
}
for _, attachment := range t.Attachments {
attachmentFile, exists := storedFiles[attachment.File.ID]
if !exists {
log.Debugf(logPrefix+"Could not find attachment file %d for attachment %d", attachment.File.ID, attachment.ID)
continue
}
af, err := attachmentFile.Open()
if err != nil {
return fmt.Errorf("could not open attachment %d for reading: %w", attachment.ID, err)
}
var buf bytes.Buffer
if _, err := buf.ReadFrom(af); err != nil {
return fmt.Errorf("could not read attachment %d: %w", attachment.ID, err)
}
attachment.ID = 0
attachment.File.ID = 0
attachment.File.FileContent = buf.Bytes()
}
}
for _, p := range projects {
err = addDetailsToProjectAndChildren(p, storedFiles)
if err != nil {
return err
}
}
err = migration.InsertFromStructure(parent, user)
err = migration.InsertFromStructure(projects, user)
if err != nil {
return fmt.Errorf("could not insert data: %w", err)
}
@ -207,3 +201,64 @@ func (v *FileMigrator) Migrate(user *user.User, file io.ReaderAt, size int64) er
return s.Commit()
}
func addDetailsToProjectAndChildren(p *models.ProjectWithTasksAndBuckets, storedFiles map[int64]*zip.File) (err error) {
err = addDetailsToProject(p, storedFiles)
if err != nil {
return err
}
for _, cp := range p.ChildProjects {
err = addDetailsToProjectAndChildren(cp, storedFiles)
if err != nil {
return
}
}
return
}
func addDetailsToProject(l *models.ProjectWithTasksAndBuckets, storedFiles map[int64]*zip.File) (err error) {
if b, exists := storedFiles[l.BackgroundFileID]; exists {
bf, err := b.Open()
if err != nil {
return fmt.Errorf("could not open project background file %d for reading: %w", l.BackgroundFileID, err)
}
var buf bytes.Buffer
if _, err := buf.ReadFrom(bf); err != nil {
return fmt.Errorf("could not read project background file %d: %w", l.BackgroundFileID, err)
}
l.BackgroundInformation = &buf
}
for _, t := range l.Tasks {
for _, label := range t.Labels {
label.ID = 0
}
for _, comment := range t.Comments {
comment.ID = 0
}
for _, attachment := range t.Attachments {
attachmentFile, exists := storedFiles[attachment.File.ID]
if !exists {
log.Debugf(logPrefix+"Could not find attachment file %d for attachment %d", attachment.File.ID, attachment.ID)
continue
}
af, err := attachmentFile.Open()
if err != nil {
return fmt.Errorf("could not open attachment %d for reading: %w", attachment.ID, err)
}
var buf bytes.Buffer
if _, err := buf.ReadFrom(af); err != nil {
return fmt.Errorf("could not read attachment %d: %w", attachment.ID, err)
}
attachment.ID = 0
attachment.File.ID = 0
attachment.File.FileContent = buf.Bytes()
}
}
return
}

106
pkg/modules/migration/vikunja-file/vikunja_test.go
View File

@ -27,49 +27,71 @@ import (
)
func TestVikunjaFileMigrator_Migrate(t *testing.T) {
db.LoadAndAssertFixtures(t)
t.Run("migrate successfully", func(t *testing.T) {
db.LoadAndAssertFixtures(t)
m := &FileMigrator{}
u := &user.User{ID: 1}
m := &FileMigrator{}
u := &user.User{ID: 1}
f, err := os.Open(config.ServiceRootpath.GetString() + "/pkg/modules/migration/vikunja-file/export.zip")
if err != nil {
t.Fatalf("Could not open file: %s", err)
}
defer f.Close()
s, err := f.Stat()
if err != nil {
t.Fatalf("Could not stat file: %s", err)
}
f, err := os.Open(config.ServiceRootpath.GetString() + "/pkg/modules/migration/vikunja-file/export.zip")
if err != nil {
t.Fatalf("Could not open file: %s", err)
}
defer f.Close()
s, err := f.Stat()
if err != nil {
t.Fatalf("Could not stat file: %s", err)
}
err = m.Migrate(u, f, s.Size())
assert.NoError(t, err)
db.AssertExists(t, "projects", map[string]interface{}{
"title": "Test project",
"owner_id": u.ID,
}, false)
db.AssertExists(t, "projects", map[string]interface{}{
"title": "A project with a background",
"owner_id": u.ID,
}, false)
db.AssertExists(t, "tasks", map[string]interface{}{
"title": "Some other task",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "task_comments", map[string]interface{}{
"comment": "This is a comment",
"author_id": u.ID,
}, false)
db.AssertExists(t, "files", map[string]interface{}{
"name": "cristiano-mozzillo-v3d5uBB26yA-unsplash.jpg",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "labels", map[string]interface{}{
"title": "test",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "buckets", map[string]interface{}{
"title": "Test Bucket",
"created_by_id": u.ID,
}, false)
err = m.Migrate(u, f, s.Size())
assert.NoError(t, err)
db.AssertExists(t, "projects", map[string]interface{}{
"title": "test project",
"owner_id": u.ID,
}, false)
db.AssertExists(t, "projects", map[string]interface{}{
"title": "Inbox",
"owner_id": u.ID,
}, false)
db.AssertExists(t, "tasks", map[string]interface{}{
"title": "some other task",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "task_comments", map[string]interface{}{
"comment": "This is a comment",
"author_id": u.ID,
}, false)
db.AssertExists(t, "files", map[string]interface{}{
"name": "grant-whitty-546453-unsplash.jpg",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "labels", map[string]interface{}{
"title": "test",
"created_by_id": u.ID,
}, false)
db.AssertExists(t, "buckets", map[string]interface{}{
"title": "Test Bucket",
"created_by_id": u.ID,
}, false)
})
t.Run("should not accept an old import", func(t *testing.T) {
db.LoadAndAssertFixtures(t)
m := &FileMigrator{}
u := &user.User{ID: 1}
f, err := os.Open(config.ServiceRootpath.GetString() + "/pkg/modules/migration/vikunja-file/export_pre_0.21.0.zip")
if err != nil {
t.Fatalf("Could not open file: %s", err)
}
defer f.Close()
s, err := f.Stat()
if err != nil {
t.Fatalf("Could not stat file: %s", err)
}
err = m.Migrate(u, f, s.Size())
assert.Error(t, err)
assert.ErrorContainsf(t, err, "export was created with an older version", "Invalid error message")
})
}