From 5acf40d86ece3eb3f6bb8886d271b373b6b0e79f Mon Sep 17 00:00:00 2001 From: kolaente Date: Tue, 13 Feb 2024 16:45:10 +0100 Subject: [PATCH] feat: add backup compression --- .gitignore | 1 + README.md | 4 ++++ config.go | 7 +++++++ dump_mysql.go | 2 +- dump_postgres.go | 2 +- go.mod | 22 +++++++++++++++++++--- save.go | 25 +++++++++++++++++++++---- 7 files changed, 54 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 9129bed..9e576fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .idea/ docker-db-backup *.sql +*.gz ./backups diff --git a/README.md b/README.md index f8ddc02..0316571 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,10 @@ Default: `12` If provided, the tool will do an empty GET request to this URL to indicate it successfully completed the backup job. You can use this with other tools to monitor if backups are completed as they should. +### `BACKUP_COMPRESS` + +If set provided and set to `true`, all backups will be compressed using gzip. + ## Building from source This project uses go modules, so you'll need at least go 1.11 to compile it. diff --git a/config.go b/config.go index 7201fee..bead4d7 100644 --- a/config.go +++ b/config.go @@ -18,6 +18,7 @@ type conf struct { Schedule string MaxBackups int CompletionWebhookURL string + CompressBackups bool } var ( @@ -30,6 +31,7 @@ const ( envSchedule = `BACKUP_SCHEDULE` envMax = `BACKUP_MAX` envCompletionWebhookURL = `BACKUP_COMPLETION_WEBHOOK_URL` + envCompressBackups = `BACKUP_COMPRESS` ) func init() { @@ -66,6 +68,11 @@ func init() { if has { config.CompletionWebhookURL = webhookURL } + + compress, has := os.LookupEnv(envCompressBackups) + if has { + config.CompressBackups = compress == "1" || compress == "true" + } } func updateFullBackupPath() { diff --git a/dump_mysql.go b/dump_mysql.go index 38a0d61..4711c41 100644 --- a/dump_mysql.go +++ b/dump_mysql.go @@ -63,5 +63,5 @@ func (m *MysqlDumper) Dump(c *client.Client) error { args := m.buildDumpArgs() - return runAndSaveCommandInContainer(getDumpFilename(m.Container.Name), c, m.Container, "mysqldump", args...) + return runAndSaveCommandInContainer(c, m.Container, "mysqldump", args...) } diff --git a/dump_postgres.go b/dump_postgres.go index 4023dc3..60d0a1a 100644 --- a/dump_postgres.go +++ b/dump_postgres.go @@ -26,5 +26,5 @@ func (d *PostgresDumper) Dump(c *client.Client) error { user = u } - return runAndSaveCommandInContainer(getDumpFilename(d.Container.Name), c, d.Container, "pg_dumpall", "-U", user) + return runAndSaveCommandInContainer(c, d.Container, "pg_dumpall", "-U", user) } diff --git a/go.mod b/go.mod index 05177b2..0971561 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,31 @@ module kolaente.dev/konrad/docker-db-backup -go 1.16 +go 1.21 require ( - github.com/containerd/containerd v1.5.5 // indirect github.com/docker/docker v20.10.8+incompatible + github.com/robfig/cron/v3 v3.0.1 +) + +require ( + github.com/Microsoft/go-winio v0.4.17 // indirect + github.com/containerd/containerd v1.5.5 // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect - github.com/robfig/cron/v3 v3.0.1 + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect + golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect google.golang.org/grpc v1.40.0 // indirect + google.golang.org/protobuf v1.26.0 // indirect ) diff --git a/save.go b/save.go index 012cb55..cd05ca4 100644 --- a/save.go +++ b/save.go @@ -1,6 +1,7 @@ package main import ( + "compress/gzip" "context" "fmt" "io" @@ -10,16 +11,21 @@ import ( "github.com/docker/docker/client" ) -func runAndSaveCommandInContainer(filename string, c *client.Client, container *types.ContainerJSON, command string, args ...string) error { +func runAndSaveCommandInContainer(c *client.Client, container *types.ContainerJSON, command string, args ...string) error { ctx := context.Background() - config := types.ExecConfig{ + filename := getDumpFilename(container.Name) + if config.CompressBackups { + filename += ".gz" + } + + containerConfig := types.ExecConfig{ AttachStderr: true, AttachStdout: true, Cmd: append([]string{command}, args...), } - r, err := c.ContainerExecCreate(ctx, container.ID, config) + r, err := c.ContainerExecCreate(ctx, container.ID, containerConfig) if err != nil { return err } @@ -36,7 +42,18 @@ func runAndSaveCommandInContainer(filename string, c *client.Client, container * } defer f.Close() - _, err = io.Copy(f, resp.Reader) + var target io.Writer = f + + if config.CompressBackups { + gw, err := gzip.NewWriterLevel(f, gzip.BestCompression) + if err != nil { + return err + } + defer gw.Close() + target = gw + } + + _, err = io.Copy(target, resp.Reader) if err != nil { return err }