Postgresql support

docs/README.md

@@ -84,10 +84,15 @@ Features:
 * **multiple file-sets** supported. This means you can choose to only sync your database, but not your binary resources/assets.
 * **Speed Optimized**: publicly available binary assets are not zipped extra; but the already-public files are simply downloaded.
   Resources which already exist locally and have the same file size and modification date are never re-downloaded.
-* **no extra SQL client needed**: We package a custom implementation of `mysqldump` into the binary.
+* **no extra SQL client needed**: We package a custom implementation of `mysqldump`/`pg_dump` into the binary - no
+  `mysqldump`, `pg_dump` or `psql` binary is ever shelled out to.
   * currently supported databases:
-    * **MySQL**
-    * (Postgres support planned)
+    * **MySQL / MariaDB**
+    * **PostgreSQL** - full schema reconstruction (columns/types, NOT NULL, defaults, PK/UNIQUE/CHECK/FK constraints,
+      indexes, sequence/identity state) from `pg_catalog`, with data streamed via `COPY ... FROM stdin`. Not (yet)
+      dumped: `CREATE TYPE`/`CREATE EXTENSION` (enum *values* round-trip, but the type/extension must already exist
+      on the target), non-`public` schemas, materialized views, partitioned tables, table inheritance,
+      triggers/functions, comments, and GRANTs/ownership.
 * **auto-cleanup**: remove dumps when tool is stopped
 
 # Installation

docs/architecture.md

@@ -56,8 +56,11 @@ The destination server will only fetch data from the source; and never the other
 on the source server (e.g. the production system), the user needs to log in, and then invoke the `synco-lite` executable. This does:
 
 * install synco-source via a shell script
-* Detect which framework is used. E.g. for Flow/Neos or Symfony, synco then knows how to create a database dump (e.g. for MySQL, using go-mysqldump or
-  pingcap dumpling; and for Postgres some pgx based solution??)
+* Detect which framework is used. E.g. for Flow/Neos or Symfony, synco then knows how to create a database dump
+  (for MySQL/MariaDB, using our vendored `go_mysqldump`; for PostgreSQL, using `pkg/util/postgres` + `pgx`, which
+  reconstructs full schema DDL from `pg_catalog` and streams data via `COPY ... FROM stdin` - see the "known
+  limitations" note in `docs/README.md` for what isn't (yet) covered on the Postgres side: custom types/extensions,
+  non-`public` schemas, materialized views, partitioned tables, triggers/functions, comments, GRANTs)
 * Publish a metadata file and encrypt it which shows the current status.
 * Create the database dump and encrypt it.
 * Create a file mapping for data/persistent in flow - as we do not need to re-compress static assets which are available online.

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/dop251/goja v0.0.0-20221003171542-5ea1285e6c91
 	github.com/dustin/go-humanize v1.0.1
 	github.com/go-sql-driver/mysql v1.10.0
+	github.com/jackc/pgx/v5 v5.10.0
 	github.com/jamf/go-mysqldump v0.7.1
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/manifoldco/promptui v0.9.0
@@ -57,8 +58,12 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gookit/color v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/lib/pq v1.10.7 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

go.sum

@@ -108,6 +108,14 @@ github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA=
 github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
+github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jamf/go-mysqldump v0.7.1 h1:JuEjzzKX51Bn9urjciXSqvmCGAxAwH3IaN3+nuplf+o=
 github.com/jamf/go-mysqldump v0.7.1/go.mod h1:YWqhOv9PfioqsO59t/DziO8gFEHw8G2vV6qBlFCdHIM=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -130,6 +138,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
+github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=
 github.com/lithammer/fuzzysearch v1.1.8/go.mod h1:IdqeyBClc3FFqSzYq/MXESsS4S0FsZ5ajtkr5xPLts4=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=

pkg/common/commonServe/databaseDump.go

@@ -7,31 +7,56 @@ import (
 	"github.com/sandstorm/synco/v2/pkg/common/dto"
 	"github.com/sandstorm/synco/v2/pkg/serve"
 	"github.com/sandstorm/synco/v2/pkg/util/mysql"
+	"github.com/sandstorm/synco/v2/pkg/util/postgres"
 )
 
+// fileSetTypeFor maps a DB driver to the dto.FileSetType used to record its
+// dump in the transfer session's metadata. Unrecognized/unset drivers
+// default to mysql, matching the pre-Postgres-support behavior.
+func fileSetTypeFor(driver common.DbDriver) dto.FileSetType {
+	if driver == common.DbDriverPostgres {
+		return dto.TYPE_POSTGRESDUMP
+	}
+	return dto.TYPE_MYSQLDUMP
+}
+
 func DatabaseDump(transferSession *serve.TransferSession, dbCredentials *common.DbCredentials, whereClauseForTables map[string]string) *sql.DB {
 	// 2) DATABASE DUMP
 	// basically the way it works is:
-	// mysql.CreateDump --> age.Encrypt --> write to file.
+	// mysql.CreateDump / postgres.CreateDump --> age.Encrypt --> write to file.
 	// but because this is based on streams, we need to construct it the other way around:
 	// 1st: open the target file
 	// 2nd: init age.Encrypt
-	// 3rd: do mysql dump (which feeds the Writer)
+	// 3rd: do the DB dump (which feeds the Writer)
 	wc, err := transferSession.EncryptToFile("dump.sql.enc")
+	if err != nil {
+		pterm.Fatal.Printfln("could not open encrypted dump file: %s", err)
+	}
 	fileSet := &dto.FileSet{
 		Name: "dbDump",
-		Type: dto.TYPE_MYSQLDUMP,
-		MysqlDump: &dto.FileSetMysqlDump{
-			FileName: "dump.sql.enc",
-		},
+		Type: fileSetTypeFor(dbCredentials.Driver),
 	}
 
 	// 2b) the actual DB dump. also finishes writing.
-	db, err := mysql.CreateDump(dbCredentials, wc, whereClauseForTables)
+	var db *sql.DB
+	switch fileSet.Type {
+	case dto.TYPE_POSTGRESDUMP:
+		fileSet.PostgresDump = &dto.FileSetPostgresDump{FileName: "dump.sql.enc"}
+		db, err = postgres.CreateDump(dbCredentials, wc, whereClauseForTables)
+	default:
+		fileSet.MysqlDump = &dto.FileSetMysqlDump{FileName: "dump.sql.enc"}
+		db, err = mysql.CreateDump(dbCredentials, wc, whereClauseForTables)
+	}
 	if err != nil {
 		pterm.Fatal.Printfln("could not create SQL dump: %s", err)
 	}
-	fileSet.MysqlDump.SizeBytes = wc.Size()
+
+	sizeBytes := wc.Size()
+	if fileSet.PostgresDump != nil {
+		fileSet.PostgresDump.SizeBytes = sizeBytes
+	} else {
+		fileSet.MysqlDump.SizeBytes = sizeBytes
+	}
 	transferSession.Meta.FileSets = append(transferSession.Meta.FileSets, fileSet)
 	err = transferSession.UpdateMetadata()
 	if err != nil {

pkg/common/commonServe/databaseDump_test.go

@@ -0,0 +1,29 @@
+package commonServe
+
+import (
+	"testing"
+
+	"github.com/sandstorm/synco/v2/pkg/common"
+	"github.com/sandstorm/synco/v2/pkg/common/dto"
+)
+
+func TestFileSetTypeForDriver(t *testing.T) {
+	tests := []struct {
+		name   string
+		driver common.DbDriver
+		want   dto.FileSetType
+	}{
+		{name: "postgres", driver: common.DbDriverPostgres, want: dto.TYPE_POSTGRESDUMP},
+		{name: "mysql", driver: common.DbDriverMysql, want: dto.TYPE_MYSQLDUMP},
+		{name: "unset driver defaults to mysql", driver: "", want: dto.TYPE_MYSQLDUMP},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := fileSetTypeFor(tt.driver)
+			if got != tt.want {
+				t.Errorf("fileSetTypeFor(%q) = %q, want %q", tt.driver, got, tt.want)
+			}
+		})
+	}
+}

pkg/common/types.go

@@ -16,7 +16,15 @@ type ServeFramework interface {
 	Serve(metadata *serve.TransferSession)
 }
 
+type DbDriver string
+
+const (
+	DbDriverMysql    DbDriver = "mysql"
+	DbDriverPostgres DbDriver = "postgres"
+)
+
 type DbCredentials struct {
+	Driver   DbDriver
 	Host     string
 	Port     int
 	User     string

pkg/frameworks/flowServe/flowServe.go

@@ -121,11 +121,24 @@ type flowPersistenceBackendOptions struct {
 }
 
 func (fp *flowPersistenceBackendOptions) ToDbCredentials() *common.DbCredentials {
-	port := 3306
+	driver := common.DbDriverMysql
+	defaultPort := 3306
+	switch fp.Driver {
+	case "pdo_pgsql":
+		driver = common.DbDriverPostgres
+		defaultPort = 5432
+	case "pdo_mysql", "":
+		// already defaulted above
+	default:
+		pterm.Warning.Printfln("unrecognized Neos.Flow.persistence.backendOptions.driver '%s', falling back to mysql", fp.Driver)
+	}
+
+	port := defaultPort
 	if len(fp.Port) != 0 {
 		port, _ = strconv.Atoi(fp.Port)
 	}
 	return &common.DbCredentials{
+		Driver:   driver,
 		Host:     fp.Host,
 		Port:     port,
 		User:     fp.User,

pkg/frameworks/flowServe/flowServer_test.go

@@ -1,6 +1,10 @@
 package flowServe
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/sandstorm/synco/v2/pkg/common"
+)
 
 type generateS3ResourcesPathTest struct {
 	filename            string
@@ -54,6 +58,34 @@ var generateS3ResourcesPathTests = []generateS3ResourcesPathTest{
 	},
 }
 
+func TestToDbCredentialsDetectsDriverAndPortDefault(t *testing.T) {
+	tests := []struct {
+		name       string
+		driver     string
+		port       string
+		wantDriver common.DbDriver
+		wantPort   int
+	}{
+		{name: "pdo_pgsql defaults to port 5432", driver: "pdo_pgsql", port: "", wantDriver: common.DbDriverPostgres, wantPort: 5432},
+		{name: "pdo_mysql defaults to port 3306", driver: "pdo_mysql", port: "", wantDriver: common.DbDriverMysql, wantPort: 3306},
+		{name: "unrecognized driver falls back to mysql", driver: "pdo_sqlite", port: "", wantDriver: common.DbDriverMysql, wantPort: 3306},
+		{name: "explicit port overrides the postgres default", driver: "pdo_pgsql", port: "6543", wantDriver: common.DbDriverPostgres, wantPort: 6543},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			opts := flowPersistenceBackendOptions{Driver: tt.driver, Port: tt.port}
+			creds := opts.ToDbCredentials()
+			if creds.Driver != tt.wantDriver {
+				t.Errorf("Driver = %q, want %q", creds.Driver, tt.wantDriver)
+			}
+			if creds.Port != tt.wantPort {
+				t.Errorf("Port = %d, want %d", creds.Port, tt.wantPort)
+			}
+		})
+	}
+}
+
 func TestGenerateS3ResourcesPaths(t *testing.T) {
 	for _, tt := range generateS3ResourcesPathTests {
 		path := generateS3ResourcePublicPath(&tt.targetConfiguration, tt.resourceSha1, tt.filename)

pkg/frameworks/laravelServe/laravelServe.go

@@ -58,11 +58,24 @@ func (ldo *laravelDatabaseOptions) ToDbCredentials() *common.DbCredentials {
 		pterm.Warning.Printfln("Could not extract DB connection, WILL NOT INCLUDE DB DUMP.")
 		return nil
 	}
-	port := 3306
+	driver := common.DbDriverMysql
+	defaultPort := 3306
+	switch connection.Driver {
+	case "pgsql":
+		driver = common.DbDriverPostgres
+		defaultPort = 5432
+	case "mysql", "":
+		// already defaulted above
+	default:
+		pterm.Warning.Printfln("unrecognized database driver '%s', falling back to mysql", connection.Driver)
+	}
+
+	port := defaultPort
 	if len(connection.Port) != 0 {
 		port, _ = strconv.Atoi(connection.Port)
 	}
 	return &common.DbCredentials{
+		Driver:   driver,
 		Host:     connection.Host,
 		Port:     port,
 		User:     connection.Username,

pkg/frameworks/laravelServe/laravelServe_test.go

@@ -0,0 +1,40 @@
+package laravelServe
+
+import (
+	"testing"
+
+	"github.com/sandstorm/synco/v2/pkg/common"
+)
+
+func TestToDbCredentialsDetectsDriverAndPortDefault(t *testing.T) {
+	tests := []struct {
+		name       string
+		driver     string
+		port       string
+		wantDriver common.DbDriver
+		wantPort   int
+	}{
+		{name: "pgsql defaults to port 5432", driver: "pgsql", port: "", wantDriver: common.DbDriverPostgres, wantPort: 5432},
+		{name: "mysql defaults to port 3306", driver: "mysql", port: "", wantDriver: common.DbDriverMysql, wantPort: 3306},
+		{name: "unrecognized driver falls back to mysql", driver: "sqlite", port: "", wantDriver: common.DbDriverMysql, wantPort: 3306},
+		{name: "explicit port overrides the postgres default", driver: "pgsql", port: "6543", wantDriver: common.DbDriverPostgres, wantPort: 6543},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			opts := laravelDatabaseOptions{
+				Default: "default",
+				Connections: map[string]laravelDatabaseConnectionOptions{
+					"default": {Driver: tt.driver, Port: tt.port},
+				},
+			}
+			creds := opts.ToDbCredentials()
+			if creds.Driver != tt.wantDriver {
+				t.Errorf("Driver = %q, want %q", creds.Driver, tt.wantDriver)
+			}
+			if creds.Port != tt.wantPort {
+				t.Errorf("Port = %d, want %d", creds.Port, tt.wantPort)
+			}
+		})
+	}
+}

pkg/receive/cmd/receive-cmd.go

@@ -76,6 +76,8 @@ var ReceiveCmd = &cobra.Command{
 			switch fileSet.Type {
 			case dto.TYPE_MYSQLDUMP:
 				err = downloadMysqldump(receiveSession, fileSet)
+			case dto.TYPE_POSTGRESDUMP:
+				err = downloadPostgresdump(receiveSession, fileSet)
 			case dto.TYPE_PUBLICFILES:
 				err = downloadPublicFiles(receiveSession, fileSet)
 			case dto.TYPE_PRIVATE_ENCRYPTED_FILES:
@@ -218,6 +220,10 @@ func downloadMysqldump(receiveSession *receive.ReceiveSession, fileSet *dto.File
 	return receiveSession.DumpAndDecryptFileWithProgressBar(fileSet.MysqlDump.FileName, fileSet.Name+".sql")
 }
 
+func downloadPostgresdump(receiveSession *receive.ReceiveSession, fileSet *dto.FileSet) error {
+	return receiveSession.DumpAndDecryptFileWithProgressBar(fileSet.PostgresDump.FileName, fileSet.Name+".sql")
+}
+
 func downloadPublicFiles(receiveSession *receive.ReceiveSession, fileSet *dto.FileSet) error {
 	indexFileName := fileSet.Name + ".index.json"
 	err := receiveSession.DumpAndDecryptFileWithProgressBar(fileSet.PublicFiles.IndexFileName, indexFileName)