feat: PostgreSQL driver support

This commit is contained in:
fsylvain committed 2026-10-06 13:35:10 +02:00
1 parent eb9bfabed5
commit dc5d12348c
24 files changed
+1585 -75

No files matched your search

@@ -13,9 +13,14 @@ const (
DatabaseDriverSQLite = "sqlite"
DatabaseDriverMySQL = "mysql"
DatabaseDriverPostgres = "postgres"
databaseDriverPostgresAlias = "postgresql"
DefaultDatabaseDriver = DatabaseDriverSQLite
DefaultDatabasePort = "3306"
DefaultPostgresPort = "5432"
DefaultDatabaseTLS = "false"
DefaultDatabaseSSLMode = "disable"
DefaultMaxOpenConns = "10"
DefaultMaxIdleConns = "5"
DefaultConnMaxLifetimeSeconds = "300"
@@ -26,6 +31,7 @@ const (
envDatabasePassword = "DB_PASSWORD"
envDatabaseName = "DB_NAME"
envDatabaseTLS = "DB_TLS"
envDatabaseSSLMode = "DB_SSLMODE"
envDatabaseMaxOpenConns = "DB_MAX_OPEN_CONNS"
envDatabaseMaxIdleConns = "DB_MAX_IDLE_CONNS"
envDatabaseConnMaxLifetimeSecs = "DB_CONN_MAX_LIFETIME_SECONDS"
@@ -35,6 +41,8 @@ var mysqlRequiredEnvVars = []string{envDatabaseHost, envDatabaseUser, envDatabas
var DatabaseTLSModes = []string{"false", "true", "skip-verify", "preferred"}
var DatabaseSSLModes = []string{"disable", "require", "verify-ca", "verify-full"}
type DatabaseSettings struct {
Driver string
Host string
@@ -43,13 +51,22 @@ type DatabaseSettings struct {
Password string
Name string
TLS string
SSLMode string
MaxOpenConns int
MaxIdleConns int
ConnMaxLifetime time.Duration
}
func NormalizeDatabaseDriver(driver string) string {
if driver == databaseDriverPostgresAlias {
return DatabaseDriverPostgres
}
return driver
}
func DatabaseRequiredEnvVars(driver string) []string {
if driver == DatabaseDriverMySQL {
switch NormalizeDatabaseDriver(driver) {
case DatabaseDriverMySQL, DatabaseDriverPostgres:
return slices.Clone(mysqlRequiredEnvVars)
}
return nil
@@ -65,18 +82,27 @@ func MissingEnvVarError(names []string, lookup func(string) (string, bool)) erro
}
func databaseDriverFromEnv() string {
return envOrDefault(EnvDatabaseDriver, DefaultDatabaseDriver)
return NormalizeDatabaseDriver(envOrDefault(EnvDatabaseDriver, DefaultDatabaseDriver))
}
func databaseSettingsFromEnv() (DatabaseSettings, error) {
driver := databaseDriverFromEnv()
if driver != DatabaseDriverMySQL {
if driver != DatabaseDriverMySQL && driver != DatabaseDriverPostgres {
return DatabaseSettings{Driver: driver}, nil
}
tls := envOrDefault(envDatabaseTLS, DefaultDatabaseTLS)
if !slices.Contains(DatabaseTLSModes, tls) {
return DatabaseSettings{}, fmt.Errorf("%s must be one of %s, got %q", envDatabaseTLS, strings.Join(DatabaseTLSModes, ", "), tls)
defaultPort, tls, sslMode := DefaultDatabasePort, "", ""
if driver == DatabaseDriverPostgres {
defaultPort = DefaultPostgresPort
sslMode = envOrDefault(envDatabaseSSLMode, DefaultDatabaseSSLMode)
if !slices.Contains(DatabaseSSLModes, sslMode) {
return DatabaseSettings{}, fmt.Errorf("%s must be one of %s, got %q", envDatabaseSSLMode, strings.Join(DatabaseSSLModes, ", "), sslMode)
}
} else {
tls = envOrDefault(envDatabaseTLS, DefaultDatabaseTLS)
if !slices.Contains(DatabaseTLSModes, tls) {
return DatabaseSettings{}, fmt.Errorf("%s must be one of %s, got %q", envDatabaseTLS, strings.Join(DatabaseTLSModes, ", "), tls)
}
}
maxOpen, err := nonNegativeIntEnv(envDatabaseMaxOpenConns, DefaultMaxOpenConns)
if err != nil {
@@ -94,11 +120,12 @@ func databaseSettingsFromEnv() (DatabaseSettings, error) {
return DatabaseSettings{
Driver: driver,
Host: os.Getenv(envDatabaseHost),
Port: envOrDefault(envDatabasePort, DefaultDatabasePort),
Port: envOrDefault(envDatabasePort, defaultPort),
User: os.Getenv(envDatabaseUser),
Password: os.Getenv(envDatabasePassword),
Name: os.Getenv(envDatabaseName),
TLS: tls,
SSLMode: sslMode,
MaxOpenConns: maxOpen,
MaxIdleConns: maxIdle,
ConnMaxLifetime: time.Duration(lifetimeSeconds) * time.Second,
@@ -26,8 +26,9 @@ type Driver interface {
}
var drivers = map[string]Driver{
DriverSQLite: sqliteDriver{},
DriverMySQL: mysqlDriver{},
DriverSQLite: sqliteDriver{},
DriverMySQL: mysqlDriver{},
DriverPostgres: postgresDriver{},
}
type Connection struct {
@@ -16,12 +16,13 @@ type DumpBackup struct {
open func() (DumpSource, error)
directory string
now func() time.Time
dialect dumpDialect
}
var _ domain.DatabaseBackup = &DumpBackup{}
func NewDumpBackup(open func() (DumpSource, error), directory string, now func() time.Time) *DumpBackup {
return &DumpBackup{open: open, directory: directory, now: now}
return &DumpBackup{open: open, directory: directory, now: now, dialect: mysqlDumpDialect}
}
func (b *DumpBackup) Backup() (string, error) {
@@ -63,7 +64,7 @@ func (b *DumpBackup) writePartial(partial string) error {
return err
}
buffered := bufio.NewWriter(file)
if err := WriteDump(buffered, source); err != nil {
if err := writeDump(buffered, source, b.dialect); err != nil {
_ = file.Close()
return err
}
@@ -17,18 +17,44 @@ func NewMySQLBackup(db *gorm.DB, directory string, now func() time.Time) *DumpBa
return NewDumpBackup(func() (DumpSource, error) { return openMySQLDumpSource(db) }, directory, now)
}
type txDumpSource struct {
tx *gorm.DB
quote func(name string) string
convert func(value any) []byte
}
func openTxDumpSource(db *gorm.DB, quote func(name string) string, convert func(value any) []byte) (*txDumpSource, error) {
tx := db.Begin(&sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
if tx.Error != nil {
return nil, tx.Error
}
return &txDumpSource{tx: tx, quote: quote, convert: convert}, nil
}
func (s *txDumpSource) Rows(table string) (DumpRows, error) {
rows, err := s.tx.Raw("SELECT * FROM " + s.quote(table)).Rows()
if err != nil {
return nil, err
}
return &sqlDumpRows{rows: rows, convert: s.convert}, nil
}
func (s *txDumpSource) Close() error {
return s.tx.Rollback().Error
}
type mysqlDumpSource struct {
tx *gorm.DB
*txDumpSource
}
var _ DumpSource = &mysqlDumpSource{}
func openMySQLDumpSource(db *gorm.DB) (*mysqlDumpSource, error) {
tx := db.Begin(&sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
if tx.Error != nil {
return nil, tx.Error
source, err := openTxDumpSource(db, quoteIdentifier, dumpBytes)
if err != nil {
return nil, err
}
return &mysqlDumpSource{tx: tx}, nil
return &mysqlDumpSource{source}, nil
}
func (s *mysqlDumpSource) Tables() ([]string, error) {
@@ -46,25 +72,14 @@ func (s *mysqlDumpSource) CreateStatement(table string) (string, error) {
return statement, nil
}
func (s *mysqlDumpSource) Rows(table string) (DumpRows, error) {
rows, err := s.tx.Raw("SELECT * FROM " + quoteIdentifier(table)).Rows()
if err != nil {
return nil, err
}
return &mysqlDumpRows{rows: rows}, nil
type sqlDumpRows struct {
rows *sql.Rows
convert func(value any) []byte
}
func (s *mysqlDumpSource) Close() error {
return s.tx.Rollback().Error
}
var _ DumpRows = &sqlDumpRows{}
type mysqlDumpRows struct {
rows *sql.Rows
}
var _ DumpRows = &mysqlDumpRows{}
func (r *mysqlDumpRows) Columns() ([]DumpColumn, error) {
func (r *sqlDumpRows) Columns() ([]DumpColumn, error) {
types, err := r.rows.ColumnTypes()
if err != nil {
return nil, err
@@ -76,11 +91,11 @@ func (r *mysqlDumpRows) Columns() ([]DumpColumn, error) {
return columns, nil
}
func (r *mysqlDumpRows) Next() bool {
func (r *sqlDumpRows) Next() bool {
return r.rows.Next()
}
func (r *mysqlDumpRows) Values() ([][]byte, error) {
func (r *sqlDumpRows) Values() ([][]byte, error) {
columnCount, err := r.rows.Columns()
if err != nil {
return nil, err
@@ -95,16 +110,16 @@ func (r *mysqlDumpRows) Values() ([][]byte, error) {
}
values := make([][]byte, len(scanned))
for index, value := range scanned {
values[index] = dumpBytes(value)
values[index] = r.convert(value)
}
return values, nil
}
func (r *mysqlDumpRows) Err() error {
func (r *sqlDumpRows) Err() error {
return r.rows.Err()
}
func (r *mysqlDumpRows) Close() error {
func (r *sqlDumpRows) Close() error {
return r.rows.Close()
}
@@ -0,0 +1,108 @@
package database
import (
domainUpdate "RenewCMS/internal/domain/update"
"fmt"
"net"
"net/url"
"reflect"
"time"
"github.com/jackc/pgx/v5/pgconn"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
const (
DriverPostgres = "postgres"
postgresTimeZone = "UTC"
postgresConnectTimeoutSeconds = 5
postgresScheme = "postgres"
postgresUTCCallback = "renewcms:postgres_utc"
gormQueryCallback = "gorm:query"
)
var timeType = reflect.TypeOf(time.Time{})
type postgresDriver struct{}
type PostgresConfig struct {
Host string
Port string
User string
Password string
Name string
SSLMode string
}
func PostgresDSN(config PostgresConfig) string {
dsn := url.URL{
Scheme: postgresScheme,
User: url.UserPassword(config.User, config.Password),
Host: net.JoinHostPort(config.Host, config.Port),
Path: "/" + config.Name,
}
query := url.Values{}
query.Set("sslmode", config.SSLMode)
query.Set("timezone", postgresTimeZone)
query.Set("connect_timeout", fmt.Sprint(postgresConnectTimeoutSeconds))
dsn.RawQuery = query.Encode()
return dsn.String()
}
func (postgresDriver) Open(source string) (*gorm.DB, error) {
parsed, err := pgconn.ParseConfig(source)
if err != nil {
return nil, fmt.Errorf("the postgres connection settings are invalid")
}
db, err := gorm.Open(postgres.Open(source), &gorm.Config{})
if err != nil {
address := net.JoinHostPort(parsed.Host, fmt.Sprint(parsed.Port))
return nil, fmt.Errorf("cannot use the postgres server at %s: %s", address, redact(err.Error(), parsed.Password))
}
if err := db.Callback().Query().After(gormQueryCallback).Register(postgresUTCCallback, readTimesInUTC); err != nil {
return nil, err
}
return db, nil
}
func readTimesInUTC(db *gorm.DB) {
if db.Error != nil || db.Statement.Dest == nil {
return
}
convertTimesToUTC(reflect.ValueOf(db.Statement.Dest))
}
func convertTimesToUTC(value reflect.Value) {
switch value.Kind() {
case reflect.Pointer, reflect.Interface:
if !value.IsNil() {
convertTimesToUTC(value.Elem())
}
case reflect.Slice, reflect.Array:
for index := 0; index < value.Len(); index++ {
convertTimesToUTC(value.Index(index))
}
case reflect.Struct:
convertStructTimesToUTC(value)
}
}
func convertStructTimesToUTC(value reflect.Value) {
if value.Type() == timeType {
if value.CanSet() {
value.Set(reflect.ValueOf(value.Interface().(time.Time).UTC()))
}
return
}
for index := 0; index < value.NumField(); index++ {
if value.Type().Field(index).IsExported() {
convertTimesToUTC(value.Field(index))
}
}
}
func (postgresDriver) NewBackup(db *gorm.DB, directory string, now func() time.Time) domainUpdate.DatabaseBackup {
return NewPostgresBackup(db, directory, now)
}
@@ -0,0 +1,189 @@
package database
import (
"fmt"
"strings"
"time"
"gorm.io/gorm"
)
const (
postgresSchema = "public"
postgresTablesQuery = "SELECT table_name FROM information_schema.tables WHERE table_schema = ? AND table_type = 'BASE TABLE' ORDER BY table_name"
postgresColumnsQuery = `SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS data_type, a.attnotnull AS not_null, COALESCE(pg_get_expr(d.adbin, d.adrelid), '') AS default_value, a.attidentity::text AS identity
FROM pg_attribute a
JOIN pg_class c ON c.oid = a.attrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum
WHERE n.nspname = ? AND c.relname = ? AND a.attnum > 0 AND NOT a.attisdropped
ORDER BY a.attnum`
postgresConstraintsQuery = `SELECT con.conname AS name, pg_get_constraintdef(con.oid) AS definition
FROM pg_constraint con
JOIN pg_class c ON c.oid = con.conrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = ? AND c.relname = ? AND con.contype IN ('p', 'u', 'c')
ORDER BY CASE con.contype WHEN 'p' THEN 0 ELSE 1 END, con.conname`
postgresIndexesQuery = `SELECT i.indexdef
FROM pg_indexes i
WHERE i.schemaname = ? AND i.tablename = ?
AND i.indexname NOT IN (
SELECT con.conname FROM pg_constraint con
JOIN pg_class c ON c.oid = con.conrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = ? AND c.relname = ?
)
ORDER BY i.indexname`
postgresIdentityColumn = "id"
postgresTimeLayout = "2006-01-02 15:04:05.999999999Z07:00"
postgresByteaType = "BYTEA"
)
var postgresNumericTypes = map[string]bool{
"INT2": true, "INT4": true, "INT8": true, "FLOAT4": true, "FLOAT8": true, "NUMERIC": true, "OID": true,
}
var postgresSerialTypes = map[string]string{
"smallint": "smallserial",
"integer": "serial",
"bigint": "bigserial",
}
var postgresDumpDialect = dumpDialect{
preamble: []string{
"SET client_encoding = 'UTF8';",
"SET standard_conforming_strings = on;",
"SET TimeZone = 'UTC';",
},
quote: quotePostgresIdentifier,
literal: postgresLiteral,
tableTrailer: postgresSequenceReset,
}
func NewPostgresBackup(db *gorm.DB, directory string, now func() time.Time) *DumpBackup {
backup := NewDumpBackup(func() (DumpSource, error) { return openPostgresDumpSource(db) }, directory, now)
backup.dialect = postgresDumpDialect
return backup
}
type postgresDumpSource struct {
*txDumpSource
}
var _ DumpSource = &postgresDumpSource{}
func openPostgresDumpSource(db *gorm.DB) (*postgresDumpSource, error) {
source, err := openTxDumpSource(db, quotePostgresIdentifier, postgresDumpBytes)
if err != nil {
return nil, err
}
return &postgresDumpSource{source}, nil
}
func (s *postgresDumpSource) Tables() ([]string, error) {
var tables []string
err := s.tx.Raw(postgresTablesQuery, postgresSchema).Scan(&tables).Error
return tables, err
}
type postgresColumn struct {
Name string
DataType string
NotNull bool
DefaultValue string
Identity string
}
type postgresConstraint struct {
Name string
Definition string
}
func (s *postgresDumpSource) CreateStatement(table string) (string, error) {
var columns []postgresColumn
if err := s.tx.Raw(postgresColumnsQuery, postgresSchema, table).Scan(&columns).Error; err != nil {
return "", err
}
var constraints []postgresConstraint
if err := s.tx.Raw(postgresConstraintsQuery, postgresSchema, table).Scan(&constraints).Error; err != nil {
return "", err
}
var indexes []string
if err := s.tx.Raw(postgresIndexesQuery, postgresSchema, table, postgresSchema, table).Scan(&indexes).Error; err != nil {
return "", err
}
var definitions []string
for _, column := range columns {
definitions = append(definitions, " "+postgresColumnDefinition(column))
}
for _, constraint := range constraints {
definitions = append(definitions, fmt.Sprintf(" CONSTRAINT %s %s", quotePostgresIdentifier(constraint.Name), constraint.Definition))
}
statements := []string{fmt.Sprintf("CREATE TABLE %s (\n%s\n)", quotePostgresIdentifier(table), strings.Join(definitions, ",\n"))}
statements = append(statements, indexes...)
return strings.Join(statements, ";\n"), nil
}
func postgresColumnDefinition(column postgresColumn) string {
quoted := quotePostgresIdentifier(column.Name)
serial, isSerial := postgresSerialTypes[column.DataType]
if isSerial && strings.HasPrefix(column.DefaultValue, "nextval(") {
return quoted + " " + serial + " NOT NULL"
}
if column.Identity != "" {
return quoted + " " + column.DataType + " GENERATED BY DEFAULT AS IDENTITY NOT NULL"
}
definition := quoted + " " + column.DataType
if column.DefaultValue != "" {
definition += " DEFAULT " + column.DefaultValue
}
if column.NotNull {
definition += " NOT NULL"
}
return definition
}
func postgresDumpBytes(value any) []byte {
if moment, isTime := value.(time.Time); isTime {
return []byte(moment.UTC().Format(postgresTimeLayout))
}
return dumpBytes(value)
}
func quotePostgresIdentifier(name string) string {
return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`
}
func postgresLiteral(databaseType string, value []byte) string {
if value == nil {
return "NULL"
}
upper := strings.ToUpper(databaseType)
if postgresNumericTypes[upper] && plainNumber.Match(value) {
return string(value)
}
if upper == postgresByteaType {
return fmt.Sprintf(`'\x%x'`, value)
}
return "'" + strings.ReplaceAll(string(value), "'", "''") + "'"
}
func postgresSequenceReset(table string, columns []DumpColumn) []string {
for _, column := range columns {
if column.Name != postgresIdentityColumn {
continue
}
quotedTable, quotedColumn := quotePostgresIdentifier(table), quotePostgresIdentifier(column.Name)
sequenceTable := strings.ReplaceAll(quotedTable, "'", "''")
return []string{fmt.Sprintf(
"SELECT setval(pg_get_serial_sequence('%s', '%s'), COALESCE((SELECT MAX(%s) FROM %s), 1), (SELECT COUNT(*) > 0 FROM %s));",
sequenceTable, column.Name, quotedColumn, quotedTable, quotedTable,
)}
}
return nil
}
@@ -28,14 +28,26 @@ type DumpSource interface {
Close() error
}
var dumpPreamble = []string{
"SET NAMES utf8mb4;",
"SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO';",
"SET foreign_key_checks = 0;",
type dumpDialect struct {
preamble []string
postamble []string
quote func(name string) string
literal func(databaseType string, value []byte) string
tableTrailer func(table string, columns []DumpColumn) []string
}
var dumpPostamble = []string{
"SET foreign_key_checks = 1;",
var mysqlDumpDialect = dumpDialect{
preamble: []string{
"SET NAMES utf8mb4;",
"SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO';",
"SET foreign_key_checks = 0;",
},
postamble: []string{
"SET foreign_key_checks = 1;",
},
quote: quoteIdentifier,
literal: sqlLiteral,
tableTrailer: func(string, []DumpColumn) []string { return nil },
}
var plainNumber = regexp.MustCompile(`^-?[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?$`)
@@ -50,7 +62,11 @@ var textEscapes = strings.NewReplacer(
)
func WriteDump(output io.Writer, source DumpSource) error {
if err := writeStatements(output, dumpPreamble); err != nil {
return writeDump(output, source, mysqlDumpDialect)
}
func writeDump(output io.Writer, source DumpSource, dialect dumpDialect) error {
if err := writeStatements(output, dialect.preamble); err != nil {
return err
}
tables, err := source.Tables()
@@ -58,11 +74,11 @@ func WriteDump(output io.Writer, source DumpSource) error {
return err
}
for _, table := range tables {
if err := writeTable(output, source, table); err != nil {
if err := writeTable(output, source, table, dialect); err != nil {
return err
}
}
return writeStatements(output, dumpPostamble)
return writeStatements(output, dialect.postamble)
}
func writeStatements(output io.Writer, statements []string) error {
@@ -74,7 +90,7 @@ func writeStatements(output io.Writer, statements []string) error {
return nil
}
func writeTable(output io.Writer, source DumpSource, table string) error {
func writeTable(output io.Writer, source DumpSource, table string, dialect dumpDialect) error {
create, err := source.CreateStatement(table)
if err != nil {
return err
@@ -87,39 +103,42 @@ func writeTable(output io.Writer, source DumpSource, table string) error {
return err
}
defer rows.Close()
return writeRows(output, table, rows)
return writeRows(output, table, rows, dialect)
}
func writeRows(output io.Writer, table string, rows DumpRows) error {
func writeRows(output io.Writer, table string, rows DumpRows, dialect dumpDialect) error {
columns, err := rows.Columns()
if err != nil {
return err
}
insertPrefix := insertStatementPrefix(table, columns)
insertPrefix := insertStatementPrefix(table, columns, dialect)
for rows.Next() {
values, err := rows.Values()
if err != nil {
return err
}
if _, err := fmt.Fprintf(output, "%s%s);\n", insertPrefix, joinLiterals(columns, values)); err != nil {
if _, err := fmt.Fprintf(output, "%s%s);\n", insertPrefix, joinLiterals(columns, values, dialect)); err != nil {
return err
}
}
return rows.Err()
if err := rows.Err(); err != nil {
return err
}
return writeStatements(output, dialect.tableTrailer(table, columns))
}
func insertStatementPrefix(table string, columns []DumpColumn) string {
func insertStatementPrefix(table string, columns []DumpColumn, dialect dumpDialect) string {
names := make([]string, len(columns))
for index, column := range columns {
names[index] = quoteIdentifier(column.Name)
names[index] = dialect.quote(column.Name)
}
return fmt.Sprintf("INSERT INTO %s (%s) VALUES (", quoteIdentifier(table), strings.Join(names, ","))
return fmt.Sprintf("INSERT INTO %s (%s) VALUES (", dialect.quote(table), strings.Join(names, ","))
}
func joinLiterals(columns []DumpColumn, values [][]byte) string {
func joinLiterals(columns []DumpColumn, values [][]byte, dialect dumpDialect) string {
literals := make([]string, len(columns))
for index, column := range columns {
literals[index] = sqlLiteral(column.Type, values[index])
literals[index] = dialect.literal(column.Type, values[index])
}
return strings.Join(literals, ",")
}