Mount PgBouncer certificates from one projected volume

PgBouncer v1.16 is able to reload TLS certificates, but we mounted the
files outside of the one directory monitored by the reload sidecar.
Rather than create more sidecars, add the certificates to the existing
projected volume.

Issue: [sc-12361]
See: https://www.pgbouncer.org/changelog.html#pgbouncer-116x
See: pgbouncer/pgbouncer@cfbb8a3ae7a0

Author: cbandy

docs/content/tutorial/administrative-tasks.md

@@ -52,7 +52,9 @@ When changing the PostgreSQL certificate authority, make sure to update
 [`customReplicationTLSSecret`]({{< relref "/tutorial/customize-cluster.md" >}}#customize-tls) as well.
 {{% /notice %}}
 
-PgBouncer needs to be restarted after its certificates change.
+PGO automatically notifies PgBouncer when there are changes to the contents of
+PgBouncer certificate Secrets. Recent PgBouncer versions load those changes
+without downtime, but versions prior to 1.16.0 need to be restarted manually.
 There are a few ways to do it:
 
 1. Store the new certificates in a new Secret. Edit the PostgresCluster object

internal/pgbouncer/assertions_test.go

@@ -16,16 +16,17 @@
 package pgbouncer
 
 import (
+	"strings"
+
 	"gotest.tools/v3/assert/cmp"
 	"sigs.k8s.io/yaml"
 )
 
-func marshalEquals(actual interface{}, expected string) cmp.Comparison {
+// marshalMatches converts actual to YAML and compares that to expected.
+func marshalMatches(actual interface{}, expected string) cmp.Comparison {
 	b, err := yaml.Marshal(actual)
-	return func() cmp.Result {
-		if err != nil {
-			return cmp.ResultFromError(err)
-		}
-		return cmp.DeepEqual(string(b), expected)()
+	if err != nil {
+		return func() cmp.Result { return cmp.ResultFromError(err) }
 	}
+	return cmp.DeepEqual(string(b), strings.Trim(expected, "\t\n")+"\n")
 }

internal/pgbouncer/certificates.go

@@ -20,23 +20,24 @@ import (
 )
 
 const (
-	certBackendDirectory  = configDirectory + "/~postgres-operator-backend"
-	certFrontendDirectory = configDirectory + "/~postgres-operator-frontend"
+	tlsAuthoritySecretKey   = "ca.crt"
+	tlsCertificateSecretKey = corev1.TLSCertKey
+	tlsPrivateKeySecretKey  = corev1.TLSPrivateKeyKey
 
-	certBackendAuthorityAbsolutePath   = certBackendDirectory + "/" + certBackendAuthorityProjectionPath
-	certBackendAuthorityProjectionPath = "ca.crt"
+	certBackendAuthorityAbsolutePath   = configDirectory + "/" + certBackendAuthorityProjectionPath
+	certBackendAuthorityProjectionPath = "~postgres-operator/backend-ca.crt"
 
-	certFrontendAuthorityAbsolutePath  = certFrontendDirectory + "/" + certFrontendAuthorityProjectionPath
-	certFrontendPrivateKeyAbsolutePath = certFrontendDirectory + "/" + certFrontendPrivateKeyProjectionPath
-	certFrontendAbsolutePath           = certFrontendDirectory + "/" + certFrontendProjectionPath
+	certFrontendAuthorityAbsolutePath  = configDirectory + "/" + certFrontendAuthorityProjectionPath
+	certFrontendPrivateKeyAbsolutePath = configDirectory + "/" + certFrontendPrivateKeyProjectionPath
+	certFrontendAbsolutePath           = configDirectory + "/" + certFrontendProjectionPath
 
-	certFrontendAuthorityProjectionPath  = "ca.crt"
-	certFrontendPrivateKeyProjectionPath = "tls.key"
-	certFrontendProjectionPath           = "tls.crt"
+	certFrontendAuthorityProjectionPath  = "~postgres-operator/frontend-ca.crt"
+	certFrontendPrivateKeyProjectionPath = "~postgres-operator/frontend-tls.key"
+	certFrontendProjectionPath           = "~postgres-operator/frontend-tls.crt"
 
-	certFrontendAuthoritySecretKey  = "pgbouncer-frontend.ca-roots" // #nosec G101 this is a name, not a credential
-	certFrontendPrivateKeySecretKey = "pgbouncer-frontend.key"      // #nosec G101 this is a name, not a credential
-	certFrontendSecretKey           = "pgbouncer-frontend.crt"      // #nosec G101 this is a name, not a credential
+	certFrontendAuthoritySecretKey  = "pgbouncer-frontend.ca-roots"
+	certFrontendPrivateKeySecretKey = "pgbouncer-frontend.key"
+	certFrontendSecretKey           = "pgbouncer-frontend.crt"
 )
 
 // backendAuthority creates a volume projection of the PostgreSQL server
@@ -46,11 +47,20 @@ func backendAuthority(postgres *corev1.SecretProjection) corev1.VolumeProjection
 	result := postgres.DeepCopy()
 
 	for i := range result.Items {
-		if result.Items[i].Path == certBackendAuthorityProjectionPath {
+		// The PostgreSQL server projection expects Path to match typical Keys.
+		if result.Items[i].Path == tlsAuthoritySecretKey {
+			result.Items[i].Path = certBackendAuthorityProjectionPath
 			items = append(items, result.Items[i])
 		}
 	}
 
+	if len(items) == 0 {
+		items = []corev1.KeyToPath{{
+			Key:  tlsAuthoritySecretKey,
+			Path: certBackendAuthorityProjectionPath,
+		}}
+	}
+
 	result.Items = items
 	return corev1.VolumeProjection{Secret: result}
 }
@@ -59,10 +69,8 @@ func backendAuthority(postgres *corev1.SecretProjection) corev1.VolumeProjection
 func frontendCertificate(
 	custom *corev1.SecretProjection, secret *corev1.Secret,
 ) corev1.VolumeProjection {
-	result := custom
-
-	if result == nil {
-		result = &corev1.SecretProjection{
+	if custom == nil {
+		return corev1.VolumeProjection{Secret: &corev1.SecretProjection{
 			LocalObjectReference: corev1.LocalObjectReference{
 				Name: secret.Name,
 			},
@@ -80,8 +88,53 @@ func frontendCertificate(
 					Path: certFrontendProjectionPath,
 				},
 			},
+		}}
+	}
+
+	// The custom projection may have more or less than the three items we need
+	// to mount. Search for items that have the Path we expect and mount them at
+	// the path we need. When no items are specified, the Key serves as the Path.
+
+	// TODO(cbandy): A more structured field or validating webhook would ensure
+	// that the necessary values are specified.
+
+	var items []corev1.KeyToPath
+	result := custom.DeepCopy()
+
+	for i := range result.Items {
+		// The custom projection expects Path to match typical Keys.
+		switch result.Items[i].Path {
+		case tlsAuthoritySecretKey:
+			result.Items[i].Path = certFrontendAuthorityProjectionPath
+			items = append(items, result.Items[i])
+
+		case tlsCertificateSecretKey:
+			result.Items[i].Path = certFrontendProjectionPath
+			items = append(items, result.Items[i])
+
+		case tlsPrivateKeySecretKey:
+			result.Items[i].Path = certFrontendPrivateKeyProjectionPath
+			items = append(items, result.Items[i])
+		}
+	}
+
+	if len(items) == 0 {
+		items = []corev1.KeyToPath{
+			{
+				Key:  tlsAuthoritySecretKey,
+				Path: certFrontendAuthorityProjectionPath,
+			},
+			{
+				Key:  tlsPrivateKeySecretKey,
+				Path: certFrontendPrivateKeyProjectionPath,
+			},
+			{
+				Key:  tlsCertificateSecretKey,
+				Path: certFrontendProjectionPath,
+			},
 		}
 	}
 
+	result.Items = items
 	return corev1.VolumeProjection{Secret: result}
 }

internal/pgbouncer/certificates_test.go

@@ -16,62 +16,91 @@
 package pgbouncer
 
 import (
-	"strings"
 	"testing"
 
 	"gotest.tools/v3/assert"
 	corev1 "k8s.io/api/core/v1"
 )
 
 func TestBackendAuthority(t *testing.T) {
+	// No items; assume Key matches Path.
 	projection := &corev1.SecretProjection{
 		LocalObjectReference: corev1.LocalObjectReference{Name: "some-name"},
-		Items: []corev1.KeyToPath{
-			{Key: "some-crt-key", Path: "tls.crt"},
-			{Key: "some-ca-key", Path: "ca.crt"},
-		},
 	}
+	assert.Assert(t, marshalMatches(backendAuthority(projection), `
+secret:
+  items:
+  - key: ca.crt
+    path: ~postgres-operator/backend-ca.crt
+  name: some-name
+	`))
 
-	assert.Assert(t, marshalEquals(backendAuthority(projection), strings.Trim(`
+	// Some items; use only the CA Path.
+	projection.Items = []corev1.KeyToPath{
+		{Key: "some-crt-key", Path: "tls.crt"},
+		{Key: "some-ca-key", Path: "ca.crt"},
+	}
+	assert.Assert(t, marshalMatches(backendAuthority(projection), `
 secret:
   items:
   - key: some-ca-key
-    path: ca.crt
+    path: ~postgres-operator/backend-ca.crt
   name: some-name
-	`, "\t\n")+"\n"))
+	`))
 }
 
 func TestFrontendCertificate(t *testing.T) {
 	secret := new(corev1.Secret)
 	secret.Name = "op-secret"
 
 	t.Run("Generated", func(t *testing.T) {
-		assert.Assert(t, marshalEquals(frontendCertificate(nil, secret), strings.Trim(`
+		assert.Assert(t, marshalMatches(frontendCertificate(nil, secret), `
 secret:
   items:
   - key: pgbouncer-frontend.ca-roots
-    path: ca.crt
+    path: ~postgres-operator/frontend-ca.crt
   - key: pgbouncer-frontend.key
-    path: tls.key
+    path: ~postgres-operator/frontend-tls.key
   - key: pgbouncer-frontend.crt
-    path: tls.crt
+    path: ~postgres-operator/frontend-tls.crt
   name: op-secret
-		`, "\t\n")+"\n"))
+		`))
 	})
 
 	t.Run("Custom", func(t *testing.T) {
 		custom := new(corev1.SecretProjection)
 		custom.Name = "some-other"
+
+		// No items; assume Key matches Path.
+		assert.Assert(t, marshalMatches(frontendCertificate(custom, secret), `
+secret:
+  items:
+  - key: ca.crt
+    path: ~postgres-operator/frontend-ca.crt
+  - key: tls.key
+    path: ~postgres-operator/frontend-tls.key
+  - key: tls.crt
+    path: ~postgres-operator/frontend-tls.crt
+  name: some-other
+		`))
+
+		// Some items; use only the TLS Paths.
 		custom.Items = []corev1.KeyToPath{
 			{Key: "any", Path: "thing"},
+			{Key: "some-ca-key", Path: "ca.crt"},
+			{Key: "some-cert-key", Path: "tls.crt"},
+			{Key: "some-key-key", Path: "tls.key"},
 		}
-
-		assert.Assert(t, marshalEquals(frontendCertificate(custom, secret), strings.Trim(`
+		assert.Assert(t, marshalMatches(frontendCertificate(custom, secret), `
 secret:
   items:
-  - key: any
-    path: thing
+  - key: some-ca-key
+    path: ~postgres-operator/frontend-ca.crt
+  - key: some-cert-key
+    path: ~postgres-operator/frontend-tls.crt
+  - key: some-key-key
+    path: ~postgres-operator/frontend-tls.key
   name: some-other
-		`, "\t\n")+"\n"))
+		`))
 	})
 }

internal/pgbouncer/config_test.go

@@ -77,15 +77,15 @@ verbose = 0
 auth_file = /etc/pgbouncer/~postgres-operator/users.txt
 auth_query = SELECT username, password from pgbouncer.get_auth($1)
 auth_user = _crunchypgbouncer
-client_tls_ca_file = /etc/pgbouncer/~postgres-operator-frontend/ca.crt
-client_tls_cert_file = /etc/pgbouncer/~postgres-operator-frontend/tls.crt
-client_tls_key_file = /etc/pgbouncer/~postgres-operator-frontend/tls.key
+client_tls_ca_file = /etc/pgbouncer/~postgres-operator/frontend-ca.crt
+client_tls_cert_file = /etc/pgbouncer/~postgres-operator/frontend-tls.crt
+client_tls_key_file = /etc/pgbouncer/~postgres-operator/frontend-tls.key
 client_tls_sslmode = require
 conffile = /etc/pgbouncer/~postgres-operator.ini
 ignore_startup_parameters = extra_float_digits
 listen_addr = *
 listen_port = 8888
-server_tls_ca_file = /etc/pgbouncer/~postgres-operator-backend/ca.crt
+server_tls_ca_file = /etc/pgbouncer/~postgres-operator/backend-ca.crt
 server_tls_sslmode = verify-full
 unix_socket_dir =
 
@@ -120,15 +120,15 @@ verbose = whomp
 auth_file = /etc/pgbouncer/~postgres-operator/users.txt
 auth_query = SELECT username, password from pgbouncer.get_auth($1)
 auth_user = _crunchypgbouncer
-client_tls_ca_file = /etc/pgbouncer/~postgres-operator-frontend/ca.crt
-client_tls_cert_file = /etc/pgbouncer/~postgres-operator-frontend/tls.crt
-client_tls_key_file = /etc/pgbouncer/~postgres-operator-frontend/tls.key
+client_tls_ca_file = /etc/pgbouncer/~postgres-operator/frontend-ca.crt
+client_tls_cert_file = /etc/pgbouncer/~postgres-operator/frontend-tls.crt
+client_tls_key_file = /etc/pgbouncer/~postgres-operator/frontend-tls.key
 client_tls_sslmode = require
 conffile = /etc/pgbouncer/~postgres-operator.ini
 ignore_startup_parameters = custom
 listen_addr = *
 listen_port = 8888
-server_tls_ca_file = /etc/pgbouncer/~postgres-operator-backend/ca.crt
+server_tls_ca_file = /etc/pgbouncer/~postgres-operator/backend-ca.crt
 server_tls_sslmode = verify-full
 unix_socket_dir =
 
@@ -154,7 +154,7 @@ func TestPodConfigFiles(t *testing.T) {
 
 	t.Run("Default", func(t *testing.T) {
 		projections := podConfigFiles(config, configmap, secret)
-		assert.Assert(t, marshalEquals(projections, strings.Trim(`
+		assert.Assert(t, marshalMatches(projections, `
 - configMap:
     items:
     - key: pgbouncer-empty
@@ -170,7 +170,7 @@ func TestPodConfigFiles(t *testing.T) {
     - key: pgbouncer-users.txt
       path: ~postgres-operator/users.txt
     name: some-shh
-		`, "\t\n")+"\n"))
+		`))
 	})
 
 	t.Run("CustomFiles", func(t *testing.T) {
@@ -187,7 +187,7 @@ func TestPodConfigFiles(t *testing.T) {
 		}
 
 		projections := podConfigFiles(config, configmap, secret)
-		assert.Assert(t, marshalEquals(projections, strings.Trim(`
+		assert.Assert(t, marshalMatches(projections, `
 - configMap:
     items:
     - key: pgbouncer-empty
@@ -210,7 +210,7 @@ func TestPodConfigFiles(t *testing.T) {
     - key: pgbouncer-users.txt
       path: ~postgres-operator/users.txt
     name: some-shh
-		`, "\t\n")+"\n"))
+		`))
 	})
 }