From 1369923c3bd62f15be66c24adfa5a354be67df7d Mon Sep 17 00:00:00 2001 From: Ivan Misic Date: Sun, 23 Mar 2025 12:54:12 +0100 Subject: [PATCH 1/4] devops: add pgo kustomize --- .gitignore | 1 + helm/install/values.yaml | 4 +- uc2c/kustomize/base/configmap.yaml | 17 +++++++ uc2c/kustomize/base/job.yaml | 47 +++++++++++++++++ uc2c/kustomize/base/kustomization.yaml | 7 +++ uc2c/kustomize/base/postgrescluster.yaml | 35 +++++++++++++ .../overlays/ccps/kustomization.yaml | 14 ++++++ .../overlays/ccps/postgrescluster-patch.yaml | 50 +++++++++++++++++++ .../overlays/staging/kustomization.yaml | 14 ++++++ .../staging/postgrescluster-patch.yaml | 50 +++++++++++++++++++ .../overlays/test/kustomization.yaml | 14 ++++++ .../overlays/test/postgrescluster-patch.yaml | 4 ++ 12 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 uc2c/kustomize/base/configmap.yaml create mode 100644 uc2c/kustomize/base/job.yaml create mode 100644 uc2c/kustomize/base/kustomization.yaml create mode 100644 uc2c/kustomize/base/postgrescluster.yaml create mode 100644 uc2c/kustomize/overlays/ccps/kustomization.yaml create mode 100644 uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml create mode 100644 uc2c/kustomize/overlays/staging/kustomization.yaml create mode 100644 uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml create mode 100644 uc2c/kustomize/overlays/test/kustomization.yaml create mode 100644 uc2c/kustomize/overlays/test/postgrescluster-patch.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..9f11b755 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea/ diff --git a/helm/install/values.yaml b/helm/install/values.yaml index 8b0f1a42..f466ebdf 100644 --- a/helm/install/values.yaml +++ b/helm/install/values.yaml @@ -49,7 +49,7 @@ pgoControllerLeaseName: cpk-leader-election-lease # replicas sets the number of PGO instances. # Warning: This should only be greater than 1 if pgoControllerLeaseName is set! -replicas: 1 +replicas: 2 # imagePullSecretNames is a list of secret names to use for pulling controller images. # More info: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod @@ -88,4 +88,4 @@ resources: # Override environment variables entirely !!DANGEROUS!! # envOverride: # - name: PG_DEBUG -# value: "true" \ No newline at end of file +# value: "true" diff --git a/uc2c/kustomize/base/configmap.yaml b/uc2c/kustomize/base/configmap.yaml new file mode 100644 index 00000000..f26cb9ad --- /dev/null +++ b/uc2c/kustomize/base/configmap.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: safety-advisor-pg-init-script-config +data: + init.sql: | + -- Grant read-only access to existing and future tables + GRANT CONNECT ON DATABASE safety_advisor TO appsmith; + GRANT USAGE ON SCHEMA public TO appsmith; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO appsmith; + + -- Optional: write access to selected tables + -- GRANT INSERT, UPDATE ON important_table TO appsmith; + + -- Ensure all future tables are readable + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT ON TABLES TO appsmith; diff --git a/uc2c/kustomize/base/job.yaml b/uc2c/kustomize/base/job.yaml new file mode 100644 index 00000000..8ee7d4a7 --- /dev/null +++ b/uc2c/kustomize/base/job.yaml @@ -0,0 +1,47 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: safety-advisor-pg-init-script-job +spec: + template: + spec: + restartPolicy: Never + containers: + - name: psql + image: postgres:16 + command: ["sh", "-c"] + args: + - | + echo "Waiting for DB..."; + until pg_isready -h $PGHOST -p $PGPORT -U $PGUSER -d safety_advisor; do sleep 2; done; + echo "Running init.sql..."; + psql -h $PGHOST -p $PGPORT -U $PGUSER -d safety_advisor -f /sql/init.sql + env: + - name: PGHOST + valueFrom: + secretKeyRef: + name: safety-advisor-pg-pguser-postgres + key: host + - name: PGPORT + valueFrom: + secretKeyRef: + name: safety-advisor-pg-pguser-postgres + key: port + - name: PGUSER + valueFrom: + secretKeyRef: + name: safety-advisor-pg-pguser-postgres + key: user + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: safety-advisor-pg-pguser-postgres + key: password + volumeMounts: + - name: sql-script + mountPath: /sql + volumes: + - name: sql-script + configMap: + name: safety-advisor-pg-init-script-config + diff --git a/uc2c/kustomize/base/kustomization.yaml b/uc2c/kustomize/base/kustomization.yaml new file mode 100644 index 00000000..ece723ec --- /dev/null +++ b/uc2c/kustomize/base/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - postgrescluster.yaml + - configmap.yaml + - job.yaml diff --git a/uc2c/kustomize/base/postgrescluster.yaml b/uc2c/kustomize/base/postgrescluster.yaml new file mode 100644 index 00000000..28255ce4 --- /dev/null +++ b/uc2c/kustomize/base/postgrescluster.yaml @@ -0,0 +1,35 @@ +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: safety-advisor-pg +spec: + postgresVersion: 16 + + instances: + - name: primary + replicas: 1 + dataVolumeClaimSpec: + accessModes: ["ReadWriteOnce"] + storageClassName: microk8s-hostpath + resources: + requests: + storage: 8Gi + resources: + requests: + cpu: "250m" + memory: "512Mi" + limits: + cpu: "500m" + memory: "1Gi" + users: + - name: postgres + - name: safety-advisor + databases: ["safety_advisor"] + - name: appsmith + databases: ["safety_advisor"] + + patroni: + dynamicConfiguration: + postgresql: + parameters: + max_connections: "300" diff --git a/uc2c/kustomize/overlays/ccps/kustomization.yaml b/uc2c/kustomize/overlays/ccps/kustomization.yaml new file mode 100644 index 00000000..8aeb0932 --- /dev/null +++ b/uc2c/kustomize/overlays/ccps/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: safety-advisor-ccps + +resources: + - ../../base + +patches: + - path: postgrescluster-patch.yaml + target: + group: postgres-operator.crunchydata.com + version: v1beta1 + kind: PostgresCluster + name: safety-advisor-pg \ No newline at end of file diff --git a/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml b/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml new file mode 100644 index 00000000..098e5bab --- /dev/null +++ b/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml @@ -0,0 +1,50 @@ +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: safety-advisor-pg +spec: + instances: + - name: primary + replicas: 1 + dataVolumeClaimSpec: + accessModes: ["ReadWriteOnce"] + storageClassName: microk8s-hostpath + resources: + requests: + storage: 32Gi + resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "1" + memory: "2Gi" + + patroni: + dynamicConfiguration: + postgresql: + parameters: + max_connections: "500" + wal_level: replica + archive_mode: "on" + archive_timeout: "60s" + + backups: + pgbackrest: + configuration: + - secret: + name: pgo-s3-creds + global: + repo1-path: /pgbackrest/crunchy-pg-operator/safety-advisor/safety-advisor-ccps + repo1-retention-full: "14" + repo1-retention-full-type: time + repos: + - name: repo1 + schedules: + full: "0 1 * * 0" + differential: "0 1 * * 1-6" + s3: + bucket: uc2civo + endpoint: s3.amazonaws.com + region: us-east-2 + diff --git a/uc2c/kustomize/overlays/staging/kustomization.yaml b/uc2c/kustomize/overlays/staging/kustomization.yaml new file mode 100644 index 00000000..78195516 --- /dev/null +++ b/uc2c/kustomize/overlays/staging/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: safety-advisor + +resources: + - ../../base + +patches: + - path: postgrescluster-patch.yaml + target: + group: postgres-operator.crunchydata.com + version: v1beta1 + kind: PostgresCluster + name: safety-advisor-pg \ No newline at end of file diff --git a/uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml b/uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml new file mode 100644 index 00000000..7327ff7b --- /dev/null +++ b/uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml @@ -0,0 +1,50 @@ +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: safety-advisor-pg +spec: + instances: + - name: primary + replicas: 1 + dataVolumeClaimSpec: + accessModes: ["ReadWriteOnce"] + storageClassName: microk8s-hostpath + resources: + requests: + storage: 32Gi + resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "1" + memory: "2Gi" + + patroni: + dynamicConfiguration: + postgresql: + parameters: + max_connections: "500" + wal_level: replica + archive_mode: "on" + archive_timeout: "60s" + + backups: + pgbackrest: + configuration: + - secret: + name: pgo-s3-creds + global: + repo1-path: /pgbackrest/crunchy-pg-operator/safety-advisor/safety-advisor-staging + repo1-retention-full: "14" + repo1-retention-full-type: time + repos: + - name: repo1 + schedules: + full: "0 1 * * 0" + differential: "0 1 * * 1-6" + s3: + bucket: uc2civo + endpoint: s3.amazonaws.com + region: us-east-2 + diff --git a/uc2c/kustomize/overlays/test/kustomization.yaml b/uc2c/kustomize/overlays/test/kustomization.yaml new file mode 100644 index 00000000..b3fbace8 --- /dev/null +++ b/uc2c/kustomize/overlays/test/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: safety-advisor-test + +resources: + - ../../base + +patches: + - path: postgrescluster-patch.yaml + target: + group: postgres-operator.crunchydata.com + version: v1beta1 + kind: PostgresCluster + name: safety-advisor-pg \ No newline at end of file diff --git a/uc2c/kustomize/overlays/test/postgrescluster-patch.yaml b/uc2c/kustomize/overlays/test/postgrescluster-patch.yaml new file mode 100644 index 00000000..8be96ece --- /dev/null +++ b/uc2c/kustomize/overlays/test/postgrescluster-patch.yaml @@ -0,0 +1,4 @@ +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: safety-advisor-pg From bf6bbe5b9487d609aab5877a93458c1b8c60eb5c Mon Sep 17 00:00:00 2001 From: Ivan Misic Date: Thu, 27 Mar 2025 08:36:20 +0100 Subject: [PATCH 2/4] devops: change s3 paths --- uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml | 2 +- uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml b/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml index 098e5bab..3e2b8a9a 100644 --- a/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml +++ b/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml @@ -35,7 +35,7 @@ spec: - secret: name: pgo-s3-creds global: - repo1-path: /pgbackrest/crunchy-pg-operator/safety-advisor/safety-advisor-ccps + repo1-path: /pgbackrest/safety-advisor-pg-staging/repo1 repo1-retention-full: "14" repo1-retention-full-type: time repos: diff --git a/uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml b/uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml index 7327ff7b..3e2b8a9a 100644 --- a/uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml +++ b/uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml @@ -35,7 +35,7 @@ spec: - secret: name: pgo-s3-creds global: - repo1-path: /pgbackrest/crunchy-pg-operator/safety-advisor/safety-advisor-staging + repo1-path: /pgbackrest/safety-advisor-pg-staging/repo1 repo1-retention-full: "14" repo1-retention-full-type: time repos: From a1c9acc47257e987c0426783afdb2bb03bc7c059 Mon Sep 17 00:00:00 2001 From: Ivan Misic Date: Thu, 27 Mar 2025 09:57:48 +0100 Subject: [PATCH 3/4] devops: fix config for s3 secrets --- .gitignore | 1 + uc2c/kustomize/base/kustomization.yaml | 8 ++++++++ uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml | 5 ++--- .../kustomize/overlays/staging/postgrescluster-patch.yaml | 5 ++--- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 9f11b755..4ddc4f5d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .idea/ +uc2c/kustomize/base/s3.conf diff --git a/uc2c/kustomize/base/kustomization.yaml b/uc2c/kustomize/base/kustomization.yaml index ece723ec..6bc5d41a 100644 --- a/uc2c/kustomize/base/kustomization.yaml +++ b/uc2c/kustomize/base/kustomization.yaml @@ -1,6 +1,14 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +secretGenerator: +- name: pgo-s3-creds + files: + - s3.conf + +generatorOptions: + disableNameSuffixHash: true + resources: - postgrescluster.yaml - configmap.yaml diff --git a/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml b/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml index 3e2b8a9a..86da0115 100644 --- a/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml +++ b/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml @@ -32,8 +32,8 @@ spec: backups: pgbackrest: configuration: - - secret: - name: pgo-s3-creds + - secret: + name: pgo-s3-creds global: repo1-path: /pgbackrest/safety-advisor-pg-staging/repo1 repo1-retention-full: "14" @@ -47,4 +47,3 @@ spec: bucket: uc2civo endpoint: s3.amazonaws.com region: us-east-2 - diff --git a/uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml b/uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml index 3e2b8a9a..86da0115 100644 --- a/uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml +++ b/uc2c/kustomize/overlays/staging/postgrescluster-patch.yaml @@ -32,8 +32,8 @@ spec: backups: pgbackrest: configuration: - - secret: - name: pgo-s3-creds + - secret: + name: pgo-s3-creds global: repo1-path: /pgbackrest/safety-advisor-pg-staging/repo1 repo1-retention-full: "14" @@ -47,4 +47,3 @@ spec: bucket: uc2civo endpoint: s3.amazonaws.com region: us-east-2 - From 256d5de8f50261fd31761c3e03a4efc97ae806d2 Mon Sep 17 00:00:00 2001 From: Ivan Misic Date: Thu, 27 Mar 2025 10:09:44 +0100 Subject: [PATCH 4/4] devops: fix repo clash --- uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml b/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml index 86da0115..de20a386 100644 --- a/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml +++ b/uc2c/kustomize/overlays/ccps/postgrescluster-patch.yaml @@ -35,7 +35,7 @@ spec: - secret: name: pgo-s3-creds global: - repo1-path: /pgbackrest/safety-advisor-pg-staging/repo1 + repo1-path: /pgbackrest/safety-advisor-pg-ccps/repo1 repo1-retention-full: "14" repo1-retention-full-type: time repos: