Move pgbackrest-restore test to Kyverno Chainsaw

.github/workflows/test.yaml

@@ -144,10 +144,12 @@ jobs:
             --env 'RELATED_IMAGE_COLLECTOR=registry.developers.crunchydata.com/crunchydata/postgres-operator:ubi9-5.8.2-0' \
             --env 'PGO_FEATURE_GATES=TablespaceVolumes=true,OpenTelemetryLogs=true,OpenTelemetryMetrics=true' \
             --name 'postgres-operator' localhost/postgres-operator
-      - name: Install kuttl
-        run: |
-          curl -Lo /usr/local/bin/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v0.13.0/kubectl-kuttl_0.13.0_linux_x86_64
-          chmod +x /usr/local/bin/kubectl-kuttl
+
+      - run: |
+          make check-chainsaw && exit
+          failed=$?
+          echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::'
+          exit $failed
 
       - run: make generate-kuttl
         env:
@@ -161,8 +163,6 @@ jobs:
           failed=$?
           echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::'
           exit $failed
-        env:
-          KUTTL: kubectl-kuttl
 
       - name: Stop PGO
         run: docker stop 'postgres-operator' || true

Makefile

@@ -12,6 +12,8 @@ GO_TEST ?= $(GO) test
 CONTROLLER ?= $(GO) tool sigs.k8s.io/controller-tools/cmd/controller-gen
 
 # Run tests using the latest tools.
+CHAINSAW ?= $(GO) run github.com/kyverno/chainsaw@latest
+CHAINSAW_TEST ?= $(CHAINSAW) test
 ENVTEST ?= $(GO) run sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
 KUTTL ?= $(GO) run github.com/kudobuilder/kuttl/cmd/kubectl-kuttl@latest
 KUTTL_TEST ?= $(KUTTL) test
@@ -170,6 +172,17 @@ check-envtest-existing: createnamespaces
 		$(GO_TEST) -count=1 -cover -p=1 ./...
 	kubectl delete -k ./config/dev
 
+# Expects operator to be running
+#
+# Chainsaw runs with a single kubectl context named "chainsaw".
+# If you experience `cluster "minikube" does not exist`, try `MINIKUBE_PROFILE=chainsaw`.
+#
+# https://kyverno.github.io/chainsaw/latest/operations/script#kubeconfig
+#
+.PHONY: check-chainsaw
+check-chainsaw:
+	$(CHAINSAW_TEST) --config testing/chainsaw/e2e/config.yaml --values testing/chainsaw/e2e/values.yaml testing/chainsaw/e2e
+
 # Expects operator to be running
 #
 # KUTTL runs with a single kubectl context named "cluster".

testing/chainsaw/e2e/config.yaml

@@ -0,0 +1,12 @@
+apiVersion: chainsaw.kyverno.io/v1alpha2
+kind: Configuration
+metadata:
+  name: end-to-end
+spec:
+  namespace:
+    template:
+      metadata:
+        labels: { postgres-operator-test: chainsaw }
+  timeouts:
+    assert: 3m
+    cleanup: 3m

testing/chainsaw/e2e/pgbackrest-restore/README.md

@@ -0,0 +1,5 @@
+# pgbackrest-restore
+
+This [chainsaw](https://github.com/kyverno/chainsaw) suite tests that CPK can clone and restore through pgBackRest backups.
+
+...other material here...

testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml

@@ -0,0 +1,201 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: pgbackrest-restore
+  labels:
+    pgbackrest: ''
+spec:
+  failFast: true
+  bindings:
+    - name: postgres
+      value:
+        version: (to_number(as_string($values.versions.postgres)))
+
+    - name: psql
+      value:
+        image: ($values.images.psql)
+        connect: { name: PGCONNECT_TIMEOUT, value: '5' }
+
+    - name: volume
+      value: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } }
+
+  steps:
+    - name: 'Create Cluster with replica, tablespace'
+      use: 
+        template: 'templates/create-cluster.yaml'
+
+    - name: 'Create Data'
+      use: 
+        template: 'templates/psql-data.yaml'
+        with:
+          bindings:
+            - name: target
+              value: original
+            - name: job
+              value: original-data            
+            - name: command
+              value: |
+                     CREATE SCHEMA IF NOT EXISTS "original";
+                     CREATE TABLE important (data) AS VALUES ('treasure');
+                     CREATE TABLE cows (name) TABLESPACE barn AS VALUES ('nellie');
+
+    - name: 'Create Backup #1'
+      use: 
+        template: 'templates/create-backup.yaml'
+        with:
+          bindings:
+            - name: annotation
+              value: one
+
+    - name: 'Clone Cluster #1'
+      skipDelete: true
+      use:
+        template: 'templates/clone-cluster.yaml'
+        with:
+          bindings:
+            - name: name
+              value: clone-one
+
+    - name: 'Verify Data on Clone #1'
+      use:
+        template: 'templates/psql-data.yaml'
+        with:
+          bindings:
+            - name: target
+              value: clone-one
+            - name: job
+              value: clone-one-data
+            - name: command
+              value: |
+                      DO $$$$
+                      DECLARE
+                        restored jsonb;
+                      BEGIN
+                        SELECT jsonb_agg(important) INTO restored FROM important;
+                        ASSERT restored = '[{"data":"treasure"}]', format('got %L', restored);
+                        SELECT jsonb_agg(cows) INTO restored FROM cows;
+                        ASSERT restored = '[{"name":"nellie"}]', format('got %L', restored);
+                      END $$$$;
+
+    - name: 'Delete Cluster #1'
+      description: >
+        Delete this clone in the background to free up resources
+      try:
+        - delete:
+            deletionPropagationPolicy: Background
+            expect: [{ check: { (`true`): true } }]
+            ref:
+              apiVersion: postgres-operator.crunchydata.com/v1beta1
+              kind: PostgresCluster
+              name: clone-one
+
+    - name: 'Restart Cluster'
+      description: >
+        Sets a timestamp and restarts the cluster, using the timestamp for comparison
+      use: 
+        template: 'templates/restart-cluster.yaml'
+
+    - name: 'Update Data'
+      use:
+        template: 'templates/psql-data.yaml'
+        with:
+          bindings:
+            - name: target
+              value: original
+            - name: job
+              value: original-more-data
+            - name: command
+              value: INSERT INTO important (data) VALUES ('water'), ('socks');
+
+    - name: 'Verify WAL backup'
+      use:
+        template: 'templates/verify-backup.yaml'
+
+    - name: 'Create Backup #2'
+      use: 
+        template: 'templates/create-backup.yaml'
+        with:
+          bindings:
+            - name: annotation
+              value: two
+
+    - name: 'Clone Cluster #2'
+      skipDelete: true
+      use:
+        template: 'templates/clone-cluster.yaml'
+        with:
+          bindings:
+            - name: name
+              value: clone-two
+
+    - name: 'Verify Data on Clone #2'
+      use:
+        template: 'templates/psql-data.yaml'
+        with:
+          bindings:
+            - name: target
+              value: clone-two
+            - name: job
+              value: clone-two-data
+            - name: command
+              value: |
+                      DO $$$$
+                      DECLARE
+                        restored jsonb;
+                      BEGIN
+                        SELECT jsonb_agg(important) INTO restored FROM important;
+                        ASSERT restored = '[
+                          {"data":"treasure"}, {"data":"water"}, {"data":"socks"}
+                        ]', format('got %L', restored);
+                      END $$$$;
+
+    - name: 'Delete Cluster #2'
+      description: >
+        Delete this clone in the background to free up resources
+      try:
+        - delete:
+            deletionPropagationPolicy: Background
+            expect: [{ check: { (`true`): true } }]
+            ref:
+              apiVersion: postgres-operator.crunchydata.com/v1beta1
+              kind: PostgresCluster
+              name: clone-two
+
+    - name: 'Lose Data'
+      description: >
+        Drop data and ensure that the data is dropped from the replica as well
+      use:
+        template: 'templates/lose-data.yaml'
+
+    - name: 'Point-In-Time Restore'
+      use:
+        template: 'templates/point-in-time-restore.yaml'
+
+    - name: 'Verify Primary'
+      description: >
+        Confirm that data was restored to the point-in-time and the cluster is healthy
+      use:
+        template: 'templates/psql-data.yaml'
+        with:
+          bindings:
+            - name: target
+              value: original
+            - name: job
+              value: original-pitr-primary
+            - name: command
+              value: |
+                      DO $$$$
+                      DECLARE
+                        restored jsonb;
+                      BEGIN
+                        SELECT jsonb_agg(important) INTO restored FROM important;
+                        ASSERT restored = '[
+                          {"data":"treasure"}, {"data":"water"}, {"data":"socks"}
+                        ]', format('got %L', restored);
+                      END $$$$;
+
+    - name: 'Confirm Replica'
+      description: >
+        Verify that the data has streamed and is streaming to the replica
+      use: 
+        template: 'templates/verify-replica.yaml'

testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml

@@ -0,0 +1,51 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: clone-cluster
+spec:
+  bindings:
+    - name: name
+      value: 'The name of the new PostgresCluster'
+
+  try:
+    -
+      description: >
+        Clone the cluster using a pgBackRest restore
+      apply:
+        resource:
+          apiVersion: postgres-operator.crunchydata.com/v1beta1
+          kind: PostgresCluster
+          metadata:
+            name: ($name)
+          spec:
+            dataSource:
+              postgresCluster:
+                clusterName: original
+                repoName: repo1
+            postgresVersion: ($postgres.version)
+            instances:
+              - dataVolumeClaimSpec: ($volume)
+                tablespaceVolumes:
+                  - { name: barn, dataVolumeClaimSpec: ($volume) }
+            backups:
+              pgbackrest:
+                repos:
+                  - name: repo1
+                    volume:
+                      volumeClaimSpec: ($volume)
+
+    -
+      description: >
+        Wait for the cluster to come online
+      assert:
+        resource:
+          apiVersion: postgres-operator.crunchydata.com/v1beta1
+          kind: PostgresCluster
+          metadata:
+            name: ($name)
+          status:
+            instances:
+              - name: '00'
+                replicas: 1
+                readyReplicas: 1
+                updatedReplicas: 1

testing/chainsaw/e2e/pgbackrest-restore/templates/create-backup.yaml

@@ -0,0 +1,36 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: create-backup
+spec:
+  bindings:
+    - name: annotation
+      value: 'The annotation to kick off a backup'
+  try:
+    -
+      description: >
+        Annotate the cluster to trigger a backup
+      patch:
+        resource:
+          apiVersion: postgres-operator.crunchydata.com/v1beta1
+          kind: PostgresCluster
+          metadata:
+            name: original
+            annotations:
+              postgres-operator.crunchydata.com/pgbackrest-backup: ($annotation)
+
+    -
+      description: >
+        Wait for the backup to complete
+      assert:
+        resource:
+          apiVersion: batch/v1
+          kind: Job
+          metadata:
+            annotations:
+              postgres-operator.crunchydata.com/pgbackrest-backup: ($annotation)
+            labels:
+              postgres-operator.crunchydata.com/cluster: original
+              postgres-operator.crunchydata.com/pgbackrest-backup: manual
+          status:
+            succeeded: 1