From fd701b22266a511ca5fe45b06e95b095ace1bef5 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Tue, 29 Jul 2025 15:58:56 +0200
Subject: [PATCH 01/23] add minikube scripts

---
 .evergreen-functions.yml | 18 ++++++++++++++++++
 .evergreen.yml           | 27 +++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml
index c004dd098..3cd67d524 100644
--- a/.evergreen-functions.yml
+++ b/.evergreen-functions.yml
@@ -236,6 +236,15 @@ functions:
         - ${workdir}/bin
       binary: scripts/dev/setup_evg_host.sh
 
+  setup_ibm_host: &setup_ibm_host
+    command: subprocess.exec
+    type: setup
+    params:
+      working_dir: src/github.com/mongodb/mongodb-kubernetes
+      add_to_path:
+        - ${workdir}/bin
+      binary: scripts/dev/setup_minikube_host.sh
+
   lint_repo:
     - command: subprocess.exec
       type: setup
@@ -261,6 +270,15 @@ functions:
     - *setup_evg_host
     - *python_venv
 
+  # This differs for normal evg_host as we require minikube instead of kind for
+  # IBM machines
+  setup_building_host_minikube:
+    - *switch_context
+    - *setup_aws
+    - *configure_docker_auth
+    - *setup_ibm_host
+    - *python_venv
+
   prune_docker_resources:
     - command: subprocess.exec
       type: setup
diff --git a/.evergreen.yml b/.evergreen.yml
index 209bf152a..17e882f95 100644
--- a/.evergreen.yml
+++ b/.evergreen.yml
@@ -77,6 +77,13 @@ variables:
       - func: download_kube_tools
       - func: setup_building_host
 
+  - &setup_group_ibm
+    setup_group_can_fail_task: true
+    setup_group:
+      - func: clone
+      - func: download_kube_tools
+      - func: setup_building_host_minikube
+
   - &setup_group_multi_cluster
     setup_group_can_fail_task: true
     setup_group:
@@ -1188,6 +1195,14 @@ task_groups:
       - e2e_om_ops_manager_backup
     <<: *teardown_group
 
+  - name: e2e_smoke_ibm_task_group
+    max_hosts: -1
+    <<: *setup_group_ibm
+    <<: *setup_and_teardown_task
+    tasks:
+      - e2e_om_ops_manager_backup
+    <<: *teardown_group
+
   - name: e2e_ops_manager_kind_5_0_only_task_group
     max_hosts: -1
     <<: *setup_group
@@ -1453,6 +1468,18 @@ buildvariants:
     tasks:
       - name: e2e_smoke_task_group
 
+  - name: e2e_smoke_ibm
+    display_name: e2e_smoke_ibm
+    tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ]
+    run_on:
+      - rhel9-power-large
+    allowed_requesters: [ "patch", "github_tag" ]
+#    depends_on:
+#      - name: build_test_image
+#        variant: init_test_run
+    tasks:
+      - name: e2e_smoke_ibm_task_group
+
   - name: e2e_static_smoke
     display_name: e2e_static_smoke
     tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ]

From 05cbe235bcd8af5aa2547631b891e0e0e783717f Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Tue, 29 Jul 2025 16:01:39 +0200
Subject: [PATCH 02/23] smoke test replicacset

---
 .evergreen.yml                     |  2 +-
 scripts/dev/contexts/e2e_smoke_ibm | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 scripts/dev/contexts/e2e_smoke_ibm

diff --git a/.evergreen.yml b/.evergreen.yml
index 17e882f95..2d2e113ae 100644
--- a/.evergreen.yml
+++ b/.evergreen.yml
@@ -1200,7 +1200,7 @@ task_groups:
     <<: *setup_group_ibm
     <<: *setup_and_teardown_task
     tasks:
-      - e2e_om_ops_manager_backup
+      - e2e_replica_set
     <<: *teardown_group
 
   - name: e2e_ops_manager_kind_5_0_only_task_group
diff --git a/scripts/dev/contexts/e2e_smoke_ibm b/scripts/dev/contexts/e2e_smoke_ibm
new file mode 100644
index 000000000..03384c26c
--- /dev/null
+++ b/scripts/dev/contexts/e2e_smoke_ibm
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -Eeou pipefail
+
+script_name=$(readlink -f "${BASH_SOURCE[0]}")
+script_dir=$(dirname "${script_name}")
+
+source "${script_dir}/root-context"
+
+export ops_manager_version="cloud_qa"
+
+# This is required to be able to rebuild the om image and use that image which has been rebuild
+export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev
+CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${script_dir}"/../../../.evergreen.yml | awk '{print $3}')
+export CUSTOM_OM_VERSION
+
+export CUSTOM_MDB_VERSION=6.0.5
+export CUSTOM_MDB_PREV_VERSION=5.0.7

From a1f3d90ed86606d7428f2499cfa3f140a19120c7 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Tue, 29 Jul 2025 16:26:38 +0200
Subject: [PATCH 03/23] smoke test replicacset

---
 scripts/evergreen/setup_kubectl.sh | 40 ++++++++++++++++++++++++++----
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/scripts/evergreen/setup_kubectl.sh b/scripts/evergreen/setup_kubectl.sh
index ab9066ac1..00cf975fd 100755
--- a/scripts/evergreen/setup_kubectl.sh
+++ b/scripts/evergreen/setup_kubectl.sh
@@ -3,22 +3,52 @@ set -Eeou pipefail
 
 source scripts/dev/set_env_context.sh
 
+# Detect system architecture and map to kubectl/helm architecture names
+detect_architecture() {
+    local arch
+    arch=$(uname -m)
+    
+    case "${arch}" in
+        x86_64)
+            echo "amd64"
+            ;;
+        aarch64|arm64)
+            echo "arm64"
+            ;;
+        ppc64le)
+            echo "ppc64le"
+            ;;
+        s390x)
+            echo "s390x"
+            ;;
+        *)
+            echo "Unsupported architecture: ${arch}" >&2
+            echo "Supported architectures: x86_64 (amd64), aarch64 (arm64), ppc64le, s390x" >&2
+            exit 1
+            ;;
+    esac
+}
+
+# Detect the current architecture
+ARCH=$(detect_architecture)
+echo "Detected architecture: ${ARCH}"
+
 bindir="${PROJECT_DIR}/bin"
 tmpdir="${PROJECT_DIR}/tmp"
 mkdir -p "${bindir}" "${tmpdir}"
 
-echo "Downloading latest kubectl"
-curl -s --retry 3 -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+echo "Downloading latest kubectl for ${ARCH}"
+curl -s --retry 3 -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl"
 chmod +x kubectl
 echo "kubectl version --client"
 ./kubectl version --client
 mv kubectl "${bindir}"
 
-echo "Downloading helm"
+echo "Downloading helm for ${ARCH}"
 helm_archive="${tmpdir}/helm.tgz"
 helm_version="v3.17.1"
-curl -s https://get.helm.sh/helm-${helm_version}-linux-amd64.tar.gz --output "${helm_archive}"
+curl -s https://get.helm.sh/helm-${helm_version}-linux-${ARCH}.tar.gz --output "${helm_archive}"
 
 tar xfz "${helm_archive}" -C "${tmpdir}" &> /dev/null
-mv "${tmpdir}/linux-amd64/helm" "${bindir}"
+mv "${tmpdir}/linux-${ARCH}/helm" "${bindir}"
 "${bindir}"/helm version

From 2d335f398673fff6ad51eeaeac3a1d0b29199292 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Tue, 29 Jul 2025 16:48:48 +0200
Subject: [PATCH 04/23] add minikube and make aws multiarch

---
 scripts/evergreen/setup_aws.sh          |  37 +++-
 scripts/minikube/install-docker.sh      | 108 +++++++++++
 scripts/minikube/install-minikube.sh    | 190 ++++++++++++++++++++
 scripts/minikube/minikube_host.sh       | 226 ++++++++++++++++++++++++
 scripts/minikube/setup_minikube_host.sh | 115 ++++++++++++
 5 files changed, 675 insertions(+), 1 deletion(-)
 create mode 100755 scripts/minikube/install-docker.sh
 create mode 100755 scripts/minikube/install-minikube.sh
 create mode 100755 scripts/minikube/minikube_host.sh
 create mode 100755 scripts/minikube/setup_minikube_host.sh

diff --git a/scripts/evergreen/setup_aws.sh b/scripts/evergreen/setup_aws.sh
index 931eb0a36..072900639 100755
--- a/scripts/evergreen/setup_aws.sh
+++ b/scripts/evergreen/setup_aws.sh
@@ -3,6 +3,40 @@ set -Eeou pipefail
 
 source scripts/dev/set_env_context.sh
 
+# Detect system architecture and map to AWS CLI architecture names
+detect_aws_architecture() {
+    local arch
+    arch=$(uname -m)
+
+    case "${arch}" in
+        x86_64)
+            echo "x86_64"
+            ;;
+        aarch64|arm64)
+            echo "aarch64"
+            ;;
+        ppc64le)
+            echo "Skipping AWS CLI installation: ppc64le (IBM Power) architecture is not supported by AWS CLI v2." >&2
+            echo "AWS CLI v2 only supports: x86_64 (amd64), aarch64 (arm64)" >&2
+            exit 0
+            ;;
+        s390x)
+            echo "Skipping AWS CLI installation: s390x (IBM Z) architecture is not supported by AWS CLI v2." >&2
+            echo "AWS CLI v2 only supports: x86_64 (amd64), aarch64 (arm64)" >&2
+            exit 0
+            ;;
+        *)
+            echo "Skipping AWS CLI installation: Unsupported architecture: ${arch}" >&2
+            echo "AWS CLI v2 only supports: x86_64 (amd64), aarch64 (arm64)" >&2
+            exit 0
+            ;;
+    esac
+}
+
+# Detect the current architecture
+ARCH=$(detect_aws_architecture)
+echo "Detected architecture: ${ARCH} (AWS CLI v2 supported)"
+
 INSTALL_DIR="${workdir:?}/.local/lib/aws"
 BIN_LOCATION="${workdir}/bin"
 
@@ -11,7 +45,8 @@ mkdir -p "${BIN_LOCATION}"
 tmpdir=$(mktemp -d)
 cd "${tmpdir}"
 
-curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+echo "Downloading AWS CLI v2 for ${ARCH}..."
+curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip"
 unzip awscliv2.zip &> /dev/null
 
 docker_dir="/home/${USER}/.docker"
diff --git a/scripts/minikube/install-docker.sh b/scripts/minikube/install-docker.sh
new file mode 100755
index 000000000..04ae3f7d8
--- /dev/null
+++ b/scripts/minikube/install-docker.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+set -Eeou pipefail
+
+# Script to install Docker on s390x architecture (specifically for RHEL/Ubuntu based systems)
+
+print_usage() {
+    echo "Usage: $0 [options]"
+    echo "Options:"
+    echo "  -h, --help    Show this help message"
+    echo "  -u, --user    Username to add to docker group (optional)"
+    echo ""
+    echo "This script installs Docker on s390x architecture systems."
+}
+
+DOCKER_USER=""
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -h|--help)
+            print_usage
+            exit 0
+            ;;
+        -u|--user)
+            DOCKER_USER="$2"
+            shift 2
+            ;;
+        *)
+            echo "Unknown option: $1"
+            print_usage
+            exit 1
+            ;;
+    esac
+done
+
+echo "Installing Docker on s390x architecture..."
+
+# Detect OS
+if [[ -f /etc/redhat-release ]]; then
+    OS_TYPE="rhel"
+elif [[ -f /etc/debian_version ]]; then
+    OS_TYPE="debian"
+else
+    echo "Unsupported OS. This script supports RHEL/CentOS and Ubuntu/Debian."
+    exit 1
+fi
+
+# Install Docker based on OS
+if [[ "$OS_TYPE" == "rhel" ]]; then
+    echo "Detected RHEL/CentOS system..."
+    
+    # Remove any existing Docker packages
+    sudo yum remove -y docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-engine || true
+    
+    # Install required packages (some may not exist on newer RHEL versions)
+    sudo yum install -y yum-utils || echo "yum-utils already installed or unavailable"
+    sudo yum install -y device-mapper-persistent-data lvm2 || echo "device-mapper packages may not be available on this system"
+    
+    # Add Docker repository
+    sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+    
+    # Install Docker CE
+    sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+    
+elif [[ "$OS_TYPE" == "debian" ]]; then
+    echo "Detected Ubuntu/Debian system..."
+    
+    # Remove any existing Docker packages
+    sudo apt-get remove -y docker docker-engine docker.io containerd runc || true
+    
+    # Update package index
+    sudo apt-get update
+    
+    # Install required packages
+    sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release
+    
+    # Add Docker's official GPG key
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+    
+    # Set up stable repository
+    echo "deb [arch=s390x signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+    
+    # Update package index again
+    sudo apt-get update
+    
+    # Install Docker CE
+    sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+fi
+
+# Start and enable Docker service
+sudo systemctl start docker
+sudo systemctl enable docker
+
+# Add user to docker group if specified
+if [[ -n "$DOCKER_USER" ]]; then
+    echo "Adding user '$DOCKER_USER' to docker group..."
+    sudo usermod -aG docker "$DOCKER_USER"
+    echo "Note: User '$DOCKER_USER' needs to log out and log back in for group membership to take effect."
+fi
+
+# Verify installation
+echo "Verifying Docker installation..."
+sudo docker --version
+sudo docker run --rm hello-world
+
+echo "Docker installation completed successfully!"
+echo ""
+echo "If you added a user to the docker group, they need to log out and log back in."
+echo "You can also run 'newgrp docker' to apply the group membership in the current session."
\ No newline at end of file
diff --git a/scripts/minikube/install-minikube.sh b/scripts/minikube/install-minikube.sh
new file mode 100755
index 000000000..3e6dd4e13
--- /dev/null
+++ b/scripts/minikube/install-minikube.sh
@@ -0,0 +1,190 @@
+#!/usr/bin/env bash
+set -Eeou pipefail
+
+# Script to install and configure minikube on s390x architecture
+
+print_usage() {
+    echo "Usage: $0 [options]"
+    echo "Options:"
+    echo "  -h, --help              Show this help message"
+    echo "  -v, --version VERSION   Minikube version to install (default: latest)"
+    echo "  -k, --kubernetes VER    Kubernetes version (default: latest stable)"
+    echo "  -m, --memory MEMORY     Memory allocation (default: 8192mb)"
+    echo "  -c, --cpus CPUS         CPU allocation (default: 4)"
+    echo "  --profile PROFILE       Minikube profile name (default: minikube)"
+    echo "  --start                 Start minikube after installation"
+    echo ""
+    echo "This script installs minikube on s390x architecture and configures it for MongoDB Kubernetes e2e testing."
+}
+
+MINIKUBE_VERSION="latest"
+K8S_VERSION=""
+MEMORY="8192"
+CPUS="4"
+PROFILE="minikube"
+START_MINIKUBE="false"
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -h|--help)
+            print_usage
+            exit 0
+            ;;
+        -v|--version)
+            MINIKUBE_VERSION="$2"
+            shift 2
+            ;;
+        -k|--kubernetes)
+            K8S_VERSION="$2"
+            shift 2
+            ;;
+        -m|--memory)
+            MEMORY="$2"
+            shift 2
+            ;;
+        -c|--cpus)
+            CPUS="$2"
+            shift 2
+            ;;
+        --profile)
+            PROFILE="$2"
+            shift 2
+            ;;
+        --start)
+            START_MINIKUBE="true"
+            shift
+            ;;
+        *)
+            echo "Unknown option: $1"
+            print_usage
+            exit 1
+            ;;
+    esac
+done
+
+echo "Installing minikube on s390x architecture..."
+
+# Verify Docker is installed
+if ! command -v docker &> /dev/null; then
+    echo "Error: Docker is required but not installed. Please install Docker first."
+    echo "You can use the install-docker.sh script in this directory."
+    exit 1
+fi
+
+# Verify Docker is running
+if ! docker info &> /dev/null; then
+    echo "Error: Docker is not running. Please start Docker service."
+    exit 1
+fi
+
+# Install kubectl if not present
+if ! command -v kubectl &> /dev/null; then
+    echo "Installing kubectl..."
+    
+    # Get the latest kubectl version if not specified
+    if [[ -z "$K8S_VERSION" ]]; then
+        K8S_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt)
+    fi
+    
+    # Download kubectl for s390x
+    curl -LO "https://dl.k8s.io/release/${K8S_VERSION}/bin/linux/s390x/kubectl"
+    
+    # Verify the binary
+    curl -LO "https://dl.k8s.io/${K8S_VERSION}/bin/linux/s390x/kubectl.sha256"
+    echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check
+    
+    # Install kubectl
+    sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+    rm -f kubectl kubectl.sha256
+    
+    echo "kubectl installed successfully"
+fi
+
+# Install minikube
+echo "Installing minikube..."
+
+if [[ "$MINIKUBE_VERSION" == "latest" ]]; then
+    # Get the latest minikube version
+    MINIKUBE_VERSION=$(curl -s https://api.github.com/repos/kubernetes/minikube/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
+fi
+
+# Download minikube for s390x
+curl -Lo minikube "https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-s390x"
+
+# Make it executable and install
+chmod +x minikube
+sudo install minikube /usr/local/bin/
+
+# Clean up
+rm -f minikube
+
+echo "Minikube ${MINIKUBE_VERSION} installed successfully"
+
+# Configure minikube for MongoDB Kubernetes testing
+echo "Configuring minikube for MongoDB Kubernetes e2e testing..."
+
+# Set default driver to docker
+minikube config set driver docker
+
+# Configure resource limits
+minikube config set memory "${MEMORY}mb"
+minikube config set cpus "${CPUS}"
+
+# Enable required addons for testing
+ADDONS=(
+    "storage-provisioner"
+    "default-storageclass"
+    "volumesnapshots"
+    "csi-hostpath-driver"
+)
+
+echo "Minikube configuration completed."
+
+if [[ "$START_MINIKUBE" == "true" ]]; then
+    echo "Starting minikube cluster with profile '${PROFILE}'..."
+    
+    # Start minikube with specific configuration for MongoDB testing
+    minikube start \
+        --profile="${PROFILE}" \
+        --driver=docker \
+        --memory="${MEMORY}mb" \
+        --cpus="${CPUS}" \
+        --disk-size=50g \
+        --extra-config=kubelet.authentication-token-webhook=true \
+        --extra-config=kubelet.authorization-mode=Webhook \
+        --extra-config=scheduler.bind-address=0.0.0.0 \
+        --extra-config=controller-manager.bind-address=0.0.0.0 \
+        ${K8S_VERSION:+--kubernetes-version=$K8S_VERSION}
+    
+    # Wait for cluster to be ready
+    echo "Waiting for cluster to be ready..."
+    kubectl wait --for=condition=Ready nodes --all --timeout=300s
+    
+    # Enable addons
+    for addon in "${ADDONS[@]}"; do
+        echo "Enabling addon: $addon"
+        minikube addons enable "$addon" --profile="${PROFILE}" || true
+    done
+    
+    # Create directories that MongoDB tests expect (similar to kind setup)
+    echo "Setting up test directories..."
+    minikube ssh --profile="${PROFILE}" -- 'sudo mkdir -p /opt/data/mongo-data-{0..2} /opt/data/mongo-logs-{0..2}'
+    minikube ssh --profile="${PROFILE}" -- 'sudo chmod 777 /opt/data/mongo-data-* /opt/data/mongo-logs-*'
+    
+    echo "Minikube cluster started successfully!"
+    echo ""
+    echo "To use this cluster:"
+    echo "  export KUBECONFIG=\$(minikube kubeconfig --profile=${PROFILE})"
+    echo "  kubectl get nodes"
+    echo ""
+    echo "To stop the cluster:"
+    echo "  minikube stop --profile=${PROFILE}"
+else
+    echo ""
+    echo "Minikube installed but not started."
+    echo "To start minikube later, run:"
+    echo "  minikube start --profile=${PROFILE} --driver=docker --memory=${MEMORY}mb --cpus=${CPUS}"
+fi
+
+echo ""
+echo "Installation completed successfully!"
\ No newline at end of file
diff --git a/scripts/minikube/minikube_host.sh b/scripts/minikube/minikube_host.sh
new file mode 100755
index 000000000..ab410d7bc
--- /dev/null
+++ b/scripts/minikube/minikube_host.sh
@@ -0,0 +1,226 @@
+#!/usr/bin/env bash
+
+# This is a helper script for running tests on s390x Hosts.
+# It allows to configure minikube clusters and expose remote API servers on a local machine to
+# enable local development while running minikube cluster on s390x instance.
+# Run "minikube_host.sh help" command to see the full usage.
+# Similar to evg_host.sh but uses minikube instead of kind.
+
+set -Eeou pipefail
+
+test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x
+
+source scripts/dev/set_env_context.sh
+source scripts/funcs/printing
+
+if [[ -z "${S390_HOST_NAME}" ]]; then
+  echo "S390_HOST_NAME env var is missing"
+  echo "Set it to your s390x host connection string (e.g., user@hostname)"
+  exit 1
+fi
+
+get_host_url() {
+  echo "${S390_HOST_NAME}"
+}
+
+cmd=${1-""}
+
+if [[ "${cmd}" != "" && "${cmd}" != "help" ]]; then
+  host_url=$(get_host_url)
+fi
+
+kubeconfig_path="${HOME}/.operator-dev/s390-host.kubeconfig"
+
+configure() {
+  shift 1
+  arch=${1-"$(uname -m)"}
+
+  echo "Configuring minikube host ${S390_HOST_NAME} (${host_url}) with architecture ${arch}"
+
+  if [[ "${cmd}" == "configure" && ! "${arch}" =~ ^(s390x|ppc64le|x86_64|aarch64)$ ]]; then
+    echo "'configure' command supports the following architectures: s390x, ppc64le, x86_64, aarch64"
+    exit 1
+  fi
+
+  ssh -T -q "${host_url}" "sudo chown \$(whoami):\$(whoami) ~/.docker || true; mkdir -p ~/.docker"
+  if [[ -f "${HOME}/.docker/config.json" ]]; then
+    echo "Copying local ~/.docker/config.json authorization credentials to s390x host"
+    jq '. | with_entries(select(.key == "auths"))' "${HOME}/.docker/config.json" | ssh -T -q "${host_url}" 'cat > ~/.docker/config.json'
+  fi
+
+  sync
+
+  ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; scripts/dev/switch_context.sh root-context; scripts/minikube/setup_minikube_host.sh ${arch}"
+}
+
+sync() {
+  rsync --verbose --archive --compress --human-readable --recursive --progress \
+  --delete --delete-excluded --max-size=1000000 --prune-empty-dirs \
+  -e ssh \
+  --include-from=.rsyncinclude \
+  --exclude-from=.gitignore \
+  --exclude-from=.rsyncignore \
+  ./ "${host_url}:~/mongodb-kubernetes/"
+
+  rsync --verbose --no-links --recursive --prune-empty-dirs --archive --compress --human-readable \
+    --max-size=1000000 \
+    -e ssh \
+    ~/.operator-dev/ \
+    "${host_url}:~/.operator-dev" &
+
+  wait
+}
+
+remote-prepare-local-e2e-run() {
+  set -x
+  sync
+  cmd make switch context=e2e_mdb_kind_ubi_cloudqa
+  cmd scripts/dev/prepare_local_e2e_run.sh
+  rsync --verbose --no-links --recursive --prune-empty-dirs --archive --compress --human-readable \
+    --max-size=1000000 \
+    -e ssh \
+    "${host_url}:~/mongodb-kubernetes/.multi_cluster_local_test_files" \
+    ./ &
+  scp "${host_url}:~/.operator-dev/multicluster_kubeconfig" "${KUBE_CONFIG_PATH}" &
+
+  wait
+}
+
+get-kubeconfig() {
+  # For minikube, we need to get the kubeconfig and certificates
+  echo "Getting kubeconfig from minikube on s390x host..."
+  local profile=${MINIKUBE_PROFILE:-mongodb-e2e}
+
+  # Create local minikube directory structure
+  mkdir -p "${HOME}/.minikube/profiles/${profile}"
+
+  # Copy certificates from remote host
+  echo "Copying minikube certificates..."
+  scp "${host_url}:~/.minikube/ca.crt" "${HOME}/.minikube/"
+  scp "${host_url}:~/.minikube/profiles/${profile}/client.crt" "${HOME}/.minikube/profiles/${profile}/"
+  scp "${host_url}:~/.minikube/profiles/${profile}/client.key" "${HOME}/.minikube/profiles/${profile}/"
+
+  # Get kubeconfig and update paths to local ones
+  ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; export KUBE_ENVIRONMENT_NAME=minikube; export MINIKUBE_PROFILE=${profile}; kubectl config view --raw" > "${kubeconfig_path}"
+
+  # Update certificate paths to local paths
+  sed -i '' "s|/home/cloud-user/.minikube|${HOME}/.minikube|g" "${kubeconfig_path}"
+
+  # Update server addresses to use localhost for tunneling
+  sed -i '' "s|https://192.168.[0-9]*.[0-9]*:\([0-9]*\)|https://127.0.0.1:\1|g" "${kubeconfig_path}"
+
+  echo "Copied minikube kubeconfig and certificates to ${kubeconfig_path}"
+}
+
+recreate-minikube-cluster() {
+  shift 1
+  profile_name=${1:-mongodb-e2e}
+  configure "$(uname -m)" 2>&1| prepend "minikube_host.sh configure"
+  echo "Recreating minikube cluster ${profile_name} on ${S390_HOST_NAME} (${host_url})..."
+  # shellcheck disable=SC2088
+  ssh -T "${host_url}" "cd ~/mongodb-kubernetes; export KUBE_ENVIRONMENT_NAME=minikube; export MINIKUBE_PROFILE=${profile_name}; minikube delete --profile=${profile_name} || true; minikube start --profile=${profile_name} --driver=docker --memory=8192mb --cpus=4"
+  echo "Copying kubeconfig to ${kubeconfig_path}"
+  get-kubeconfig
+}
+
+tunnel() {
+  shift 1
+  echo "Setting up tunnel for minikube cluster..."
+  local profile=${MINIKUBE_PROFILE:-mongodb-e2e}
+
+  # Get the minikube API server port from remote host
+  local api_port
+  api_port=$(ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; export MINIKUBE_PROFILE=${profile}; minikube ip --profile=${profile} 2>/dev/null && echo ':8443' | tr -d '\n'")
+
+  if [[ -z "${api_port}" ]]; then
+    echo "Could not determine minikube API server details. Is the cluster running?"
+    return 1
+  fi
+
+  # Extract just the port (8443)
+  local port="8443"
+  echo "Forwarding localhost:${port} to minikube cluster API server"
+
+  # Forward the API server port through minikube
+  set -x
+  # shellcheck disable=SC2029
+  ssh -L "${port}:$(ssh -T -q "${host_url}" "export MINIKUBE_PROFILE=${profile}; minikube ip --profile=${profile}"):${port}" "${host_url}" "$@"
+  set +x
+}
+
+retry_with_sleep() {
+  shift 1
+  cmd=$1
+  local sleep_time
+  sleep_time=5
+
+  while true; do
+    ${cmd} || true
+    echo "Retrying command after ${sleep_time} of sleep: ${cmd}"
+    sleep 5;
+  done
+}
+
+ssh_to_host() {
+  shift 1
+  # shellcheck disable=SC2029
+  ssh "$@" "${host_url}"
+}
+
+upload-my-ssh-private-key() {
+    ssh -T -q "${host_url}" "mkdir -p ~/.ssh"
+    scp "${HOME}/.ssh/id_rsa" "${host_url}:~/.ssh/id_rsa"
+    scp "${HOME}/.ssh/id_rsa.pub" "${host_url}:~/.ssh/id_rsa.pub"
+    ssh -T -q "${host_url}" "chmod 700 ~/.ssh && chown -R \$(whoami):\$(whoami) ~/.ssh"
+}
+
+cmd() {
+  if [[ "$1" == "cmd" ]]; then
+    shift 1
+  fi
+
+  cmd="cd ~/mongodb-kubernetes; $*"
+  ssh -T -q "${host_url}" "${cmd}"
+}
+
+usage() {
+  echo "USAGE:
+  minikube_host.sh <command>
+
+PREREQUISITES:
+  - s390x host with SSH access
+  - define S390_HOST_NAME env var (e.g., export S390_HOST_NAME=user@hostname)
+  - SSH key-based authentication configured
+
+COMMANDS:
+  configure <architecture>           installs on a host: calls sync, switches context, installs necessary software (auto-detects arch)
+  sync                              rsync of project directory
+  recreate-minikube-cluster <profile>  recreates minikube cluster with specific profile and executes get-kubeconfig
+  remote-prepare-local-e2e-run       executes prepare-local-e2e on the remote host
+  get-kubeconfig                     copies remote minikube kubeconfig locally to ~/.operator-dev/s390-host.kubeconfig
+  tunnel [args]                      creates ssh session with tunneling to all API servers
+  ssh [args]                         creates ssh session passing optional arguments to ssh
+  cmd [command with args]            execute command as if being on s390x host
+  upload-my-ssh-private-key          uploads your ssh keys (~/.ssh/id_rsa) to s390x host
+  help                               this message
+
+EXAMPLES:
+  export S390_HOST_NAME=user@ibmz8
+  minikube_host.sh tunnel
+  minikube_host.sh cmd 'make e2e test=replica_set'
+"
+}
+
+case ${cmd} in
+configure) configure "$@" ;;
+recreate-minikube-cluster) recreate-minikube-cluster "$@" ;;
+get-kubeconfig) get-kubeconfig ;;
+remote-prepare-local-e2e-run) remote-prepare-local-e2e-run ;;
+ssh) ssh_to_host "$@" ;;
+tunnel) retry_with_sleep tunnel "$@" ;;
+sync) sync ;;
+cmd) cmd "$@" ;;
+upload-my-ssh-private-key) upload-my-ssh-private-key ;;
+help) usage ;;
+*) usage ;;
+esac
diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh
new file mode 100755
index 000000000..7628c388c
--- /dev/null
+++ b/scripts/minikube/setup_minikube_host.sh
@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+# this script downloads necessary tooling for alternative architectures (s390x, ppc64le) using minikube (similar to setup_evg_host.sh)
+
+set -Eeou pipefail
+
+check_disk_space() {
+  echo "Checking available disk space..."
+  local available_gb
+  available_gb=$(df / | awk 'NR==2 {print int($4/1024/1024)}')
+
+  if [[ $available_gb -lt 5 ]]; then
+    echo "ERROR: Insufficient disk space. Available: ${available_gb}GB, Required: 5GB minimum"
+    echo "Please clean up disk space before continuing:"
+    echo "  sudo dnf clean all"
+    echo "  sudo rm -rf /var/cache/dnf/* /tmp/* /var/tmp/*"
+    echo "  docker system prune -af"
+    return 1
+  fi
+
+  echo "Disk space check passed: ${available_gb}GB available"
+}
+
+set_limits() {
+  echo "Increasing fs.inotify.max_user_instances"
+  sudo sysctl -w fs.inotify.max_user_instances=8192
+
+  echo "Increasing fs.inotify.max_user_watches"
+  sudo sysctl -w fs.inotify.max_user_watches=10485760
+
+  echo "Increasing the number of open files"
+  nofile_max=$(cat /proc/sys/fs/nr_open)
+  nproc_max=$(ulimit -u)
+  sudo tee -a /etc/security/limits.conf <<EOF
+  root hard nofile ${nofile_max}
+  root hard nproc ${nproc_max}
+  root soft nofile ${nofile_max}
+  root soft nproc ${nproc_max}
+  * hard nofile ${nofile_max}
+  * hard nproc ${nproc_max}
+  * soft nofile ${nofile_max}
+  * soft nproc ${nproc_max}
+EOF
+}
+
+# retrieve arch variable off the shell command line
+ARCH=${1-"$(uname -m)"}
+
+# Validate architecture
+case "${ARCH}" in
+  s390x|ppc64le|x86_64|aarch64)
+    echo "Setting up minikube host for architecture: ${ARCH}"
+    ;;
+  *)
+    echo "ERROR: Unsupported architecture: ${ARCH}"
+    echo "Supported architectures: s390x, ppc64le, x86_64, aarch64"
+    exit 1
+    ;;
+esac
+
+download_minikube() {
+  echo "Downloading minikube for ${ARCH}..."
+  scripts/minikube/install-minikube.sh
+}
+
+download_docker() {
+  echo "Installing Docker for ${ARCH}..."
+  scripts/minikube/install-docker.sh --user "$(whoami)"
+}
+
+# Setup group for minikube based on architecture
+setup_group() {
+  echo "Setting up group configuration for ${ARCH}..."
+
+  # Set SETUP_GROUP evergreen variable based on architecture
+  case "${ARCH}" in
+    s390x)
+      export SETUP_GROUP="s390x_minikube"
+      ;;
+    ppc64le)
+      export SETUP_GROUP="ppc64le_minikube"
+      ;;
+    x86_64)
+      export SETUP_GROUP="x86_64_minikube"
+      ;;
+    aarch64)
+      export SETUP_GROUP="arm64_minikube"
+      ;;
+    *)
+      export SETUP_GROUP="unknown_minikube"
+      ;;
+  esac
+
+  echo "SETUP_GROUP set to: ${SETUP_GROUP}"
+
+  # Add to bashrc for persistence
+  echo "export SETUP_GROUP=${SETUP_GROUP}" >> ~/.bashrc
+}
+
+check_disk_space
+set_limits
+setup_group
+
+download_minikube &
+download_docker &
+
+wait
+
+echo "Setting up minikube environment variables..."
+echo 'export KUBE_ENVIRONMENT_NAME=minikube' >> ~/.bashrc
+echo 'export MINIKUBE_PROFILE=${MINIKUBE_PROFILE:-mongodb-e2e}' >> ~/.bashrc
+echo 'export KUBECONFIG=$(minikube kubeconfig --profile=${MINIKUBE_PROFILE:-mongodb-e2e} 2>/dev/null || echo ~/.kube/config)' >> ~/.bashrc
+
+echo "Minikube host setup completed successfully for ${ARCH}!"
+echo "SETUP_GROUP: ${SETUP_GROUP}"

From 46f0998d2335fad9311734cbede5b14e6864415d Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Wed, 30 Jul 2025 13:52:17 +0200
Subject: [PATCH 05/23] make jq multiarch and all other scriupts as well

---
 .evergreen-functions.yml                 |  15 +--
 .evergreen.yml                           |   1 -
 scripts/dev/recreate_python_venv.sh      |  87 ++++++++++++++--
 scripts/evergreen/setup_aws.sh           | 125 +++++++++++++++++------
 scripts/evergreen/setup_jq.sh            |   4 +-
 scripts/evergreen/setup_minikube_host.sh |  66 ++++++++++++
 scripts/minikube/minikube_host.sh        |  12 +--
 7 files changed, 254 insertions(+), 56 deletions(-)
 create mode 100755 scripts/evergreen/setup_minikube_host.sh

diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml
index 3cd67d524..71837ab97 100644
--- a/.evergreen-functions.yml
+++ b/.evergreen-functions.yml
@@ -243,7 +243,7 @@ functions:
       working_dir: src/github.com/mongodb/mongodb-kubernetes
       add_to_path:
         - ${workdir}/bin
-      binary: scripts/dev/setup_minikube_host.sh
+      binary: scripts/minikube/setup_minikube_host.sh
 
   lint_repo:
     - command: subprocess.exec
@@ -271,13 +271,16 @@ functions:
     - *python_venv
 
   # This differs for normal evg_host as we require minikube instead of kind for
-  # IBM machines
+  # IBM machines and install aws cli via pip instead
   setup_building_host_minikube:
     - *switch_context
-    - *setup_aws
-    - *configure_docker_auth
-    - *setup_ibm_host
-    - *python_venv
+    - command: subprocess.exec
+      type: setup
+      params:
+        working_dir: src/github.com/mongodb/mongodb-kubernetes
+        add_to_path:
+          - ${workdir}/bin
+        command: scripts/evergreen/setup_minikube_host.sh
 
   prune_docker_resources:
     - command: subprocess.exec
diff --git a/.evergreen.yml b/.evergreen.yml
index 2d2e113ae..31655847c 100644
--- a/.evergreen.yml
+++ b/.evergreen.yml
@@ -81,7 +81,6 @@ variables:
     setup_group_can_fail_task: true
     setup_group:
       - func: clone
-      - func: download_kube_tools
       - func: setup_building_host_minikube
 
   - &setup_group_multi_cluster
diff --git a/scripts/dev/recreate_python_venv.sh b/scripts/dev/recreate_python_venv.sh
index fb1f9ab8f..a5388e40f 100755
--- a/scripts/dev/recreate_python_venv.sh
+++ b/scripts/dev/recreate_python_venv.sh
@@ -4,26 +4,101 @@
 
 set -Eeou pipefail
 
+# Parse command line arguments
+INSTALL_REQUIREMENTS=true
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --skip-requirements)
+            INSTALL_REQUIREMENTS=false
+            shift
+            ;;
+        -h|--help)
+            echo "Usage: $0 [--skip-requirements]"
+            echo "  --skip-requirements    Skip installing requirements.txt"
+            echo "  -h, --help            Show this help message"
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $1"
+            echo "Use -h or --help for usage information"
+            exit 1
+            ;;
+    esac
+done
+
 source scripts/dev/set_env_context.sh
 
+# Ensure Python 3.10 is available, install if needed
+ensure_python310() {
+    echo "Checking current Python version..." >&2
+    
+    # Check if current python is 3.10
+    if command -v python3 &> /dev/null; then
+        local version
+        if version=$(python3 --version 2>&1) && [[ "${version}" == *"Python 3.10"* ]]; then
+            echo "Found Python 3.10: ${version}" >&2
+            echo "python3"
+            return 0
+        else
+            echo "Current python3 version: ${version}" >&2
+        fi
+    fi
+    
+    # Try to install Python 3.10 using pyenv if available
+    if command -v pyenv &> /dev/null; then
+        echo "Python 3.10 not found. Attempting to install via pyenv..." >&2
+        
+        # Check if any 3.10 version is already installed
+        if pyenv versions --bare | grep -q "^3\.10\."; then
+            local installed_version
+            installed_version=$(pyenv versions --bare | grep "^3\.10\." | head -1)
+            echo "Found existing pyenv Python 3.10: ${installed_version}" >&2
+            pyenv global "${installed_version}"
+            echo "python3"
+            return 0
+        fi
+        
+        # Install latest Python 3.10
+        local latest_310
+        latest_310=$(pyenv install --list | grep -E "^[[:space:]]*3\.10\.[0-9]+$" | tail -1 | xargs)
+        if [[ -n "${latest_310}" ]]; then
+            echo "Installing Python ${latest_310} via pyenv..." >&2
+            if pyenv install "${latest_310}"; then
+                pyenv global "${latest_310}"
+                echo "python3"
+                return 0
+            fi
+        fi
+    fi
+    
+    echo "Error: No suitable Python 3.10 installation found and unable to install via pyenv." >&2
+    echo "Please ensure Python 3.10 is installed or pyenv is available." >&2
+    return 1
+}
+
 if [[ -d "${PROJECT_DIR}"/venv ]]; then
   echo "Removing venv..."
   cd "${PROJECT_DIR}"
   rm -rf "venv"
 fi
 
-# in our EVG hosts, python versions are always in /opt/python
-python_bin="/opt/python/${PYTHON_VERSION}/bin/python3"
-if [[ "$(uname)" == "Darwin" ]]; then
-  python_bin="python${PYTHON_VERSION}"
-fi
+# Ensure Python 3.10 is available
+python_bin=$(ensure_python310)
 
 echo "Using python from the following path: ${python_bin}"
 
 "${python_bin}" -m venv venv
 source venv/bin/activate
 pip install --upgrade pip
-pip install -r requirements.txt
+
+if [[ "${INSTALL_REQUIREMENTS}" == "true" ]]; then
+    echo "Installing requirements.txt..."
+    pip install -r requirements.txt
+else
+    echo "Skipping requirements.txt installation (--skip-requirements flag used)"
+fi
+
 echo "Python venv was recreated successfully."
 echo "Current python path: $(which python)"
 python --version
diff --git a/scripts/evergreen/setup_aws.sh b/scripts/evergreen/setup_aws.sh
index 072900639..5563a8a30 100755
--- a/scripts/evergreen/setup_aws.sh
+++ b/scripts/evergreen/setup_aws.sh
@@ -3,51 +3,116 @@ set -Eeou pipefail
 
 source scripts/dev/set_env_context.sh
 
-# Detect system architecture and map to AWS CLI architecture names
-detect_aws_architecture() {
+# Detect system architecture
+detect_architecture() {
     local arch
     arch=$(uname -m)
+    echo "Detected architecture: ${arch}" >&2
+    echo "${arch}"
+}
+
+# Install AWS CLI v2 via binary download (for x86_64 and aarch64)
+install_aws_cli_binary() {
+    local arch="$1"
+    echo "Installing AWS CLI v2 via binary download for ${arch}..."
 
+    # Map architecture names for AWS CLI download URLs
+    local aws_arch
     case "${arch}" in
         x86_64)
-            echo "x86_64"
+            aws_arch="x86_64"
             ;;
         aarch64|arm64)
-            echo "aarch64"
-            ;;
-        ppc64le)
-            echo "Skipping AWS CLI installation: ppc64le (IBM Power) architecture is not supported by AWS CLI v2." >&2
-            echo "AWS CLI v2 only supports: x86_64 (amd64), aarch64 (arm64)" >&2
-            exit 0
-            ;;
-        s390x)
-            echo "Skipping AWS CLI installation: s390x (IBM Z) architecture is not supported by AWS CLI v2." >&2
-            echo "AWS CLI v2 only supports: x86_64 (amd64), aarch64 (arm64)" >&2
-            exit 0
+            aws_arch="aarch64"
             ;;
         *)
-            echo "Skipping AWS CLI installation: Unsupported architecture: ${arch}" >&2
-            echo "AWS CLI v2 only supports: x86_64 (amd64), aarch64 (arm64)" >&2
-            exit 0
+            echo "Error: Unsupported architecture for binary installation: ${arch}" >&2
+            return 1
             ;;
     esac
+
+    # Download and install AWS CLI v2
+    local temp_dir
+    temp_dir=$(mktemp -d)
+    cd "${temp_dir}"
+
+    echo "Downloading AWS CLI v2 for ${aws_arch}..."
+    curl -s "https://awscli.amazonaws.com/awscli-exe-linux-${aws_arch}.zip" -o "awscliv2.zip"
+    
+    unzip -q awscliv2.zip
+    sudo ./aws/install --update
+
+    # Clean up
+    cd - > /dev/null
+    rm -rf "${temp_dir}"
+
+    # Verify installation
+    if command -v aws &> /dev/null; then
+        echo "AWS CLI v2 installed successfully:"
+        aws --version
+    else
+        echo "Error: AWS CLI v2 installation failed" >&2
+        return 1
+    fi
 }
 
-# Detect the current architecture
-ARCH=$(detect_aws_architecture)
-echo "Detected architecture: ${ARCH} (AWS CLI v2 supported)"
+# Install AWS CLI v1 via pip (for IBM architectures: ppc64le, s390x)
+install_aws_cli_pip() {
+    echo "Installing AWS CLI v1 via pip (for IBM architectures)..."
+
+    # Ensure pip is available
+    if ! command -v pip3 &> /dev/null && ! command -v pip &> /dev/null; then
+        echo "Error: pip is not available. Please install Python and pip first." >&2
+        return 1
+    fi
 
-INSTALL_DIR="${workdir:?}/.local/lib/aws"
-BIN_LOCATION="${workdir}/bin"
+    # Use pip3 if available, otherwise pip
+    local pip_cmd="pip3"
+    if ! command -v pip3 &> /dev/null; then
+        pip_cmd="pip"
+    fi
 
-mkdir -p "${BIN_LOCATION}"
+    echo "Installing AWS CLI using ${pip_cmd}..."
+    ${pip_cmd} install --user awscli
 
-tmpdir=$(mktemp -d)
-cd "${tmpdir}"
+    # Add ~/.local/bin to PATH if not already there (where pip --user installs)
+    if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then
+        export PATH="$HOME/.local/bin:$PATH"
+        echo "Added ~/.local/bin to PATH"
+    fi
+
+    # Verify installation
+    if command -v aws &> /dev/null; then
+        echo "AWS CLI v1 installed successfully:"
+        aws --version
+    else
+        echo "Error: AWS CLI v1 installation failed or not found in PATH" >&2
+        return 1
+    fi
+}
+
+# Main installation logic
+install_aws_cli() {
+    local arch
+    arch=$(detect_architecture)
+
+    case "${arch}" in
+        ppc64le|s390x)
+            echo "IBM architecture detected (${arch}). Using pip installation..."
+            install_aws_cli_pip
+            ;;
+        x86_64|aarch64|arm64)
+            echo "Standard architecture detected (${arch}). Using binary installation..."
+            install_aws_cli_binary "${arch}"
+            ;;
+        *)
+            echo "Warning: Unknown architecture ${arch}. Falling back to pip installation..."
+            install_aws_cli_pip
+            ;;
+    esac
+}
 
-echo "Downloading AWS CLI v2 for ${ARCH}..."
-curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip"
-unzip awscliv2.zip &> /dev/null
+install_aws_cli
 
 docker_dir="/home/${USER}/.docker"
 if [[ ! -d "${docker_dir}" ]]; then
@@ -56,7 +121,5 @@ fi
 
 sudo chown "${USER}":"${USER}" "${docker_dir}" -R
 sudo chmod g+rwx "${docker_dir}" -R
-sudo ./aws/install --bin-dir "${BIN_LOCATION}" --install-dir "${INSTALL_DIR}" --update
-cd -
 
-rm -rf "${tmpdir}"
+echo "AWS CLI setup completed successfully."
diff --git a/scripts/evergreen/setup_jq.sh b/scripts/evergreen/setup_jq.sh
index e21d4a07e..23ad3e723 100755
--- a/scripts/evergreen/setup_jq.sh
+++ b/scripts/evergreen/setup_jq.sh
@@ -10,4 +10,6 @@ set -Eeou pipefail
 source scripts/dev/set_env_context.sh
 source scripts/funcs/install
 
-download_and_install_binary "${PROJECT_DIR:-.}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64"
+arch=$(uname -m)
+
+download_and_install_binary "${PROJECT_DIR:-.}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.8.1/jq-linux-${arch}"
diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh
new file mode 100755
index 000000000..eb2e7906c
--- /dev/null
+++ b/scripts/evergreen/setup_minikube_host.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+# Consolidated setup script for minikube host with multi-architecture support
+# This script groups all the setup steps needed for IBM machines and other architectures
+# Can be run on static hosts for testing and verification
+
+source scripts/dev/set_env_context.sh
+set -Eeou pipefail
+
+echo "=========================================="
+echo "Setting up minikube host with multi-architecture support"
+echo "Architecture: $(uname -m)"
+echo "OS: $(uname -s)"
+echo "=========================================="
+
+# Function to run a setup step with error handling and logging
+run_setup_step() {
+    local step_name="$1"
+    shift
+    local script_command=("$@")
+
+    echo ""
+    echo ">>> Running: ${step_name}"
+    echo ">>> Command: ${script_command[*]}"
+
+    local script_path="${script_command[0]}"
+    if [[ -f "${script_path}" ]]; then
+        if "${script_command[@]}"; then
+            echo "✅ ${step_name} completed successfully"
+        else
+            echo "❌ ${step_name} failed"
+            exit 1
+        fi
+    else
+        echo "❌ Script not found: ${script_path}"
+        exit 1
+    fi
+}
+
+# Setup Python environment (needed for AWS CLI pip installation)
+run_setup_step "Python Virtual Environment" "scripts/dev/recreate_python_venv.sh" "--skip-requirements"
+
+run_setup_step "AWS CLI Setup" "scripts/evergreen/setup_aws.sh"
+
+run_setup_step "kubectl and helm Setup" "scripts/evergreen/setup_kubectl.sh"
+
+run_setup_step "jq Setup" "scripts/evergreen/setup_jq.sh"
+
+run_setup_step "IBM Host Setup" "scripts/minikube/setup_minikube_host.sh"
+
+run_setup_step "Docker Authentication" "scripts/dev/configure_docker_auth.sh"
+
+echo ""
+echo "=========================================="
+echo "✅ Minikube host setup completed successfully!"
+echo "=========================================="
+echo ""
+echo "Installed tools summary:"
+echo "- Python: $(python --version 2>/dev/null || python3 --version 2>/dev/null || echo 'Not found')"
+echo "- AWS CLI: $(aws --version 2>/dev/null || echo 'Not found')"
+echo "- kubectl: $(kubectl version --client 2>/dev/null || echo 'Not found')"
+echo "- helm: $(helm version --short 2>/dev/null || echo 'Not found')"
+echo "- jq: $(jq --version 2>/dev/null || echo 'Not found')"
+echo "- Docker: $(docker --version 2>/dev/null || echo 'Not found')"
+echo ""
+echo "Setup complete! Host is ready for minikube operations."
diff --git a/scripts/minikube/minikube_host.sh b/scripts/minikube/minikube_host.sh
index ab410d7bc..c383ac09c 100755
--- a/scripts/minikube/minikube_host.sh
+++ b/scripts/minikube/minikube_host.sh
@@ -32,16 +32,6 @@ fi
 kubeconfig_path="${HOME}/.operator-dev/s390-host.kubeconfig"
 
 configure() {
-  shift 1
-  arch=${1-"$(uname -m)"}
-
-  echo "Configuring minikube host ${S390_HOST_NAME} (${host_url}) with architecture ${arch}"
-
-  if [[ "${cmd}" == "configure" && ! "${arch}" =~ ^(s390x|ppc64le|x86_64|aarch64)$ ]]; then
-    echo "'configure' command supports the following architectures: s390x, ppc64le, x86_64, aarch64"
-    exit 1
-  fi
-
   ssh -T -q "${host_url}" "sudo chown \$(whoami):\$(whoami) ~/.docker || true; mkdir -p ~/.docker"
   if [[ -f "${HOME}/.docker/config.json" ]]; then
     echo "Copying local ~/.docker/config.json authorization credentials to s390x host"
@@ -50,7 +40,7 @@ configure() {
 
   sync
 
-  ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; scripts/dev/switch_context.sh root-context; scripts/minikube/setup_minikube_host.sh ${arch}"
+  ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; scripts/dev/switch_context.sh root-context; scripts/minikube/setup_minikube_host.sh "
 }
 
 sync() {

From 433cdc17eaa8eb06ac72d150a6950fe29f528c15 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Wed, 30 Jul 2025 14:02:49 +0200
Subject: [PATCH 06/23] fix jq

---
 scripts/evergreen/setup_jq.sh | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/scripts/evergreen/setup_jq.sh b/scripts/evergreen/setup_jq.sh
index 23ad3e723..1f260883a 100755
--- a/scripts/evergreen/setup_jq.sh
+++ b/scripts/evergreen/setup_jq.sh
@@ -10,6 +10,32 @@ set -Eeou pipefail
 source scripts/dev/set_env_context.sh
 source scripts/funcs/install
 
-arch=$(uname -m)
+# Detect and map architecture for jq releases
+detect_jq_architecture() {
+    local arch
+    arch=$(uname -m)
+    
+    case "${arch}" in
+        x86_64)
+            echo "amd64"
+            ;;
+        aarch64|arm64)
+            echo "arm64"
+            ;;
+        ppc64le)
+            echo "ppc64el"  # jq uses ppc64el instead of ppc64le
+            ;;
+        s390x)
+            echo "s390x"
+            ;;
+        *)
+            echo "Error: Unsupported architecture for jq: ${arch}" >&2
+            exit 1
+            ;;
+    esac
+}
 
-download_and_install_binary "${PROJECT_DIR:-.}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.8.1/jq-linux-${arch}"
+jq_arch=$(detect_jq_architecture)
+echo "Detected architecture: $(uname -m), using jq architecture: ${jq_arch}"
+
+download_and_install_binary "${PROJECT_DIR:-.}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.8.1/jq-linux-${jq_arch}"

From 161252561c1f457b636d2dc8827f065fc84f0ba5 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Wed, 30 Jul 2025 15:35:54 +0200
Subject: [PATCH 07/23] fix docker, minikube etc

---
 .evergreen.yml                           |   8 +-
 scripts/evergreen/e2e/e2e.sh             |   2 +-
 scripts/evergreen/setup_minikube_host.sh |  18 ++
 scripts/minikube/install-docker.sh       |  58 ++-----
 scripts/minikube/install-minikube.sh     | 200 ++++-------------------
 scripts/minikube/setup_minikube_host.sh  |  67 ++++----
 6 files changed, 106 insertions(+), 247 deletions(-)

diff --git a/.evergreen.yml b/.evergreen.yml
index 31655847c..98eaf8a75 100644
--- a/.evergreen.yml
+++ b/.evergreen.yml
@@ -1197,7 +1197,13 @@ task_groups:
   - name: e2e_smoke_ibm_task_group
     max_hosts: -1
     <<: *setup_group_ibm
-    <<: *setup_and_teardown_task
+    setup_task_can_fail_task: true
+    setup_task:
+      - func: cleanup_exec_environment
+    teardown_task_can_fail_task: true
+    teardown_task:
+      - func: upload_e2e_logs
+      - func: teardown_kubernetes_environment
     tasks:
       - e2e_replica_set
     <<: *teardown_group
diff --git a/scripts/evergreen/e2e/e2e.sh b/scripts/evergreen/e2e/e2e.sh
index 12f839b86..eb6140a22 100755
--- a/scripts/evergreen/e2e/e2e.sh
+++ b/scripts/evergreen/e2e/e2e.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -Eeou pipefail
+set -Eeoux pipefail
 
 start_time=$(date +%s)
 
diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh
index eb2e7906c..c96cfe9c5 100755
--- a/scripts/evergreen/setup_minikube_host.sh
+++ b/scripts/evergreen/setup_minikube_host.sh
@@ -50,6 +50,24 @@ run_setup_step "IBM Host Setup" "scripts/minikube/setup_minikube_host.sh"
 
 run_setup_step "Docker Authentication" "scripts/dev/configure_docker_auth.sh"
 
+# Setup Kubernetes cluster after Docker is properly configured
+echo ""
+echo ">>> Setting up Kubernetes cluster"
+if [[ "${KUBE_ENVIRONMENT_NAME:-}" == "kind" ]]; then
+    run_setup_step "Kind Kubernetes Cluster" "scripts/dev/recreate_kind_cluster.sh" "kind"
+elif [[ "${KUBE_ENVIRONMENT_NAME:-}" == "minikube" ]]; then
+    echo ">>> Running: Minikube Kubernetes Cluster"
+    echo ">>> Command: minikube start --profile=${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4"
+    if minikube start --profile="${MINIKUBE_PROFILE:-mongodb-e2e}" --driver=docker --memory=8192mb --cpus=4; then
+        echo "✅ Minikube Kubernetes Cluster completed successfully"
+    else
+        echo "❌ Minikube Kubernetes Cluster failed"
+        exit 1
+    fi
+else
+    echo "⚠️  No Kubernetes environment specified (KUBE_ENVIRONMENT_NAME not set)"
+fi
+
 echo ""
 echo "=========================================="
 echo "✅ Minikube host setup completed successfully!"
diff --git a/scripts/minikube/install-docker.sh b/scripts/minikube/install-docker.sh
index 04ae3f7d8..164d94c3a 100755
--- a/scripts/minikube/install-docker.sh
+++ b/scripts/minikube/install-docker.sh
@@ -1,25 +1,10 @@
 #!/usr/bin/env bash
 set -Eeou pipefail
 
-# Script to install Docker on s390x architecture (specifically for RHEL/Ubuntu based systems)
-
-print_usage() {
-    echo "Usage: $0 [options]"
-    echo "Options:"
-    echo "  -h, --help    Show this help message"
-    echo "  -u, --user    Username to add to docker group (optional)"
-    echo ""
-    echo "This script installs Docker on s390x architecture systems."
-}
-
 DOCKER_USER=""
 
 while [[ $# -gt 0 ]]; do
     case $1 in
-        -h|--help)
-            print_usage
-            exit 0
-            ;;
         -u|--user)
             DOCKER_USER="$2"
             shift 2
@@ -32,7 +17,7 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 
-echo "Installing Docker on s390x architecture..."
+echo "Installing Docker"
 
 # Detect OS
 if [[ -f /etc/redhat-release ]]; then
@@ -47,41 +32,31 @@ fi
 # Install Docker based on OS
 if [[ "$OS_TYPE" == "rhel" ]]; then
     echo "Detected RHEL/CentOS system..."
-    
-    # Remove any existing Docker packages
-    sudo yum remove -y docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-engine || true
-    
-    # Install required packages (some may not exist on newer RHEL versions)
-    sudo yum install -y yum-utils || echo "yum-utils already installed or unavailable"
-    sudo yum install -y device-mapper-persistent-data lvm2 || echo "device-mapper packages may not be available on this system"
-    
+
     # Add Docker repository
     sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
-    
+
     # Install Docker CE
     sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
-    
+
 elif [[ "$OS_TYPE" == "debian" ]]; then
     echo "Detected Ubuntu/Debian system..."
-    
-    # Remove any existing Docker packages
-    sudo apt-get remove -y docker docker-engine docker.io containerd runc || true
-    
+
     # Update package index
     sudo apt-get update
-    
+
     # Install required packages
     sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release
-    
+
     # Add Docker's official GPG key
     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-    
+
     # Set up stable repository
     echo "deb [arch=s390x signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-    
+
     # Update package index again
     sudo apt-get update
-    
+
     # Install Docker CE
     sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
 fi
@@ -100,9 +75,12 @@ fi
 # Verify installation
 echo "Verifying Docker installation..."
 sudo docker --version
-sudo docker run --rm hello-world
 
-echo "Docker installation completed successfully!"
-echo ""
-echo "If you added a user to the docker group, they need to log out and log back in."
-echo "You can also run 'newgrp docker' to apply the group membership in the current session."
\ No newline at end of file
+# Test docker access with newgrp (temporary group membership)
+echo "Testing Docker access..."
+if newgrp docker -c 'docker ps' >/dev/null 2>&1; then
+  echo "✅ Docker access confirmed"
+else
+  echo "⚠️  Docker group membership requires logout/login to take effect"
+  echo "Continuing with setup..."
+fi
diff --git a/scripts/minikube/install-minikube.sh b/scripts/minikube/install-minikube.sh
index 3e6dd4e13..903351960 100755
--- a/scripts/minikube/install-minikube.sh
+++ b/scripts/minikube/install-minikube.sh
@@ -1,73 +1,35 @@
 #!/usr/bin/env bash
 set -Eeou pipefail
 
-# Script to install and configure minikube on s390x architecture
-
-print_usage() {
-    echo "Usage: $0 [options]"
-    echo "Options:"
-    echo "  -h, --help              Show this help message"
-    echo "  -v, --version VERSION   Minikube version to install (default: latest)"
-    echo "  -k, --kubernetes VER    Kubernetes version (default: latest stable)"
-    echo "  -m, --memory MEMORY     Memory allocation (default: 8192mb)"
-    echo "  -c, --cpus CPUS         CPU allocation (default: 4)"
-    echo "  --profile PROFILE       Minikube profile name (default: minikube)"
-    echo "  --start                 Start minikube after installation"
-    echo ""
-    echo "This script installs minikube on s390x architecture and configures it for MongoDB Kubernetes e2e testing."
-}
-
-MINIKUBE_VERSION="latest"
-K8S_VERSION=""
-MEMORY="8192"
-CPUS="4"
-PROFILE="minikube"
-START_MINIKUBE="false"
-
-while [[ $# -gt 0 ]]; do
-    case $1 in
-        -h|--help)
-            print_usage
-            exit 0
-            ;;
-        -v|--version)
-            MINIKUBE_VERSION="$2"
-            shift 2
-            ;;
-        -k|--kubernetes)
-            K8S_VERSION="$2"
-            shift 2
-            ;;
-        -m|--memory)
-            MEMORY="$2"
-            shift 2
-            ;;
-        -c|--cpus)
-            CPUS="$2"
-            shift 2
-            ;;
-        --profile)
-            PROFILE="$2"
-            shift 2
-            ;;
-        --start)
-            START_MINIKUBE="true"
-            shift
-            ;;
-        *)
-            echo "Unknown option: $1"
-            print_usage
-            exit 1
-            ;;
-    esac
-done
-
-echo "Installing minikube on s390x architecture..."
+source scripts/dev/set_env_context.sh
+
+# Detect architecture
+ARCH=$(uname -m)
+case "${ARCH}" in
+    x86_64)
+        MINIKUBE_ARCH="amd64"
+        ;;
+    aarch64)
+        MINIKUBE_ARCH="arm64"
+        ;;
+    ppc64le)
+        MINIKUBE_ARCH="ppc64le"
+        ;;
+    s390x)
+        MINIKUBE_ARCH="s390x"
+        ;;
+    *)
+        echo "Error: Unsupported architecture: ${ARCH}"
+        echo "Supported architectures: x86_64, aarch64, ppc64le, s390x"
+        exit 1
+        ;;
+esac
+
+echo "Installing minikube on ${ARCH} architecture..."
 
 # Verify Docker is installed
 if ! command -v docker &> /dev/null; then
     echo "Error: Docker is required but not installed. Please install Docker first."
-    echo "You can use the install-docker.sh script in this directory."
     exit 1
 fi
 
@@ -77,114 +39,12 @@ if ! docker info &> /dev/null; then
     exit 1
 fi
 
-# Install kubectl if not present
-if ! command -v kubectl &> /dev/null; then
-    echo "Installing kubectl..."
-    
-    # Get the latest kubectl version if not specified
-    if [[ -z "$K8S_VERSION" ]]; then
-        K8S_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt)
-    fi
-    
-    # Download kubectl for s390x
-    curl -LO "https://dl.k8s.io/release/${K8S_VERSION}/bin/linux/s390x/kubectl"
-    
-    # Verify the binary
-    curl -LO "https://dl.k8s.io/${K8S_VERSION}/bin/linux/s390x/kubectl.sha256"
-    echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check
-    
-    # Install kubectl
-    sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
-    rm -f kubectl kubectl.sha256
-    
-    echo "kubectl installed successfully"
-fi
-
 # Install minikube
-echo "Installing minikube..."
-
-if [[ "$MINIKUBE_VERSION" == "latest" ]]; then
-    # Get the latest minikube version
-    MINIKUBE_VERSION=$(curl -s https://api.github.com/repos/kubernetes/minikube/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
-fi
-
-# Download minikube for s390x
-curl -Lo minikube "https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-s390x"
+echo "Installing minikube for ${ARCH}..."
 
-# Make it executable and install
-chmod +x minikube
-sudo install minikube /usr/local/bin/
+MINIKUBE_VERSION=$(curl -s https://api.github.com/repos/kubernetes/minikube/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
 
-# Clean up
-rm -f minikube
-
-echo "Minikube ${MINIKUBE_VERSION} installed successfully"
-
-# Configure minikube for MongoDB Kubernetes testing
-echo "Configuring minikube for MongoDB Kubernetes e2e testing..."
-
-# Set default driver to docker
-minikube config set driver docker
-
-# Configure resource limits
-minikube config set memory "${MEMORY}mb"
-minikube config set cpus "${CPUS}"
-
-# Enable required addons for testing
-ADDONS=(
-    "storage-provisioner"
-    "default-storageclass"
-    "volumesnapshots"
-    "csi-hostpath-driver"
-)
-
-echo "Minikube configuration completed."
-
-if [[ "$START_MINIKUBE" == "true" ]]; then
-    echo "Starting minikube cluster with profile '${PROFILE}'..."
-    
-    # Start minikube with specific configuration for MongoDB testing
-    minikube start \
-        --profile="${PROFILE}" \
-        --driver=docker \
-        --memory="${MEMORY}mb" \
-        --cpus="${CPUS}" \
-        --disk-size=50g \
-        --extra-config=kubelet.authentication-token-webhook=true \
-        --extra-config=kubelet.authorization-mode=Webhook \
-        --extra-config=scheduler.bind-address=0.0.0.0 \
-        --extra-config=controller-manager.bind-address=0.0.0.0 \
-        ${K8S_VERSION:+--kubernetes-version=$K8S_VERSION}
-    
-    # Wait for cluster to be ready
-    echo "Waiting for cluster to be ready..."
-    kubectl wait --for=condition=Ready nodes --all --timeout=300s
-    
-    # Enable addons
-    for addon in "${ADDONS[@]}"; do
-        echo "Enabling addon: $addon"
-        minikube addons enable "$addon" --profile="${PROFILE}" || true
-    done
-    
-    # Create directories that MongoDB tests expect (similar to kind setup)
-    echo "Setting up test directories..."
-    minikube ssh --profile="${PROFILE}" -- 'sudo mkdir -p /opt/data/mongo-data-{0..2} /opt/data/mongo-logs-{0..2}'
-    minikube ssh --profile="${PROFILE}" -- 'sudo chmod 777 /opt/data/mongo-data-* /opt/data/mongo-logs-*'
-    
-    echo "Minikube cluster started successfully!"
-    echo ""
-    echo "To use this cluster:"
-    echo "  export KUBECONFIG=\$(minikube kubeconfig --profile=${PROFILE})"
-    echo "  kubectl get nodes"
-    echo ""
-    echo "To stop the cluster:"
-    echo "  minikube stop --profile=${PROFILE}"
-else
-    echo ""
-    echo "Minikube installed but not started."
-    echo "To start minikube later, run:"
-    echo "  minikube start --profile=${PROFILE} --driver=docker --memory=${MEMORY}mb --cpus=${CPUS}"
-fi
+# Download minikube for detected architecture
+download_and_install_binary "${PROJECT_DIR:-.}/bin" minikube "https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-${MINIKUBE_ARCH}"
 
-echo ""
-echo "Installation completed successfully!"
\ No newline at end of file
+echo "Minikube ${MINIKUBE_VERSION} installed successfully for ${ARCH}"
diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh
index 7628c388c..a8c4cf70c 100755
--- a/scripts/minikube/setup_minikube_host.sh
+++ b/scripts/minikube/setup_minikube_host.sh
@@ -65,51 +65,48 @@ download_minikube() {
 
 download_docker() {
   echo "Installing Docker for ${ARCH}..."
+
   scripts/minikube/install-docker.sh --user "$(whoami)"
 }
 
-# Setup group for minikube based on architecture
-setup_group() {
-  echo "Setting up group configuration for ${ARCH}..."
-
-  # Set SETUP_GROUP evergreen variable based on architecture
-  case "${ARCH}" in
-    s390x)
-      export SETUP_GROUP="s390x_minikube"
-      ;;
-    ppc64le)
-      export SETUP_GROUP="ppc64le_minikube"
-      ;;
-    x86_64)
-      export SETUP_GROUP="x86_64_minikube"
-      ;;
-    aarch64)
-      export SETUP_GROUP="arm64_minikube"
-      ;;
-    *)
-      export SETUP_GROUP="unknown_minikube"
-      ;;
-  esac
-
-  echo "SETUP_GROUP set to: ${SETUP_GROUP}"
-
-  # Add to bashrc for persistence
-  echo "export SETUP_GROUP=${SETUP_GROUP}" >> ~/.bashrc
+start_minikube() {
+  echo "Starting minikube cluster..."
+  local profile=${MINIKUBE_PROFILE:-mongodb-e2e}
+
+  if minikube start --profile="${profile}" --driver=docker --memory=8192mb --cpus=4; then
+    echo "✅ Minikube cluster started successfully"
+
+    # Test cluster connectivity
+    if kubectl --kubeconfig="$(minikube kubeconfig --profile="${profile}")" get nodes >/dev/null 2>&1; then
+      echo "✅ Cluster connectivity verified"
+    else
+      echo "⚠️  Cluster connectivity test failed - may need manual intervention"
+    fi
+  else
+    echo "⚠️  Minikube start failed - likely due to docker permissions"
+    echo "This will be resolved after logout/login"
+  fi
 }
 
 check_disk_space
 set_limits
-setup_group
-
-download_minikube &
 download_docker &
+download_minikube &
 
 wait
 
-echo "Setting up minikube environment variables..."
-echo 'export KUBE_ENVIRONMENT_NAME=minikube' >> ~/.bashrc
-echo 'export MINIKUBE_PROFILE=${MINIKUBE_PROFILE:-mongodb-e2e}' >> ~/.bashrc
-echo 'export KUBECONFIG=$(minikube kubeconfig --profile=${MINIKUBE_PROFILE:-mongodb-e2e} 2>/dev/null || echo ~/.kube/config)' >> ~/.bashrc
+# Start minikube cluster
+start_minikube
 
 echo "Minikube host setup completed successfully for ${ARCH}!"
-echo "SETUP_GROUP: ${SETUP_GROUP}"
+
+# Final status
+echo ""
+echo "=========================================="
+echo "✅ Setup Summary"
+echo "=========================================="
+echo "Architecture: ${ARCH}"
+echo "Minikube Profile: ${MINIKUBE_PROFILE:-mongodb-e2e}"
+echo ""
+echo "If docker permissions failed, logout and login, then run:"
+echo "  minikube start --profile=\${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4"

From bf8e64ae74eacb2a6721418cd9fb63de64cc1a03 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Wed, 30 Jul 2025 15:57:20 +0200
Subject: [PATCH 08/23] remove docker group management

---
 scripts/evergreen/setup_minikube_host.sh | 18 ++++++---------
 scripts/minikube/install-docker.sh       | 11 ++++++---
 scripts/minikube/setup_minikube_host.sh  | 29 ++----------------------
 3 files changed, 17 insertions(+), 41 deletions(-)

diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh
index c96cfe9c5..36aa423ee 100755
--- a/scripts/evergreen/setup_minikube_host.sh
+++ b/scripts/evergreen/setup_minikube_host.sh
@@ -53,17 +53,13 @@ run_setup_step "Docker Authentication" "scripts/dev/configure_docker_auth.sh"
 # Setup Kubernetes cluster after Docker is properly configured
 echo ""
 echo ">>> Setting up Kubernetes cluster"
-if [[ "${KUBE_ENVIRONMENT_NAME:-}" == "kind" ]]; then
-    run_setup_step "Kind Kubernetes Cluster" "scripts/dev/recreate_kind_cluster.sh" "kind"
-elif [[ "${KUBE_ENVIRONMENT_NAME:-}" == "minikube" ]]; then
-    echo ">>> Running: Minikube Kubernetes Cluster"
-    echo ">>> Command: minikube start --profile=${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4"
-    if minikube start --profile="${MINIKUBE_PROFILE:-mongodb-e2e}" --driver=docker --memory=8192mb --cpus=4; then
-        echo "✅ Minikube Kubernetes Cluster completed successfully"
-    else
-        echo "❌ Minikube Kubernetes Cluster failed"
-        exit 1
-    fi
+echo ">>> Command: minikube start --profile=${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4"
+if minikube start --profile="${MINIKUBE_PROFILE:-mongodb-e2e}" --driver=docker --memory=8192mb --cpus=4; then
+    echo "✅ Minikube Kubernetes Cluster completed successfully"
+else
+    echo "❌ Minikube Kubernetes Cluster failed"
+    exit 1
+fi
 else
     echo "⚠️  No Kubernetes environment specified (KUBE_ENVIRONMENT_NAME not set)"
 fi
diff --git a/scripts/minikube/install-docker.sh b/scripts/minikube/install-docker.sh
index 164d94c3a..f645a6a97 100755
--- a/scripts/minikube/install-docker.sh
+++ b/scripts/minikube/install-docker.sh
@@ -76,11 +76,16 @@ fi
 echo "Verifying Docker installation..."
 sudo docker --version
 
-# Test docker access with newgrp (temporary group membership)
+# Test docker access
 echo "Testing Docker access..."
-if newgrp docker -c 'docker ps' >/dev/null 2>&1; then
+if docker ps >/dev/null 2>&1; then
   echo "✅ Docker access confirmed"
 else
-  echo "⚠️  Docker group membership requires logout/login to take effect"
+  echo "⚠️  Docker access test failed - checking if running as root..."
+  if [[ $(id -u) -eq 0 ]]; then
+    echo "Running as root - Docker should work"
+  else
+    echo "Docker group membership may require logout/login to take effect"
+  fi
   echo "Continuing with setup..."
 fi
diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh
index a8c4cf70c..bd2ac72bf 100755
--- a/scripts/minikube/setup_minikube_host.sh
+++ b/scripts/minikube/setup_minikube_host.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 # this script downloads necessary tooling for alternative architectures (s390x, ppc64le) using minikube (similar to setup_evg_host.sh)
+source scripts/dev/set_env_context.sh
 
 set -Eeou pipefail
 
@@ -65,27 +66,7 @@ download_minikube() {
 
 download_docker() {
   echo "Installing Docker for ${ARCH}..."
-
-  scripts/minikube/install-docker.sh --user "$(whoami)"
-}
-
-start_minikube() {
-  echo "Starting minikube cluster..."
-  local profile=${MINIKUBE_PROFILE:-mongodb-e2e}
-
-  if minikube start --profile="${profile}" --driver=docker --memory=8192mb --cpus=4; then
-    echo "✅ Minikube cluster started successfully"
-
-    # Test cluster connectivity
-    if kubectl --kubeconfig="$(minikube kubeconfig --profile="${profile}")" get nodes >/dev/null 2>&1; then
-      echo "✅ Cluster connectivity verified"
-    else
-      echo "⚠️  Cluster connectivity test failed - may need manual intervention"
-    fi
-  else
-    echo "⚠️  Minikube start failed - likely due to docker permissions"
-    echo "This will be resolved after logout/login"
-  fi
+  scripts/minikube/install-docker.sh
 }
 
 check_disk_space
@@ -95,9 +76,6 @@ download_minikube &
 
 wait
 
-# Start minikube cluster
-start_minikube
-
 echo "Minikube host setup completed successfully for ${ARCH}!"
 
 # Final status
@@ -107,6 +85,3 @@ echo "✅ Setup Summary"
 echo "=========================================="
 echo "Architecture: ${ARCH}"
 echo "Minikube Profile: ${MINIKUBE_PROFILE:-mongodb-e2e}"
-echo ""
-echo "If docker permissions failed, logout and login, then run:"
-echo "  minikube start --profile=\${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4"

From 540b4200a160fea89a7837b3fbd19b3b9e6b6379 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Wed, 30 Jul 2025 18:07:25 +0200
Subject: [PATCH 09/23] installation works, minikube starts but docker doesn't
 start

---
 scripts/evergreen/setup_minikube_host.sh |  7 ++---
 scripts/minikube/install-docker.sh       | 40 +++++++++---------------
 scripts/minikube/install-minikube.sh     | 14 +--------
 scripts/minikube/setup_minikube_host.sh  | 32 +++++++++++++++++++
 4 files changed, 50 insertions(+), 43 deletions(-)

diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh
index 36aa423ee..ffc571d3b 100755
--- a/scripts/evergreen/setup_minikube_host.sh
+++ b/scripts/evergreen/setup_minikube_host.sh
@@ -5,7 +5,7 @@
 # Can be run on static hosts for testing and verification
 
 source scripts/dev/set_env_context.sh
-set -Eeou pipefail
+set -Eeoux pipefail
 
 echo "=========================================="
 echo "Setting up minikube host with multi-architecture support"
@@ -54,15 +54,14 @@ run_setup_step "Docker Authentication" "scripts/dev/configure_docker_auth.sh"
 echo ""
 echo ">>> Setting up Kubernetes cluster"
 echo ">>> Command: minikube start --profile=${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4"
+
+# Start minikube cluster for CI
 if minikube start --profile="${MINIKUBE_PROFILE:-mongodb-e2e}" --driver=docker --memory=8192mb --cpus=4; then
     echo "✅ Minikube Kubernetes Cluster completed successfully"
 else
     echo "❌ Minikube Kubernetes Cluster failed"
     exit 1
 fi
-else
-    echo "⚠️  No Kubernetes environment specified (KUBE_ENVIRONMENT_NAME not set)"
-fi
 
 echo ""
 echo "=========================================="
diff --git a/scripts/minikube/install-docker.sh b/scripts/minikube/install-docker.sh
index f645a6a97..770c75e5a 100755
--- a/scripts/minikube/install-docker.sh
+++ b/scripts/minikube/install-docker.sh
@@ -1,21 +1,7 @@
 #!/usr/bin/env bash
 set -Eeou pipefail
 
-DOCKER_USER=""
-
-while [[ $# -gt 0 ]]; do
-    case $1 in
-        -u|--user)
-            DOCKER_USER="$2"
-            shift 2
-            ;;
-        *)
-            echo "Unknown option: $1"
-            print_usage
-            exit 1
-            ;;
-    esac
-done
+source scripts/dev/set_env_context.sh
 
 echo "Installing Docker"
 
@@ -65,11 +51,15 @@ fi
 sudo systemctl start docker
 sudo systemctl enable docker
 
-# Add user to docker group if specified
-if [[ -n "$DOCKER_USER" ]]; then
-    echo "Adding user '$DOCKER_USER' to docker group..."
-    sudo usermod -aG docker "$DOCKER_USER"
-    echo "Note: User '$DOCKER_USER' needs to log out and log back in for group membership to take effect."
+# Add current CI user to docker group and change socket permissions
+current_user=$(whoami)
+if [[ "$current_user" != "root" ]]; then
+    echo "Adding CI user '$current_user' to docker group..."
+    sudo usermod -aG docker "$current_user"
+    
+    # For CI: Change docker socket permissions to allow immediate access
+    echo "Setting docker socket permissions for CI..."
+    sudo chmod 666 /var/run/docker.sock
 fi
 
 # Verify installation
@@ -81,11 +71,9 @@ echo "Testing Docker access..."
 if docker ps >/dev/null 2>&1; then
   echo "✅ Docker access confirmed"
 else
-  echo "⚠️  Docker access test failed - checking if running as root..."
-  if [[ $(id -u) -eq 0 ]]; then
-    echo "Running as root - Docker should work"
-  else
-    echo "Docker group membership may require logout/login to take effect"
+  echo "❌ Docker access failed - CI may not work properly"
+  echo "Trying with sudo..."
+  if sudo docker ps >/dev/null 2>&1; then
+    echo "⚠️  Docker only works with sudo"
   fi
-  echo "Continuing with setup..."
 fi
diff --git a/scripts/minikube/install-minikube.sh b/scripts/minikube/install-minikube.sh
index 903351960..b82c829b6 100755
--- a/scripts/minikube/install-minikube.sh
+++ b/scripts/minikube/install-minikube.sh
@@ -2,6 +2,7 @@
 set -Eeou pipefail
 
 source scripts/dev/set_env_context.sh
+source scripts/funcs/install
 
 # Detect architecture
 ARCH=$(uname -m)
@@ -26,19 +27,6 @@ case "${ARCH}" in
 esac
 
 echo "Installing minikube on ${ARCH} architecture..."
-
-# Verify Docker is installed
-if ! command -v docker &> /dev/null; then
-    echo "Error: Docker is required but not installed. Please install Docker first."
-    exit 1
-fi
-
-# Verify Docker is running
-if ! docker info &> /dev/null; then
-    echo "Error: Docker is not running. Please start Docker service."
-    exit 1
-fi
-
 # Install minikube
 echo "Installing minikube for ${ARCH}..."
 
diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh
index bd2ac72bf..a455fca57 100755
--- a/scripts/minikube/setup_minikube_host.sh
+++ b/scripts/minikube/setup_minikube_host.sh
@@ -76,6 +76,36 @@ download_minikube &
 
 wait
 
+echo ""
+echo ">>> Verifying minikube installation..."
+if command -v minikube &> /dev/null; then
+    minikube_version=$(minikube version --short 2>/dev/null || minikube version 2>/dev/null | head -n1)
+    echo "✅ Minikube installed successfully: ${minikube_version}"
+else
+    echo "❌ Minikube installation failed - minikube command not found"
+    echo "Please check the installation logs above for errors"
+    exit 1
+fi
+
+echo ""
+echo ">>> Verifying docker installation..."
+if command -v docker &> /dev/null; then
+    docker_version=$(docker --version 2>/dev/null)
+    echo "✅ Docker installed successfully: ${docker_version}"
+
+    # Check if docker service is running
+    if systemctl is-active --quiet docker 2>/dev/null || docker info &>/dev/null; then
+        echo "✅ Docker service is running"
+    else
+        echo "⚠️  Docker is installed but service may not be running"
+        echo "You may need to start docker service: sudo systemctl start docker"
+    fi
+else
+    echo "❌ Docker installation failed - docker command not found"
+    echo "Please check the installation logs above for errors"
+    exit 1
+fi
+
 echo "Minikube host setup completed successfully for ${ARCH}!"
 
 # Final status
@@ -85,3 +115,5 @@ echo "✅ Setup Summary"
 echo "=========================================="
 echo "Architecture: ${ARCH}"
 echo "Minikube Profile: ${MINIKUBE_PROFILE:-mongodb-e2e}"
+echo "Minikube: ${minikube_version}"
+echo "Docker: ${docker_version}"

From fd8db45fc696456e28e5a4f8011cf6be770b8548 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Thu, 31 Jul 2025 16:16:09 +0200
Subject: [PATCH 10/23] podman support

---
 scripts/dev/configure_docker_auth.sh     | 100 ++++++++---
 scripts/evergreen/setup_minikube_host.sh |  21 +--
 scripts/minikube/install-docker.sh       |  79 ---------
 scripts/minikube/install-minikube.sh     |  41 ++++-
 scripts/minikube/minikube_host.sh        |  22 +--
 scripts/minikube/setup_minikube_host.sh  | 212 ++++++++++++++++++-----
 6 files changed, 303 insertions(+), 172 deletions(-)
 delete mode 100755 scripts/minikube/install-docker.sh

diff --git a/scripts/dev/configure_docker_auth.sh b/scripts/dev/configure_docker_auth.sh
index dfcb14f0b..74e0d9abc 100755
--- a/scripts/dev/configure_docker_auth.sh
+++ b/scripts/dev/configure_docker_auth.sh
@@ -8,7 +8,33 @@ source scripts/funcs/checks
 source scripts/funcs/printing
 source scripts/funcs/kubernetes
 
+# Detect available container runtime
+detect_container_runtime() {
+  if command -v podman &> /dev/null && podman info &> /dev/null; then
+    CONTAINER_RUNTIME="podman"
+    CONFIG_PATH="${HOME}/.config/containers/auth.json"
+    mkdir -p "$(dirname "${CONFIG_PATH}")"
+    echo "Using Podman for container authentication"
+    return 0
+  elif command -v docker &> /dev/null; then
+    CONTAINER_RUNTIME="docker"
+    CONFIG_PATH="${HOME}/.docker/config.json"
+    mkdir -p "$(dirname "${CONFIG_PATH}")"
+    echo "Using Docker for container authentication"
+    return 0
+  else
+    echo "Error: Neither Docker nor Podman is available"
+    exit 1
+  fi
+}
+
 check_docker_daemon_is_running() {
+  if [[ "${CONTAINER_RUNTIME}" == "podman" ]]; then
+    # Podman doesn't require a daemon
+    echo "Using Podman (no daemon required)"
+    return 0
+  fi
+
   if [[ "$(uname -s)" != "Linux" ]]; then
     echo "Skipping docker daemon check when not running in Linux"
     return 0
@@ -34,71 +60,95 @@ check_docker_daemon_is_running() {
 remove_element() {
   config_option="${1}"
   tmpfile=$(mktemp)
-  jq 'del(.'"${config_option}"')' ~/.docker/config.json >"${tmpfile}"
-  cp "${tmpfile}" ~/.docker/config.json
+  
+  # Initialize config file if it doesn't exist
+  if [[ ! -f "${CONFIG_PATH}" ]]; then
+    echo '{}' > "${CONFIG_PATH}"
+  fi
+  
+  jq 'del(.'"${config_option}"')' "${CONFIG_PATH}" >"${tmpfile}"
+  cp "${tmpfile}" "${CONFIG_PATH}"
   rm "${tmpfile}"
 }
 
-# This is the script which performs docker authentication to different registries that we use (so far ECR and RedHat)
-# As the result of this login the ~/.docker/config.json will have all the 'auth' information necessary to work with docker registries
+# Container runtime login wrapper
+container_login() {
+  local username="$1"
+  local registry="$2"
+  
+  if [[ "${CONTAINER_RUNTIME}" == "podman" ]]; then
+    podman login --username "${username}" --password-stdin "${registry}"
+  else
+    docker login --username "${username}" --password-stdin "${registry}"
+  fi
+}
+
+# This is the script which performs container authentication to different registries that we use (so far ECR and RedHat)
+# As the result of this login the config file will have all the 'auth' information necessary to work with container registries
+
+# Detect container runtime and set appropriate config path
+detect_container_runtime
 
 check_docker_daemon_is_running
 
-if [[ -f ~/.docker/config.json ]]; then
+if [[ -f "${CONFIG_PATH}" ]]; then
   if [[ "${RUNNING_IN_EVG:-"false"}" != "true" ]]; then
-    # Check if login is actually required by making a HEAD request to ECR using existing Docker config
-    echo "Checking if Docker credentials are valid..."
-    ecr_auth=$(jq -r '.auths."268558157000.dkr.ecr.us-east-1.amazonaws.com".auth // empty' ~/.docker/config.json)
+    # Check if login is actually required by making a HEAD request to ECR using existing credentials
+    echo "Checking if container registry credentials are valid..."
+    ecr_auth=$(jq -r '.auths."268558157000.dkr.ecr.us-east-1.amazonaws.com".auth // empty' "${CONFIG_PATH}")
 
     if [[ -n "${ecr_auth}" ]]; then
       http_status=$(curl --head -s -o /dev/null -w "%{http_code}" --max-time 3 "https://268558157000.dkr.ecr.us-east-1.amazonaws.com/v2/dev/mongodb-kubernetes/manifests/latest" \
         -H "Authorization: Basic ${ecr_auth}" 2>/dev/null || echo "error/timeout")
 
       if [[ "${http_status}" != "401" && "${http_status}" != "403" && "${http_status}" != "error/timeout" ]]; then
-        echo "Docker credentials are up to date - not performing the new login!"
+        echo "Container registry credentials are up to date - not performing the new login!"
         exit
       fi
-      echo "Docker login required (HTTP status: ${http_status})"
+      echo "Container login required (HTTP status: ${http_status})"
     else
-      echo "No ECR credentials found in Docker config - login required"
+      echo "No ECR credentials found in container config - login required"
     fi
   fi
 
-  title "Performing docker login to ECR registries"
+  title "Performing container login to ECR registries"
 
-  # There could be some leftovers on Evergreen
-  if grep -q "credsStore" ~/.docker/config.json; then
-    remove_element "credsStore"
-  fi
-  if grep -q "credHelpers" ~/.docker/config.json; then
-    remove_element "credHelpers"
+  # There could be some leftovers on Evergreen (Docker-specific, skip for Podman)
+  if [[ "${CONTAINER_RUNTIME}" == "docker" ]]; then
+    if grep -q "credsStore" "${CONFIG_PATH}"; then
+      remove_element "credsStore"
+    fi
+    if grep -q "credHelpers" "${CONFIG_PATH}"; then
+      remove_element "credHelpers"
+    fi
   fi
 fi
 
 
 echo "$(aws --version)}"
 
-aws ecr get-login-password --region "us-east-1" | docker login --username AWS --password-stdin 268558157000.dkr.ecr.us-east-1.amazonaws.com
+aws ecr get-login-password --region "us-east-1" | container_login "AWS" "268558157000.dkr.ecr.us-east-1.amazonaws.com"
 
 # by default docker tries to store credentials in an external storage (e.g. OS keychain) - not in the config.json
 # We need to store it as base64 string in config.json instead so we need to remove the "credsStore" element
-if grep -q "credsStore" ~/.docker/config.json; then
+# This is Docker-specific behavior, Podman stores credentials directly in auth.json
+if [[ "${CONTAINER_RUNTIME}" == "docker" ]] && grep -q "credsStore" "${CONFIG_PATH}"; then
   remove_element "credsStore"
 
   # login again to store the credentials into the config.json
-  aws ecr get-login-password --region "us-east-1" | docker login --username AWS --password-stdin 268558157000.dkr.ecr.us-east-1.amazonaws.com
+  aws ecr get-login-password --region "us-east-1" | container_login "AWS" "268558157000.dkr.ecr.us-east-1.amazonaws.com"
 fi
 
-aws ecr get-login-password --region "eu-west-1" | docker login --username AWS --password-stdin 268558157000.dkr.ecr.eu-west-1.amazonaws.com
+aws ecr get-login-password --region "eu-west-1" | container_login "AWS" "268558157000.dkr.ecr.eu-west-1.amazonaws.com"
 
 if [[ -n "${COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON:-}" ]]; then
   # log in to quay.io for the mongodb/mongodb-search-community private repo
   # TODO remove once we switch to the official repo in Public Preview
   quay_io_auth_file=$(mktemp)
-  docker_configjson_tmp=$(mktemp)
+  config_tmp=$(mktemp)
   echo "${COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON}" | base64 -d > "${quay_io_auth_file}"
-  jq -s '.[0] * .[1]' "${quay_io_auth_file}" ~/.docker/config.json > "${docker_configjson_tmp}"
-  mv "${docker_configjson_tmp}" ~/.docker/config.json
+  jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}"
+  mv "${config_tmp}" "${CONFIG_PATH}"
   rm "${quay_io_auth_file}"
 fi
 
diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh
index ffc571d3b..9d4884530 100755
--- a/scripts/evergreen/setup_minikube_host.sh
+++ b/scripts/evergreen/setup_minikube_host.sh
@@ -46,22 +46,14 @@ run_setup_step "kubectl and helm Setup" "scripts/evergreen/setup_kubectl.sh"
 
 run_setup_step "jq Setup" "scripts/evergreen/setup_jq.sh"
 
-run_setup_step "IBM Host Setup" "scripts/minikube/setup_minikube_host.sh"
+run_setup_step "Minikube Host Setup with Container Runtime Detection" "scripts/minikube/setup_minikube_host.sh"
 
-run_setup_step "Docker Authentication" "scripts/dev/configure_docker_auth.sh"
+run_setup_step "Container Registry Authentication" "scripts/dev/configure_docker_auth.sh"
 
-# Setup Kubernetes cluster after Docker is properly configured
+# The minikube cluster is already started by the setup_minikube_host.sh script
 echo ""
-echo ">>> Setting up Kubernetes cluster"
-echo ">>> Command: minikube start --profile=${MINIKUBE_PROFILE:-mongodb-e2e} --driver=docker --memory=8192mb --cpus=4"
-
-# Start minikube cluster for CI
-if minikube start --profile="${MINIKUBE_PROFILE:-mongodb-e2e}" --driver=docker --memory=8192mb --cpus=4; then
-    echo "✅ Minikube Kubernetes Cluster completed successfully"
-else
-    echo "❌ Minikube Kubernetes Cluster failed"
-    exit 1
-fi
+echo ">>> Minikube cluster startup completed by setup_minikube_host.sh"
+echo "✅ Minikube cluster is ready for use"
 
 echo ""
 echo "=========================================="
@@ -74,6 +66,7 @@ echo "- AWS CLI: $(aws --version 2>/dev/null || echo 'Not found')"
 echo "- kubectl: $(kubectl version --client 2>/dev/null || echo 'Not found')"
 echo "- helm: $(helm version --short 2>/dev/null || echo 'Not found')"
 echo "- jq: $(jq --version 2>/dev/null || echo 'Not found')"
-echo "- Docker: $(docker --version 2>/dev/null || echo 'Not found')"
+echo "- Container Runtime: $(command -v podman &>/dev/null && echo "Podman $(podman --version 2>/dev/null)" || command -v docker &>/dev/null && echo "Docker $(docker --version 2>/dev/null)" || echo "Not found")"
+echo "- Minikube: $(./bin/minikube version --short 2>/dev/null || echo 'Not found')"
 echo ""
 echo "Setup complete! Host is ready for minikube operations."
diff --git a/scripts/minikube/install-docker.sh b/scripts/minikube/install-docker.sh
deleted file mode 100755
index 770c75e5a..000000000
--- a/scripts/minikube/install-docker.sh
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/usr/bin/env bash
-set -Eeou pipefail
-
-source scripts/dev/set_env_context.sh
-
-echo "Installing Docker"
-
-# Detect OS
-if [[ -f /etc/redhat-release ]]; then
-    OS_TYPE="rhel"
-elif [[ -f /etc/debian_version ]]; then
-    OS_TYPE="debian"
-else
-    echo "Unsupported OS. This script supports RHEL/CentOS and Ubuntu/Debian."
-    exit 1
-fi
-
-# Install Docker based on OS
-if [[ "$OS_TYPE" == "rhel" ]]; then
-    echo "Detected RHEL/CentOS system..."
-
-    # Add Docker repository
-    sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
-
-    # Install Docker CE
-    sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
-
-elif [[ "$OS_TYPE" == "debian" ]]; then
-    echo "Detected Ubuntu/Debian system..."
-
-    # Update package index
-    sudo apt-get update
-
-    # Install required packages
-    sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release
-
-    # Add Docker's official GPG key
-    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-
-    # Set up stable repository
-    echo "deb [arch=s390x signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-
-    # Update package index again
-    sudo apt-get update
-
-    # Install Docker CE
-    sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
-fi
-
-# Start and enable Docker service
-sudo systemctl start docker
-sudo systemctl enable docker
-
-# Add current CI user to docker group and change socket permissions
-current_user=$(whoami)
-if [[ "$current_user" != "root" ]]; then
-    echo "Adding CI user '$current_user' to docker group..."
-    sudo usermod -aG docker "$current_user"
-    
-    # For CI: Change docker socket permissions to allow immediate access
-    echo "Setting docker socket permissions for CI..."
-    sudo chmod 666 /var/run/docker.sock
-fi
-
-# Verify installation
-echo "Verifying Docker installation..."
-sudo docker --version
-
-# Test docker access
-echo "Testing Docker access..."
-if docker ps >/dev/null 2>&1; then
-  echo "✅ Docker access confirmed"
-else
-  echo "❌ Docker access failed - CI may not work properly"
-  echo "Trying with sudo..."
-  if sudo docker ps >/dev/null 2>&1; then
-    echo "⚠️  Docker only works with sudo"
-  fi
-fi
diff --git a/scripts/minikube/install-minikube.sh b/scripts/minikube/install-minikube.sh
index b82c829b6..527b27543 100755
--- a/scripts/minikube/install-minikube.sh
+++ b/scripts/minikube/install-minikube.sh
@@ -27,12 +27,49 @@ case "${ARCH}" in
 esac
 
 echo "Installing minikube on ${ARCH} architecture..."
+
+# Install crictl (container runtime CLI)
+echo "Installing crictl for ${ARCH}..."
+CRICTL_VERSION=$(curl -s https://api.github.com/repos/kubernetes-sigs/cri-tools/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
+
+# Download and extract crictl tar.gz
+mkdir -p "${PROJECT_DIR:-.}/bin"
+CRICTL_URL="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${MINIKUBE_ARCH}.tar.gz"
+echo "Downloading ${CRICTL_URL}"
+TEMP_DIR=$(mktemp -d)
+curl --retry 3 --silent -L "${CRICTL_URL}" -o "${TEMP_DIR}/crictl.tar.gz"
+tar -xzf "${TEMP_DIR}/crictl.tar.gz" -C "${TEMP_DIR}/"
+chmod +x "${TEMP_DIR}/crictl"
+mv "${TEMP_DIR}/crictl" "${PROJECT_DIR:-.}/bin/crictl"
+rm -rf "${TEMP_DIR}"
+echo "Installed crictl to ${PROJECT_DIR:-.}/bin"
+
+# Also install crictl system-wide so minikube can find it
+echo "Installing crictl system-wide..."
+if [[ -f "${PROJECT_DIR:-.}/bin/crictl" ]]; then
+    # Install to both /usr/local/bin and /usr/bin for better PATH coverage
+    sudo cp "${PROJECT_DIR:-.}/bin/crictl" /usr/local/bin/crictl
+    sudo cp "${PROJECT_DIR:-.}/bin/crictl" /usr/bin/crictl
+    sudo chmod +x /usr/local/bin/crictl
+    sudo chmod +x /usr/bin/crictl
+    echo "✅ crictl installed to /usr/local/bin/ and /usr/bin/"
+    
+    # Verify installation
+    if command -v crictl >/dev/null 2>&1; then
+        echo "✅ crictl is now available in PATH: $(which crictl)"
+        echo "✅ crictl version: $(crictl --version 2>/dev/null || echo 'version check failed')"
+    else
+        echo "⚠️  crictl installed but not found in PATH"
+    fi
+else
+    echo "⚠️  crictl not found in project bin, minikube may have issues"
+fi
+
 # Install minikube
 echo "Installing minikube for ${ARCH}..."
-
 MINIKUBE_VERSION=$(curl -s https://api.github.com/repos/kubernetes/minikube/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
 
 # Download minikube for detected architecture
 download_and_install_binary "${PROJECT_DIR:-.}/bin" minikube "https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-${MINIKUBE_ARCH}"
 
-echo "Minikube ${MINIKUBE_VERSION} installed successfully for ${ARCH}"
+echo "Crictl ${CRICTL_VERSION} and Minikube ${MINIKUBE_VERSION} installed successfully for ${ARCH}"
diff --git a/scripts/minikube/minikube_host.sh b/scripts/minikube/minikube_host.sh
index c383ac09c..09c5b80f7 100755
--- a/scripts/minikube/minikube_host.sh
+++ b/scripts/minikube/minikube_host.sh
@@ -79,19 +79,18 @@ remote-prepare-local-e2e-run() {
 get-kubeconfig() {
   # For minikube, we need to get the kubeconfig and certificates
   echo "Getting kubeconfig from minikube on s390x host..."
-  local profile=${MINIKUBE_PROFILE:-mongodb-e2e}
 
   # Create local minikube directory structure
-  mkdir -p "${HOME}/.minikube/profiles/${profile}"
+  mkdir -p "${HOME}/.minikube"
 
   # Copy certificates from remote host
   echo "Copying minikube certificates..."
   scp "${host_url}:~/.minikube/ca.crt" "${HOME}/.minikube/"
-  scp "${host_url}:~/.minikube/profiles/${profile}/client.crt" "${HOME}/.minikube/profiles/${profile}/"
-  scp "${host_url}:~/.minikube/profiles/${profile}/client.key" "${HOME}/.minikube/profiles/${profile}/"
+  scp "${host_url}:~/.minikube/client.crt" "${HOME}/.minikube/"
+  scp "${host_url}:~/.minikube/client.key" "${HOME}/.minikube/"
 
   # Get kubeconfig and update paths to local ones
-  ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; export KUBE_ENVIRONMENT_NAME=minikube; export MINIKUBE_PROFILE=${profile}; kubectl config view --raw" > "${kubeconfig_path}"
+  ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; export KUBE_ENVIRONMENT_NAME=minikube; kubectl config view --raw" > "${kubeconfig_path}"
 
   # Update certificate paths to local paths
   sed -i '' "s|/home/cloud-user/.minikube|${HOME}/.minikube|g" "${kubeconfig_path}"
@@ -103,12 +102,10 @@ get-kubeconfig() {
 }
 
 recreate-minikube-cluster() {
-  shift 1
-  profile_name=${1:-mongodb-e2e}
   configure "$(uname -m)" 2>&1| prepend "minikube_host.sh configure"
-  echo "Recreating minikube cluster ${profile_name} on ${S390_HOST_NAME} (${host_url})..."
+  echo "Recreating minikube cluster on ${S390_HOST_NAME} (${host_url})..."
   # shellcheck disable=SC2088
-  ssh -T "${host_url}" "cd ~/mongodb-kubernetes; export KUBE_ENVIRONMENT_NAME=minikube; export MINIKUBE_PROFILE=${profile_name}; minikube delete --profile=${profile_name} || true; minikube start --profile=${profile_name} --driver=docker --memory=8192mb --cpus=4"
+  ssh -T "${host_url}" "cd ~/mongodb-kubernetes; export KUBE_ENVIRONMENT_NAME=minikube; minikube delete || true; minikube start --driver=podman --memory=8192mb --cpus=4"
   echo "Copying kubeconfig to ${kubeconfig_path}"
   get-kubeconfig
 }
@@ -116,11 +113,10 @@ recreate-minikube-cluster() {
 tunnel() {
   shift 1
   echo "Setting up tunnel for minikube cluster..."
-  local profile=${MINIKUBE_PROFILE:-mongodb-e2e}
 
   # Get the minikube API server port from remote host
   local api_port
-  api_port=$(ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; export MINIKUBE_PROFILE=${profile}; minikube ip --profile=${profile} 2>/dev/null && echo ':8443' | tr -d '\n'")
+  api_port=$(ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; minikube ip 2>/dev/null && echo ':8443' | tr -d '\n'")
 
   if [[ -z "${api_port}" ]]; then
     echo "Could not determine minikube API server details. Is the cluster running?"
@@ -134,7 +130,7 @@ tunnel() {
   # Forward the API server port through minikube
   set -x
   # shellcheck disable=SC2029
-  ssh -L "${port}:$(ssh -T -q "${host_url}" "export MINIKUBE_PROFILE=${profile}; minikube ip --profile=${profile}"):${port}" "${host_url}" "$@"
+  ssh -L "${port}:$(ssh -T -q "${host_url}" "minikube ip"):${port}" "${host_url}" "$@"
   set +x
 }
 
@@ -185,7 +181,7 @@ PREREQUISITES:
 COMMANDS:
   configure <architecture>           installs on a host: calls sync, switches context, installs necessary software (auto-detects arch)
   sync                              rsync of project directory
-  recreate-minikube-cluster <profile>  recreates minikube cluster with specific profile and executes get-kubeconfig
+  recreate-minikube-cluster          recreates minikube cluster and executes get-kubeconfig
   remote-prepare-local-e2e-run       executes prepare-local-e2e on the remote host
   get-kubeconfig                     copies remote minikube kubeconfig locally to ~/.operator-dev/s390-host.kubeconfig
   tunnel [args]                      creates ssh session with tunneling to all API servers
diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh
index a455fca57..8726e8e3f 100755
--- a/scripts/minikube/setup_minikube_host.sh
+++ b/scripts/minikube/setup_minikube_host.sh
@@ -5,23 +5,6 @@ source scripts/dev/set_env_context.sh
 
 set -Eeou pipefail
 
-check_disk_space() {
-  echo "Checking available disk space..."
-  local available_gb
-  available_gb=$(df / | awk 'NR==2 {print int($4/1024/1024)}')
-
-  if [[ $available_gb -lt 5 ]]; then
-    echo "ERROR: Insufficient disk space. Available: ${available_gb}GB, Required: 5GB minimum"
-    echo "Please clean up disk space before continuing:"
-    echo "  sudo dnf clean all"
-    echo "  sudo rm -rf /var/cache/dnf/* /tmp/* /var/tmp/*"
-    echo "  docker system prune -af"
-    return 1
-  fi
-
-  echo "Disk space check passed: ${available_gb}GB available"
-}
-
 set_limits() {
   echo "Increasing fs.inotify.max_user_instances"
   sudo sysctl -w fs.inotify.max_user_instances=8192
@@ -64,17 +47,125 @@ download_minikube() {
   scripts/minikube/install-minikube.sh
 }
 
-download_docker() {
-  echo "Installing Docker for ${ARCH}..."
-  scripts/minikube/install-docker.sh
+# Fix crictl for ppc64le kicbase images
+fix_crictl_ppc64le() {
+  if [[ "${ARCH}" == "ppc64le" ]]; then
+    echo ">>> Applying ppc64le crictl fix for podman driver..."
+
+    # Ensure crictl is available on host
+    if ! command -v crictl &> /dev/null; then
+      echo "❌ crictl not found on host - this is required for ppc64le fix"
+      return 1
+    fi
+
+    # Wait for minikube container to be created
+    local container_name="minikube"
+    local max_wait=60
+    local waited=0
+
+    echo "Waiting for minikube container '${container_name}' to be ready..."
+    while ! podman ps --format '{{.Names}}' | grep -q "^${container_name}$" && [[ $waited -lt $max_wait ]]; do
+      sleep 2
+      waited=$((waited + 2))
+    done
+
+    if [[ $waited -ge $max_wait ]]; then
+      echo "⚠️  Timeout waiting for minikube container - crictl fix may be needed manually"
+      return 1
+    fi
+
+    # Copy crictl from host to container using podman
+    echo "Copying crictl to minikube container using podman..."
+    if podman cp "$(which crictl)" "${container_name}:/usr/bin/crictl"; then
+      podman exec "${container_name}" chmod +x /usr/bin/crictl
+      echo "✅ crictl fix applied successfully with podman"
+
+      # Verify the fix
+      if podman exec "${container_name}" crictl --version &>/dev/null; then
+        echo "✅ crictl is now working in minikube container"
+        return 0
+      else
+        echo "⚠️  crictl copy succeeded but binary may not be working"
+        return 1
+      fi
+    else
+      echo "❌ Failed to copy crictl to container via podman"
+      return 1
+    fi
+  fi
+  return 0
 }
 
-check_disk_space
-set_limits
-download_docker &
-download_minikube &
+# Start minikube with podman driver
+start_minikube_cluster() {
+  echo ">>> Starting minikube cluster with podman driver..."
+
+  local start_args=("--driver=podman")
+
+  # Add Calico CNI for better compatibility
+  start_args+=("--cni=calico")
+
+  # For ppc64le, we need to handle the crictl fix
+  if [[ "${ARCH}" == "ppc64le" ]]; then
+    echo "Starting minikube (ppc64le requires crictl fix for podman)..."
+
+    # Start minikube in background to let container initialize
+    timeout 120 "${PROJECT_DIR:-.}/bin/minikube" start "${start_args[@]}" &
+    local minikube_pid=$!
 
-wait
+    # Wait a bit for container to be created
+    sleep 15
+
+    # Apply crictl fix while minikube is starting
+    if fix_crictl_ppc64le; then
+      echo "✅ crictl fix applied, waiting for minikube to complete..."
+    else
+      echo "⚠️  crictl fix failed, but continuing..."
+    fi
+
+    # Wait for minikube to finish
+    if wait $minikube_pid; then
+      echo "✅ Minikube started successfully with crictl fix"
+    else
+      echo "❌ Minikube failed to start"
+      return 1
+    fi
+  else
+    # Standard minikube start
+    if "${PROJECT_DIR:-.}/bin/minikube" start "${start_args[@]}"; then
+      echo "✅ Minikube started successfully"
+    else
+      echo "❌ Minikube failed to start"
+      return 1
+    fi
+  fi
+}
+
+setup_podman() {
+  echo "Setting up podman for ${ARCH}..."
+
+  # Check if podman is already available
+  if command -v podman &> /dev/null; then
+    echo "✅ Podman already installed"
+  else
+    echo "Installing podman..."
+    sudo dnf install -y podman
+  fi
+
+  # Configure podman
+  echo "Configuring Podman for ${ARCH}..."
+
+  # Start podman service if not running
+  systemctl --user enable podman.socket 2>/dev/null || true
+  systemctl --user start podman.socket 2>/dev/null || true
+
+  echo "✅ Podman configured successfully"
+}
+
+# Setup podman and container runtime
+setup_podman
+set_limits
+download_minikube
 
 echo ""
 echo ">>> Verifying minikube installation..."
@@ -88,24 +179,61 @@ else
 fi
 
 echo ""
-echo ">>> Verifying docker installation..."
-if command -v docker &> /dev/null; then
-    docker_version=$(docker --version 2>/dev/null)
-    echo "✅ Docker installed successfully: ${docker_version}"
-
-    # Check if docker service is running
-    if systemctl is-active --quiet docker 2>/dev/null || docker info &>/dev/null; then
-        echo "✅ Docker service is running"
+echo ">>> Verifying podman installation..."
+if command -v podman &> /dev/null; then
+    podman_version=$(podman --version 2>/dev/null)
+    echo "✅ Podman installed successfully: ${podman_version}"
+
+    # Check podman info
+    if podman info &>/dev/null; then
+        echo "✅ Podman is working correctly"
     else
-        echo "⚠️  Docker is installed but service may not be running"
-        echo "You may need to start docker service: sudo systemctl start docker"
+        echo "⚠️  Podman may need additional configuration"
     fi
 else
-    echo "❌ Docker installation failed - docker command not found"
-    echo "Please check the installation logs above for errors"
+    echo "❌ Podman installation failed - podman command not found"
     exit 1
 fi
 
+echo ""
+echo ">>> Verifying crictl is available in PATH..."
+if command -v crictl &> /dev/null; then
+    crictl_version=$(crictl --version 2>/dev/null || echo "crictl available")
+    crictl_path=$(which crictl 2>/dev/null || echo "unknown path")
+    echo "✅ crictl found in PATH: ${crictl_version}"
+    echo "✅ crictl location: ${crictl_path}"
+else
+    echo "❌ crictl not found in PATH - this may cause minikube issues"
+    echo "Checking if crictl exists in project bin..."
+    if [[ -f "${PROJECT_DIR:-.}/bin/crictl" ]]; then
+        echo "Found crictl in project bin, installing to system directories..."
+        sudo cp "${PROJECT_DIR:-.}/bin/crictl" /usr/local/bin/crictl
+        sudo cp "${PROJECT_DIR:-.}/bin/crictl" /usr/bin/crictl
+        sudo chmod +x /usr/local/bin/crictl
+        sudo chmod +x /usr/bin/crictl
+        echo "✅ crictl installed to /usr/local/bin/ and /usr/bin/"
+
+        # Force PATH refresh and verify
+        hash -r 2>/dev/null || true
+        if command -v crictl &> /dev/null; then
+            echo "✅ crictl now available: $(which crictl)"
+        else
+            echo "⚠️  crictl installation may not be working properly"
+        fi
+    else
+        echo "❌ crictl not found in project bin either"
+    fi
+fi
+
+# Start the minikube cluster
+start_minikube_cluster
+
+# Update kubectl context to point to the running cluster
+echo ""
+echo ">>> Updating kubectl context for minikube cluster..."
+"${PROJECT_DIR:-.}/bin/minikube" update-context
+echo "✅ Kubectl context updated successfully"
+
 echo "Minikube host setup completed successfully for ${ARCH}!"
 
 # Final status
@@ -114,6 +242,12 @@ echo "=========================================="
 echo "✅ Setup Summary"
 echo "=========================================="
 echo "Architecture: ${ARCH}"
-echo "Minikube Profile: ${MINIKUBE_PROFILE:-mongodb-e2e}"
+echo "Container Runtime: podman"
+echo "Minikube Driver: podman"
+echo "Minikube: Default cluster"
 echo "Minikube: ${minikube_version}"
-echo "Docker: ${docker_version}"
+echo "Podman: ${podman_version}"
+echo "CNI: Calico"
+if [[ "${ARCH}" == "ppc64le" ]]; then
+    echo "Special Config: ppc64le crictl fix applied for podman"
+fi

From 284c7e19beed1ab20b1e2ebd619b0efeef6aa25f Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Thu, 31 Jul 2025 16:21:51 +0200
Subject: [PATCH 11/23] z series support

---
 .evergreen.yml                          | 14 +++++++++++++-
 scripts/minikube/setup_minikube_host.sh |  4 ++--
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/.evergreen.yml b/.evergreen.yml
index 98eaf8a75..fdcb4eaae 100644
--- a/.evergreen.yml
+++ b/.evergreen.yml
@@ -1473,7 +1473,7 @@ buildvariants:
     tasks:
       - name: e2e_smoke_task_group
 
-  - name: e2e_smoke_ibm
+  - name: e2e_smoke_ibm_power
     display_name: e2e_smoke_ibm
     tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ]
     run_on:
@@ -1481,6 +1481,18 @@ buildvariants:
     allowed_requesters: [ "patch", "github_tag" ]
 #    depends_on:
 #      - name: build_test_image
+#        variant: init_test_run
+    tasks:
+      - name: e2e_smoke_ibm_task_group
+
+  - name: e2e_smoke_ibm_z
+    display_name: e2e_smoke_ibm
+    tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ]
+    run_on:
+      - rhel9-zseries-large
+    allowed_requesters: [ "patch", "github_tag" ]
+#    depends_on:
+#      - name: build_test_image
 #        variant: init_test_run
     tasks:
       - name: e2e_smoke_ibm_task_group
diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh
index 8726e8e3f..cf38652dc 100755
--- a/scripts/minikube/setup_minikube_host.sh
+++ b/scripts/minikube/setup_minikube_host.sh
@@ -102,8 +102,8 @@ start_minikube_cluster() {
 
   local start_args=("--driver=podman")
 
-  # Add Calico CNI for better compatibility
-  start_args+=("--cni=calico")
+  # Use default bridge CNI to avoid Docker Hub rate limiting issues
+  # start_args+=("--cni=bridge")
 
   # For ppc64le, we need to handle the crictl fix
   if [[ "${ARCH}" == "ppc64le" ]]; then

From 00439c0e67f64d59e62da3a7a34a8977ef8c7435 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Thu, 31 Jul 2025 18:32:27 +0200
Subject: [PATCH 12/23] custom container with working power critcl

---
 scripts/minikube/setup_minikube_host.sh | 246 ++++++++++++++----------
 1 file changed, 141 insertions(+), 105 deletions(-)

diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh
index cf38652dc..f4af288c0 100755
--- a/scripts/minikube/setup_minikube_host.sh
+++ b/scripts/minikube/setup_minikube_host.sh
@@ -47,51 +47,97 @@ download_minikube() {
   scripts/minikube/install-minikube.sh
 }
 
-# Fix crictl for ppc64le kicbase images
-fix_crictl_ppc64le() {
+# Setup local registry and build custom kicbase image for ppc64le with crictl
+setup_local_registry_and_custom_image() {
   if [[ "${ARCH}" == "ppc64le" ]]; then
-    echo ">>> Applying ppc64le crictl fix for podman driver..."
-
-    # Ensure crictl is available on host
-    if ! command -v crictl &> /dev/null; then
-      echo "❌ crictl not found on host - this is required for ppc64le fix"
-      return 1
+    echo ">>> Setting up local registry and custom kicbase image for ppc64le..."
+    
+    # Check if local registry is running
+    if ! podman ps --filter "name=registry" --format "{{.Names}}" | grep -q "^registry$"; then
+      echo "Starting local container registry on port 5000..."
+      podman run -d -p 5000:5000 --name registry --restart=always docker.io/library/registry:2 || {
+        echo "Registry might already exist, trying to start it..."
+        podman start registry || {
+          echo "Removing existing registry and creating new one..."
+          podman rm -f registry 2>/dev/null || true
+          podman run -d -p 5000:5000 --name registry --restart=always docker.io/library/registry:2
+        }
+      }
+      
+      # Wait for registry to be ready
+      echo "Waiting for registry to be ready..."
+      for i in {1..30}; do
+        if curl -s http://localhost:5000/v2/_catalog >/dev/null 2>&1; then
+          break
+        fi
+        sleep 1
+      done
+    else
+      echo "✅ Local registry already running"
     fi
-
-    # Wait for minikube container to be created
-    local container_name="minikube"
-    local max_wait=60
-    local waited=0
-
-    echo "Waiting for minikube container '${container_name}' to be ready..."
-    while ! podman ps --format '{{.Names}}' | grep -q "^${container_name}$" && [[ $waited -lt $max_wait ]]; do
-      sleep 2
-      waited=$((waited + 2))
-    done
-
-    if [[ $waited -ge $max_wait ]]; then
-      echo "⚠️  Timeout waiting for minikube container - crictl fix may be needed manually"
-      return 1
+    
+    # Configure per-user podman to trust local registry (no system-wide changes)
+    echo "Configuring per-user podman registry settings..."
+    mkdir -p ~/.config/containers
+    
+    # Create user-specific registries.conf that includes insecure local registry
+    cat > ~/.config/containers/registries.conf << 'EOF'
+unqualified-search-registries = ["registry.access.redhat.com", "registry.redhat.io", "docker.io"]
+
+[[registry]]
+location = "localhost:5000"
+insecure = true
+
+short-name-mode = "permissive"
+EOF
+    
+    # Check if custom image already exists in local registry
+    if curl -s http://localhost:5000/v2/kicbase/tags/list | grep -q "v0.0.47"; then
+      echo "✅ Custom kicbase image already exists in local registry"
+      return 0
     fi
-
-    # Copy crictl from host to container using podman
-    echo "Copying crictl to minikube container using podman..."
-    if podman cp "$(which crictl)" "${container_name}:/usr/bin/crictl"; then
-      podman exec "${container_name}" chmod +x /usr/bin/crictl
-      echo "✅ crictl fix applied successfully with podman"
-
-      # Verify the fix
-      if podman exec "${container_name}" crictl --version &>/dev/null; then
-        echo "✅ crictl is now working in minikube container"
-        return 0
-      else
-        echo "⚠️  crictl copy succeeded but binary may not be working"
-        return 1
-      fi
-    else
-      echo "❌ Failed to copy crictl to container via podman"
-      return 1
+    
+    # Build custom kicbase image with crictl
+    echo "Building custom kicbase image with crictl for ppc64le..."
+    
+    # Create build directory if it doesn't exist
+    mkdir -p "${PROJECT_DIR:-.}/scripts/minikube/kicbase"
+    
+    # Create Dockerfile for custom kicbase
+    cat > "${PROJECT_DIR:-.}/scripts/minikube/kicbase/Dockerfile" << 'EOF'
+FROM gcr.io/k8s-minikube/kicbase:v0.0.47
+
+# Install crictl for ppc64le if needed
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
+        echo "Installing crictl for ppc64le architecture..." && \
+        CRICTL_VERSION="v1.28.0" && \
+        curl -L "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-ppc64le.tar.gz" \
+            -o /tmp/crictl.tar.gz && \
+        tar -C /usr/bin -xzf /tmp/crictl.tar.gz && \
+        chmod +x /usr/bin/crictl && \
+        rm /tmp/crictl.tar.gz && \
+        echo "crictl installed successfully" && \
+        crictl --version; \
+    else \
+        echo "Not ppc64le architecture, skipping crictl installation"; \
     fi
+
+# Verify crictl is available
+RUN command -v crictl >/dev/null 2>&1 && echo "crictl is available" || echo "crictl not found"
+EOF
+    
+    # Build and push to local registry
+    echo "Building custom kicbase image..."
+    cd "${PROJECT_DIR:-.}/scripts/minikube/kicbase"
+    
+    podman build -t localhost:5000/kicbase:v0.0.47 .
+    
+    echo "Pushing custom image to local registry..."
+    podman push localhost:5000/kicbase:v0.0.47 --tls-verify=false
+    
+    cd - > /dev/null
+    
+    echo "✅ Custom kicbase image with crictl ready in local registry"
   fi
   return 0
 }
@@ -100,44 +146,36 @@ fix_crictl_ppc64le() {
 start_minikube_cluster() {
   echo ">>> Starting minikube cluster with podman driver..."
 
-  local start_args=("--driver=podman")
+  # Clean up any existing minikube state to avoid cached configuration issues
+  echo "Cleaning up any existing minikube state..."
+  if [[ -d ~/.minikube/machines/minikube ]]; then
+    echo "Removing ~/.minikube/machines/minikube directory..."
+    rm -rf ~/.minikube/machines/minikube
+  fi
+  
+  # Delete any existing minikube cluster to start fresh
+  echo "Ensuring clean minikube state..."
+  "${PROJECT_DIR:-.}/bin/minikube" delete 2>/dev/null || true
 
-  # Use default bridge CNI to avoid Docker Hub rate limiting issues
-  # start_args+=("--cni=bridge")
+  local start_args=("--driver=podman")
 
-  # For ppc64le, we need to handle the crictl fix
+  # Use custom kicbase image for ppc64le with crictl included
   if [[ "${ARCH}" == "ppc64le" ]]; then
-    echo "Starting minikube (ppc64le requires crictl fix for podman)..."
-
-    # Start minikube in background to let container initialize
-    timeout 120 "${PROJECT_DIR:-.}/bin/minikube" start "${start_args[@]}" &
-    local minikube_pid=$!
-
-    # Wait a bit for container to be created
-    sleep 15
+    echo "Using custom kicbase image for ppc64le with crictl..."
+    start_args+=("--base-image=localhost:5000/kicbase:v0.0.47")
+  fi
 
-    # Apply crictl fix while minikube is starting
-    if fix_crictl_ppc64le; then
-      echo "✅ crictl fix applied, waiting for minikube to complete..."
-    else
-      echo "⚠️  crictl fix failed, but continuing..."
-    fi
+  # Use default bridge CNI to avoid Docker Hub rate limiting issues
+  # start_args+=("--cni=bridge")
 
-    # Wait for minikube to finish
-    if wait $minikube_pid; then
-      echo "✅ Minikube started successfully with crictl fix"
-    else
-      echo "❌ Minikube failed to start"
-      return 1
-    fi
+  echo "Starting minikube with args: ${start_args[*]}"
+  if "${PROJECT_DIR:-.}/bin/minikube" start "${start_args[@]}"; then
+    echo "✅ Minikube started successfully"
   else
-    # Standard minikube start
-    if "${PROJECT_DIR:-.}/bin/minikube" start "${start_args[@]}"; then
-      echo "✅ Minikube started successfully"
-    else
-      echo "❌ Minikube failed to start"
-      return 1
-    fi
+    echo "❌ Minikube failed to start"
+    echo "Minikube logs:"
+    "${PROJECT_DIR:-.}/bin/minikube" logs | tail -20
+    return 1
   fi
 }
 
@@ -152,14 +190,31 @@ setup_podman() {
     sudo dnf install -y podman
   fi
 
-  # Configure podman
-  echo "Configuring Podman for ${ARCH}..."
+  # Configure podman for CI environment
+  echo "Configuring Podman for ${ARCH} CI environment..."
+
+  # Enable lingering for the current user to fix systemd session issues
+  current_user=$(whoami)
+  current_uid=$(id -u)
+  echo "Enabling systemd lingering for user ${current_user} (UID: ${current_uid})"
+  sudo loginctl enable-linger "${current_uid}" 2>/dev/null || true
+
+  # Configure podman to use cgroupfs instead of systemd in CI
+  mkdir -p ~/.config/containers
+  cat > ~/.config/containers/containers.conf << EOF
+[containers]
+cgroup_manager = "cgroupfs"
+events_logger = "file"
+
+[engine]
+cgroup_manager = "cgroupfs"
+EOF
 
   # Start podman service if not running
   systemctl --user enable podman.socket 2>/dev/null || true
   systemctl --user start podman.socket 2>/dev/null || true
 
-  echo "✅ Podman configured successfully"
+  echo "✅ Podman configured successfully for CI"
 }
 
 # Setup podman and container runtime
@@ -167,6 +222,9 @@ setup_podman
 set_limits
 download_minikube
 
+# Setup local registry and custom kicbase image for ppc64le if needed
+setup_local_registry_and_custom_image
+
 echo ""
 echo ">>> Verifying minikube installation..."
 if command -v minikube &> /dev/null; then
@@ -195,34 +253,12 @@ else
     exit 1
 fi
 
-echo ""
-echo ">>> Verifying crictl is available in PATH..."
-if command -v crictl &> /dev/null; then
-    crictl_version=$(crictl --version 2>/dev/null || echo "crictl available")
-    crictl_path=$(which crictl 2>/dev/null || echo "unknown path")
-    echo "✅ crictl found in PATH: ${crictl_version}"
-    echo "✅ crictl location: ${crictl_path}"
+if [[ "${ARCH}" == "ppc64le" ]]; then
+    echo ""
+    echo ">>> Note: crictl will be patched into the minikube container after startup"
 else
-    echo "❌ crictl not found in PATH - this may cause minikube issues"
-    echo "Checking if crictl exists in project bin..."
-    if [[ -f "${PROJECT_DIR:-.}/bin/crictl" ]]; then
-        echo "Found crictl in project bin, installing to system directories..."
-        sudo cp "${PROJECT_DIR:-.}/bin/crictl" /usr/local/bin/crictl
-        sudo cp "${PROJECT_DIR:-.}/bin/crictl" /usr/bin/crictl
-        sudo chmod +x /usr/local/bin/crictl
-        sudo chmod +x /usr/bin/crictl
-        echo "✅ crictl installed to /usr/local/bin/ and /usr/bin/"
-
-        # Force PATH refresh and verify
-        hash -r 2>/dev/null || true
-        if command -v crictl &> /dev/null; then
-            echo "✅ crictl now available: $(which crictl)"
-        else
-            echo "⚠️  crictl installation may not be working properly"
-        fi
-    else
-        echo "❌ crictl not found in project bin either"
-    fi
+    echo ""
+    echo ">>> Using standard kicbase image (crictl included for x86_64/aarch64/s390x)"
 fi
 
 # Start the minikube cluster
@@ -247,7 +283,7 @@ echo "Minikube Driver: podman"
 echo "Minikube: Default cluster"
 echo "Minikube: ${minikube_version}"
 echo "Podman: ${podman_version}"
-echo "CNI: Calico"
+echo "CNI: bridge (default)"
 if [[ "${ARCH}" == "ppc64le" ]]; then
-    echo "Special Config: ppc64le crictl fix applied for podman"
+    echo "Special Config: Custom kicbase image with crictl via local registry"
 fi

From c675c83bda19fa5431c407ce06a3750e487f6c8e Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Thu, 31 Jul 2025 18:33:59 +0200
Subject: [PATCH 13/23] custom container with working power critcl

---
 .evergreen.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.evergreen.yml b/.evergreen.yml
index fdcb4eaae..fd3dc78b0 100644
--- a/.evergreen.yml
+++ b/.evergreen.yml
@@ -1474,7 +1474,7 @@ buildvariants:
       - name: e2e_smoke_task_group
 
   - name: e2e_smoke_ibm_power
-    display_name: e2e_smoke_ibm
+    display_name: e2e_smoke_ibm_power
     tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ]
     run_on:
       - rhel9-power-large
@@ -1486,7 +1486,7 @@ buildvariants:
       - name: e2e_smoke_ibm_task_group
 
   - name: e2e_smoke_ibm_z
-    display_name: e2e_smoke_ibm
+    display_name: e2e_smoke_ibm_z
     tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ]
     run_on:
       - rhel9-zseries-large

From 37c31de408372163ffb6fefdf8e53a9ab7d05014 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Thu, 31 Jul 2025 21:40:47 +0200
Subject: [PATCH 14/23] add insecure

---
 scripts/minikube/setup_minikube_host.sh | 44 ++++++++++++++-----------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh
index f4af288c0..88afd424f 100755
--- a/scripts/minikube/setup_minikube_host.sh
+++ b/scripts/minikube/setup_minikube_host.sh
@@ -51,7 +51,7 @@ download_minikube() {
 setup_local_registry_and_custom_image() {
   if [[ "${ARCH}" == "ppc64le" ]]; then
     echo ">>> Setting up local registry and custom kicbase image for ppc64le..."
-    
+
     # Check if local registry is running
     if ! podman ps --filter "name=registry" --format "{{.Names}}" | grep -q "^registry$"; then
       echo "Starting local container registry on port 5000..."
@@ -63,7 +63,7 @@ setup_local_registry_and_custom_image() {
           podman run -d -p 5000:5000 --name registry --restart=always docker.io/library/registry:2
         }
       }
-      
+
       # Wait for registry to be ready
       echo "Waiting for registry to be ready..."
       for i in {1..30}; do
@@ -75,13 +75,18 @@ setup_local_registry_and_custom_image() {
     else
       echo "✅ Local registry already running"
     fi
-    
-    # Configure per-user podman to trust local registry (no system-wide changes)
-    echo "Configuring per-user podman registry settings..."
-    mkdir -p ~/.config/containers
-    
-    # Create user-specific registries.conf that includes insecure local registry
-    cat > ~/.config/containers/registries.conf << 'EOF'
+
+    # Configure system-wide podman to trust local registry (with backup)
+    echo "Configuring system registries.conf to trust local registry..."
+
+    # Backup existing registries.conf if it exists
+    if [[ -f /etc/containers/registries.conf ]]; then
+      echo "Backing up existing registries.conf..."
+      sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.minikube-backup
+    fi
+
+    # Create a clean registries.conf that includes insecure local registry
+    sudo tee /etc/containers/registries.conf << 'EOF'
 unqualified-search-registries = ["registry.access.redhat.com", "registry.redhat.io", "docker.io"]
 
 [[registry]]
@@ -90,19 +95,19 @@ insecure = true
 
 short-name-mode = "permissive"
 EOF
-    
+
     # Check if custom image already exists in local registry
     if curl -s http://localhost:5000/v2/kicbase/tags/list | grep -q "v0.0.47"; then
       echo "✅ Custom kicbase image already exists in local registry"
       return 0
     fi
-    
+
     # Build custom kicbase image with crictl
     echo "Building custom kicbase image with crictl for ppc64le..."
-    
+
     # Create build directory if it doesn't exist
     mkdir -p "${PROJECT_DIR:-.}/scripts/minikube/kicbase"
-    
+
     # Create Dockerfile for custom kicbase
     cat > "${PROJECT_DIR:-.}/scripts/minikube/kicbase/Dockerfile" << 'EOF'
 FROM gcr.io/k8s-minikube/kicbase:v0.0.47
@@ -125,18 +130,18 @@ RUN if [ "$(uname -m)" = "ppc64le" ]; then \
 # Verify crictl is available
 RUN command -v crictl >/dev/null 2>&1 && echo "crictl is available" || echo "crictl not found"
 EOF
-    
+
     # Build and push to local registry
     echo "Building custom kicbase image..."
     cd "${PROJECT_DIR:-.}/scripts/minikube/kicbase"
-    
+
     podman build -t localhost:5000/kicbase:v0.0.47 .
-    
+
     echo "Pushing custom image to local registry..."
     podman push localhost:5000/kicbase:v0.0.47 --tls-verify=false
-    
+
     cd - > /dev/null
-    
+
     echo "✅ Custom kicbase image with crictl ready in local registry"
   fi
   return 0
@@ -152,7 +157,7 @@ start_minikube_cluster() {
     echo "Removing ~/.minikube/machines/minikube directory..."
     rm -rf ~/.minikube/machines/minikube
   fi
-  
+
   # Delete any existing minikube cluster to start fresh
   echo "Ensuring clean minikube state..."
   "${PROJECT_DIR:-.}/bin/minikube" delete 2>/dev/null || true
@@ -163,6 +168,7 @@ start_minikube_cluster() {
   if [[ "${ARCH}" == "ppc64le" ]]; then
     echo "Using custom kicbase image for ppc64le with crictl..."
     start_args+=("--base-image=localhost:5000/kicbase:v0.0.47")
+    start_args+=("--insecure-registry=localhost:5000")
   fi
 
   # Use default bridge CNI to avoid Docker Hub rate limiting issues

From 148779d4c1053b09b69ef39bfe83c7b9d1379dc7 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Thu, 31 Jul 2025 23:43:02 +0200
Subject: [PATCH 15/23] add empty config handling and broken podman making

---
 .evergreen.yml                          | 4 ++--
 scripts/dev/configure_docker_auth.sh    | 5 +++++
 scripts/minikube/setup_minikube_host.sh | 7 +++++++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/.evergreen.yml b/.evergreen.yml
index fd3dc78b0..2275ab0e4 100644
--- a/.evergreen.yml
+++ b/.evergreen.yml
@@ -1477,7 +1477,7 @@ buildvariants:
     display_name: e2e_smoke_ibm_power
     tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ]
     run_on:
-      - rhel9-power-large
+      - rhel9-power-small
     allowed_requesters: [ "patch", "github_tag" ]
 #    depends_on:
 #      - name: build_test_image
@@ -1489,7 +1489,7 @@ buildvariants:
     display_name: e2e_smoke_ibm_z
     tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ]
     run_on:
-      - rhel9-zseries-large
+      - rhel9-zseries-small
     allowed_requesters: [ "patch", "github_tag" ]
 #    depends_on:
 #      - name: build_test_image
diff --git a/scripts/dev/configure_docker_auth.sh b/scripts/dev/configure_docker_auth.sh
index 74e0d9abc..2e90328ed 100755
--- a/scripts/dev/configure_docker_auth.sh
+++ b/scripts/dev/configure_docker_auth.sh
@@ -91,6 +91,11 @@ detect_container_runtime
 
 check_docker_daemon_is_running
 
+# Initialize config file if it doesn't exist
+if [[ ! -f "${CONFIG_PATH}" ]]; then
+  echo '{}' > "${CONFIG_PATH}"
+fi
+
 if [[ -f "${CONFIG_PATH}" ]]; then
   if [[ "${RUNNING_IN_EVG:-"false"}" != "true" ]]; then
     # Check if login is actually required by making a HEAD request to ECR using existing credentials
diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh
index 88afd424f..cf39e8f4d 100755
--- a/scripts/minikube/setup_minikube_host.sh
+++ b/scripts/minikube/setup_minikube_host.sh
@@ -191,6 +191,13 @@ setup_podman() {
   # Check if podman is already available
   if command -v podman &> /dev/null; then
     echo "✅ Podman already installed"
+    
+    # Reset podman if it's in an invalid state
+    if podman info 2>&1 | grep -q "invalid internal status"; then
+      echo "Resetting podman due to invalid internal status..."
+      podman system migrate || true
+      podman system reset --force || true
+    fi
   else
     echo "Installing podman..."
     sudo dnf install -y podman

From 10811d7440721f34ceeb90a231630ec4ea466a12 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Thu, 31 Jul 2025 23:48:57 +0200
Subject: [PATCH 16/23] handle edge cases

---
 .../{e2e_smoke_ibm => e2e_smoke_ibm_power}     |  0
 scripts/dev/contexts/e2e_smoke_ibm_z           | 18 ++++++++++++++++++
 2 files changed, 18 insertions(+)
 rename scripts/dev/contexts/{e2e_smoke_ibm => e2e_smoke_ibm_power} (100%)
 create mode 100644 scripts/dev/contexts/e2e_smoke_ibm_z

diff --git a/scripts/dev/contexts/e2e_smoke_ibm b/scripts/dev/contexts/e2e_smoke_ibm_power
similarity index 100%
rename from scripts/dev/contexts/e2e_smoke_ibm
rename to scripts/dev/contexts/e2e_smoke_ibm_power
diff --git a/scripts/dev/contexts/e2e_smoke_ibm_z b/scripts/dev/contexts/e2e_smoke_ibm_z
new file mode 100644
index 000000000..03384c26c
--- /dev/null
+++ b/scripts/dev/contexts/e2e_smoke_ibm_z
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -Eeou pipefail
+
+script_name=$(readlink -f "${BASH_SOURCE[0]}")
+script_dir=$(dirname "${script_name}")
+
+source "${script_dir}/root-context"
+
+export ops_manager_version="cloud_qa"
+
+# This is required to be able to rebuild the om image and use that image which has been rebuild
+export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev
+CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${script_dir}"/../../../.evergreen.yml | awk '{print $3}')
+export CUSTOM_OM_VERSION
+
+export CUSTOM_MDB_VERSION=6.0.5
+export CUSTOM_MDB_PREV_VERSION=5.0.7

From a4cc24fdc7eb8f7f2e351745c52f2aaa1c4b463b Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Fri, 1 Aug 2025 17:00:36 +0200
Subject: [PATCH 17/23] use python wheel locacl build, unify teardown for
 minikube, use run for cloudqa

---
 .evergreen-functions.yml                      |  11 +-
 .evergreen.yml                                |  12 +-
 scripts/dev/configure_docker_auth.sh          |  74 +++++---
 scripts/dev/contexts/e2e_smoke_ibm_power      |   1 +
 scripts/dev/contexts/e2e_smoke_ibm_z          |   1 +
 scripts/dev/recreate_python_venv.sh           |  90 ++++------
 .../evergreen/setup_kubernetes_environment.sh |   2 +
 scripts/evergreen/setup_minikube_host.sh      |   4 +-
 .../teardown_kubernetes_environment.sh        |   8 +-
 scripts/minikube/setup_minikube_host.sh       | 161 +++++++-----------
 10 files changed, 159 insertions(+), 205 deletions(-)

diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml
index 71837ab97..c67634152 100644
--- a/.evergreen-functions.yml
+++ b/.evergreen-functions.yml
@@ -236,15 +236,6 @@ functions:
         - ${workdir}/bin
       binary: scripts/dev/setup_evg_host.sh
 
-  setup_ibm_host: &setup_ibm_host
-    command: subprocess.exec
-    type: setup
-    params:
-      working_dir: src/github.com/mongodb/mongodb-kubernetes
-      add_to_path:
-        - ${workdir}/bin
-      binary: scripts/minikube/setup_minikube_host.sh
-
   lint_repo:
     - command: subprocess.exec
       type: setup
@@ -349,7 +340,7 @@ functions:
         working_dir: src/github.com/mongodb/mongodb-kubernetes
         script: |
           source .generated/context.export.env
-          scripts/evergreen/e2e/setup_cloud_qa.py create
+          scripts/evergreen/run_python.sh scripts/evergreen/e2e/setup_cloud_qa.py create
     # The additional switch is needed, since we now have created the needed OM exports.
     - *switch_context
 
diff --git a/.evergreen.yml b/.evergreen.yml
index 2275ab0e4..136212703 100644
--- a/.evergreen.yml
+++ b/.evergreen.yml
@@ -140,7 +140,7 @@ variables:
 
   - &teardown_group
     teardown_group:
-      - func: prune_docker_resources
+      - func: teardown_kubernetes_environment
       - func: run_retry_script
 
   - &base_om7_dependency
@@ -1197,13 +1197,7 @@ task_groups:
   - name: e2e_smoke_ibm_task_group
     max_hosts: -1
     <<: *setup_group_ibm
-    setup_task_can_fail_task: true
-    setup_task:
-      - func: cleanup_exec_environment
-    teardown_task_can_fail_task: true
-    teardown_task:
-      - func: upload_e2e_logs
-      - func: teardown_kubernetes_environment
+    <<: *setup_and_teardown_task_cloudqa
     tasks:
       - e2e_replica_set
     <<: *teardown_group
@@ -1477,7 +1471,7 @@ buildvariants:
     display_name: e2e_smoke_ibm_power
     tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ]
     run_on:
-      - rhel9-power-small
+      - rhel9-power-large
     allowed_requesters: [ "patch", "github_tag" ]
 #    depends_on:
 #      - name: build_test_image
diff --git a/scripts/dev/configure_docker_auth.sh b/scripts/dev/configure_docker_auth.sh
index 2e90328ed..4edf68817 100755
--- a/scripts/dev/configure_docker_auth.sh
+++ b/scripts/dev/configure_docker_auth.sh
@@ -10,11 +10,12 @@ source scripts/funcs/kubernetes
 
 # Detect available container runtime
 detect_container_runtime() {
-  if command -v podman &> /dev/null && podman info &> /dev/null; then
+  if command -v podman &> /dev/null && (podman info &> /dev/null || sudo podman info &> /dev/null); then
     CONTAINER_RUNTIME="podman"
-    CONFIG_PATH="${HOME}/.config/containers/auth.json"
-    mkdir -p "$(dirname "${CONFIG_PATH}")"
-    echo "Using Podman for container authentication"
+    # Use root's auth.json since minikube uses sudo podman
+    CONFIG_PATH="/root/.config/containers/auth.json"
+    sudo mkdir -p "$(dirname "${CONFIG_PATH}")"
+    echo "Using Podman for container authentication (sudo mode)"
     return 0
   elif command -v docker &> /dev/null; then
     CONTAINER_RUNTIME="docker"
@@ -60,14 +61,23 @@ check_docker_daemon_is_running() {
 remove_element() {
   config_option="${1}"
   tmpfile=$(mktemp)
-  
+
   # Initialize config file if it doesn't exist
   if [[ ! -f "${CONFIG_PATH}" ]]; then
-    echo '{}' > "${CONFIG_PATH}"
+    if [[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]]; then
+      echo '{}' | sudo tee "${CONFIG_PATH}" > /dev/null
+    else
+      echo '{}' > "${CONFIG_PATH}"
+    fi
+  fi
+
+  if [[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]]; then
+    sudo "${PROJECT_DIR:-.}/bin/jq" 'del(.'"${config_option}"')' "${CONFIG_PATH}" >"${tmpfile}"
+    sudo cp "${tmpfile}" "${CONFIG_PATH}"
+  else
+    "${PROJECT_DIR:-.}/bin/jq" 'del(.'"${config_option}"')' "${CONFIG_PATH}" >"${tmpfile}"
+    cp "${tmpfile}" "${CONFIG_PATH}"
   fi
-  
-  jq 'del(.'"${config_option}"')' "${CONFIG_PATH}" >"${tmpfile}"
-  cp "${tmpfile}" "${CONFIG_PATH}"
   rm "${tmpfile}"
 }
 
@@ -75,9 +85,9 @@ remove_element() {
 container_login() {
   local username="$1"
   local registry="$2"
-  
+
   if [[ "${CONTAINER_RUNTIME}" == "podman" ]]; then
-    podman login --username "${username}" --password-stdin "${registry}"
+    sudo podman login --username "${username}" --password-stdin "${registry}"
   else
     docker login --username "${username}" --password-stdin "${registry}"
   fi
@@ -93,14 +103,22 @@ check_docker_daemon_is_running
 
 # Initialize config file if it doesn't exist
 if [[ ! -f "${CONFIG_PATH}" ]]; then
-  echo '{}' > "${CONFIG_PATH}"
+  if [[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]]; then
+    echo '{}' | sudo tee "${CONFIG_PATH}" > /dev/null
+  else
+    echo '{}' > "${CONFIG_PATH}"
+  fi
 fi
 
 if [[ -f "${CONFIG_PATH}" ]]; then
   if [[ "${RUNNING_IN_EVG:-"false"}" != "true" ]]; then
     # Check if login is actually required by making a HEAD request to ECR using existing credentials
     echo "Checking if container registry credentials are valid..."
-    ecr_auth=$(jq -r '.auths."268558157000.dkr.ecr.us-east-1.amazonaws.com".auth // empty' "${CONFIG_PATH}")
+    if [[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]]; then
+      ecr_auth=$(sudo "${PROJECT_DIR:-.}/bin/jq" -r '.auths."268558157000.dkr.ecr.us-east-1.amazonaws.com".auth // empty' "${CONFIG_PATH}")
+    else
+      ecr_auth=$("${PROJECT_DIR:-.}/bin/jq" -r '.auths."268558157000.dkr.ecr.us-east-1.amazonaws.com".auth // empty' "${CONFIG_PATH}")
+    fi
 
     if [[ -n "${ecr_auth}" ]]; then
       http_status=$(curl --head -s -o /dev/null -w "%{http_code}" --max-time 3 "https://268558157000.dkr.ecr.us-east-1.amazonaws.com/v2/dev/mongodb-kubernetes/manifests/latest" \
@@ -120,11 +138,20 @@ if [[ -f "${CONFIG_PATH}" ]]; then
 
   # There could be some leftovers on Evergreen (Docker-specific, skip for Podman)
   if [[ "${CONTAINER_RUNTIME}" == "docker" ]]; then
-    if grep -q "credsStore" "${CONFIG_PATH}"; then
-      remove_element "credsStore"
-    fi
-    if grep -q "credHelpers" "${CONFIG_PATH}"; then
-      remove_element "credHelpers"
+    if [[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]]; then
+      if sudo grep -q "credsStore" "${CONFIG_PATH}"; then
+        remove_element "credsStore"
+      fi
+      if sudo grep -q "credHelpers" "${CONFIG_PATH}"; then
+        remove_element "credHelpers"
+      fi
+    else
+      if grep -q "credsStore" "${CONFIG_PATH}"; then
+        remove_element "credsStore"
+      fi
+      if grep -q "credHelpers" "${CONFIG_PATH}"; then
+        remove_element "credHelpers"
+      fi
     fi
   fi
 fi
@@ -137,7 +164,7 @@ aws ecr get-login-password --region "us-east-1" | container_login "AWS" "2685581
 # by default docker tries to store credentials in an external storage (e.g. OS keychain) - not in the config.json
 # We need to store it as base64 string in config.json instead so we need to remove the "credsStore" element
 # This is Docker-specific behavior, Podman stores credentials directly in auth.json
-if [[ "${CONTAINER_RUNTIME}" == "docker" ]] && grep -q "credsStore" "${CONFIG_PATH}"; then
+if [[ "${CONTAINER_RUNTIME}" == "docker" ]] && (([[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]] && sudo grep -q "credsStore" "${CONFIG_PATH}") || ([[ "${CONFIG_PATH}" != "/root/.config/containers/auth.json" ]] && grep -q "credsStore" "${CONFIG_PATH}")); then
   remove_element "credsStore"
 
   # login again to store the credentials into the config.json
@@ -152,8 +179,13 @@ if [[ -n "${COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON:-}" ]]; then
   quay_io_auth_file=$(mktemp)
   config_tmp=$(mktemp)
   echo "${COMMUNITY_PRIVATE_PREVIEW_PULLSECRET_DOCKERCONFIGJSON}" | base64 -d > "${quay_io_auth_file}"
-  jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}"
-  mv "${config_tmp}" "${CONFIG_PATH}"
+  if [[ "${CONFIG_PATH}" == "/root/.config/containers/auth.json" ]]; then
+    sudo jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}"
+    sudo mv "${config_tmp}" "${CONFIG_PATH}"
+  else
+    jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}"
+    mv "${config_tmp}" "${CONFIG_PATH}"
+  fi
   rm "${quay_io_auth_file}"
 fi
 
diff --git a/scripts/dev/contexts/e2e_smoke_ibm_power b/scripts/dev/contexts/e2e_smoke_ibm_power
index 03384c26c..4ba998050 100644
--- a/scripts/dev/contexts/e2e_smoke_ibm_power
+++ b/scripts/dev/contexts/e2e_smoke_ibm_power
@@ -16,3 +16,4 @@ export CUSTOM_OM_VERSION
 
 export CUSTOM_MDB_VERSION=6.0.5
 export CUSTOM_MDB_PREV_VERSION=5.0.7
+export KUBE_ENVIRONMENT_NAME=minikube
diff --git a/scripts/dev/contexts/e2e_smoke_ibm_z b/scripts/dev/contexts/e2e_smoke_ibm_z
index 03384c26c..4ba998050 100644
--- a/scripts/dev/contexts/e2e_smoke_ibm_z
+++ b/scripts/dev/contexts/e2e_smoke_ibm_z
@@ -16,3 +16,4 @@ export CUSTOM_OM_VERSION
 
 export CUSTOM_MDB_VERSION=6.0.5
 export CUSTOM_MDB_PREV_VERSION=5.0.7
+export KUBE_ENVIRONMENT_NAME=minikube
diff --git a/scripts/dev/recreate_python_venv.sh b/scripts/dev/recreate_python_venv.sh
index a5388e40f..bd8b411db 100755
--- a/scripts/dev/recreate_python_venv.sh
+++ b/scripts/dev/recreate_python_venv.sh
@@ -4,76 +4,54 @@
 
 set -Eeou pipefail
 
-# Parse command line arguments
-INSTALL_REQUIREMENTS=true
+ensure_required_python() {
+    local required_version="${PYTHON_VERSION:-3.10}"
+    local major_minor
+    major_minor=$(echo "${required_version}" | grep -oE '^[0-9]+\.[0-9]+')
 
-while [[ $# -gt 0 ]]; do
-    case $1 in
-        --skip-requirements)
-            INSTALL_REQUIREMENTS=false
-            shift
-            ;;
-        -h|--help)
-            echo "Usage: $0 [--skip-requirements]"
-            echo "  --skip-requirements    Skip installing requirements.txt"
-            echo "  -h, --help            Show this help message"
-            exit 0
-            ;;
-        *)
-            echo "Unknown option: $1"
-            echo "Use -h or --help for usage information"
-            exit 1
-            ;;
-    esac
-done
+    echo "Checking for Python ${required_version} (${major_minor}.x)..." >&2
 
-source scripts/dev/set_env_context.sh
-
-# Ensure Python 3.10 is available, install if needed
-ensure_python310() {
-    echo "Checking current Python version..." >&2
-    
-    # Check if current python is 3.10
+    # Check if current python matches required version
     if command -v python3 &> /dev/null; then
         local version
-        if version=$(python3 --version 2>&1) && [[ "${version}" == *"Python 3.10"* ]]; then
-            echo "Found Python 3.10: ${version}" >&2
+        if version=$(python3 --version 2>&1) && [[ "${version}" == *"Python ${major_minor}"* ]]; then
+            echo "Found Python ${major_minor}: ${version}" >&2
             echo "python3"
             return 0
         else
             echo "Current python3 version: ${version}" >&2
         fi
     fi
-    
-    # Try to install Python 3.10 using pyenv if available
+
+    # Try to install required Python version using pyenv if available
     if command -v pyenv &> /dev/null; then
-        echo "Python 3.10 not found. Attempting to install via pyenv..." >&2
-        
-        # Check if any 3.10 version is already installed
-        if pyenv versions --bare | grep -q "^3\.10\."; then
+        echo "Python ${major_minor} not found. Attempting to install via pyenv..." >&2
+
+        # Check if any version in the required series is already installed
+        if pyenv versions --bare | grep -q "^${major_minor}\."; then
             local installed_version
-            installed_version=$(pyenv versions --bare | grep "^3\.10\." | head -1)
-            echo "Found existing pyenv Python 3.10: ${installed_version}" >&2
+            installed_version=$(pyenv versions --bare | grep "^${major_minor}\." | head -1)
+            echo "Found existing pyenv Python ${major_minor}: ${installed_version}" >&2
             pyenv global "${installed_version}"
             echo "python3"
             return 0
         fi
-        
-        # Install latest Python 3.10
-        local latest_310
-        latest_310=$(pyenv install --list | grep -E "^[[:space:]]*3\.10\.[0-9]+$" | tail -1 | xargs)
-        if [[ -n "${latest_310}" ]]; then
-            echo "Installing Python ${latest_310} via pyenv..." >&2
-            if pyenv install "${latest_310}"; then
-                pyenv global "${latest_310}"
+
+        # Install latest version in the required series
+        local latest_version
+        latest_version=$(pyenv install --list | grep -E "^[[:space:]]*${major_minor}\.[0-9]+$" | tail -1 | xargs)
+        if [[ -n "${latest_version}" ]]; then
+            echo "Installing Python ${latest_version} via pyenv..." >&2
+            if pyenv install "${latest_version}"; then
+                pyenv global "${latest_version}"
                 echo "python3"
                 return 0
             fi
         fi
     fi
-    
-    echo "Error: No suitable Python 3.10 installation found and unable to install via pyenv." >&2
-    echo "Please ensure Python 3.10 is installed or pyenv is available." >&2
+
+    echo "Error: No suitable Python ${major_minor} installation found and unable to install via pyenv." >&2
+    echo "Please ensure Python ${major_minor} is installed or pyenv is available." >&2
     return 1
 }
 
@@ -83,22 +61,16 @@ if [[ -d "${PROJECT_DIR}"/venv ]]; then
   rm -rf "venv"
 fi
 
-# Ensure Python 3.10 is available
-python_bin=$(ensure_python310)
+# Ensure required Python version is available
+python_bin=$(ensure_required_python)
 
 echo "Using python from the following path: ${python_bin}"
 
 "${python_bin}" -m venv venv
 source venv/bin/activate
 pip install --upgrade pip
-
-if [[ "${INSTALL_REQUIREMENTS}" == "true" ]]; then
-    echo "Installing requirements.txt..."
-    pip install -r requirements.txt
-else
-    echo "Skipping requirements.txt installation (--skip-requirements flag used)"
-fi
-
+echo "Installing requirements.txt..."
+pip install -r requirements.txt
 echo "Python venv was recreated successfully."
 echo "Current python path: $(which python)"
 python --version
diff --git a/scripts/evergreen/setup_kubernetes_environment.sh b/scripts/evergreen/setup_kubernetes_environment.sh
index 707231c9f..6edaad50d 100755
--- a/scripts/evergreen/setup_kubernetes_environment.sh
+++ b/scripts/evergreen/setup_kubernetes_environment.sh
@@ -30,6 +30,8 @@ elif [ "${KUBE_ENVIRONMENT_NAME}" = "kind" ] || [ "${KUBE_ENVIRONMENT_NAME}" = "
     scripts/dev/recreate_kind_cluster.sh "kind"
 elif [[ "${KUBE_ENVIRONMENT_NAME}" = "multi" && "${CLUSTER_TYPE}" == "kind" ]]; then
     scripts/dev/recreate_kind_clusters.sh
+elif [[ "${KUBE_ENVIRONMENT_NAME}" = "minikube" ]]; then
+    echo "Nothing to do for minikube"
 else
     echo "KUBE_ENVIRONMENT_NAME not recognized"
     echo "value is <<${KUBE_ENVIRONMENT_NAME}>>. If empty it means it was not set"
diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh
index 9d4884530..f4f5d643b 100755
--- a/scripts/evergreen/setup_minikube_host.sh
+++ b/scripts/evergreen/setup_minikube_host.sh
@@ -38,7 +38,9 @@ run_setup_step() {
 }
 
 # Setup Python environment (needed for AWS CLI pip installation)
-run_setup_step "Python Virtual Environment" "scripts/dev/recreate_python_venv.sh" "--skip-requirements"
+
+export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1
+run_setup_step "Python Virtual Environment" "scripts/dev/recreate_python_venv.sh"
 
 run_setup_step "AWS CLI Setup" "scripts/evergreen/setup_aws.sh"
 
diff --git a/scripts/evergreen/teardown_kubernetes_environment.sh b/scripts/evergreen/teardown_kubernetes_environment.sh
index e5e2bd869..ec0c59966 100755
--- a/scripts/evergreen/teardown_kubernetes_environment.sh
+++ b/scripts/evergreen/teardown_kubernetes_environment.sh
@@ -1,10 +1,16 @@
 #!/usr/bin/env bash
 
-set -Eeou pipefail
+set -Eeoux pipefail
 
 source scripts/dev/set_env_context.sh
 
 if [ "${KUBE_ENVIRONMENT_NAME}" = "kind" ]; then
+    docker system prune -a -f
     echo "Deleting Kind cluster"
     kind delete clusters --all
 fi
+
+if [ "${KUBE_ENVIRONMENT_NAME}" = "minikube" ]; then
+    echo "Deleting minikube cluster"
+    "${PROJECT_DIR:-.}/bin/minikube" delete
+fi
diff --git a/scripts/minikube/setup_minikube_host.sh b/scripts/minikube/setup_minikube_host.sh
index cf39e8f4d..16f1c1c31 100755
--- a/scripts/minikube/setup_minikube_host.sh
+++ b/scripts/minikube/setup_minikube_host.sh
@@ -30,17 +30,7 @@ EOF
 # retrieve arch variable off the shell command line
 ARCH=${1-"$(uname -m)"}
 
-# Validate architecture
-case "${ARCH}" in
-  s390x|ppc64le|x86_64|aarch64)
-    echo "Setting up minikube host for architecture: ${ARCH}"
-    ;;
-  *)
-    echo "ERROR: Unsupported architecture: ${ARCH}"
-    echo "Supported architectures: s390x, ppc64le, x86_64, aarch64"
-    exit 1
-    ;;
-esac
+echo "Setting up minikube host for architecture: ${ARCH}"
 
 download_minikube() {
   echo "Downloading minikube for ${ARCH}..."
@@ -52,17 +42,23 @@ setup_local_registry_and_custom_image() {
   if [[ "${ARCH}" == "ppc64le" ]]; then
     echo ">>> Setting up local registry and custom kicbase image for ppc64le..."
 
-    # Check if local registry is running
-    if ! podman ps --filter "name=registry" --format "{{.Names}}" | grep -q "^registry$"; then
+    # Check if local registry is running (with fallback for namespace issues)
+    registry_running=false
+    if curl -s http://localhost:5000/v2/_catalog >/dev/null 2>&1; then
+      echo "Registry detected via HTTP check (podman ps failed)"
+      registry_running=true
+    fi
+
+    if ! $registry_running; then
       echo "Starting local container registry on port 5000..."
-      podman run -d -p 5000:5000 --name registry --restart=always docker.io/library/registry:2 || {
-        echo "Registry might already exist, trying to start it..."
-        podman start registry || {
-          echo "Removing existing registry and creating new one..."
-          podman rm -f registry 2>/dev/null || true
-          podman run -d -p 5000:5000 --name registry --restart=always docker.io/library/registry:2
-        }
-      }
+
+      # Clean up any existing registry first
+      sudo podman rm -f registry 2>/dev/null || true
+
+      if ! sudo podman run -d -p 5000:5000 --name registry --restart=always docker.io/library/registry:2; then
+        echo "❌ Failed to start local registry - trying alternative approach"
+        exit 1
+      fi
 
       # Wait for registry to be ready
       echo "Waiting for registry to be ready..."
@@ -76,73 +72,63 @@ setup_local_registry_and_custom_image() {
       echo "✅ Local registry already running"
     fi
 
-    # Configure system-wide podman to trust local registry (with backup)
-    echo "Configuring system registries.conf to trust local registry..."
-
-    # Backup existing registries.conf if it exists
-    if [[ -f /etc/containers/registries.conf ]]; then
-      echo "Backing up existing registries.conf..."
-      sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.minikube-backup
-    fi
-
-    # Create a clean registries.conf that includes insecure local registry
-    sudo tee /etc/containers/registries.conf << 'EOF'
-unqualified-search-registries = ["registry.access.redhat.com", "registry.redhat.io", "docker.io"]
+    # Configure podman to trust local registry (both user and root level for minikube)
+    echo "Configuring registries.conf to trust local registry..."
 
+    # User-level config
+    mkdir -p ~/.config/containers
+    cat > ~/.config/containers/registries.conf << 'EOF'
 [[registry]]
 location = "localhost:5000"
 insecure = true
+EOF
 
-short-name-mode = "permissive"
+    # Root-level config (since minikube uses sudo podman)
+    sudo mkdir -p /root/.config/containers
+    sudo tee /root/.config/containers/registries.conf << 'EOF' >/dev/null
+[[registry]]
+location = "localhost:5000"
+insecure = true
 EOF
 
-    # Check if custom image already exists in local registry
+    echo "✅ Registry configuration created for both user and root"
+    custom_image_tag="localhost:5000/kicbase:v0.0.47"
+
+    # Determine image tag
+    custom_image_tag="localhost:5000/kicbase:v0.0.47"
     if curl -s http://localhost:5000/v2/kicbase/tags/list | grep -q "v0.0.47"; then
-      echo "✅ Custom kicbase image already exists in local registry"
+      echo "Custom kicbase image already exists in local registry"
       return 0
     fi
 
     # Build custom kicbase image with crictl
     echo "Building custom kicbase image with crictl for ppc64le..."
 
-    # Create build directory if it doesn't exist
+    # Build custom kicbase image
     mkdir -p "${PROJECT_DIR:-.}/scripts/minikube/kicbase"
-
-    # Create Dockerfile for custom kicbase
     cat > "${PROJECT_DIR:-.}/scripts/minikube/kicbase/Dockerfile" << 'EOF'
 FROM gcr.io/k8s-minikube/kicbase:v0.0.47
-
-# Install crictl for ppc64le if needed
 RUN if [ "$(uname -m)" = "ppc64le" ]; then \
-        echo "Installing crictl for ppc64le architecture..." && \
         CRICTL_VERSION="v1.28.0" && \
         curl -L "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-ppc64le.tar.gz" \
             -o /tmp/crictl.tar.gz && \
         tar -C /usr/bin -xzf /tmp/crictl.tar.gz && \
         chmod +x /usr/bin/crictl && \
-        rm /tmp/crictl.tar.gz && \
-        echo "crictl installed successfully" && \
-        crictl --version; \
-    else \
-        echo "Not ppc64le architecture, skipping crictl installation"; \
+        rm /tmp/crictl.tar.gz; \
     fi
-
-# Verify crictl is available
-RUN command -v crictl >/dev/null 2>&1 && echo "crictl is available" || echo "crictl not found"
 EOF
 
-    # Build and push to local registry
-    echo "Building custom kicbase image..."
     cd "${PROJECT_DIR:-.}/scripts/minikube/kicbase"
-
-    podman build -t localhost:5000/kicbase:v0.0.47 .
-
-    echo "Pushing custom image to local registry..."
-    podman push localhost:5000/kicbase:v0.0.47 --tls-verify=false
-
-    cd - > /dev/null
-
-    echo "✅ Custom kicbase image with crictl ready in local registry"
+    sudo podman build -t "${custom_image_tag}" . || {
+      echo "Failed to build custom image"
+      return 1
+    }
+    sudo podman push "${custom_image_tag}" --tls-verify=false || {
+      echo "Failed to push to registry"
+      return 1
+    }
+    cd - >/dev/null
+    echo "Custom kicbase image ready: ${custom_image_tag}"
   fi
   return 0
 }
@@ -158,15 +144,14 @@ start_minikube_cluster() {
     rm -rf ~/.minikube/machines/minikube
   fi
 
-  # Delete any existing minikube cluster to start fresh
   echo "Ensuring clean minikube state..."
   "${PROJECT_DIR:-.}/bin/minikube" delete 2>/dev/null || true
 
   local start_args=("--driver=podman")
 
-  # Use custom kicbase image for ppc64le with crictl included
   if [[ "${ARCH}" == "ppc64le" ]]; then
     echo "Using custom kicbase image for ppc64le with crictl..."
+
     start_args+=("--base-image=localhost:5000/kicbase:v0.0.47")
     start_args+=("--insecure-registry=localhost:5000")
   fi
@@ -191,26 +176,17 @@ setup_podman() {
   # Check if podman is already available
   if command -v podman &> /dev/null; then
     echo "✅ Podman already installed"
-    
-    # Reset podman if it's in an invalid state
-    if podman info 2>&1 | grep -q "invalid internal status"; then
-      echo "Resetting podman due to invalid internal status..."
-      podman system migrate || true
-      podman system reset --force || true
-    fi
-  else
-    echo "Installing podman..."
-    sudo dnf install -y podman
-  fi
 
-  # Configure podman for CI environment
-  echo "Configuring Podman for ${ARCH} CI environment..."
+    # Diagnose podman state
+    echo "=== Podman Diagnostics ==="
+    echo "User: $(whoami), UID: $(id -u)"
+    echo "User namespace support: $(cat /proc/self/uid_map 2>/dev/null || echo 'not available')"
+    echo "Systemctl user status:"
+    systemctl --user status podman.socket 2>/dev/null || echo "podman.socket not active"
+    echo "Running 'sudo podman info' command..."
+    sudo podman info 2>&1
+  fi
 
-  # Enable lingering for the current user to fix systemd session issues
-  current_user=$(whoami)
-  current_uid=$(id -u)
-  echo "Enabling systemd lingering for user ${current_user} (UID: ${current_uid})"
-  sudo loginctl enable-linger "${current_uid}" 2>/dev/null || true
 
   # Configure podman to use cgroupfs instead of systemd in CI
   mkdir -p ~/.config/containers
@@ -223,11 +199,6 @@ events_logger = "file"
 cgroup_manager = "cgroupfs"
 EOF
 
-  # Start podman service if not running
-  systemctl --user enable podman.socket 2>/dev/null || true
-  systemctl --user start podman.socket 2>/dev/null || true
-
-  echo "✅ Podman configured successfully for CI"
 }
 
 # Setup podman and container runtime
@@ -249,23 +220,6 @@ else
     exit 1
 fi
 
-echo ""
-echo ">>> Verifying podman installation..."
-if command -v podman &> /dev/null; then
-    podman_version=$(podman --version 2>/dev/null)
-    echo "✅ Podman installed successfully: ${podman_version}"
-
-    # Check podman info
-    if podman info &>/dev/null; then
-        echo "✅ Podman is working correctly"
-    else
-        echo "⚠️  Podman may need additional configuration"
-    fi
-else
-    echo "❌ Podman installation failed - podman command not found"
-    exit 1
-fi
-
 if [[ "${ARCH}" == "ppc64le" ]]; then
     echo ""
     echo ">>> Note: crictl will be patched into the minikube container after startup"
@@ -295,7 +249,6 @@ echo "Container Runtime: podman"
 echo "Minikube Driver: podman"
 echo "Minikube: Default cluster"
 echo "Minikube: ${minikube_version}"
-echo "Podman: ${podman_version}"
 echo "CNI: bridge (default)"
 if [[ "${ARCH}" == "ppc64le" ]]; then
     echo "Special Config: Custom kicbase image with crictl via local registry"

From d72017af3825f5f99393a87e9d40b49037a53aff Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Fri, 1 Aug 2025 17:27:27 +0200
Subject: [PATCH 18/23] handle kube secret for podman

---
 scripts/funcs/kubernetes | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/scripts/funcs/kubernetes b/scripts/funcs/kubernetes
index 5377d8927..d4426ef3c 100644
--- a/scripts/funcs/kubernetes
+++ b/scripts/funcs/kubernetes
@@ -98,12 +98,25 @@ create_image_registries_secret() {
     context=$1
     namespace=$2
     secret_name=$3
+    
+    # Detect the correct config file path based on container runtime
+    local config_file
+    if command -v podman &> /dev/null && (podman info &> /dev/null || sudo podman info &> /dev/null); then
+      # For Podman, use root's auth.json since minikube uses sudo podman
+      config_file="/root/.config/containers/auth.json"
+      echo "Using Podman config: ${config_file}"
+    else
+      # For Docker, use standard docker config
+      config_file="${HOME}/.docker/config.json"
+      echo "Using Docker config: ${config_file}"
+    fi
+    
     # shellcheck disable=SC2154
     if kubectl --context "${context}" get namespace "${namespace}"; then
       kubectl --context "${context}" -n "${namespace}" delete secret "${secret_name}" --ignore-not-found
       echo "${context}: Creating ${namespace}/${secret_name} pull secret"
       kubectl --context "${context}" -n "${namespace}" create secret generic "${secret_name}" \
-        --from-file=.dockerconfigjson="${HOME}/.docker/config.json" --type=kubernetes.io/dockerconfigjson
+        --from-file=.dockerconfigjson="${config_file}" --type=kubernetes.io/dockerconfigjson
     else
       echo "Skipping creating pull secret in ${context}/${namespace}. The namespace doesn't exist yet."
     fi

From 02073269560d6b9413690a3e08aea7f24f57562d Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Fri, 1 Aug 2025 17:38:59 +0200
Subject: [PATCH 19/23] handle path

---
 scripts/dev/configure_docker_auth.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/dev/configure_docker_auth.sh b/scripts/dev/configure_docker_auth.sh
index 4edf68817..a36896970 100755
--- a/scripts/dev/configure_docker_auth.sh
+++ b/scripts/dev/configure_docker_auth.sh
@@ -87,7 +87,7 @@ container_login() {
   local registry="$2"
 
   if [[ "${CONTAINER_RUNTIME}" == "podman" ]]; then
-    sudo podman login --username "${username}" --password-stdin "${registry}"
+    sudo podman login --authfile "${CONFIG_PATH}" --username "${username}" --password-stdin "${registry}"
   else
     docker login --username "${username}" --password-stdin "${registry}"
   fi

From 0bd74d88434f361420c2f29da20b304ef6351155 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Fri, 1 Aug 2025 18:05:12 +0200
Subject: [PATCH 20/23] handle auth issue for secret

---
 scripts/funcs/kubernetes | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/scripts/funcs/kubernetes b/scripts/funcs/kubernetes
index d4426ef3c..11250422d 100644
--- a/scripts/funcs/kubernetes
+++ b/scripts/funcs/kubernetes
@@ -101,10 +101,17 @@ create_image_registries_secret() {
     
     # Detect the correct config file path based on container runtime
     local config_file
+    local temp_config_file=""
     if command -v podman &> /dev/null && (podman info &> /dev/null || sudo podman info &> /dev/null); then
       # For Podman, use root's auth.json since minikube uses sudo podman
       config_file="/root/.config/containers/auth.json"
       echo "Using Podman config: ${config_file}"
+      
+      # Create a temporary copy that the current user can read
+      temp_config_file=$(mktemp)
+      sudo cp "${config_file}" "${temp_config_file}"
+      sudo chown "$(whoami):$(whoami)" "${temp_config_file}"
+      config_file="${temp_config_file}"
     else
       # For Docker, use standard docker config
       config_file="${HOME}/.docker/config.json"
@@ -120,6 +127,11 @@ create_image_registries_secret() {
     else
       echo "Skipping creating pull secret in ${context}/${namespace}. The namespace doesn't exist yet."
     fi
+    
+    # Clean up temporary file
+    if [[ -n "${temp_config_file}" ]] && [[ -f "${temp_config_file}" ]]; then
+      rm -f "${temp_config_file}"
+    fi
   }
 
   echo "Creating/updating pull secret from docker configured file"

From 5cfdd82897e053dc1c641d69a6b72fa34efa00df Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Fri, 1 Aug 2025 19:38:51 +0200
Subject: [PATCH 21/23] use lucian evg and remove debug mode

---
 scripts/dev/contexts/evg-private-context             | 8 ++++++++
 scripts/evergreen/e2e/e2e.sh                         | 2 +-
 scripts/evergreen/setup_minikube_host.sh             | 2 +-
 scripts/evergreen/teardown_kubernetes_environment.sh | 2 +-
 4 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/scripts/dev/contexts/evg-private-context b/scripts/dev/contexts/evg-private-context
index 8f25842e8..8c1dfd672 100644
--- a/scripts/dev/contexts/evg-private-context
+++ b/scripts/dev/contexts/evg-private-context
@@ -122,3 +122,11 @@ export cognito_workload_url="${cognito_workload_url}"
 export cognito_workload_user_id="${cognito_workload_user_id}"
 
 export MDB_UPDATE_LICENSES=true
+
+export BASE_REPO_URL="268558157000.dkr.ecr.us-east-1.amazonaws.com/lucian.tosa"
+export REGISTRY="${BASE_REPO_URL}"
+export INIT_DATABASE_IMAGE_REPOSITORY="${BASE_REPO_URL}/mongodb-kubernetes-init-database"
+export OPERATOR_REGISTRY=${BASE_REPO_URL}
+export DATABASE_REGISTRY=${BASE_REPO_URL}
+export INIT_DATABASE_REGISTRY=${BASE_REPO_URL}
+
diff --git a/scripts/evergreen/e2e/e2e.sh b/scripts/evergreen/e2e/e2e.sh
index eb6140a22..12f839b86 100755
--- a/scripts/evergreen/e2e/e2e.sh
+++ b/scripts/evergreen/e2e/e2e.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -Eeoux pipefail
+set -Eeou pipefail
 
 start_time=$(date +%s)
 
diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh
index f4f5d643b..cc85dbcd4 100755
--- a/scripts/evergreen/setup_minikube_host.sh
+++ b/scripts/evergreen/setup_minikube_host.sh
@@ -5,7 +5,7 @@
 # Can be run on static hosts for testing and verification
 
 source scripts/dev/set_env_context.sh
-set -Eeoux pipefail
+set -Eeou pipefail
 
 echo "=========================================="
 echo "Setting up minikube host with multi-architecture support"
diff --git a/scripts/evergreen/teardown_kubernetes_environment.sh b/scripts/evergreen/teardown_kubernetes_environment.sh
index ec0c59966..a9babfbed 100755
--- a/scripts/evergreen/teardown_kubernetes_environment.sh
+++ b/scripts/evergreen/teardown_kubernetes_environment.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -Eeoux pipefail
+set -Eeou pipefail
 
 source scripts/dev/set_env_context.sh
 

From 7c0cfbd3e2ba1dfeb17d1f3beec0c59a464730be Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Mon, 4 Aug 2025 11:07:17 +0200
Subject: [PATCH 22/23] fix z and workdir setup

---
 .evergreen-functions.yml                 | 15 ++++++++-------
 scripts/evergreen/setup_jq.sh            |  5 ++---
 scripts/evergreen/setup_minikube_host.sh |  2 +-
 scripts/funcs/printing                   | 23 +++++++++++++++++++++++
 4 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml
index c67634152..8cf892a2b 100644
--- a/.evergreen-functions.yml
+++ b/.evergreen-functions.yml
@@ -51,6 +51,13 @@ functions:
 
   ### Setup Functions ###
 
+  setup_jq: &setup_jq
+    command: subprocess.exec
+    type: setup
+    params:
+      working_dir: src/github.com/mongodb/mongodb-kubernetes
+      binary: scripts/evergreen/setup_jq.sh
+
   setup_context: &setup_context # Running the first switch is important to fill the workdir and other important initial env vars
     command: shell.exec
     type: setup
@@ -103,6 +110,7 @@ functions:
       type: setup
       params:
         command: "git config --global user.email 'kubernetes-hosted-team@mongodb.com'"
+    - *setup_jq # we need jq in the context
     - *setup_context
 
   setup_kubectl: &setup_kubectl
@@ -112,13 +120,6 @@ functions:
       working_dir: src/github.com/mongodb/mongodb-kubernetes
       binary: scripts/evergreen/setup_kubectl.sh
 
-  setup_jq: &setup_jq
-    command: subprocess.exec
-    type: setup
-    params:
-      working_dir: src/github.com/mongodb/mongodb-kubernetes
-      binary: scripts/evergreen/setup_jq.sh
-
   setup_shellcheck:
     command: subprocess.exec
     type: setup
diff --git a/scripts/evergreen/setup_jq.sh b/scripts/evergreen/setup_jq.sh
index 1f260883a..8905d4d46 100755
--- a/scripts/evergreen/setup_jq.sh
+++ b/scripts/evergreen/setup_jq.sh
@@ -7,14 +7,13 @@
 
 set -Eeou pipefail
 
-source scripts/dev/set_env_context.sh
 source scripts/funcs/install
 
 # Detect and map architecture for jq releases
 detect_jq_architecture() {
     local arch
     arch=$(uname -m)
-    
+
     case "${arch}" in
         x86_64)
             echo "amd64"
@@ -38,4 +37,4 @@ detect_jq_architecture() {
 jq_arch=$(detect_jq_architecture)
 echo "Detected architecture: $(uname -m), using jq architecture: ${jq_arch}"
 
-download_and_install_binary "${PROJECT_DIR:-.}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.8.1/jq-linux-${jq_arch}"
+download_and_install_binary "${PROJECT_DIR:-${workdir:-.}}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.8.1/jq-linux-${jq_arch}"
diff --git a/scripts/evergreen/setup_minikube_host.sh b/scripts/evergreen/setup_minikube_host.sh
index cc85dbcd4..f4f5d643b 100755
--- a/scripts/evergreen/setup_minikube_host.sh
+++ b/scripts/evergreen/setup_minikube_host.sh
@@ -5,7 +5,7 @@
 # Can be run on static hosts for testing and verification
 
 source scripts/dev/set_env_context.sh
-set -Eeou pipefail
+set -Eeoux pipefail
 
 echo "=========================================="
 echo "Setting up minikube host with multi-architecture support"
diff --git a/scripts/funcs/printing b/scripts/funcs/printing
index 0ee4b5ec0..7d55e5af9 100644
--- a/scripts/funcs/printing
+++ b/scripts/funcs/printing
@@ -21,3 +21,26 @@ prepend() {
 
 export RED='\033[0;31m'
 export NO_COLOR='\033[0m'
+
+run_setup_step() {
+    local step_name="$1"
+    shift
+    local script_command=("$@")
+
+    echo ""
+    echo ">>> Running: ${step_name}"
+    echo ">>> Command: ${script_command[*]}"
+
+    local script_path="${script_command[0]}"
+    if [[ -f "${script_path}" ]]; then
+        if "${script_command[@]}"; then
+            echo "✅ ${step_name} completed successfully"
+        else
+            echo "❌ ${step_name} failed"
+            exit 1
+        fi
+    else
+        echo "❌ Script not found: ${script_path}"
+        exit 1
+    fi
+}

From 718d3cc409e2f3596ebc322924fce60eb26059b4 Mon Sep 17 00:00:00 2001
From: Nam Nguyen <nam.nguyen@mongodb.com>
Date: Mon, 4 Aug 2025 11:07:32 +0200
Subject: [PATCH 23/23] fix z and workdir setup

---
 scripts/funcs/printing | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/scripts/funcs/printing b/scripts/funcs/printing
index 7d55e5af9..0ee4b5ec0 100644
--- a/scripts/funcs/printing
+++ b/scripts/funcs/printing
@@ -21,26 +21,3 @@ prepend() {
 
 export RED='\033[0;31m'
 export NO_COLOR='\033[0m'
-
-run_setup_step() {
-    local step_name="$1"
-    shift
-    local script_command=("$@")
-
-    echo ""
-    echo ">>> Running: ${step_name}"
-    echo ">>> Command: ${script_command[*]}"
-
-    local script_path="${script_command[0]}"
-    if [[ -f "${script_path}" ]]; then
-        if "${script_command[@]}"; then
-            echo "✅ ${step_name} completed successfully"
-        else
-            echo "❌ ${step_name} failed"
-            exit 1
-        fi
-    else
-        echo "❌ Script not found: ${script_path}"
-        exit 1
-    fi
-}
