diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
index 7a73b63..4433626 100644
--- a/.github/workflows/lint.yaml
+++ b/.github/workflows/lint.yaml
@@ -17,12 +17,12 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.22
+          go-version: 1.23
 
       - name: Install Helm
         uses: azure/setup-helm@v4
         with:
-          version: v3.14.4
+          version: v3.17.1
 
       - name: Install yq
         run: |
diff --git a/.github/workflows/nightly-build.yaml b/.github/workflows/nightly-build.yaml
index 40a6935..e64f92c 100644
--- a/.github/workflows/nightly-build.yaml
+++ b/.github/workflows/nightly-build.yaml
@@ -21,7 +21,7 @@ jobs:
       - name: Install Helm
         uses: azure/setup-helm@v4
         with:
-          version: v3.14.4
+          version: v3.17.1
 
       - name: Install yq
         run: |
@@ -31,11 +31,10 @@ jobs:
       - name: make build
         run: |
           make build > output.log 2>&1
-        continue-on-error: true
+        continue-on-error: false
 
       - name: Upload script output
-        if: failure()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: script-output
           path: output.log
diff --git a/.gitignore b/.gitignore
index 949b791..834eed3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+.idea
 charts/
 build/
 scratch
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f97b441..8bb9049 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## v0.3.0
+
+- Adding prebuilt workspace dashboard & alerts
+
 ## v0.2.1
 
 - Upgraded subcharts
@@ -23,4 +27,4 @@
 
 ## v0.0.1
 
-- Initial release
\ No newline at end of file
+- Initial release
diff --git a/CODEOWNERS b/CODEOWNERS
new file mode 100644
index 0000000..4521512
--- /dev/null
+++ b/CODEOWNERS
@@ -0,0 +1 @@
+* @dannykopping
\ No newline at end of file
diff --git a/README.gotmpl b/README.gotmpl
index 840296e..411d638 100644
--- a/README.gotmpl
+++ b/README.gotmpl
@@ -199,6 +199,22 @@ stringData:
   password: "<your password>" # this matches the "passwordKey" field above
 ```
 
+To add an Ingress for Grafana, define this in your `values.yaml`:
+
+```yaml
+grafana:
+  grafana.ini:
+    server:
+      domain: observability.example.com
+      root_url: "%(protocol)s://%(domain)s/grafana"
+      serve_from_sub_path: true
+  ingress:
+    enabled: true
+    hosts:
+      - "observability.example.com"
+    path: "/"
+```
+
 ## Subcharts
 
 {{ template "chart.requirementsTable" . }}
diff --git a/README.md b/README.md
index f479746..733eb70 100644
--- a/README.md
+++ b/README.md
@@ -199,14 +199,31 @@ stringData:
   password: "<your password>" # this matches the "passwordKey" field above
 ```
 
+To add an Ingress for Grafana, define this in your `values.yaml`:
+
+```yaml
+grafana:
+  grafana.ini:
+    server:
+      domain: observability.example.com
+      root_url: "%(protocol)s://%(domain)s/grafana"
+      serve_from_sub_path: true
+  ingress:
+    enabled: true
+    hosts:
+      - "observability.example.com"
+    path: "/"
+```
+
 ## Subcharts
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://grafana.github.io/helm-charts | grafana | ^v7.3.7 |
-| https://grafana.github.io/helm-charts | grafana-agent(grafana-agent) | ^0.37.0 |
-| https://grafana.github.io/helm-charts | loki | ^v6.3.4 |
-| https://prometheus-community.github.io/helm-charts | prometheus | ^v25.18.0 |
+| https://grafana.github.io/helm-charts | grafana | ~v7.3.7 |
+| https://grafana.github.io/helm-charts | grafana-agent(grafana-agent) | ~0.37.0 |
+| https://grafana.github.io/helm-charts | loki | ~v6.7.3 |
+| https://grafana.github.io/helm-charts | pyroscope | ~v1.14.1 |
+| https://prometheus-community.github.io/helm-charts | prometheus | ~v25.24.1 |
 
 Each subchart can be disabled by setting the `enabled` field to `false`.
 
@@ -228,7 +245,7 @@ values which are defined [here](https://github.com/grafana/helm-charts/tree/main
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| global.coder.alerts | object | `{"coderd":{"groups":{"CPU":{"delay":"10m","enabled":true,"period":"10m","thresholds":{"critical":0.9,"warning":0.8}},"Memory":{"delay":"10m","enabled":true,"thresholds":{"critical":0.9,"warning":0.8}},"Replicas":{"delay":"5m","enabled":true,"thresholds":{"critical":1,"notify":3,"warning":2}},"Restarts":{"delay":"1m","enabled":true,"period":"10m","thresholds":{"critical":3,"notify":1,"warning":2}},"WorkspaceBuildFailures":{"delay":"10m","enabled":true,"period":"10m","thresholds":{"critical":10,"notify":2,"warning":5}}}},"enterprise":{"groups":{"Licences":{"delay":"1m","enabled":true,"thresholds":{"critical":1,"warning":0.9}}}},"provisionerd":{"groups":{"Replicas":{"delay":"5m","enabled":true,"thresholds":{"critical":1,"notify":3,"warning":2}}}}}` | alerts for the various aspects of Coder |
+| global.coder.alerts | object | `{"coderd":{"groups":{"CPU":{"delay":"10m","enabled":true,"period":"10m","thresholds":{"critical":0.9,"warning":0.8}},"IneligiblePrebuilds":{"delay":"10m","enabled":true,"thresholds":{"notify":1}},"Memory":{"delay":"10m","enabled":true,"thresholds":{"critical":0.9,"warning":0.8}},"Replicas":{"delay":"5m","enabled":true,"thresholds":{"critical":1,"notify":3,"warning":2}},"Restarts":{"delay":"1m","enabled":true,"period":"10m","thresholds":{"critical":3,"notify":1,"warning":2}},"UnprovisionedPrebuiltWorkspaces":{"delay":"10m","enabled":true,"thresholds":{"warn":1}},"WorkspaceBuildFailures":{"delay":"10m","enabled":true,"period":"10m","thresholds":{"critical":10,"notify":2,"warning":5}}}},"enterprise":{"groups":{"Licences":{"delay":"1m","enabled":true,"thresholds":{"critical":1,"warning":0.9}}}},"provisionerd":{"groups":{"Replicas":{"delay":"5m","enabled":true,"thresholds":{"critical":1,"notify":3,"warning":2}}}}}` | alerts for the various aspects of Coder |
 | global.coder.coderdSelector | string | `"pod=~`coder.*`, pod!~`.*provisioner.*`"` | series selector for Prometheus/Loki to locate provisioner pods. ensure this uses backticks for quotes! |
 | global.coder.controlPlaneNamespace | string | `"coder"` | the namespace into which the control plane has been deployed. |
 | global.coder.externalProvisionersNamespace | string | `"coder"` | the namespace into which any external provisioners have been deployed. |
@@ -242,14 +259,16 @@ values which are defined [here](https://github.com/grafana/helm-charts/tree/main
 | global.dashboards.timerange | string | `"12h"` | how far back dashboards should look |
 | global.externalScheme | string | `"http"` |  |
 | global.externalZone | string | `"svc.cluster.local"` |  |
-| global.postgres | object | `{"alerts":{"groups":{"Basic":{"delay":"1m","enabled":true},"Connections":{"delay":"5m","enabled":true,"thresholds":{"critical":0.9,"notify":0.5,"warning":0.8}},"Notifications":{"delay":"15m","enabled":true,"thresholds":{"critical":0.9,"notify":0.5,"warning":0.8}}}},"database":"coder","hostname":"localhost","mountSecret":"secret-postgres","password":null,"port":5432,"sslmode":"disable","username":"coder"}` | postgres connection information NOTE: these settings are global so we can parameterise some values which get rendered by subcharts |
+| global.postgres | object | `{"alerts":{"groups":{"Basic":{"delay":"1m","enabled":true},"Connections":{"delay":"5m","enabled":true,"thresholds":{"critical":0.9,"notify":0.5,"warning":0.8}},"Notifications":{"delay":"15m","enabled":true,"thresholds":{"critical":0.9,"notify":0.5,"warning":0.8}}}},"database":"coder","exporter":{"image":"quay.io/prometheuscommunity/postgres-exporter"},"hostname":"localhost","mountSecret":"secret-postgres","password":null,"port":5432,"sslmode":"disable","sslrootcert":null,"username":"coder","volumeMounts":[],"volumes":[]}` | postgres connection information NOTE: these settings are global so we can parameterise some values which get rendered by subcharts |
 | global.postgres.alerts | object | `{"groups":{"Basic":{"delay":"1m","enabled":true},"Connections":{"delay":"5m","enabled":true,"thresholds":{"critical":0.9,"notify":0.5,"warning":0.8}},"Notifications":{"delay":"15m","enabled":true,"thresholds":{"critical":0.9,"notify":0.5,"warning":0.8}}}}` | alerts for postgres |
-| global.telemetry | object | `{"metrics":{"scrape_interval":"15s","scrape_timeout":"12s"}}` | control telemetry collection |
+| global.telemetry | object | `{"metrics":{"scrape_interval":"15s","scrape_timeout":"12s"},"profiling":{"scrape_interval":"60s","scrape_timeout":"60s"}}` | control telemetry collection |
 | global.telemetry.metrics | object | `{"scrape_interval":"15s","scrape_timeout":"12s"}` | control metric collection |
 | global.telemetry.metrics.scrape_interval | string | `"15s"` | how often the collector will scrape discovered pods |
 | global.telemetry.metrics.scrape_timeout | string | `"12s"` | how long a request will be allowed to wait before being canceled |
+| global.telemetry.profiling.scrape_interval | string | `"60s"` | how often the collector will scrape pprof endpoints |
+| global.telemetry.profiling.scrape_timeout | string | `"60s"` | how long a request will be allowed to wait before being canceled |
 | global.zone | string | `"svc"` |  |
-| grafana-agent.agent.clustering.enabled | bool | `false` |  |
+| grafana-agent.agent.clustering.enabled | bool | `true` |  |
 | grafana-agent.agent.configMap.create | bool | `false` |  |
 | grafana-agent.agent.configMap.key | string | `"config.river"` |  |
 | grafana-agent.agent.configMap.name | string | `"collector-config"` |  |
@@ -313,30 +332,38 @@ values which are defined [here](https://github.com/grafana/helm-charts/tree/main
 | grafana.datasources."datasources.yaml".apiVersion | int | `1` |  |
 | grafana.datasources."datasources.yaml".datasources[0].access | string | `"proxy"` |  |
 | grafana.datasources."datasources.yaml".datasources[0].editable | bool | `false` |  |
-| grafana.datasources."datasources.yaml".datasources[0].isDefault | bool | `true` |  |
-| grafana.datasources."datasources.yaml".datasources[0].name | string | `"metrics"` |  |
+| grafana.datasources."datasources.yaml".datasources[0].isDefault | bool | `false` |  |
+| grafana.datasources."datasources.yaml".datasources[0].name | string | `"pyroscope"` |  |
 | grafana.datasources."datasources.yaml".datasources[0].timeout | string | `"{{ add $.Values.global.dashboards.queryTimeout 5 }}"` |  |
-| grafana.datasources."datasources.yaml".datasources[0].type | string | `"prometheus"` |  |
-| grafana.datasources."datasources.yaml".datasources[0].uid | string | `"prometheus"` |  |
-| grafana.datasources."datasources.yaml".datasources[0].url | string | `"http://prometheus.{{ .Release.Namespace }}.{{ $.Values.global.zone }}"` |  |
+| grafana.datasources."datasources.yaml".datasources[0].type | string | `"grafana-pyroscope-datasource"` |  |
+| grafana.datasources."datasources.yaml".datasources[0].uid | string | `"pyroscope"` |  |
+| grafana.datasources."datasources.yaml".datasources[0].url | string | `"http://pyroscope.{{ .Release.Namespace }}.{{ $.Values.global.zone }}:4040"` |  |
 | grafana.datasources."datasources.yaml".datasources[1].access | string | `"proxy"` |  |
 | grafana.datasources."datasources.yaml".datasources[1].editable | bool | `false` |  |
-| grafana.datasources."datasources.yaml".datasources[1].isDefault | bool | `false` |  |
-| grafana.datasources."datasources.yaml".datasources[1].name | string | `"logs"` |  |
+| grafana.datasources."datasources.yaml".datasources[1].isDefault | bool | `true` |  |
+| grafana.datasources."datasources.yaml".datasources[1].name | string | `"metrics"` |  |
 | grafana.datasources."datasources.yaml".datasources[1].timeout | string | `"{{ add $.Values.global.dashboards.queryTimeout 5 }}"` |  |
-| grafana.datasources."datasources.yaml".datasources[1].type | string | `"loki"` |  |
-| grafana.datasources."datasources.yaml".datasources[1].uid | string | `"loki"` |  |
-| grafana.datasources."datasources.yaml".datasources[1].url | string | `"http://loki-gateway.{{ .Release.Namespace }}.{{ $.Values.global.zone }}"` |  |
+| grafana.datasources."datasources.yaml".datasources[1].type | string | `"prometheus"` |  |
+| grafana.datasources."datasources.yaml".datasources[1].uid | string | `"prometheus"` |  |
+| grafana.datasources."datasources.yaml".datasources[1].url | string | `"http://prometheus.{{ .Release.Namespace }}.{{ $.Values.global.zone }}"` |  |
+| grafana.datasources."datasources.yaml".datasources[2].access | string | `"proxy"` |  |
 | grafana.datasources."datasources.yaml".datasources[2].editable | bool | `false` |  |
 | grafana.datasources."datasources.yaml".datasources[2].isDefault | bool | `false` |  |
-| grafana.datasources."datasources.yaml".datasources[2].jsonData.sslmode | string | `"{{ .Values.global.postgres.sslmode }}"` |  |
-| grafana.datasources."datasources.yaml".datasources[2].name | string | `"postgres"` |  |
-| grafana.datasources."datasources.yaml".datasources[2].secureJsonData.password | string | `"{{ if .Values.global.postgres.password }}{{ .Values.global.postgres.password }}{{ else }}$PGPASSWORD{{ end }}"` |  |
+| grafana.datasources."datasources.yaml".datasources[2].name | string | `"logs"` |  |
 | grafana.datasources."datasources.yaml".datasources[2].timeout | string | `"{{ add $.Values.global.dashboards.queryTimeout 5 }}"` |  |
-| grafana.datasources."datasources.yaml".datasources[2].type | string | `"postgres"` |  |
-| grafana.datasources."datasources.yaml".datasources[2].uid | string | `"postgres"` |  |
-| grafana.datasources."datasources.yaml".datasources[2].url | string | `"{{ .Values.global.postgres.hostname }}:{{ .Values.global.postgres.port }}"` |  |
-| grafana.datasources."datasources.yaml".datasources[2].user | string | `"{{ .Values.global.postgres.username }}"` |  |
+| grafana.datasources."datasources.yaml".datasources[2].type | string | `"loki"` |  |
+| grafana.datasources."datasources.yaml".datasources[2].uid | string | `"loki"` |  |
+| grafana.datasources."datasources.yaml".datasources[2].url | string | `"http://loki-gateway.{{ .Release.Namespace }}.{{ $.Values.global.zone }}"` |  |
+| grafana.datasources."datasources.yaml".datasources[3].editable | bool | `false` |  |
+| grafana.datasources."datasources.yaml".datasources[3].isDefault | bool | `false` |  |
+| grafana.datasources."datasources.yaml".datasources[3].jsonData.sslmode | string | `"{{ .Values.global.postgres.sslmode }}"` |  |
+| grafana.datasources."datasources.yaml".datasources[3].name | string | `"postgres"` |  |
+| grafana.datasources."datasources.yaml".datasources[3].secureJsonData.password | string | `"{{ if .Values.global.postgres.password }}{{ .Values.global.postgres.password }}{{ else }}$PGPASSWORD{{ end }}"` |  |
+| grafana.datasources."datasources.yaml".datasources[3].timeout | string | `"{{ add $.Values.global.dashboards.queryTimeout 5 }}"` |  |
+| grafana.datasources."datasources.yaml".datasources[3].type | string | `"postgres"` |  |
+| grafana.datasources."datasources.yaml".datasources[3].uid | string | `"postgres"` |  |
+| grafana.datasources."datasources.yaml".datasources[3].url | string | `"{{ .Values.global.postgres.hostname }}:{{ .Values.global.postgres.port }}"` |  |
+| grafana.datasources."datasources.yaml".datasources[3].user | string | `"{{ .Values.global.postgres.username }}"` |  |
 | grafana.deploymentStrategy.type | string | `"Recreate"` |  |
 | grafana.enabled | bool | `true` |  |
 | grafana.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION | bool | `true` |  |
@@ -360,7 +387,12 @@ values which are defined [here](https://github.com/grafana/helm-charts/tree/main
 | grafana.extraConfigmapMounts[4].mountPath | string | `"/var/lib/grafana/dashboards/coder/4"` |  |
 | grafana.extraConfigmapMounts[4].name | string | `"dashboards-workspace-detail"` |  |
 | grafana.extraConfigmapMounts[4].readOnly | bool | `false` |  |
+| grafana.extraConfigmapMounts[5].configMap | string | `"dashboards-prebuilds"` |  |
+| grafana.extraConfigmapMounts[5].mountPath | string | `"/var/lib/grafana/dashboards/coder/5"` |  |
+| grafana.extraConfigmapMounts[5].name | string | `"dashboards-prebuilds"` |  |
+| grafana.extraConfigmapMounts[5].readOnly | bool | `false` |  |
 | grafana.fullnameOverride | string | `"grafana"` |  |
+| grafana.image.tag | string | `"10.4.19"` |  |
 | grafana.persistence.enabled | bool | `true` |  |
 | grafana.persistence.size | string | `"10Gi"` |  |
 | grafana.replicas | int | `1` |  |
@@ -464,6 +496,17 @@ values which are defined [here](https://github.com/grafana/helm-charts/tree/main
 | prometheus.server.service.type | string | `"ClusterIP"` |  |
 | prometheus.server.statefulSet.enabled | bool | `true` |  |
 | prometheus.serverFiles."prometheus.yml".rule_files[0] | string | `"/etc/config/alerts/*.yaml"` |  |
-| prometheus.serverFiles."prometheus.yml".scrape_configs | string | `nil` |  |
+| prometheus.serverFiles."prometheus.yml".scrape_configs | list | `[]` |  |
 | prometheus.testFramework.enabled | bool | `false` |  |
+| pyroscope.alloy.enabled | bool | `false` |  |
+| pyroscope.enabled | bool | `false` |  |
+| pyroscope.pyroscope.extraArgs."log.level" | string | `"info"` |  |
+| pyroscope.pyroscope.fullnameOverride | string | `"pyroscope"` |  |
+| pyroscope.pyroscope.persistence.enabled | bool | `true` |  |
+| pyroscope.pyroscope.persistence.size | string | `"10Gi"` |  |
+| pyroscope.pyroscope.replicaCount | int | `1` |  |
+| pyroscope.pyroscope.service.port | int | `4040` |  |
+| pyroscope.pyroscope.service.type | string | `"ClusterIP"` |  |
+| runbookViewer.image | string | `"dannyben/madness"` |  |
+| sqlExporter.image | string | `"burningalchemist/sql_exporter"` |  |
 
diff --git a/TESTING_KIND.md b/TESTING_KIND.md
new file mode 100644
index 0000000..6a3e1f3
--- /dev/null
+++ b/TESTING_KIND.md
@@ -0,0 +1,113 @@
+# Using kind to test observability
+
+<details>
+  <summary>If using nix</summary>
+
+    `nix-shell -p kind kubernetes-helm kubectl`
+
+</details>
+
+To test the observability chart locally without a kubernetes cluster, you can use [`kind` (Kubernetes in Docker)](https://kind.sigs.k8s.io/). This allows you to create a local Kubernetes cluster that can be used for testing purposes.
+
+```bash
+kind create cluster --name observability
+# To clean everything up, you can delete the cluster with:
+# kind delete cluster --name observability
+```
+
+First a `coder` deployment is required, as the observability stack is designed to monitor `coder` deployments.
+
+## Launch `coder` in a namespace
+
+Install `coder` to the namespaced `coder` following these docs: https://coder.com/docs/install/kubernetes
+
+In the `values.yaml` file, set `CODER_ACCESS_URL` to the empty string to get a public `try` url. Also turn on prometheus scraping.
+```yaml
+coder:
+  podAnnotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "2112"
+    pyroscope.io/scrape: "true"
+    pyroscope.io/port: '6060'
+  env:
+    - name: CODER_PPROF_ADDRESS
+      value: "0.0.0.0:6060"
+    - name: CODER_PPROF_ENABLE
+      value: "true"
+    - name: CODER_PROMETHEUS_ENABLE
+      value: "true"
+    - name: CODER_PG_CONNECTION_URL
+      valueFrom:
+        secretKeyRef:
+          name: coder-db-url
+          key: url
+    - name: CODER_OAUTH2_GITHUB_DEFAULT_PROVIDER_ENABLE
+      value: "false"
+    - name: CODER_ACCESS_URL
+      # Keep this an empty string to get a public `try` url
+      value: ""
+```
+
+### Verify `coder` installation
+
+Check the pods are running:
+
+```bash
+$ kubectl get pods -n coder
+NAME                   READY   STATUS    RESTARTS   AGE
+coder-76d47f8b-qhjvj   1/1     Running   0          28s
+postgresql-0           1/1     Running   0          76m
+```
+
+### Access `coder`
+
+The IP address of `coder` will be behind the docker container hosting the kind cluster. You can get the IP address of the coder deployment like so:
+
+```bash
+$ docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' <kind_cluster_container_name>
+172.18.0.2
+```
+
+And to know what port to access, find the load balancer service:
+
+```bash
+$ kubectl get svc -n coder
+NAME            TYPE           CLUSTER-IP     EXTERNAL-IP   PORT(S)        AGE
+coder           LoadBalancer   10.96.255.11   <pending>     80:31332/TCP   12m
+```
+
+So coder can be accessed at `http://172.18.0.2:31332`. After making an account, the public URL will be available at http://172.18.0.2:31332/deployment/overview. This will give you an https publicly accessible URL that you can use to access the coder deployment, and not have to use the IP address directly.
+
+# Install the observability stack (testing the chart!)
+
+Install the local observability chart using Helm into its own namespace.
+
+```bash
+# Set kubectl default namespace to `coder-observability` for easier usage
+kubectl config set-context --current --namespace=coder-observability
+# This will install the local observability chart into the `coder-observability` namespace
+
+helm install --namespace coder-observability --create-namespace observe .
+```
+
+## To update
+
+```bash
+helm upgrade --namespace coder-observability --create-namespace observe .
+```
+
+Sometimes the config maps do not take. A quick way to reset the helm installation is just to uninstall and install.
+
+```bash
+helm uninstall observe
+# Install using the `helm install ...` command from above.
+```
+
+## To port-forward services
+
+To view the monitoring services, you can port forward their UI's.
+
+- Grafana: `kubectl -n coder-observability port-forward svc/grafana 3000:80`
+- Prometheus: `kubectl -n coder-observability port-forward svc/prometheus 3001:80`
+- Pyroscope: `kubectl port-forward svc/pyroscope 3002:4040`
+- Grafana Agent: `kubectl port-forward svc/grafana-agent 3003:80`
\ No newline at end of file
diff --git a/coder-observability/Chart.lock b/coder-observability/Chart.lock
index 2aeb46c..e72a227 100644
--- a/coder-observability/Chart.lock
+++ b/coder-observability/Chart.lock
@@ -1,15 +1,18 @@
 dependencies:
+- name: pyroscope
+  repository: https://grafana.github.io/helm-charts
+  version: 1.14.1
 - name: grafana
   repository: https://grafana.github.io/helm-charts
   version: 7.3.12
 - name: prometheus
   repository: https://prometheus-community.github.io/helm-charts
-  version: 25.24.1
+  version: 25.24.2
 - name: loki
   repository: https://grafana.github.io/helm-charts
-  version: 6.7.3
+  version: 6.7.4
 - name: grafana-agent
   repository: https://grafana.github.io/helm-charts
   version: 0.37.0
-digest: sha256:10a5d2b617b691e0ed87ca9e31c86618e05ca3b8031ddb3b417610f47e8bb069
-generated: "2024-07-26T10:52:20.819468+02:00"
+digest: sha256:5a5f27f74bbf34848da9c1bab508d3b33fda19789016c2eda9608dcd6373921d
+generated: "2025-08-04T13:28:59.433447595-05:00"
diff --git a/coder-observability/Chart.yaml b/coder-observability/Chart.yaml
index 8de1651..c03e4e8 100644
--- a/coder-observability/Chart.yaml
+++ b/coder-observability/Chart.yaml
@@ -5,23 +5,27 @@ description: Gain insights into your Coder deployment
 type: application
 version: 0.1.0
 dependencies:
+  - name: pyroscope
+    condition: pyroscope.enabled
+    repository: https://grafana.github.io/helm-charts
+    version: '~v1.14.1'
   - name: grafana
     condition: grafana.enabled
     repository: https://grafana.github.io/helm-charts
-    version: '^v7.3.7'
+    version: '~v7.3.7'
   - name: prometheus
     condition: prometheus.enabled
     repository: https://prometheus-community.github.io/helm-charts
-    version: '^v25.18.0'
+    version: '~v25.24.1'
   - name: loki
     condition: loki.enabled
     repository: https://grafana.github.io/helm-charts
-    version: '^v6.3.4'
+    version: '~v6.7.3'
   - name: grafana-agent
     alias: grafana-agent
     condition: grafana-agent.enabled
     repository: https://grafana.github.io/helm-charts
-    version: '^0.37.0'
+    version: '~0.37.0'
 maintainers:
   - name: Coder Technologies, Inc.
     url: https://github.com/coder/observability/issues
diff --git a/coder-observability/runbooks/coderd.md b/coder-observability/runbooks/coderd.md
index 62c80f5..4a42444 100644
--- a/coder-observability/runbooks/coderd.md
+++ b/coder-observability/runbooks/coderd.md
@@ -76,3 +76,60 @@ Terraform plugin.
 Your Enterprise license is approaching or has exceeded the number of seats purchased.
 
 Please contact your Coder sales contact, or visit https://coder.com/contact/sales.
+
+## CoderdIneligiblePrebuilds
+
+Prebuilds only become eligible to be claimed by users once the workspace's agent is a) running and b) all of its startup
+scripts have completed.
+
+If a prebuilt workspace is not eligible, view its agent logs to diagnose the problem.
+
+## CoderdUnprovisionedPrebuiltWorkspaces
+
+The number of running prebuilt workspaces is lower than the desired instances. This could be for several reasons,
+ordered by likehood:
+
+### Experiment/License
+
+The prebuilds feature is currently gated behind an experiment *and* a premium license.
+
+Ensure that the prebuilds experiment is enabled with `CODER_EXPERIMENTS=workspace-prebuilds`, and that you have a premium
+license added.
+
+### Preset Validation Issue
+
+Templates which have prebuilds configured will require a configured preset defined, with ALL of the required parameters
+set in the preset. If any of these are missing, or any of the parameters - as defined - fail validation, then the prebuilds
+subsystem will refuse to attempt a workspace build.
+
+Consult the coderd logs for more information; look out for errors or warnings from the prebuilds subsystem.
+
+### Template Misconfiguration or Error
+
+Prebuilt workspaces cannot be provisioned due to some issue at `terraform apply`-time. This could be due to misconfigured
+cloud resources, improper authorization, or any number of other issues.
+
+Visit the Workspaces page, change the search term to `owner:prebuilds`, and view on the previously failed builds. The
+error will likely be quite obvious.
+
+### Provisioner Latency
+
+If your provisioners are overloaded and cannot process provisioner jobs quickly enough, prebuilt workspaces may be affected.
+There is no prioritization at present for prebuilt workspace jobs.
+
+Ensure your provisioners are appropriately resources (i.e. you have enough instances) to handle the concurrent build demand.
+
+### Use of Workspace Tags
+
+If you are using `coder_workspace_tags` ([docs](https://coder.com/docs/admin/templates/extending-templates/workspace-tags))
+in your template, chances are you do not have any provisioners running or they are under-resourced (see **Provisioner Latency**).
+
+Ensure your running provisioners are configured with your desired tags.
+
+### Reconciliation Loop Issue
+
+The prebuilds subsystem runs a _reconciliation loop_ which monitors the state of prebuilt workspaces to ensure the desired
+number of instances are present at all times. Workspace Prebuilds is currently a BETA feature and so there could be a bug
+in this _reconciliation loop_, which should be reported to Coder.
+
+Examine your coderd logs for any errors or warnings relating to prebuilds.
\ No newline at end of file
diff --git a/coder-observability/templates/_collector-config.tpl b/coder-observability/templates/_collector-config.tpl
index 555065c..1e3e4e7 100644
--- a/coder-observability/templates/_collector-config.tpl
+++ b/coder-observability/templates/_collector-config.tpl
@@ -74,6 +74,55 @@ discovery.relabel "pod_metrics" {
   {{- end }}
 }
 
+discovery.relabel "pod_pprof" {
+  targets = discovery.kubernetes.pods.targets
+  {{ $agent.commonRelabellings | nindent 6 }}
+  // The relabeling allows the actual pod scrape endpoint to be configured via the
+  // following annotations:
+  //
+  // * `pyroscope.io/scrape`: Only scrape pods that have a value of `true`.
+  // * `pyroscope.io/application-name`: Name of the application being profiled.
+  // * `pyroscope.io/scheme`: If the metrics endpoint is secured then you will need
+  // to set this to `https` & most likely set the `tls_config` of the scrape config.
+  // * `pyroscope.io/port`: Scrape the pod on the indicated port.
+  //
+  // Kubernetes labels will be added as Pyroscope labels on metrics via the
+  // `labelmap` relabeling action.
+  rule {
+    source_labels = ["__meta_kubernetes_pod_annotation_pyroscope_io_scrape"]
+    action = "keep"
+    regex = "true"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_annotation_pyroscope_io_application_name"]
+    action = "replace"
+    target_label = "__name__"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_annotation_pyroscope_io_scheme"]
+    action = "replace"
+    regex = "(https?)"
+    target_label = "__scheme__"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_annotation_pyroscope_io_port", "__meta_kubernetes_pod_ip"]
+    action = "replace"
+    regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
+    replacement = "[$2]:$1"
+    target_label = "__address__"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_annotation_pyroscope_io_port", "__meta_kubernetes_pod_ip"]
+    action = "replace"
+    regex = "(\\d+);((([0-9]+?)(\\.|$)){4})"
+    replacement = "$2:$1"
+    target_label = "__address__"
+  }
+  {{- if $agent.podMetricsRelabelRules -}}
+  {{ $agent.podMetricsRelabelRules | trim | nindent 2 }}
+  {{- end }}
+}
+
 local.file_match "pod_logs" {
   path_targets = discovery.relabel.pod_logs.output
 }
@@ -154,6 +203,26 @@ loki.write "loki" {
   }
 }
 
+{{ if .Values.pyroscope.enabled }}
+pyroscope.scrape "pods" {
+  targets = discovery.relabel.pod_pprof.output
+  forward_to = [pyroscope.write.pods.receiver]
+
+  scrape_interval = "{{ .Values.global.telemetry.profiling.scrape_interval }}"
+  scrape_timeout = "{{ .Values.global.telemetry.profiling.scrape_timeout }}"
+
+  clustering {
+    enabled = true
+  }
+}
+
+pyroscope.write "pods" {
+  endpoint {
+    url = "http://{{ include "pyroscope.fullname" .Subcharts.pyroscope }}.{{ .Release.Namespace }}.{{ .Values.global.zone }}:{{ .Values.pyroscope.pyroscope.service.port }}"
+  }
+}
+{{ end }}
+
 prometheus.scrape "pods" {
   targets = discovery.relabel.pod_metrics.output
   forward_to = [prometheus.relabel.pods.receiver]
diff --git a/coder-observability/templates/_helpers.tpl b/coder-observability/templates/_helpers.tpl
index a182b3d..0d8578d 100644
--- a/coder-observability/templates/_helpers.tpl
+++ b/coder-observability/templates/_helpers.tpl
@@ -63,10 +63,22 @@ Create the name of the service account to use
 
 {{/* Postgres connector string */}}
 {{- define "postgres-connector-string" -}}
-{{- if .Values.global.postgres.password -}}
-postgresql://{{ .Values.global.postgres.username }}:{{ urlquery .Values.global.postgres.password }}@{{ .Values.global.postgres.hostname }}:{{ .Values.global.postgres.port }}/postgres?sslmode={{ .Values.global.postgres.sslmode }}
-{{- else if .Values.global.postgres.mountSecret -}}
-postgresql://{{ .Values.global.postgres.username }}@{{ .Values.global.postgres.hostname }}:{{ .Values.global.postgres.port }}/postgres?sslmode={{ .Values.global.postgres.sslmode }}
+{{- if and .Values.global.postgres.password (eq .Values.global.postgres.sslmode "disable") -}}
+postgresql://{{ .Values.global.postgres.username }}:{{ urlquery .Values.global.postgres.password }}@{{ .Values.global.postgres.hostname }}:{{ .Values.global.postgres.port }}/{{ .Values.global.postgres.database }}?sslmode={{ .Values.global.postgres.sslmode }}
+{{- else if and .Values.global.postgres.password (ne .Values.global.postgres.sslmode "disable") -}}
+{{- if .Values.global.postgres.sslrootcert -}}
+postgresql://{{ .Values.global.postgres.username }}:{{ urlquery .Values.global.postgres.password }}@{{ .Values.global.postgres.hostname }}:{{ .Values.global.postgres.port }}/{{ .Values.global.postgres.database }}?sslmode={{ .Values.global.postgres.sslmode }}&sslrootcert={{ .Values.global.postgres.sslrootcert }}
+{{- else -}}
+postgresql://{{ .Values.global.postgres.username }}:{{ urlquery .Values.global.postgres.password }}@{{ .Values.global.postgres.hostname }}:{{ .Values.global.postgres.port }}/{{ .Values.global.postgres.database }}?sslmode={{ .Values.global.postgres.sslmode }}
+{{- end -}}
+{{- else if and .Values.global.postgres.mountSecret (eq .Values.global.postgres.sslmode "disable") -}}
+postgresql://{{ .Values.global.postgres.username }}@{{ .Values.global.postgres.hostname }}:{{ .Values.global.postgres.port }}/{{ .Values.global.postgres.database }}?sslmode={{ .Values.global.postgres.sslmode }}
+{{- else if and .Values.global.postgres.mountSecret (ne .Values.global.postgres.sslmode "disable") -}}
+{{- if .Values.global.postgres.sslrootcert -}}
+postgresql://{{ .Values.global.postgres.username }}@{{ .Values.global.postgres.hostname }}:{{ .Values.global.postgres.port }}/{{ .Values.global.postgres.database }}?sslmode={{ .Values.global.postgres.sslmode }}&sslrootcert={{ .Values.global.postgres.sslrootcert }}
+{{- else -}}
+postgresql://{{ .Values.global.postgres.username }}@{{ .Values.global.postgres.hostname }}:{{ .Values.global.postgres.port }}/{{ .Values.global.postgres.database }}?sslmode={{ .Values.global.postgres.sslmode }}
+{{- end -}}
 {{- else -}}
 {{ fail "either postgres.password or postgres.mountSecret must be defined" }}
 {{- end -}}
diff --git a/coder-observability/templates/configmap-prometheus-alerts.yaml b/coder-observability/templates/configmap-prometheus-alerts.yaml
index eec7171..bf9bcc4 100644
--- a/coder-observability/templates/configmap-prometheus-alerts.yaml
+++ b/coder-observability/templates/configmap-prometheus-alerts.yaml
@@ -4,7 +4,7 @@ metadata:
   name: metrics-alerts
   namespace: {{ .Release.Namespace }}
 data:
-  {{- $service := dict "service" "coder" -}}
+  {{- $service := dict "service" "coderd" -}}
 
   {{- with .Values.global.coder.alerts.coderd }} {{/* start-section */}}
   coderd.yaml: |-
@@ -104,6 +104,47 @@ data:
   {{- end }}
   {{- end }}
 
+  {{- with .groups.IneligiblePrebuilds }}
+  {{- $group := . }}
+  {{- if .enabled }}
+    - name: Coderd Ineligible Prebuilds
+      rules:
+      {{ $alert := "CoderdIneligiblePrebuilds" }}
+      {{- range $severity, $threshold := .thresholds }}
+      - alert: {{ $alert }}
+        expr: max by (template_name, preset_name) (coderd_prebuilt_workspaces_running - coderd_prebuilt_workspaces_eligible) > 0
+        for: {{ $group.delay }}
+        annotations:
+          summary: >
+            {{ `{{ $value }}` }} prebuilt workspace(s) are currently ineligible for claiming for the "{{ `{{ $labels.template_name }}` }}" template and "{{ `{{ $labels.preset_name }}` }}" preset.
+            This usually indicates that the agent has not started correctly, or is still running its startup scripts after an extended period of time.
+        labels:
+          severity: {{ $severity }}
+          runbook_url: {{ template "runbook-url" (deepCopy $ | merge (dict "alert" $alert) $service) }}
+      {{- end }}
+  {{- end }}
+  {{- end }}
+
+  {{- with .groups.UnprovisionedPrebuiltWorkspaces }}
+  {{- $group := . }}
+  {{- if .enabled }}
+    - name: Coderd Unprovisioned Prebuilt Workspaces
+      rules:
+      {{ $alert := "CoderdUnprovisionedPrebuiltWorkspaces" }}
+      {{- range $severity, $threshold := .thresholds }}
+      - alert: {{ $alert }}
+        expr: max by (template_name, preset_name) (coderd_prebuilt_workspaces_desired - coderd_prebuilt_workspaces_running) > 0
+        for: {{ $group.delay }}
+        annotations:
+          summary: >
+            {{ `{{ $value }}` }} prebuilt workspace(s) not yet been provisioned for the "{{ `{{ $labels.template_name }}` }}" template and "{{ `{{ $labels.preset_name }}` }}" preset.
+        labels:
+          severity: {{ $severity }}
+          runbook_url: {{ template "runbook-url" (deepCopy $ | merge (dict "alert" $alert) $service) }}
+      {{- end }}
+  {{- end }}
+  {{- end }}
+
   {{- end }} {{/* end-section */}}
 
 
diff --git a/coder-observability/templates/dashboards/_dashboards_prebuilds.json.tpl b/coder-observability/templates/dashboards/_dashboards_prebuilds.json.tpl
new file mode 100644
index 0000000..938b501
--- /dev/null
+++ b/coder-observability/templates/dashboards/_dashboards_prebuilds.json.tpl
@@ -0,0 +1,1050 @@
+{{ define "prebuilds-dashboard.json" }}
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 10,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "orange",
+                  "index": 2,
+                  "text": "Not enabled"
+                },
+                "1": {
+                  "color": "green",
+                  "index": 0,
+                  "text": "Enabled"
+                }
+              },
+              "type": "value"
+            },
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "color": "orange",
+                  "index": 1,
+                  "text": "Not enabled"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "id": 15,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "text": {
+          "valueSize": 15
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "min(coderd_experiments{experiment=\"workspace-prebuilds\"})",
+          "instant": true,
+          "legendFormat": "__auto",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Experiment enabled?",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "text",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 4,
+        "y": 0
+      },
+      "id": 49,
+      "interval": "30s",
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "orientation": "vertical",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.3",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(max(coderd_prebuilt_workspaces_desired) by (template_name, preset_name)) or vector(0)",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Desired",
+          "range": false,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(max(coderd_prebuilt_workspaces_running) by (template_name, preset_name)) or vector(0)",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Running",
+          "range": false,
+          "refId": "D"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(max(coderd_prebuilt_workspaces_eligible) by (template_name, preset_name)) or vector(0)",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Eligible",
+          "range": false,
+          "refId": "E"
+        }
+      ],
+      "title": "Current: Global",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "text",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 8,
+        "y": 0
+      },
+      "id": 48,
+      "interval": "30s",
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "orientation": "vertical",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.3",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(max by (template_name, preset_name) (coderd_prebuilt_workspaces_created_total)) or vector(0)",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Created",
+          "range": false,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(max by (template_name, preset_name) (coderd_prebuilt_workspaces_failed_total)) or vector(0)",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Failed",
+          "range": false,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(max by (template_name, preset_name) (coderd_prebuilt_workspaces_claimed_total)) or vector(0)",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Claimed",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "All Time: Global",
+      "type": "stat"
+    },
+    {
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "id": 2,
+      "panels": [],
+      "repeat": "template",
+      "repeatDirection": "h",
+      "title": "$template",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "text",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 4,
+        "x": 0,
+        "y": 5
+      },
+      "id": 31,
+      "interval": "30s",
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "orientation": "vertical",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.3",
+      "repeat": "preset",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "max(coderd_prebuilt_workspaces_desired{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Desired",
+          "range": false,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "max(coderd_prebuilt_workspaces_running{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Running",
+          "range": false,
+          "refId": "D"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "max(coderd_prebuilt_workspaces_eligible{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Eligible",
+          "range": false,
+          "refId": "E"
+        }
+      ],
+      "title": "Current: $preset",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMax": 10,
+            "axisSoftMin": 0,
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 18,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "fieldMinMax": false,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Desired"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "purple",
+                  "mode": "fixed"
+                }
+              },
+              {
+                "id": "custom.lineStyle",
+                "value": {
+                  "dash": [
+                    10,
+                    10
+                  ],
+                  "fill": "dash"
+                }
+              },
+              {
+                "id": "custom.fillOpacity",
+                "value": 85
+              },
+              {
+                "id": "custom.fillBelowTo",
+                "value": "Running"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Running"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "yellow",
+                  "mode": "fixed"
+                }
+              },
+              {
+                "id": "custom.fillBelowTo",
+                "value": "Eligible"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Eligible"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "green",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 4,
+        "y": 5
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.4.3",
+      "repeat": "preset",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "max(coderd_prebuilt_workspaces_desired{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "Desired",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "max(coderd_prebuilt_workspaces_running{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "legendFormat": "Running",
+          "range": true,
+          "refId": "D"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "max(coderd_prebuilt_workspaces_eligible{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "legendFormat": "Eligible",
+          "range": true,
+          "refId": "E"
+        }
+      ],
+      "title": "Pool Capacity: $preset",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMax": 10,
+            "axisSoftMin": 0,
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 13,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "fieldMinMax": false,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Failed"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "red",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Created"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "blue",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Desired"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "purple",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Running"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "yellow",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Eligible"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "green",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Claimed"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "dark-green",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 12,
+        "y": 5
+      },
+      "id": 38,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.4.3",
+      "repeat": "preset",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "floor(max(increase(coderd_prebuilt_workspaces_created_total{template_name=~\"$template\", preset_name=~\"$preset\"}[$__rate_interval]))) or vector(0)",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "legendFormat": "Created",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "floor(max(increase(coderd_prebuilt_workspaces_failed_total{template_name=~\"$template\", preset_name=~\"$preset\"}[$__rate_interval]))) or vector(0)",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "legendFormat": "Failed",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "floor(max(increase(coderd_prebuilt_workspaces_claimed_total{template_name=~\"$template\", preset_name=~\"$preset\"}[$__rate_interval]))) or vector(0)",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "legendFormat": "Claimed",
+          "range": true,
+          "refId": "F"
+        }
+      ],
+      "title": "Pool Operations: $preset",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "text",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 4,
+        "x": 20,
+        "y": 5
+      },
+      "id": 1,
+      "interval": "30s",
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "orientation": "vertical",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.3",
+      "repeat": "preset",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "max(coderd_prebuilt_workspaces_created_total{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Created",
+          "range": false,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "max(coderd_prebuilt_workspaces_failed_total{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Failed",
+          "range": false,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "max(coderd_prebuilt_workspaces_claimed_total{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Claimed",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "All Time: $preset",
+      "type": "stat"
+    }
+  ],
+  "refresh": "{{- include "dashboard-refresh" . -}}",
+  "schemaVersion": 39,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allValue": "",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "definition": "label_values(coderd_prebuilt_workspaces_desired,template_name)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Template",
+        "multi": false,
+        "name": "template",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(coderd_prebuilt_workspaces_desired,template_name)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": "",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "definition": "label_values(coderd_prebuilt_workspaces_desired{template_name=~\"$template\"},preset_name)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Preset",
+        "multi": true,
+        "name": "preset",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(coderd_prebuilt_workspaces_desired{template_name=~\"$template\"},preset_name)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-{{- include "dashboard-range" . -}}",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Prebuilds",
+  "uid": "cej6jysyme22oa",
+  "version": 13,
+  "weekStart": ""
+}
+{{ end }}
\ No newline at end of file
diff --git a/coder-observability/templates/dashboards/configmap-dashboards-prebuilds.yaml b/coder-observability/templates/dashboards/configmap-dashboards-prebuilds.yaml
new file mode 100644
index 0000000..14d5908
--- /dev/null
+++ b/coder-observability/templates/dashboards/configmap-dashboards-prebuilds.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dashboards-prebuilds
+  namespace: {{ .Release.Namespace }}
+data:
+  prebuilds.json: |- {{- include "prebuilds-dashboard.json" . | trim | nindent 4 }}
\ No newline at end of file
diff --git a/coder-observability/templates/statefulset-postgres-exporter.yaml b/coder-observability/templates/statefulset-postgres-exporter.yaml
index bcef353..a1f6e55 100644
--- a/coder-observability/templates/statefulset-postgres-exporter.yaml
+++ b/coder-observability/templates/statefulset-postgres-exporter.yaml
@@ -20,7 +20,7 @@ spec:
     spec:
       containers:
         - name: postgres-exporter
-          image: quay.io/prometheuscommunity/postgres-exporter
+          image: {{ .Values.global.postgres.exporter.image }}
           args:
             - --collector.long_running_transactions
           ports:
@@ -29,4 +29,12 @@ spec:
           env:
             - name: DATA_SOURCE_NAME
               value: '{{ include "postgres-connector-string" . }}'
-          {{ include "postgres-secret-mount" . | nindent 10 }}
\ No newline at end of file
+          {{ include "postgres-secret-mount" . | nindent 10 }}
+          {{- if .Values.global.postgres.volumeMounts }}
+          volumeMounts:
+          {{ toYaml .Values.global.postgres.volumeMounts | nindent 12 }}
+          {{- end }}
+      {{- if .Values.global.postgres.volumes }}
+      volumes:
+      {{ toYaml .Values.global.postgres.volumes | nindent 8 }}
+      {{- end }}
\ No newline at end of file
diff --git a/coder-observability/templates/statefulset-runbook-viewer.yaml b/coder-observability/templates/statefulset-runbook-viewer.yaml
index 0ab2179..64f50e4 100644
--- a/coder-observability/templates/statefulset-runbook-viewer.yaml
+++ b/coder-observability/templates/statefulset-runbook-viewer.yaml
@@ -19,7 +19,7 @@ spec:
     spec:
       containers:
         - name: madness
-          image: dannyben/madness
+          image: {{ .Values.runbookViewer.image }}
           ports:
             - containerPort: 3000
               name: madness
diff --git a/coder-observability/templates/statefulset-sql-exporter.yaml b/coder-observability/templates/statefulset-sql-exporter.yaml
index 3ef64c2..628339e 100644
--- a/coder-observability/templates/statefulset-sql-exporter.yaml
+++ b/coder-observability/templates/statefulset-sql-exporter.yaml
@@ -21,7 +21,7 @@ spec:
     spec:
       containers:
         - name: sql-exporter
-          image: burningalchemist/sql_exporter
+          image: {{ .Values.sqlExporter.image }}
           args:
             - -config.file=/cfg/config.yaml
           ports:
diff --git a/coder-observability/values.yaml b/coder-observability/values.yaml
index c66a6ac..7aea2df 100644
--- a/coder-observability/values.yaml
+++ b/coder-observability/values.yaml
@@ -76,6 +76,16 @@ global:
               notify: 2
               warning: 5
               critical: 10
+          IneligiblePrebuilds:
+            enabled: true
+            delay: 10m
+            thresholds:
+              notify: 1
+          UnprovisionedPrebuiltWorkspaces:
+            enabled: true
+            delay: 10m
+            thresholds:
+              warn: 1
       provisionerd:
         groups:
           Replicas:
@@ -103,6 +113,11 @@ global:
       scrape_interval: 15s
       # global.telemetry.metrics.scrape_timeout -- how long a request will be allowed to wait before being canceled
       scrape_timeout: 12s
+    profiling:
+      # global.telemetry.profiling.scrape_interval -- how often the collector will scrape pprof endpoints
+      scrape_interval: 60s
+      # global.telemetry.profiling.scrape_timeout -- how long a request will be allowed to wait before being canceled
+      scrape_timeout: 60s
 
   # global.postgres -- postgres connection information
   # NOTE: these settings are global so we can parameterise some values which get rendered by subcharts
@@ -113,8 +128,17 @@ global:
     password:
     database: coder
     sslmode: disable
+    # SSL root certificate path - only required when sslmode != "disable"
+    sslrootcert:
+
     # ensure that your secret has a field named `PGPASSWORD`
     mountSecret: "secret-postgres"
+    exporter:
+      image: "quay.io/prometheuscommunity/postgres-exporter"
+
+    # volumes and volumeMounts for SSL certificates
+    volumes: []
+    volumeMounts: []
 
     # global.postgres.alerts -- alerts for postgres
     alerts:
@@ -146,6 +170,12 @@ global:
     # global.dashboards.queryTimeout -- how long until a query in Grafana will timeout after
     queryTimeout: 900
 
+runbookViewer:
+  image: "dannyben/madness"
+
+sqlExporter:
+  image: "burningalchemist/sql_exporter"
+
 grafana-agent:
   enabled: true
   fullnameOverride: grafana-agent
@@ -156,7 +186,7 @@ grafana-agent:
       key: config.river
       create: false
     clustering:
-      enabled: false
+      enabled: true
     extraArgs:
       - --disable-reporting=true
     mounts:
@@ -257,6 +287,8 @@ grafana-agent:
 
 grafana:
   enabled: true
+  image:
+    tag: 10.4.19
   fullnameOverride: grafana
   useStatefulSet: true
   replicas: 1
@@ -323,6 +355,14 @@ grafana:
     datasources.yaml:
       apiVersion: 1
       datasources:
+        - name: pyroscope
+          type: grafana-pyroscope-datasource
+          url: http://pyroscope.{{ .Release.Namespace }}.{{ $.Values.global.zone }}:4040
+          isDefault: false
+          editable: false
+          access: proxy
+          timeout: '{{ add $.Values.global.dashboards.queryTimeout 5 }}'
+          uid: pyroscope
         - name: metrics
           type: prometheus
           url: http://prometheus.{{ .Release.Namespace }}.{{ $.Values.global.zone }}
@@ -406,6 +446,10 @@ grafana:
       mountPath: /var/lib/grafana/dashboards/coder/4
       configMap: dashboards-workspace-detail
       readOnly: false
+    - name: dashboards-prebuilds
+      mountPath: /var/lib/grafana/dashboards/coder/5
+      configMap: dashboards-prebuilds
+      readOnly: false
 
 prometheus:
   enabled: true
@@ -444,7 +488,7 @@ prometheus:
   serverFiles:
     prometheus.yml:
       # disables scraping of metrics by the Prometheus helm chart since this is managed by the collector
-      scrape_configs:
+      scrape_configs: []
       # use custom rule files to be able to render templates (can't do that in values.yaml, unless that value is evaluated by a tpl call)
       rule_files:
         - /etc/config/alerts/*.yaml
@@ -487,6 +531,22 @@ prometheus:
   prometheus-pushgateway:
     enabled: false
 
+pyroscope:
+  enabled: false
+  alloy:
+    enabled: false
+  pyroscope:
+    fullnameOverride: pyroscope
+    service:
+      port: 4040
+      type: ClusterIP
+    extraArgs:
+      log.level: info
+    replicaCount: 1
+    persistence:
+      enabled: true
+      size: 10Gi
+
 loki:
   enabled: true
   nameOverride: loki
diff --git a/compiled/resources.yaml b/compiled/resources.yaml
index c2d7968..44b35b9 100644
--- a/compiled/resources.yaml
+++ b/compiled/resources.yaml
@@ -205,6 +205,14 @@ data:
   datasources.yaml: |
     apiVersion: 1
     datasources:
+    - access: proxy
+      editable: false
+      isDefault: false
+      name: pyroscope
+      timeout: '905'
+      type: grafana-pyroscope-datasource
+      uid: pyroscope
+      url: http://pyroscope.coder-observability.svc:4040
     - access: proxy
       editable: false
       isDefault: true
@@ -735,306 +743,7 @@ data:
       scrape_timeout: 10s
     rule_files:
     - /etc/config/alerts/*.yaml
-    scrape_configs:
-    - job_name: prometheus
-      static_configs:
-      - targets:
-        - localhost:9090
-    - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-      job_name: kubernetes-apiservers
-      kubernetes_sd_configs:
-      - role: endpoints
-      relabel_configs:
-      - action: keep
-        regex: default;kubernetes;https
-        source_labels:
-        - __meta_kubernetes_namespace
-        - __meta_kubernetes_service_name
-        - __meta_kubernetes_endpoint_port_name
-      scheme: https
-      tls_config:
-        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        insecure_skip_verify: true
-    - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-      job_name: kubernetes-nodes
-      kubernetes_sd_configs:
-      - role: node
-      relabel_configs:
-      - action: labelmap
-        regex: __meta_kubernetes_node_label_(.+)
-      - replacement: kubernetes.default.svc:443
-        target_label: __address__
-      - regex: (.+)
-        replacement: /api/v1/nodes/$1/proxy/metrics
-        source_labels:
-        - __meta_kubernetes_node_name
-        target_label: __metrics_path__
-      scheme: https
-      tls_config:
-        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        insecure_skip_verify: true
-    - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-      job_name: kubernetes-nodes-cadvisor
-      kubernetes_sd_configs:
-      - role: node
-      relabel_configs:
-      - action: labelmap
-        regex: __meta_kubernetes_node_label_(.+)
-      - replacement: kubernetes.default.svc:443
-        target_label: __address__
-      - regex: (.+)
-        replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
-        source_labels:
-        - __meta_kubernetes_node_name
-        target_label: __metrics_path__
-      scheme: https
-      tls_config:
-        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        insecure_skip_verify: true
-    - honor_labels: true
-      job_name: kubernetes-service-endpoints
-      kubernetes_sd_configs:
-      - role: endpoints
-      relabel_configs:
-      - action: keep
-        regex: true
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_scrape
-      - action: drop
-        regex: true
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow
-      - action: replace
-        regex: (https?)
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_scheme
-        target_label: __scheme__
-      - action: replace
-        regex: (.+)
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_path
-        target_label: __metrics_path__
-      - action: replace
-        regex: (.+?)(?::\d+)?;(\d+)
-        replacement: $1:$2
-        source_labels:
-        - __address__
-        - __meta_kubernetes_service_annotation_prometheus_io_port
-        target_label: __address__
-      - action: labelmap
-        regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
-        replacement: __param_$1
-      - action: labelmap
-        regex: __meta_kubernetes_service_label_(.+)
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_namespace
-        target_label: namespace
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_service_name
-        target_label: service
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_pod_node_name
-        target_label: node
-    - honor_labels: true
-      job_name: kubernetes-service-endpoints-slow
-      kubernetes_sd_configs:
-      - role: endpoints
-      relabel_configs:
-      - action: keep
-        regex: true
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow
-      - action: replace
-        regex: (https?)
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_scheme
-        target_label: __scheme__
-      - action: replace
-        regex: (.+)
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_path
-        target_label: __metrics_path__
-      - action: replace
-        regex: (.+?)(?::\d+)?;(\d+)
-        replacement: $1:$2
-        source_labels:
-        - __address__
-        - __meta_kubernetes_service_annotation_prometheus_io_port
-        target_label: __address__
-      - action: labelmap
-        regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
-        replacement: __param_$1
-      - action: labelmap
-        regex: __meta_kubernetes_service_label_(.+)
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_namespace
-        target_label: namespace
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_service_name
-        target_label: service
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_pod_node_name
-        target_label: node
-      scrape_interval: 5m
-      scrape_timeout: 30s
-    - honor_labels: true
-      job_name: prometheus-pushgateway
-      kubernetes_sd_configs:
-      - role: service
-      relabel_configs:
-      - action: keep
-        regex: pushgateway
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_probe
-    - honor_labels: true
-      job_name: kubernetes-services
-      kubernetes_sd_configs:
-      - role: service
-      metrics_path: /probe
-      params:
-        module:
-        - http_2xx
-      relabel_configs:
-      - action: keep
-        regex: true
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_probe
-      - source_labels:
-        - __address__
-        target_label: __param_target
-      - replacement: blackbox
-        target_label: __address__
-      - source_labels:
-        - __param_target
-        target_label: instance
-      - action: labelmap
-        regex: __meta_kubernetes_service_label_(.+)
-      - source_labels:
-        - __meta_kubernetes_namespace
-        target_label: namespace
-      - source_labels:
-        - __meta_kubernetes_service_name
-        target_label: service
-    - honor_labels: true
-      job_name: kubernetes-pods
-      kubernetes_sd_configs:
-      - role: pod
-      relabel_configs:
-      - action: keep
-        regex: true
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_scrape
-      - action: drop
-        regex: true
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
-      - action: replace
-        regex: (https?)
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_scheme
-        target_label: __scheme__
-      - action: replace
-        regex: (.+)
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_path
-        target_label: __metrics_path__
-      - action: replace
-        regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})
-        replacement: '[$2]:$1'
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_port
-        - __meta_kubernetes_pod_ip
-        target_label: __address__
-      - action: replace
-        regex: (\d+);((([0-9]+?)(\.|$)){4})
-        replacement: $2:$1
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_port
-        - __meta_kubernetes_pod_ip
-        target_label: __address__
-      - action: labelmap
-        regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
-        replacement: __param_$1
-      - action: labelmap
-        regex: __meta_kubernetes_pod_label_(.+)
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_namespace
-        target_label: namespace
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_pod_name
-        target_label: pod
-      - action: drop
-        regex: Pending|Succeeded|Failed|Completed
-        source_labels:
-        - __meta_kubernetes_pod_phase
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_pod_node_name
-        target_label: node
-    - honor_labels: true
-      job_name: kubernetes-pods-slow
-      kubernetes_sd_configs:
-      - role: pod
-      relabel_configs:
-      - action: keep
-        regex: true
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
-      - action: replace
-        regex: (https?)
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_scheme
-        target_label: __scheme__
-      - action: replace
-        regex: (.+)
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_path
-        target_label: __metrics_path__
-      - action: replace
-        regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})
-        replacement: '[$2]:$1'
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_port
-        - __meta_kubernetes_pod_ip
-        target_label: __address__
-      - action: replace
-        regex: (\d+);((([0-9]+?)(\.|$)){4})
-        replacement: $2:$1
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_port
-        - __meta_kubernetes_pod_ip
-        target_label: __address__
-      - action: labelmap
-        regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
-        replacement: __param_$1
-      - action: labelmap
-        regex: __meta_kubernetes_pod_label_(.+)
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_namespace
-        target_label: namespace
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_pod_name
-        target_label: pod
-      - action: drop
-        regex: Pending|Succeeded|Failed|Completed
-        source_labels:
-        - __meta_kubernetes_pod_phase
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_pod_node_name
-        target_label: node
-      scrape_interval: 5m
-      scrape_timeout: 30s
+    scrape_configs: []
     alerting:
       alertmanagers:
       - kubernetes_sd_configs:
@@ -1067,7 +776,7 @@ metadata:
   name: collector-config
   namespace: coder-observability
 data:
-  config.river: "\n// Discover k8s nodes\ndiscovery.kubernetes \"nodes\" {\n  role = \"node\"\n}\n\n// Discover k8s pods\ndiscovery.kubernetes \"pods\" {\n  role = \"pod\"\n  selectors {\n   role  = \"pod\"\n  }\n}\n\ndiscovery.relabel \"pod_logs\" {\n  targets = discovery.kubernetes.pods.targets\n  \n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\"]\n    target_label  = \"namespace\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_name\"]\n    target_label  = \"pod\"\n  }\n  // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n  rule {\n    source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n    separator      = \"/\"\n    target_label   = \"__meta_app\"\n    action         = \"replace\"\n    regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n    replacement    = \"${1}\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n    separator     = \"/\"\n    target_label  = \"job\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_name\"]\n    target_label  = \"container\"\n  }\n  rule {\n    regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n    action  = \"labeldrop\"\n  }\n  rule {\n    regex   = \"pod_template_generation\"\n    action  = \"labeldrop\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_phase\"]\n    regex = \"Pending|Succeeded|Failed|Completed\"\n    action = \"drop\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_node_name\"]\n    action = \"replace\"\n    target_label = \"node\"\n  }\n  rule {\n    action = \"labelmap\"\n    regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n    replacement = \"__param_$1\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_uid\", \"__meta_kubernetes_pod_container_name\"]\n    separator     = \"/\"\n    action        = \"replace\"\n    replacement   = \"/var/log/pods/*$1/*.log\"\n    target_label  = \"__path__\"\n  }\n  rule {\n    action = \"replace\"\n    source_labels = [\"__meta_kubernetes_pod_container_id\"]\n    regex = \"^(\\\\w+):\\\\/\\\\/.+$\"\n    replacement = \"$1\"\n    target_label = \"tmp_container_runtime\"\n  }\n}\n\ndiscovery.relabel \"pod_metrics\" {\n  targets = discovery.kubernetes.pods.targets\n  \n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\"]\n        target_label  = \"namespace\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_name\"]\n        target_label  = \"pod\"\n      }\n      // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n      rule {\n        source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n        separator      = \"/\"\n        target_label   = \"__meta_app\"\n        action         = \"replace\"\n        regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n        replacement    = \"${1}\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n        separator     = \"/\"\n        target_label  = \"job\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_container_name\"]\n        target_label  = \"container\"\n      }\n      rule {\n        regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        regex   = \"pod_template_generation\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_phase\"]\n        regex = \"Pending|Succeeded|Failed|Completed\"\n        action = \"drop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_node_name\"]\n        action = \"replace\"\n        target_label = \"node\"\n      }\n      rule {\n        action = \"labelmap\"\n        regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n        replacement = \"__param_$1\"\n      }\n  // drop ports that do not expose Prometheus metrics, but might otherwise be exposed by a container which *also*\n  // exposes an HTTP port which exposes metrics\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_port_name\"]\n    regex         = \"grpc|http-(memberlist|console)\"\n    action        = \"drop\"\n  }\n  // adapted from the Prometheus helm chart\n  // https://github.com/prometheus-community/helm-charts/blob/862870fc3c847e32479b509e511584d5283126a3/charts/prometheus/values.yaml#L1070\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scrape\"]\n    action = \"keep\"\n    regex = \"true\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scheme\"]\n    action = \"replace\"\n    regex = \"(https?)\"\n    target_label = \"__scheme__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_path\"]\n    action = \"replace\"\n    target_label = \"__metrics_path__\"\n    regex = \"(.+)\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n    replacement = \"[$2]:$1\"\n    target_label = \"__address__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n    replacement = \"$2:$1\"\n    target_label = \"__address__\"\n  }\n}\n\nlocal.file_match \"pod_logs\" {\n  path_targets = discovery.relabel.pod_logs.output\n}\n\nloki.source.file \"pod_logs\" {\n  targets    = local.file_match.pod_logs.targets\n  forward_to = [loki.process.pod_logs.receiver]\n}\n\nloki.process \"pod_logs\" {\n  stage.match {\n    selector = \"{tmp_container_runtime=\\\"containerd\\\"}\"\n    // the cri processing stage extracts the following k/v pairs: log, stream, time, flags\n    stage.cri {}\n    // Set the extract flags and stream values as labels\n    stage.labels {\n      values = {\n        flags   = \"\",\n        stream  = \"\",\n      }\n    }\n  }\n\n  // if the label tmp_container_runtime from above is docker parse using docker\n  stage.match {\n    selector = \"{tmp_container_runtime=\\\"docker\\\"}\"\n    // the docker processing stage extracts the following k/v pairs: log, stream, time\n    stage.docker {}\n\n    // Set the extract stream value as a label\n    stage.labels {\n      values = {\n        stream  = \"\",\n      }\n    }\n  }\n\n  // drop the temporary container runtime label as it is no longer needed\n  stage.label_drop {\n    values = [\"tmp_container_runtime\"]\n  }\n\n  // parse Coder logs and extract level & logger for efficient filtering\n  stage.match {\n    selector = \"{pod=~\\\"coder.*\\\"}\" // TODO: make configurable\n\n    stage.multiline {\n      firstline     = \"^(?P<ts>\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\"\n      max_wait_time = \"10s\"\n    }\n\n    stage.regex {\n      expression = \"^(?P<ts>\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\\\\s\\\\[(?P<level>\\\\w+)\\\\]\\\\s\\\\s(?P<logger>[^:]+):\\\\s(?P<line>.+)\"\n    }\n\n    stage.timestamp {\n      source            = \"ts\"\n      format            = \"2006-01-02 15:04:05.000\"\n      action_on_failure = \"fudge\" // rather have inaccurate time than drop the log line\n    }\n\n    stage.labels {\n      values = {\n        level  = \"\",\n        logger = \"\",\n      }\n    }\n  }\n\n  forward_to = [loki.write.loki.receiver]\n}\n\nloki.write \"loki\" {\n  endpoint {\n    url = \"http://loki-gateway.coder-observability.svc/loki/api/v1/push\"\n  }\n}\n\nprometheus.scrape \"pods\" {\n  targets = discovery.relabel.pod_metrics.output\n  forward_to = [prometheus.relabel.pods.receiver]\n\n  scrape_interval = \"15s\"\n  scrape_timeout = \"12s\"\n}\n\n// These are metric_relabel_configs while discovery.relabel are relabel_configs.\n// See https://github.com/grafana/agent/blob/main/internal/converter/internal/prometheusconvert/prometheusconvert.go#L95-L106\nprometheus.relabel \"pods\" {\n  forward_to = [prometheus.remote_write.default.receiver]\n\n  // Drop kube-state-metrics' labels which clash with ours\n  rule {\n    source_labels = [\"__name__\", \"container\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"container\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"pod\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"pod\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"namespace\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"namespace\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_container\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"container\"\n    replacement   = \"$1\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_pod\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"pod\"\n    replacement   = \"$1\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_namespace\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"namespace\"\n    replacement   = \"$1\"\n  }\n  rule {\n    regex         = \"^(exported_.*|image_.*|container_id|id|uid)$\"\n    action        = \"labeldrop\"\n  }\n}\n\ndiscovery.relabel \"cadvisor\" {\n  targets = discovery.kubernetes.nodes.targets\n  rule {\n    replacement   = \"/metrics/cadvisor\"\n    target_label  = \"__metrics_path__\"\n  }\n}\n\nprometheus.scrape \"cadvisor\" {\n  targets    = discovery.relabel.cadvisor.output\n  forward_to = [ prometheus.relabel.cadvisor.receiver ]\n  scheme     = \"https\"\n  tls_config {\n    insecure_skip_verify = true\n  }\n  bearer_token_file = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n  scrape_interval   = \"15s\"\n  scrape_timeout    = \"12s\"\n}\n\nprometheus.relabel \"cadvisor\" {\n  forward_to = [ prometheus.remote_write.default.receiver ]\n\n  // Drop empty container labels, addressing https://github.com/google/cadvisor/issues/2688\n  rule {\n    source_labels = [\"__name__\",\"container\"]\n    separator = \"@\"\n    regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*)@\"\n    action = \"drop\"\n  }\n  // Drop empty image labels, addressing https://github.com/google/cadvisor/issues/2688\n  rule {\n    source_labels = [\"__name__\",\"image\"]\n    separator = \"@\"\n    regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*|container_network_.*)@\"\n    action = \"drop\"\n  }\n  // Drop irrelevant series\n  rule {\n    source_labels = [\"container\"]\n    regex         = \"^POD$\"\n    action        = \"drop\"\n  }\n  // Drop unnecessary labels\n  rule {\n    source_labels = [\"id\"]\n    target_label  = \"id\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"job\"]\n    target_label  = \"job\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"name\"]\n    target_label  = \"name\"\n    replacement   = \"\"\n  }\n}\n\nprometheus.remote_write \"default\" {\n  endpoint {\n    url =\"http://prometheus.coder-observability.svc/api/v1/write\"\n\n    // drop instance label which unnecessarily adds new series when pods are restarted, since pod IPs are dynamically assigned\n    // NOTE: \"__address__\" is mapped to \"instance\", so will contain <hostname>:<port>\n    write_relabel_config {\n      regex  = \"instance\"\n      action = \"labeldrop\"\n    }\n  }\n}"
+  config.river: "\n// Discover k8s nodes\ndiscovery.kubernetes \"nodes\" {\n  role = \"node\"\n}\n\n// Discover k8s pods\ndiscovery.kubernetes \"pods\" {\n  role = \"pod\"\n  selectors {\n   role  = \"pod\"\n  }\n}\n\ndiscovery.relabel \"pod_logs\" {\n  targets = discovery.kubernetes.pods.targets\n  \n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\"]\n    target_label  = \"namespace\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_name\"]\n    target_label  = \"pod\"\n  }\n  // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n  rule {\n    source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n    separator      = \"/\"\n    target_label   = \"__meta_app\"\n    action         = \"replace\"\n    regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n    replacement    = \"${1}\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n    separator     = \"/\"\n    target_label  = \"job\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_name\"]\n    target_label  = \"container\"\n  }\n  rule {\n    regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n    action  = \"labeldrop\"\n  }\n  rule {\n    regex   = \"pod_template_generation\"\n    action  = \"labeldrop\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_phase\"]\n    regex = \"Pending|Succeeded|Failed|Completed\"\n    action = \"drop\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_node_name\"]\n    action = \"replace\"\n    target_label = \"node\"\n  }\n  rule {\n    action = \"labelmap\"\n    regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n    replacement = \"__param_$1\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_uid\", \"__meta_kubernetes_pod_container_name\"]\n    separator     = \"/\"\n    action        = \"replace\"\n    replacement   = \"/var/log/pods/*$1/*.log\"\n    target_label  = \"__path__\"\n  }\n  rule {\n    action = \"replace\"\n    source_labels = [\"__meta_kubernetes_pod_container_id\"]\n    regex = \"^(\\\\w+):\\\\/\\\\/.+$\"\n    replacement = \"$1\"\n    target_label = \"tmp_container_runtime\"\n  }\n}\n\ndiscovery.relabel \"pod_metrics\" {\n  targets = discovery.kubernetes.pods.targets\n  \n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\"]\n        target_label  = \"namespace\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_name\"]\n        target_label  = \"pod\"\n      }\n      // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n      rule {\n        source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n        separator      = \"/\"\n        target_label   = \"__meta_app\"\n        action         = \"replace\"\n        regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n        replacement    = \"${1}\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n        separator     = \"/\"\n        target_label  = \"job\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_container_name\"]\n        target_label  = \"container\"\n      }\n      rule {\n        regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        regex   = \"pod_template_generation\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_phase\"]\n        regex = \"Pending|Succeeded|Failed|Completed\"\n        action = \"drop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_node_name\"]\n        action = \"replace\"\n        target_label = \"node\"\n      }\n      rule {\n        action = \"labelmap\"\n        regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n        replacement = \"__param_$1\"\n      }\n  // drop ports that do not expose Prometheus metrics, but might otherwise be exposed by a container which *also*\n  // exposes an HTTP port which exposes metrics\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_port_name\"]\n    regex         = \"grpc|http-(memberlist|console)\"\n    action        = \"drop\"\n  }\n  // adapted from the Prometheus helm chart\n  // https://github.com/prometheus-community/helm-charts/blob/862870fc3c847e32479b509e511584d5283126a3/charts/prometheus/values.yaml#L1070\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scrape\"]\n    action = \"keep\"\n    regex = \"true\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scheme\"]\n    action = \"replace\"\n    regex = \"(https?)\"\n    target_label = \"__scheme__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_path\"]\n    action = \"replace\"\n    target_label = \"__metrics_path__\"\n    regex = \"(.+)\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n    replacement = \"[$2]:$1\"\n    target_label = \"__address__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n    replacement = \"$2:$1\"\n    target_label = \"__address__\"\n  }\n}\n\ndiscovery.relabel \"pod_pprof\" {\n  targets = discovery.kubernetes.pods.targets\n  \n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\"]\n        target_label  = \"namespace\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_name\"]\n        target_label  = \"pod\"\n      }\n      // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n      rule {\n        source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n        separator      = \"/\"\n        target_label   = \"__meta_app\"\n        action         = \"replace\"\n        regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n        replacement    = \"${1}\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n        separator     = \"/\"\n        target_label  = \"job\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_container_name\"]\n        target_label  = \"container\"\n      }\n      rule {\n        regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        regex   = \"pod_template_generation\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_phase\"]\n        regex = \"Pending|Succeeded|Failed|Completed\"\n        action = \"drop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_node_name\"]\n        action = \"replace\"\n        target_label = \"node\"\n      }\n      rule {\n        action = \"labelmap\"\n        regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n        replacement = \"__param_$1\"\n      }\n  // The relabeling allows the actual pod scrape endpoint to be configured via the\n  // following annotations:\n  //\n  // * `pyroscope.io/scrape`: Only scrape pods that have a value of `true`.\n  // * `pyroscope.io/application-name`: Name of the application being profiled.\n  // * `pyroscope.io/scheme`: If the metrics endpoint is secured then you will need\n  // to set this to `https` & most likely set the `tls_config` of the scrape config.\n  // * `pyroscope.io/port`: Scrape the pod on the indicated port.\n  //\n  // Kubernetes labels will be added as Pyroscope labels on metrics via the\n  // `labelmap` relabeling action.\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_scrape\"]\n    action = \"keep\"\n    regex = \"true\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_application_name\"]\n    action = \"replace\"\n    target_label = \"__name__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_scheme\"]\n    action = \"replace\"\n    regex = \"(https?)\"\n    target_label = \"__scheme__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n    replacement = \"[$2]:$1\"\n    target_label = \"__address__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n    replacement = \"$2:$1\"\n    target_label = \"__address__\"\n  }\n}\n\nlocal.file_match \"pod_logs\" {\n  path_targets = discovery.relabel.pod_logs.output\n}\n\nloki.source.file \"pod_logs\" {\n  targets    = local.file_match.pod_logs.targets\n  forward_to = [loki.process.pod_logs.receiver]\n}\n\nloki.process \"pod_logs\" {\n  stage.match {\n    selector = \"{tmp_container_runtime=\\\"containerd\\\"}\"\n    // the cri processing stage extracts the following k/v pairs: log, stream, time, flags\n    stage.cri {}\n    // Set the extract flags and stream values as labels\n    stage.labels {\n      values = {\n        flags   = \"\",\n        stream  = \"\",\n      }\n    }\n  }\n\n  // if the label tmp_container_runtime from above is docker parse using docker\n  stage.match {\n    selector = \"{tmp_container_runtime=\\\"docker\\\"}\"\n    // the docker processing stage extracts the following k/v pairs: log, stream, time\n    stage.docker {}\n\n    // Set the extract stream value as a label\n    stage.labels {\n      values = {\n        stream  = \"\",\n      }\n    }\n  }\n\n  // drop the temporary container runtime label as it is no longer needed\n  stage.label_drop {\n    values = [\"tmp_container_runtime\"]\n  }\n\n  // parse Coder logs and extract level & logger for efficient filtering\n  stage.match {\n    selector = \"{pod=~\\\"coder.*\\\"}\" // TODO: make configurable\n\n    stage.multiline {\n      firstline     = \"^(?P<ts>\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\"\n      max_wait_time = \"10s\"\n    }\n\n    stage.regex {\n      expression = \"^(?P<ts>\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\\\\s\\\\[(?P<level>\\\\w+)\\\\]\\\\s\\\\s(?P<logger>[^:]+):\\\\s(?P<line>.+)\"\n    }\n\n    stage.timestamp {\n      source            = \"ts\"\n      format            = \"2006-01-02 15:04:05.000\"\n      action_on_failure = \"fudge\" // rather have inaccurate time than drop the log line\n    }\n\n    stage.labels {\n      values = {\n        level  = \"\",\n        logger = \"\",\n      }\n    }\n  }\n\n  forward_to = [loki.write.loki.receiver]\n}\n\nloki.write \"loki\" {\n  endpoint {\n    url = \"http://loki-gateway.coder-observability.svc/loki/api/v1/push\"\n  }\n}\n\n\n\nprometheus.scrape \"pods\" {\n  targets = discovery.relabel.pod_metrics.output\n  forward_to = [prometheus.relabel.pods.receiver]\n\n  scrape_interval = \"15s\"\n  scrape_timeout = \"12s\"\n}\n\n// These are metric_relabel_configs while discovery.relabel are relabel_configs.\n// See https://github.com/grafana/agent/blob/main/internal/converter/internal/prometheusconvert/prometheusconvert.go#L95-L106\nprometheus.relabel \"pods\" {\n  forward_to = [prometheus.remote_write.default.receiver]\n\n  // Drop kube-state-metrics' labels which clash with ours\n  rule {\n    source_labels = [\"__name__\", \"container\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"container\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"pod\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"pod\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"namespace\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"namespace\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_container\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"container\"\n    replacement   = \"$1\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_pod\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"pod\"\n    replacement   = \"$1\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_namespace\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"namespace\"\n    replacement   = \"$1\"\n  }\n  rule {\n    regex         = \"^(exported_.*|image_.*|container_id|id|uid)$\"\n    action        = \"labeldrop\"\n  }\n}\n\ndiscovery.relabel \"cadvisor\" {\n  targets = discovery.kubernetes.nodes.targets\n  rule {\n    replacement   = \"/metrics/cadvisor\"\n    target_label  = \"__metrics_path__\"\n  }\n}\n\nprometheus.scrape \"cadvisor\" {\n  targets    = discovery.relabel.cadvisor.output\n  forward_to = [ prometheus.relabel.cadvisor.receiver ]\n  scheme     = \"https\"\n  tls_config {\n    insecure_skip_verify = true\n  }\n  bearer_token_file = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n  scrape_interval   = \"15s\"\n  scrape_timeout    = \"12s\"\n}\n\nprometheus.relabel \"cadvisor\" {\n  forward_to = [ prometheus.remote_write.default.receiver ]\n\n  // Drop empty container labels, addressing https://github.com/google/cadvisor/issues/2688\n  rule {\n    source_labels = [\"__name__\",\"container\"]\n    separator = \"@\"\n    regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*)@\"\n    action = \"drop\"\n  }\n  // Drop empty image labels, addressing https://github.com/google/cadvisor/issues/2688\n  rule {\n    source_labels = [\"__name__\",\"image\"]\n    separator = \"@\"\n    regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*|container_network_.*)@\"\n    action = \"drop\"\n  }\n  // Drop irrelevant series\n  rule {\n    source_labels = [\"container\"]\n    regex         = \"^POD$\"\n    action        = \"drop\"\n  }\n  // Drop unnecessary labels\n  rule {\n    source_labels = [\"id\"]\n    target_label  = \"id\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"job\"]\n    target_label  = \"job\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"name\"]\n    target_label  = \"name\"\n    replacement   = \"\"\n  }\n}\n\nprometheus.remote_write \"default\" {\n  endpoint {\n    url =\"http://prometheus.coder-observability.svc/api/v1/write\"\n\n    // drop instance label which unnecessarily adds new series when pods are restarted, since pod IPs are dynamically assigned\n    // NOTE: \"__address__\" is mapped to \"instance\", so will contain <hostname>:<port>\n    write_relabel_config {\n      regex  = \"instance\"\n      action = \"labeldrop\"\n    }\n  }\n}"
 ---
 # Source: coder-observability/templates/configmap-prometheus-alerts.yaml
 apiVersion: v1
@@ -1076,8 +785,8 @@ metadata:
   name: metrics-alerts
   namespace: coder-observability
 data:
-  coderd.yaml: "groups:\n- name: CPU Usage\n  rules:\n  \n  - alert: CoderdCPUUsage\n    expr: max by (pod) (rate(container_cpu_usage_seconds_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}[10m])) / max by(pod) (kube_pod_container_resource_limits{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, resource=\"cpu\"}) > 0.9\n    for: 10m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} is using high amounts of CPU, which may impact application performance.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdcpuusage\n  - alert: CoderdCPUUsage\n    expr: max by (pod) (rate(container_cpu_usage_seconds_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}[10m])) / max by(pod) (kube_pod_container_resource_limits{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, resource=\"cpu\"}) > 0.8\n    for: 10m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} is using high amounts of CPU, which may impact application performance.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdcpuusage\n- name: Memory Usage\n  rules:\n  \n  - alert: CoderdMemoryUsage\n    expr: max by (pod) (container_memory_working_set_bytes{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) / max by (pod) (kube_pod_container_resource_limits{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, resource=\"memory\"})  > 0.9\n    for: 10m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} is using high amounts of memory, which may lead to an Out-Of-Memory (OOM) error.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdmemoryusage\n  - alert: CoderdMemoryUsage\n    expr: max by (pod) (container_memory_working_set_bytes{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) / max by (pod) (kube_pod_container_resource_limits{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, resource=\"memory\"})  > 0.8\n    for: 10m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} is using high amounts of memory, which may lead to an Out-Of-Memory (OOM) error.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdmemoryusage\n- name: Pod Restarts\n  rules:\n  \n  - alert: CoderdRestarts\n    expr: sum by(pod) (increase(kube_pod_container_status_restarts_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}[10m])) > 3\n    for: 1m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} has restarted multiple times in the last 10m, which may indicate a CrashLoop.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdrestarts\n  - alert: CoderdRestarts\n    expr: sum by(pod) (increase(kube_pod_container_status_restarts_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}[10m])) > 1\n    for: 1m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} has restarted multiple times in the last 10m, which may indicate a CrashLoop.\n    labels:\n      severity: notify\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdrestarts\n  - alert: CoderdRestarts\n    expr: sum by(pod) (increase(kube_pod_container_status_restarts_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}[10m])) > 2\n    for: 1m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} has restarted multiple times in the last 10m, which may indicate a CrashLoop.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdrestarts\n- name: Coderd Replicas\n  rules:\n  \n  - alert: CoderdReplicas\n    expr: sum(up{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) < 1\n    for: 5m\n    annotations:\n      summary: Number of alive coderd replicas is below the threshold = 1.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdreplicas\n  - alert: CoderdReplicas\n    expr: sum(up{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) < 3\n    for: 5m\n    annotations:\n      summary: Number of alive coderd replicas is below the threshold = 3.\n    labels:\n      severity: notify\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdreplicas\n  - alert: CoderdReplicas\n    expr: sum(up{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) < 2\n    for: 5m\n    annotations:\n      summary: Number of alive coderd replicas is below the threshold = 2.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdreplicas\n- name: Coderd Workspace Build Failures\n  rules:\n  \n  - alert: CoderdWorkspaceBuildFailures\n    expr: sum(increase(coderd_workspace_builds_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, status=\"failed\" }[10m])) > 10\n    for: 10m\n    annotations:\n      summary: Workspace builds have failed multiple times in the last 10m, which may indicate a broken Coder template.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdworkspacebuildfailures\n  - alert: CoderdWorkspaceBuildFailures\n    expr: sum(increase(coderd_workspace_builds_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, status=\"failed\" }[10m])) > 2\n    for: 10m\n    annotations:\n      summary: Workspace builds have failed multiple times in the last 10m, which may indicate a broken Coder template.\n    labels:\n      severity: notify\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdworkspacebuildfailures\n  - alert: CoderdWorkspaceBuildFailures\n    expr: sum(increase(coderd_workspace_builds_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, status=\"failed\" }[10m])) > 5\n    for: 10m\n    annotations:\n      summary: Workspace builds have failed multiple times in the last 10m, which may indicate a broken Coder template.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#coderdworkspacebuildfailures  "
-  provisionerd.yaml: "groups:\n- name: Provisionerd Replicas\n  rules:\n  \n  - alert: ProvisionerdReplicas\n    expr: sum(coderd_provisionerd_num_daemons{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) < 1\n    for: 5m\n    annotations:\n      summary: Number of alive provisionerd replicas is below the threshold = 1.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#provisionerdreplicas\n  - alert: ProvisionerdReplicas\n    expr: sum(coderd_provisionerd_num_daemons{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) < 3\n    for: 5m\n    annotations:\n      summary: Number of alive provisionerd replicas is below the threshold = 3.\n    labels:\n      severity: notify\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#provisionerdreplicas\n  - alert: ProvisionerdReplicas\n    expr: sum(coderd_provisionerd_num_daemons{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) < 2\n    for: 5m\n    annotations:\n      summary: Number of alive provisionerd replicas is below the threshold = 2.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coder#provisionerdreplicas  "
+  coderd.yaml: "groups:\n- name: CPU Usage\n  rules:\n  \n  - alert: CoderdCPUUsage\n    expr: max by (pod) (rate(container_cpu_usage_seconds_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}[10m])) / max by(pod) (kube_pod_container_resource_limits{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, resource=\"cpu\"}) > 0.9\n    for: 10m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} is using high amounts of CPU, which may impact application performance.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdcpuusage\n  - alert: CoderdCPUUsage\n    expr: max by (pod) (rate(container_cpu_usage_seconds_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}[10m])) / max by(pod) (kube_pod_container_resource_limits{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, resource=\"cpu\"}) > 0.8\n    for: 10m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} is using high amounts of CPU, which may impact application performance.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdcpuusage\n- name: Memory Usage\n  rules:\n  \n  - alert: CoderdMemoryUsage\n    expr: max by (pod) (container_memory_working_set_bytes{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) / max by (pod) (kube_pod_container_resource_limits{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, resource=\"memory\"})  > 0.9\n    for: 10m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} is using high amounts of memory, which may lead to an Out-Of-Memory (OOM) error.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdmemoryusage\n  - alert: CoderdMemoryUsage\n    expr: max by (pod) (container_memory_working_set_bytes{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) / max by (pod) (kube_pod_container_resource_limits{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, resource=\"memory\"})  > 0.8\n    for: 10m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} is using high amounts of memory, which may lead to an Out-Of-Memory (OOM) error.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdmemoryusage\n- name: Pod Restarts\n  rules:\n  \n  - alert: CoderdRestarts\n    expr: sum by(pod) (increase(kube_pod_container_status_restarts_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}[10m])) > 3\n    for: 1m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} has restarted multiple times in the last 10m, which may indicate a CrashLoop.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdrestarts\n  - alert: CoderdRestarts\n    expr: sum by(pod) (increase(kube_pod_container_status_restarts_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}[10m])) > 1\n    for: 1m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} has restarted multiple times in the last 10m, which may indicate a CrashLoop.\n    labels:\n      severity: notify\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdrestarts\n  - alert: CoderdRestarts\n    expr: sum by(pod) (increase(kube_pod_container_status_restarts_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}[10m])) > 2\n    for: 1m\n    annotations:\n      summary: The Coder instance {{ $labels.pod }} has restarted multiple times in the last 10m, which may indicate a CrashLoop.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdrestarts\n- name: Coderd Replicas\n  rules:\n  \n  - alert: CoderdReplicas\n    expr: sum(up{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) < 1\n    for: 5m\n    annotations:\n      summary: Number of alive coderd replicas is below the threshold = 1.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdreplicas\n  - alert: CoderdReplicas\n    expr: sum(up{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) < 3\n    for: 5m\n    annotations:\n      summary: Number of alive coderd replicas is below the threshold = 3.\n    labels:\n      severity: notify\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdreplicas\n  - alert: CoderdReplicas\n    expr: sum(up{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) < 2\n    for: 5m\n    annotations:\n      summary: Number of alive coderd replicas is below the threshold = 2.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdreplicas\n- name: Coderd Workspace Build Failures\n  rules:\n  \n  - alert: CoderdWorkspaceBuildFailures\n    expr: sum(increase(coderd_workspace_builds_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, status=\"failed\" }[10m])) > 10\n    for: 10m\n    annotations:\n      summary: Workspace builds have failed multiple times in the last 10m, which may indicate a broken Coder template.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdworkspacebuildfailures\n  - alert: CoderdWorkspaceBuildFailures\n    expr: sum(increase(coderd_workspace_builds_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, status=\"failed\" }[10m])) > 2\n    for: 10m\n    annotations:\n      summary: Workspace builds have failed multiple times in the last 10m, which may indicate a broken Coder template.\n    labels:\n      severity: notify\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdworkspacebuildfailures\n  - alert: CoderdWorkspaceBuildFailures\n    expr: sum(increase(coderd_workspace_builds_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`, status=\"failed\" }[10m])) > 5\n    for: 10m\n    annotations:\n      summary: Workspace builds have failed multiple times in the last 10m, which may indicate a broken Coder template.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdworkspacebuildfailures\n- name: Coderd Ineligible Prebuilds\n  rules:\n  \n  - alert: CoderdIneligiblePrebuilds\n    expr: max by (template_name, preset_name) (coderd_prebuilt_workspaces_running - coderd_prebuilt_workspaces_eligible) > 0\n    for: 10m\n    annotations:\n      summary: >\n        {{ $value }} prebuilt workspace(s) are currently ineligible for claiming for the \"{{ $labels.template_name }}\" template and \"{{ $labels.preset_name }}\" preset.\n        This usually indicates that the agent has not started correctly, or is still running its startup scripts after an extended period of time.\n    labels:\n      severity: notify\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdineligibleprebuilds\n- name: Coderd Unprovisioned Prebuilt Workspaces\n  rules:\n  \n  - alert: CoderdUnprovisionedPrebuiltWorkspaces\n    expr: max by (template_name, preset_name) (coderd_prebuilt_workspaces_desired - coderd_prebuilt_workspaces_running) > 0\n    for: 10m\n    annotations:\n      summary: >\n        {{ $value }} prebuilt workspace(s) not yet been provisioned for the \"{{ $labels.template_name }}\" template and \"{{ $labels.preset_name }}\" preset.\n    labels:\n      severity: warn\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#coderdunprovisionedprebuiltworkspaces  "
+  provisionerd.yaml: "groups:\n- name: Provisionerd Replicas\n  rules:\n  \n  - alert: ProvisionerdReplicas\n    expr: sum(coderd_provisionerd_num_daemons{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) < 1\n    for: 5m\n    annotations:\n      summary: Number of alive provisionerd replicas is below the threshold = 1.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#provisionerdreplicas\n  - alert: ProvisionerdReplicas\n    expr: sum(coderd_provisionerd_num_daemons{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) < 3\n    for: 5m\n    annotations:\n      summary: Number of alive provisionerd replicas is below the threshold = 3.\n    labels:\n      severity: notify\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#provisionerdreplicas\n  - alert: ProvisionerdReplicas\n    expr: sum(coderd_provisionerd_num_daemons{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}) < 2\n    for: 5m\n    annotations:\n      summary: Number of alive provisionerd replicas is below the threshold = 2.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/coderd#provisionerdreplicas  "
   enterprise.yaml: "groups:\n  - name: Licences\n    rules:\n  \n      - alert: CoderLicenseSeats\n        expr: 'max(coderd_license_active_users) / max(coderd_license_limit_users) >=1'\n        for: 1m\n        annotations:\n          summary: Your Coder enterprise licence usage is now at {{ $value | humanizePercentage }} capacity.\n        labels:\n          severity: critical\n          runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/enterprise#coderlicenseseats\n      - alert: CoderLicenseSeats\n        expr: 'max(coderd_license_active_users) / max(coderd_license_limit_users) >=0.9'\n        for: 1m\n        annotations:\n          summary: Your Coder enterprise licence usage is now at {{ $value | humanizePercentage }} capacity.\n        labels:\n          severity: warning\n          runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/enterprise#coderlicenseseats "
   postgres.yaml: "groups:\n- name: Notifications\n  rules:\n  \n  - alert: PostgresNotificationQueueFillingUp\n    expr: pg_pubsub_usage > 0.9\n    for: 15m\n    annotations:\n      summary: The postgres instance {{ $labels.instance }} has a notification that is filling up, which may impact application performance.\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/postgres#postgresnotificationqueuefillingup\n  - alert: PostgresNotificationQueueFillingUp\n    expr: pg_pubsub_usage > 0.5\n    for: 15m\n    annotations:\n      summary: The postgres instance {{ $labels.instance }} has a notification that is filling up, which may impact application performance.\n    labels:\n      severity: notify\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/postgres#postgresnotificationqueuefillingup\n  - alert: PostgresNotificationQueueFillingUp\n    expr: pg_pubsub_usage > 0.8\n    for: 15m\n    annotations:\n      summary: The postgres instance {{ $labels.instance }} has a notification that is filling up, which may impact application performance.\n    labels:\n      severity: warning\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/postgres#postgresnotificationqueuefillingup\n\n- name: Liveness\n  rules:\n  \n  - alert: PostgresDown\n    expr: pg_up == 0\n    for: 1m\n    annotations:\n      summary: The postgres instance {{ $labels.instance }} is down!\n    labels:\n      severity: critical\n      runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/postgres#postgresdown\n\n\n- name: Connections\n  rules:\n  \n    - alert: PostgresConnectionsRunningLow\n      expr: sum by (datname, instance) (pg_stat_activity_count) > on () group_left() (pg_settings_max_connections * 0.9)\n      for: 5m\n      labels:\n        summary: The postgres instance {{ $labels.instance }} is running low on connections which may impact application performance.\n        severity: critical\n        runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/postgres#postgresconnectionsrunninglow\n    - alert: PostgresConnectionsRunningLow\n      expr: sum by (datname, instance) (pg_stat_activity_count) > on () group_left() (pg_settings_max_connections * 0.5)\n      for: 5m\n      labels:\n        summary: The postgres instance {{ $labels.instance }} is running low on connections which may impact application performance.\n        severity: notify\n        runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/postgres#postgresconnectionsrunninglow\n    - alert: PostgresConnectionsRunningLow\n      expr: sum by (datname, instance) (pg_stat_activity_count) > on () group_left() (pg_settings_max_connections * 0.8)\n      for: 5m\n      labels:\n        summary: The postgres instance {{ $labels.instance }} is running low on connections which may impact application performance.\n        severity: warning\n        runbook_url: http://runbook-viewer.coder-observability.svc.cluster.local/postgres#postgresconnectionsrunninglow"
 ---
@@ -1088,9 +797,9 @@ metadata:
   name: runbooks
   namespace: coder-observability
   annotations:
-    checksum/config: 7ab4d06d5b454cc584880f9b58fb50fe85f2803bbd0021edb957c5a2d73b640e
+    checksum/config: b0c41033d0385ee3d46488f08e85bcef0d939614dcb99194e0c5913dbf0c2c33
 data:
-  coderd.md: |
+  coderd.md: |-
     # Coderd Runbooks
 
     ## CoderdCPUUsage
@@ -1169,6 +878,63 @@ data:
     Your Enterprise license is approaching or has exceeded the number of seats purchased.
 
     Please contact your Coder sales contact, or visit https://coder.com/contact/sales.
+
+    ## CoderdIneligiblePrebuilds
+
+    Prebuilds only become eligible to be claimed by users once the workspace's agent is a) running and b) all of its startup
+    scripts have completed.
+
+    If a prebuilt workspace is not eligible, view its agent logs to diagnose the problem.
+
+    ## CoderdUnprovisionedPrebuiltWorkspaces
+
+    The number of running prebuilt workspaces is lower than the desired instances. This could be for several reasons,
+    ordered by likehood:
+
+    ### Experiment/License
+
+    The prebuilds feature is currently gated behind an experiment *and* a premium license.
+
+    Ensure that the prebuilds experiment is enabled with `CODER_EXPERIMENTS=workspace-prebuilds`, and that you have a premium
+    license added.
+
+    ### Preset Validation Issue
+
+    Templates which have prebuilds configured will require a configured preset defined, with ALL of the required parameters
+    set in the preset. If any of these are missing, or any of the parameters - as defined - fail validation, then the prebuilds
+    subsystem will refuse to attempt a workspace build.
+
+    Consult the coderd logs for more information; look out for errors or warnings from the prebuilds subsystem.
+
+    ### Template Misconfiguration or Error
+
+    Prebuilt workspaces cannot be provisioned due to some issue at `terraform apply`-time. This could be due to misconfigured
+    cloud resources, improper authorization, or any number of other issues.
+
+    Visit the Workspaces page, change the search term to `owner:prebuilds`, and view on the previously failed builds. The
+    error will likely be quite obvious.
+
+    ### Provisioner Latency
+
+    If your provisioners are overloaded and cannot process provisioner jobs quickly enough, prebuilt workspaces may be affected.
+    There is no prioritization at present for prebuilt workspace jobs.
+
+    Ensure your provisioners are appropriately resources (i.e. you have enough instances) to handle the concurrent build demand.
+
+    ### Use of Workspace Tags
+
+    If you are using `coder_workspace_tags` ([docs](https://coder.com/docs/admin/templates/extending-templates/workspace-tags))
+    in your template, chances are you do not have any provisioners running or they are under-resourced (see **Provisioner Latency**).
+
+    Ensure your running provisioners are configured with your desired tags.
+
+    ### Reconciliation Loop Issue
+
+    The prebuilds subsystem runs a _reconciliation loop_ which monitors the state of prebuilt workspaces to ensure the desired
+    number of instances are present at all times. Workspace Prebuilds is currently a BETA feature and so there could be a bug
+    in this _reconciliation loop_, which should be reported to Coder.
+
+    Examine your coderd logs for any errors or warnings relating to prebuilds.
   postgres.md: |
     # Postgres Runbooks
 
@@ -1236,7 +1002,7 @@ data:
     global:
     target:
       name: postgres
-      data_source_name: 'postgresql://coder@localhost:5432/postgres?sslmode=disable'
+      data_source_name: 'postgresql://coder@localhost:5432/coder?sslmode=disable'
       collectors:
         - notify
     collectors:
@@ -2538,29 +2304,1039 @@ data:
             ]
           },
           "gridPos": {
-            "h": 3,
-            "w": 4,
+            "h": 3,
+            "w": 4,
+            "x": 12,
+            "y": 15
+          },
+          "id": 21,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": {
+              "calcs": [
+                "mean"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "showPercentChange": false,
+            "textMode": "auto",
+            "wideLayout": true
+          },
+          "pluginVersion": "10.4.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "expr": "quantile(0.9, coder_pubsub_send_latency_seconds)",
+              "instant": false,
+              "legendFormat": "Send",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "expr": "quantile(0.9, coder_pubsub_receive_latency_seconds)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Receive",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Pubsub Latency (P90)",
+          "type": "stat"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "bars",
+                "fillOpacity": 100,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 0,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "red",
+                    "value": null
+                  },
+                  {
+                    "color": "green",
+                    "value": 1
+                  }
+                ]
+              },
+              "unit": "reqps"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 6,
+            "w": 6,
+            "x": 0,
+            "y": 18
+          },
+          "id": 35,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "10.4.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum by(pod) (rate(coderd_api_requests_processed_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}[$__rate_interval]))",
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "API Requests",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "description": "",
+          "gridPos": {
+            "h": 6,
+            "w": 6,
+            "x": 6,
+            "y": 18
+          },
+          "id": 36,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "This shows the number of requests per second each `coderd` replica is handling.\n\nHeavy skewing towards a single `coderd` replica indicates faulty loadbalancing.",
+            "mode": "markdown"
+          },
+          "pluginVersion": "10.4.0",
+          "transparent": true,
+          "type": "text"
+        }
+      ],
+      "refresh": "30s",
+      "schemaVersion": 39,
+      "tags": [],
+      "templating": {
+        "list": []
+      },
+      "time": {
+        "from": "now-12h",
+        "to": "now"
+      },
+      "timepicker": {},
+      "timezone": "browser",
+      "title": "Control Plane",
+      "uid": "coderd",
+      "version": 6,
+      "weekStart": ""
+    }
+---
+# Source: coder-observability/templates/dashboards/configmap-dashboards-prebuilds.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dashboards-prebuilds
+  namespace: coder-observability
+data:
+  prebuilds.json: |-
+    {
+      "annotations": {
+        "list": [
+          {
+            "builtIn": 1,
+            "datasource": {
+              "type": "grafana",
+              "uid": "-- Grafana --"
+            },
+            "enable": true,
+            "hide": true,
+            "iconColor": "rgba(0, 211, 255, 1)",
+            "name": "Annotations & Alerts",
+            "type": "dashboard"
+          }
+        ]
+      },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "graphTooltip": 0,
+      "id": 10,
+      "links": [],
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "mappings": [
+                {
+                  "options": {
+                    "0": {
+                      "color": "orange",
+                      "index": 2,
+                      "text": "Not enabled"
+                    },
+                    "1": {
+                      "color": "green",
+                      "index": 0,
+                      "text": "Enabled"
+                    }
+                  },
+                  "type": "value"
+                },
+                {
+                  "options": {
+                    "match": "null",
+                    "result": {
+                      "color": "orange",
+                      "index": 1,
+                      "text": "Not enabled"
+                    }
+                  },
+                  "type": "special"
+                }
+              ],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 4,
+            "w": 4,
+            "x": 0,
+            "y": 0
+          },
+          "id": 15,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "none",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "showPercentChange": false,
+            "text": {
+              "valueSize": 15
+            },
+            "textMode": "auto",
+            "wideLayout": true
+          },
+          "pluginVersion": "10.4.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "expr": "min(coderd_experiments{experiment=\"workspace-prebuilds\"})",
+              "instant": true,
+              "legendFormat": "__auto",
+              "range": false,
+              "refId": "A"
+            }
+          ],
+          "title": "Experiment enabled?",
+          "type": "stat"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "fixedColor": "text",
+                "mode": "fixed"
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 4,
+            "w": 4,
+            "x": 4,
+            "y": 0
+          },
+          "id": 49,
+          "interval": "30s",
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "orientation": "vertical",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "showPercentChange": false,
+            "textMode": "auto",
+            "wideLayout": true
+          },
+          "pluginVersion": "10.4.3",
+          "repeatDirection": "v",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(max(coderd_prebuilt_workspaces_desired) by (template_name, preset_name)) or vector(0)",
+              "instant": true,
+              "interval": "",
+              "legendFormat": "Desired",
+              "range": false,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(max(coderd_prebuilt_workspaces_running) by (template_name, preset_name)) or vector(0)",
+              "hide": false,
+              "instant": true,
+              "interval": "",
+              "legendFormat": "Running",
+              "range": false,
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(max(coderd_prebuilt_workspaces_eligible) by (template_name, preset_name)) or vector(0)",
+              "hide": false,
+              "instant": true,
+              "interval": "",
+              "legendFormat": "Eligible",
+              "range": false,
+              "refId": "E"
+            }
+          ],
+          "title": "Current: Global",
+          "type": "stat"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "fixedColor": "text",
+                "mode": "fixed"
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 4,
+            "w": 4,
+            "x": 8,
+            "y": 0
+          },
+          "id": 48,
+          "interval": "30s",
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "orientation": "vertical",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "showPercentChange": false,
+            "textMode": "auto",
+            "wideLayout": true
+          },
+          "pluginVersion": "10.4.3",
+          "repeatDirection": "v",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(max by (template_name, preset_name) (coderd_prebuilt_workspaces_created_total)) or vector(0)",
+              "hide": false,
+              "instant": true,
+              "interval": "",
+              "legendFormat": "Created",
+              "range": false,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(max by (template_name, preset_name) (coderd_prebuilt_workspaces_failed_total)) or vector(0)",
+              "hide": false,
+              "instant": true,
+              "interval": "",
+              "legendFormat": "Failed",
+              "range": false,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum(max by (template_name, preset_name) (coderd_prebuilt_workspaces_claimed_total)) or vector(0)",
+              "hide": false,
+              "instant": true,
+              "interval": "",
+              "legendFormat": "Claimed",
+              "range": false,
+              "refId": "A"
+            }
+          ],
+          "title": "All Time: Global",
+          "type": "stat"
+        },
+        {
+          "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 4
+          },
+          "id": 2,
+          "panels": [],
+          "repeat": "template",
+          "repeatDirection": "h",
+          "title": "$template",
+          "type": "row"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "fixedColor": "text",
+                "mode": "fixed"
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 4,
+            "x": 0,
+            "y": 5
+          },
+          "id": 31,
+          "interval": "30s",
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "orientation": "vertical",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "showPercentChange": false,
+            "textMode": "auto",
+            "wideLayout": true
+          },
+          "pluginVersion": "10.4.3",
+          "repeat": "preset",
+          "repeatDirection": "v",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "max(coderd_prebuilt_workspaces_desired{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+              "instant": true,
+              "interval": "",
+              "legendFormat": "Desired",
+              "range": false,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "max(coderd_prebuilt_workspaces_running{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+              "hide": false,
+              "instant": true,
+              "interval": "",
+              "legendFormat": "Running",
+              "range": false,
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "max(coderd_prebuilt_workspaces_eligible{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+              "hide": false,
+              "instant": true,
+              "interval": "",
+              "legendFormat": "Eligible",
+              "range": false,
+              "refId": "E"
+            }
+          ],
+          "title": "Current: $preset",
+          "type": "stat"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMax": 10,
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 18,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "smooth",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 2,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "fieldMinMax": false,
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Desired"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "purple",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.lineStyle",
+                    "value": {
+                      "dash": [
+                        10,
+                        10
+                      ],
+                      "fill": "dash"
+                    }
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 85
+                  },
+                  {
+                    "id": "custom.fillBelowTo",
+                    "value": "Running"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Running"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "yellow",
+                      "mode": "fixed"
+                    }
+                  },
+                  {
+                    "id": "custom.fillBelowTo",
+                    "value": "Eligible"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Eligible"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "green",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 4,
+            "y": 5
+          },
+          "id": 5,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "10.4.3",
+          "repeat": "preset",
+          "repeatDirection": "v",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "expr": "max(coderd_prebuilt_workspaces_desired{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+              "instant": false,
+              "interval": "",
+              "legendFormat": "Desired",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "expr": "max(coderd_prebuilt_workspaces_running{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "legendFormat": "Running",
+              "range": true,
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "expr": "max(coderd_prebuilt_workspaces_eligible{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "legendFormat": "Eligible",
+              "range": true,
+              "refId": "E"
+            }
+          ],
+          "title": "Pool Capacity: $preset",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "axisSoftMax": 10,
+                "axisSoftMin": 0,
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 13,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "smooth",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 2,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "decimals": 0,
+              "fieldMinMax": false,
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Failed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "red",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Created"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "blue",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Desired"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "purple",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Running"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "yellow",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Eligible"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "green",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Claimed"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "dark-green",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 8,
             "x": 12,
-            "y": 15
+            "y": 5
           },
-          "id": 21,
+          "id": 38,
           "options": {
-            "colorMode": "value",
-            "graphMode": "area",
-            "justifyMode": "center",
-            "orientation": "auto",
-            "reduceOptions": {
-              "calcs": [
-                "mean"
-              ],
-              "fields": "",
-              "values": false
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
             },
-            "showPercentChange": false,
-            "textMode": "auto",
-            "wideLayout": true
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
           },
-          "pluginVersion": "10.4.0",
+          "pluginVersion": "10.4.3",
+          "repeat": "preset",
+          "repeatDirection": "v",
           "targets": [
             {
               "datasource": {
@@ -2568,11 +3344,13 @@ data:
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "quantile(0.9, coder_pubsub_send_latency_seconds)",
+              "expr": "floor(max(increase(coderd_prebuilt_workspaces_created_total{template_name=~\"$template\", preset_name=~\"$preset\"}[$__rate_interval]))) or vector(0)",
+              "hide": false,
               "instant": false,
-              "legendFormat": "Send",
+              "interval": "",
+              "legendFormat": "Created",
               "range": true,
-              "refId": "A"
+              "refId": "B"
             },
             {
               "datasource": {
@@ -2580,16 +3358,31 @@ data:
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "quantile(0.9, coder_pubsub_receive_latency_seconds)",
+              "expr": "floor(max(increase(coderd_prebuilt_workspaces_failed_total{template_name=~\"$template\", preset_name=~\"$preset\"}[$__rate_interval]))) or vector(0)",
               "hide": false,
               "instant": false,
-              "legendFormat": "Receive",
+              "interval": "",
+              "legendFormat": "Failed",
               "range": true,
-              "refId": "B"
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "expr": "floor(max(increase(coderd_prebuilt_workspaces_claimed_total{template_name=~\"$template\", preset_name=~\"$preset\"}[$__rate_interval]))) or vector(0)",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "legendFormat": "Claimed",
+              "range": true,
+              "refId": "F"
             }
           ],
-          "title": "Pubsub Latency (P90)",
-          "type": "stat"
+          "title": "Pool Operations: $preset",
+          "type": "timeseries"
         },
         {
           "datasource": {
@@ -2600,78 +3393,53 @@ data:
           "fieldConfig": {
             "defaults": {
               "color": {
-                "mode": "palette-classic"
-              },
-              "custom": {
-                "axisBorderShow": false,
-                "axisCenteredZero": false,
-                "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
-                "barAlignment": 0,
-                "drawStyle": "bars",
-                "fillOpacity": 100,
-                "gradientMode": "none",
-                "hideFrom": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": false
-                },
-                "insertNulls": false,
-                "lineInterpolation": "linear",
-                "lineWidth": 0,
-                "pointSize": 5,
-                "scaleDistribution": {
-                  "type": "linear"
-                },
-                "showPoints": "never",
-                "spanNulls": false,
-                "stacking": {
-                  "group": "A",
-                  "mode": "none"
-                },
-                "thresholdsStyle": {
-                  "mode": "off"
-                }
+                "fixedColor": "text",
+                "mode": "fixed"
               },
               "mappings": [],
               "thresholds": {
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "red",
+                    "color": "green",
                     "value": null
                   },
                   {
-                    "color": "green",
-                    "value": 1
+                    "color": "red",
+                    "value": 80
                   }
                 ]
-              },
-              "unit": "reqps"
+              }
             },
             "overrides": []
           },
           "gridPos": {
-            "h": 6,
-            "w": 6,
-            "x": 0,
-            "y": 18
+            "h": 7,
+            "w": 4,
+            "x": 20,
+            "y": 5
           },
-          "id": 35,
+          "id": 1,
+          "interval": "30s",
           "options": {
-            "legend": {
-              "calcs": [],
-              "displayMode": "list",
-              "placement": "bottom",
-              "showLegend": true
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "orientation": "vertical",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
             },
-            "tooltip": {
-              "mode": "single",
-              "sort": "none"
-            }
+            "showPercentChange": false,
+            "textMode": "auto",
+            "wideLayout": true
           },
-          "pluginVersion": "10.4.0",
+          "pluginVersion": "10.4.3",
+          "repeat": "preset",
+          "repeatDirection": "v",
           "targets": [
             {
               "datasource": {
@@ -2680,48 +3448,103 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "sum by(pod) (rate(coderd_api_requests_processed_total{pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`coder`}[$__rate_interval]))",
-              "instant": false,
-              "legendFormat": "__auto",
-              "range": true,
+              "expr": "max(coderd_prebuilt_workspaces_created_total{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+              "hide": false,
+              "instant": true,
+              "interval": "",
+              "legendFormat": "Created",
+              "range": false,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "max(coderd_prebuilt_workspaces_failed_total{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+              "hide": false,
+              "instant": true,
+              "interval": "",
+              "legendFormat": "Failed",
+              "range": false,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "max(coderd_prebuilt_workspaces_claimed_total{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)",
+              "hide": false,
+              "instant": true,
+              "interval": "",
+              "legendFormat": "Claimed",
+              "range": false,
               "refId": "A"
             }
           ],
-          "title": "API Requests",
-          "type": "timeseries"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prometheus"
-          },
-          "description": "",
-          "gridPos": {
-            "h": 6,
-            "w": 6,
-            "x": 6,
-            "y": 18
-          },
-          "id": 36,
-          "options": {
-            "code": {
-              "language": "plaintext",
-              "showLineNumbers": false,
-              "showMiniMap": false
-            },
-            "content": "This shows the number of requests per second each `coderd` replica is handling.\n\nHeavy skewing towards a single `coderd` replica indicates faulty loadbalancing.",
-            "mode": "markdown"
-          },
-          "pluginVersion": "10.4.0",
-          "transparent": true,
-          "type": "text"
+          "title": "All Time: $preset",
+          "type": "stat"
         }
       ],
       "refresh": "30s",
       "schemaVersion": 39,
       "tags": [],
       "templating": {
-        "list": []
+        "list": [
+          {
+            "allValue": "",
+            "datasource": {
+              "type": "prometheus",
+              "uid": "prometheus"
+            },
+            "definition": "label_values(coderd_prebuilt_workspaces_desired,template_name)",
+            "hide": 0,
+            "includeAll": false,
+            "label": "Template",
+            "multi": false,
+            "name": "template",
+            "options": [],
+            "query": {
+              "qryType": 1,
+              "query": "label_values(coderd_prebuilt_workspaces_desired,template_name)",
+              "refId": "PrometheusVariableQueryEditor-VariableQuery"
+            },
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 0,
+            "type": "query"
+          },
+          {
+            "allValue": "",
+            "datasource": {
+              "type": "prometheus",
+              "uid": "prometheus"
+            },
+            "definition": "label_values(coderd_prebuilt_workspaces_desired{template_name=~\"$template\"},preset_name)",
+            "hide": 0,
+            "includeAll": true,
+            "label": "Preset",
+            "multi": true,
+            "name": "preset",
+            "options": [],
+            "query": {
+              "qryType": 1,
+              "query": "label_values(coderd_prebuilt_workspaces_desired{template_name=~\"$template\"},preset_name)",
+              "refId": "PrometheusVariableQueryEditor-VariableQuery"
+            },
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 0,
+            "type": "query"
+          }
+        ]
       },
       "time": {
         "from": "now-12h",
@@ -2729,9 +3552,9 @@ data:
       },
       "timepicker": {},
       "timezone": "browser",
-      "title": "Control Plane",
-      "uid": "coderd",
-      "version": 6,
+      "title": "Prebuilds",
+      "uid": "cej6jysyme22oa",
+      "version": 13,
       "weekStart": ""
     }
 ---
@@ -9277,6 +10100,33 @@ subjects:
     name: grafana
     namespace: coder-observability
 ---
+# Source: coder-observability/charts/grafana-agent/templates/cluster_service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana-agent-cluster
+  labels:
+    app.kubernetes.io/name: grafana-agent
+    app.kubernetes.io/instance: coder-observability
+    app.kubernetes.io/managed-by: Helm
+spec:
+  type: ClusterIP
+  clusterIP: 'None'
+  selector:
+    app.kubernetes.io/name: grafana-agent
+    app.kubernetes.io/instance: coder-observability
+  ports:
+    # Do not include the -metrics suffix in the port name, otherwise metrics
+    # can be double-collected with the non-headless Service if it's also
+    # enabled.
+    #
+    # This service should only be used for clustering, and not metric
+    # collection.
+    - name: http
+      port: 80
+      targetPort: 80
+      protocol: "TCP"
+---
 # Source: coder-observability/charts/grafana-agent/templates/service.yaml
 apiVersion: v1
 kind: Service
@@ -9905,6 +10755,8 @@ spec:
             - --storage.path=/tmp/agent
             - --server.http.listen-addr=0.0.0.0:80
             - --server.http.ui-path-prefix=/
+            - --cluster.enabled=true
+            - --cluster.join-addresses=grafana-agent-cluster
             - --disable-reporting=true
           env:
             - name: AGENT_MODE
@@ -10162,7 +11014,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: 308677931777ae343a565387f5edecd66c53876ef7d120e9df3c6196c1884b30
+        checksum/config: bc7add19cdc0df1566dec1bf8f9421082357d4393124d6ea2df28d7e5888cc8a
       labels:
         app.kubernetes.io/name: loki
         app.kubernetes.io/instance: coder-observability
@@ -10246,7 +11098,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: 616b4c9b39f90d71edfd156f86c7f57751f662542a83f3e345c30496c3da7d27
+        checksum/config: 4dbe50185304244ab527314b7723d048ea4544f97d0a4d8e0192863861811005
         prometheus.io/scrape: "true"
       labels:
         app.kubernetes.io/part-of: memberlist
@@ -10432,8 +11284,8 @@ spec:
         app.kubernetes.io/name: grafana
         app.kubernetes.io/instance: coder-observability
       annotations:
-        checksum/config: 4b5f6512e962f90e1dcdfbecb3713a10f3a998745141a4fc1adfcbb4cff23282
-        checksum/dashboards-json-config: 3f59a9bfe9e7e9b7e6ca4ea81afd7bac7a8d78eadb7edbb44be4a327efd1d931
+        checksum/config: 8143dd78a48b90e972a84c3078e9812a2536a4efff7ca13f404626d5eae08ab2
+        checksum/dashboards-json-config: 010b57348b6dd1f09007330c03d22a0570022534712646511cad39a9e3cb4bb7
         checksum/sc-dashboard-provider-config: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
         kubectl.kubernetes.io/default-container: grafana
     spec:
@@ -10518,6 +11370,10 @@ spec:
               mountPath: /var/lib/grafana/dashboards/coder/4
               subPath:
               readOnly: false
+            - name: dashboards-prebuilds
+              mountPath: /var/lib/grafana/dashboards/coder/5
+              subPath:
+              readOnly: false
             - name: storage
               mountPath: "/var/lib/grafana"
             - name: config
@@ -10587,6 +11443,9 @@ spec:
         - name: dashboards-workspace-detail
           configMap:
             name: dashboards-workspace-detail
+        - name: dashboards-prebuilds
+          configMap:
+            name: dashboards-prebuilds
         - name: dashboards-infra
           configMap:
             name: grafana-dashboards-infra
@@ -10725,7 +11584,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: 616b4c9b39f90d71edfd156f86c7f57751f662542a83f3e345c30496c3da7d27
+        checksum/config: 4dbe50185304244ab527314b7723d048ea4544f97d0a4d8e0192863861811005
         prometheus.io/scrape: "true"
       labels:
         app.kubernetes.io/name: loki
@@ -11042,7 +11901,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: 616b4c9b39f90d71edfd156f86c7f57751f662542a83f3e345c30496c3da7d27
+        checksum/config: 4dbe50185304244ab527314b7723d048ea4544f97d0a4d8e0192863861811005
         prometheus.io/scrape: "true"
       labels:
         app.kubernetes.io/name: loki
@@ -11242,7 +12101,7 @@ spec:
         app.kubernetes.io/name: prometheus
         app.kubernetes.io/instance: coder-observability
         app.kubernetes.io/version: v2.53.1
-        helm.sh/chart: prometheus-25.24.1
+        helm.sh/chart: prometheus-25.24.2
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/part-of: prometheus
     spec:
@@ -11393,7 +12252,7 @@ spec:
               name: exporter
           env:
             - name: DATA_SOURCE_NAME
-              value: 'postgresql://coder@localhost:5432/postgres?sslmode=disable'
+              value: 'postgresql://coder@localhost:5432/coder?sslmode=disable'
           envFrom:
             - secretRef:
                 name: secret-postgres
@@ -11413,7 +12272,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: 7ab4d06d5b454cc584880f9b58fb50fe85f2803bbd0021edb957c5a2d73b640e
+        checksum/config: b0c41033d0385ee3d46488f08e85bcef0d939614dcb99194e0c5913dbf0c2c33
       labels:
         app: runbook-viewer
     spec:
@@ -11448,7 +12307,7 @@ spec:
     metadata:
       annotations:
         prometheus.io/scrape: 'true'
-        checksum/config: e12c0044ef2cab3438ffdc8f5e16c24c5acf5ee36dcc8bee77294f27e53ce4a2
+        checksum/config: 71bb9e7579b6e138ae28c623aa29d72025be00387da6c1b8dd5aa168c96ca1e0
       labels:
         app: sql-exporter
         app.kubernetes.io/name: "database-stats"
diff --git a/scripts/compile.sh b/scripts/compile.sh
index 13515de..a00ce4f 100755
--- a/scripts/compile.sh
+++ b/scripts/compile.sh
@@ -1,6 +1,12 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# check versions
+HELM_VERSION=3.17
+YQ_VERSION=4.42
+[[ "$(helm version)" == *v${HELM_VERSION}* ]] || { echo "Expected helm version v${HELM_VERSION} but got $(helm version)" >&2; exit 1; }
+[[ "$(yq --version)" == *v${YQ_VERSION}* ]] || { echo "Expected yq version v${YQ_VERSION} but got $(yq --version)" >&2; exit 1; }
+
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 
 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
@@ -13,7 +19,7 @@ helm dependency update coder-observability/
 # We *expect* that the versions will change in the rendered template output, so we ignore those, but
 # if there are changes to the manifests themselves then we need to fail the build to force manual review.
 helm template --namespace coder-observability -f coder-observability/values.yaml coder-observability coder-observability/ | \
-  yq 'del(.spec.template.spec.containers[].image, .metadata.labels."helm.sh/chart", .metadata.labels."app.kubernetes.io/version")' - \
+  yq e 'del(.spec.template.spec.containers[].image, .metadata.labels."helm.sh/chart", .metadata.labels."app.kubernetes.io/version")' - \
   > compiled/resources.yaml
 
 check_unstaged "compiled"
\ No newline at end of file
diff --git a/scripts/version.sh b/scripts/version.sh
index 488df3a..cb5ca3c 100755
--- a/scripts/version.sh
+++ b/scripts/version.sh
@@ -25,7 +25,7 @@ function bump_version() {
   local new_version
 
   if [[ $version == "major" ]]; then
-    new_version=$(echo $current_version | awk -F. '{print $1+1".0.0"}')
+    new_version=$(echo $current_version | sed 's/^v//' | awk -F. '{print "v" $1+1".0.0"}')
   elif [[ $version == "minor" ]]; then
     new_version=$(echo $current_version | awk -F. '{print $1"."$2+1".0"}')
   elif [[ $version == "patch" ]]; then
