From 143e4f538c9a41267225952dab0374adfd1327f6 Mon Sep 17 00:00:00 2001
From: Steven Masley <stevenmasley@gmail.com>
Date: Wed, 6 Aug 2025 14:40:27 -0500
Subject: [PATCH 1/3] chore: switch pyroscope from 80 -> 4040 (to unprivledged)

---
 TESTING_KIND.md                 |   2 +-
 coder-observability/values.yaml |   4 +-
 compiled/resources.yaml         | 190 +++++++++++++++++++++++++++++++-
 3 files changed, 191 insertions(+), 5 deletions(-)

diff --git a/TESTING_KIND.md b/TESTING_KIND.md
index ac2723f..0c7b03e 100644
--- a/TESTING_KIND.md
+++ b/TESTING_KIND.md
@@ -102,5 +102,5 @@ To view the monitoring services, you can port forward their UI's.
 
 - Grafana: `kubectl -n coder-observability port-forward svc/grafana 3000:80`
 - Prometheus: `kubectl -n coder-observability port-forward svc/prometheus 3001:80`
-- Pyroscope: `kubectl port-forward svc/pyroscope 3002:80`
+- Pyroscope: `kubectl port-forward svc/pyroscope 3002:4040`
 - Grafana Agent: `kubectl port-forward svc/grafana-agent 3003:80`
\ No newline at end of file
diff --git a/coder-observability/values.yaml b/coder-observability/values.yaml
index 546a1dc..b561cac 100644
--- a/coder-observability/values.yaml
+++ b/coder-observability/values.yaml
@@ -357,7 +357,7 @@ grafana:
       datasources:
         - name: pyroscope
           type: grafana-pyroscope-datasource
-          url: http://pyroscope.{{ .Release.Namespace }}.{{ $.Values.global.zone }}
+          url: http://pyroscope.{{ .Release.Namespace }}.{{ $.Values.global.zone }}:4040
           isDefault: false
           editable: false
           access: proxy
@@ -538,7 +538,7 @@ pyroscope:
   pyroscope:
     fullnameOverride: pyroscope
     service:
-      port: 80
+      port: 4040
       type: ClusterIP
     extraArgs:
       log.level: info
diff --git a/compiled/resources.yaml b/compiled/resources.yaml
index 4b9f536..3c8bde4 100644
--- a/compiled/resources.yaml
+++ b/compiled/resources.yaml
@@ -212,7 +212,7 @@ data:
       timeout: '905'
       type: grafana-pyroscope-datasource
       uid: pyroscope
-      url: http://pyroscope.coder-observability.svc
+      url: http://pyroscope.coder-observability.svc:4040
     - access: proxy
       editable: false
       isDefault: true
@@ -776,7 +776,11 @@ metadata:
   name: collector-config
   namespace: coder-observability
 data:
+<<<<<<< HEAD
   config.river: "\n// Discover k8s nodes\ndiscovery.kubernetes \"nodes\" {\n  role = \"node\"\n}\n\n// Discover k8s pods\ndiscovery.kubernetes \"pods\" {\n  role = \"pod\"\n  selectors {\n   role  = \"pod\"\n  }\n}\n\ndiscovery.relabel \"pod_logs\" {\n  targets = discovery.kubernetes.pods.targets\n  \n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\"]\n    target_label  = \"namespace\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_name\"]\n    target_label  = \"pod\"\n  }\n  // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n  rule {\n    source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n    separator      = \"/\"\n    target_label   = \"__meta_app\"\n    action         = \"replace\"\n    regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n    replacement    = \"${1}\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n    separator     = \"/\"\n    target_label  = \"job\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_name\"]\n    target_label  = \"container\"\n  }\n  rule {\n    regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n    action  = \"labeldrop\"\n  }\n  rule {\n    regex   = \"pod_template_generation\"\n    action  = \"labeldrop\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_phase\"]\n    regex = \"Pending|Succeeded|Failed|Completed\"\n    action = \"drop\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_node_name\"]\n    action = \"replace\"\n    target_label = \"node\"\n  }\n  rule {\n    action = \"labelmap\"\n    regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n    replacement = \"__param_$1\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_uid\", \"__meta_kubernetes_pod_container_name\"]\n    separator     = \"/\"\n    action        = \"replace\"\n    replacement   = \"/var/log/pods/*$1/*.log\"\n    target_label  = \"__path__\"\n  }\n  rule {\n    action = \"replace\"\n    source_labels = [\"__meta_kubernetes_pod_container_id\"]\n    regex = \"^(\\\\w+):\\\\/\\\\/.+$\"\n    replacement = \"$1\"\n    target_label = \"tmp_container_runtime\"\n  }\n}\n\ndiscovery.relabel \"pod_metrics\" {\n  targets = discovery.kubernetes.pods.targets\n  \n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\"]\n        target_label  = \"namespace\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_name\"]\n        target_label  = \"pod\"\n      }\n      // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n      rule {\n        source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n        separator      = \"/\"\n        target_label   = \"__meta_app\"\n        action         = \"replace\"\n        regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n        replacement    = \"${1}\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n        separator     = \"/\"\n        target_label  = \"job\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_container_name\"]\n        target_label  = \"container\"\n      }\n      rule {\n        regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        regex   = \"pod_template_generation\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_phase\"]\n        regex = \"Pending|Succeeded|Failed|Completed\"\n        action = \"drop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_node_name\"]\n        action = \"replace\"\n        target_label = \"node\"\n      }\n      rule {\n        action = \"labelmap\"\n        regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n        replacement = \"__param_$1\"\n      }\n  // drop ports that do not expose Prometheus metrics, but might otherwise be exposed by a container which *also*\n  // exposes an HTTP port which exposes metrics\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_port_name\"]\n    regex         = \"grpc|http-(memberlist|console)\"\n    action        = \"drop\"\n  }\n  // adapted from the Prometheus helm chart\n  // https://github.com/prometheus-community/helm-charts/blob/862870fc3c847e32479b509e511584d5283126a3/charts/prometheus/values.yaml#L1070\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scrape\"]\n    action = \"keep\"\n    regex = \"true\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scheme\"]\n    action = \"replace\"\n    regex = \"(https?)\"\n    target_label = \"__scheme__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_path\"]\n    action = \"replace\"\n    target_label = \"__metrics_path__\"\n    regex = \"(.+)\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n    replacement = \"[$2]:$1\"\n    target_label = \"__address__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n    replacement = \"$2:$1\"\n    target_label = \"__address__\"\n  }\n}\n\ndiscovery.relabel \"pod_pprof\" {\n  targets = discovery.kubernetes.pods.targets\n  \n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\"]\n        target_label  = \"namespace\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_name\"]\n        target_label  = \"pod\"\n      }\n      // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n      rule {\n        source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n        separator      = \"/\"\n        target_label   = \"__meta_app\"\n        action         = \"replace\"\n        regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n        replacement    = \"${1}\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n        separator     = \"/\"\n        target_label  = \"job\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_container_name\"]\n        target_label  = \"container\"\n      }\n      rule {\n        regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        regex   = \"pod_template_generation\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_phase\"]\n        regex = \"Pending|Succeeded|Failed|Completed\"\n        action = \"drop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_node_name\"]\n        action = \"replace\"\n        target_label = \"node\"\n      }\n      rule {\n        action = \"labelmap\"\n        regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n        replacement = \"__param_$1\"\n      }\n  // The relabeling allows the actual pod scrape endpoint to be configured via the\n  // following annotations:\n  //\n  // * `pyroscope.io/scrape`: Only scrape pods that have a value of `true`.\n  // * `pyroscope.io/application-name`: Name of the application being profiled.\n  // * `pyroscope.io/scheme`: If the metrics endpoint is secured then you will need\n  // to set this to `https` & most likely set the `tls_config` of the scrape config.\n  // * `pyroscope.io/port`: Scrape the pod on the indicated port.\n  //\n  // Kubernetes labels will be added as Pyroscope labels on metrics via the\n  // `labelmap` relabeling action.\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_scrape\"]\n    action = \"keep\"\n    regex = \"true\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_application_name\"]\n    action = \"replace\"\n    target_label = \"__name__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_scheme\"]\n    action = \"replace\"\n    regex = \"(https?)\"\n    target_label = \"__scheme__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n    replacement = \"[$2]:$1\"\n    target_label = \"__address__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n    replacement = \"$2:$1\"\n    target_label = \"__address__\"\n  }\n}\n\nlocal.file_match \"pod_logs\" {\n  path_targets = discovery.relabel.pod_logs.output\n}\n\nloki.source.file \"pod_logs\" {\n  targets    = local.file_match.pod_logs.targets\n  forward_to = [loki.process.pod_logs.receiver]\n}\n\nloki.process \"pod_logs\" {\n  stage.match {\n    selector = \"{tmp_container_runtime=\\\"containerd\\\"}\"\n    // the cri processing stage extracts the following k/v pairs: log, stream, time, flags\n    stage.cri {}\n    // Set the extract flags and stream values as labels\n    stage.labels {\n      values = {\n        flags   = \"\",\n        stream  = \"\",\n      }\n    }\n  }\n\n  // if the label tmp_container_runtime from above is docker parse using docker\n  stage.match {\n    selector = \"{tmp_container_runtime=\\\"docker\\\"}\"\n    // the docker processing stage extracts the following k/v pairs: log, stream, time\n    stage.docker {}\n\n    // Set the extract stream value as a label\n    stage.labels {\n      values = {\n        stream  = \"\",\n      }\n    }\n  }\n\n  // drop the temporary container runtime label as it is no longer needed\n  stage.label_drop {\n    values = [\"tmp_container_runtime\"]\n  }\n\n  // parse Coder logs and extract level & logger for efficient filtering\n  stage.match {\n    selector = \"{pod=~\\\"coder.*\\\"}\" // TODO: make configurable\n\n    stage.multiline {\n      firstline     = \"^(?P<ts>\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\"\n      max_wait_time = \"10s\"\n    }\n\n    stage.regex {\n      expression = \"^(?P<ts>\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\\\\s\\\\[(?P<level>\\\\w+)\\\\]\\\\s\\\\s(?P<logger>[^:]+):\\\\s(?P<line>.+)\"\n    }\n\n    stage.timestamp {\n      source            = \"ts\"\n      format            = \"2006-01-02 15:04:05.000\"\n      action_on_failure = \"fudge\" // rather have inaccurate time than drop the log line\n    }\n\n    stage.labels {\n      values = {\n        level  = \"\",\n        logger = \"\",\n      }\n    }\n  }\n\n  forward_to = [loki.write.loki.receiver]\n}\n\nloki.write \"loki\" {\n  endpoint {\n    url = \"http://loki-gateway.coder-observability.svc/loki/api/v1/push\"\n  }\n}\n\n\n\nprometheus.scrape \"pods\" {\n  targets = discovery.relabel.pod_metrics.output\n  forward_to = [prometheus.relabel.pods.receiver]\n\n  scrape_interval = \"15s\"\n  scrape_timeout = \"12s\"\n}\n\n// These are metric_relabel_configs while discovery.relabel are relabel_configs.\n// See https://github.com/grafana/agent/blob/main/internal/converter/internal/prometheusconvert/prometheusconvert.go#L95-L106\nprometheus.relabel \"pods\" {\n  forward_to = [prometheus.remote_write.default.receiver]\n\n  // Drop kube-state-metrics' labels which clash with ours\n  rule {\n    source_labels = [\"__name__\", \"container\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"container\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"pod\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"pod\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"namespace\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"namespace\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_container\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"container\"\n    replacement   = \"$1\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_pod\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"pod\"\n    replacement   = \"$1\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_namespace\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"namespace\"\n    replacement   = \"$1\"\n  }\n  rule {\n    regex         = \"^(exported_.*|image_.*|container_id|id|uid)$\"\n    action        = \"labeldrop\"\n  }\n}\n\ndiscovery.relabel \"cadvisor\" {\n  targets = discovery.kubernetes.nodes.targets\n  rule {\n    replacement   = \"/metrics/cadvisor\"\n    target_label  = \"__metrics_path__\"\n  }\n}\n\nprometheus.scrape \"cadvisor\" {\n  targets    = discovery.relabel.cadvisor.output\n  forward_to = [ prometheus.relabel.cadvisor.receiver ]\n  scheme     = \"https\"\n  tls_config {\n    insecure_skip_verify = true\n  }\n  bearer_token_file = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n  scrape_interval   = \"15s\"\n  scrape_timeout    = \"12s\"\n}\n\nprometheus.relabel \"cadvisor\" {\n  forward_to = [ prometheus.remote_write.default.receiver ]\n\n  // Drop empty container labels, addressing https://github.com/google/cadvisor/issues/2688\n  rule {\n    source_labels = [\"__name__\",\"container\"]\n    separator = \"@\"\n    regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*)@\"\n    action = \"drop\"\n  }\n  // Drop empty image labels, addressing https://github.com/google/cadvisor/issues/2688\n  rule {\n    source_labels = [\"__name__\",\"image\"]\n    separator = \"@\"\n    regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*|container_network_.*)@\"\n    action = \"drop\"\n  }\n  // Drop irrelevant series\n  rule {\n    source_labels = [\"container\"]\n    regex         = \"^POD$\"\n    action        = \"drop\"\n  }\n  // Drop unnecessary labels\n  rule {\n    source_labels = [\"id\"]\n    target_label  = \"id\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"job\"]\n    target_label  = \"job\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"name\"]\n    target_label  = \"name\"\n    replacement   = \"\"\n  }\n}\n\nprometheus.remote_write \"default\" {\n  endpoint {\n    url =\"http://prometheus.coder-observability.svc/api/v1/write\"\n\n    // drop instance label which unnecessarily adds new series when pods are restarted, since pod IPs are dynamically assigned\n    // NOTE: \"__address__\" is mapped to \"instance\", so will contain <hostname>:<port>\n    write_relabel_config {\n      regex  = \"instance\"\n      action = \"labeldrop\"\n    }\n  }\n}"
+=======
+  config.river: "\n// Discover k8s nodes\ndiscovery.kubernetes \"nodes\" {\n  role = \"node\"\n}\n\n// Discover k8s pods\ndiscovery.kubernetes \"pods\" {\n  role = \"pod\"\n  selectors {\n   role  = \"pod\"\n  }\n}\n\ndiscovery.relabel \"pod_logs\" {\n  targets = discovery.kubernetes.pods.targets\n  \n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\"]\n    target_label  = \"namespace\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_name\"]\n    target_label  = \"pod\"\n  }\n  // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n  rule {\n    source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n    separator      = \"/\"\n    target_label   = \"__meta_app\"\n    action         = \"replace\"\n    regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n    replacement    = \"${1}\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n    separator     = \"/\"\n    target_label  = \"job\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_name\"]\n    target_label  = \"container\"\n  }\n  rule {\n    regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n    action  = \"labeldrop\"\n  }\n  rule {\n    regex   = \"pod_template_generation\"\n    action  = \"labeldrop\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_phase\"]\n    regex = \"Pending|Succeeded|Failed|Completed\"\n    action = \"drop\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_node_name\"]\n    action = \"replace\"\n    target_label = \"node\"\n  }\n  rule {\n    action = \"labelmap\"\n    regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n    replacement = \"__param_$1\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_uid\", \"__meta_kubernetes_pod_container_name\"]\n    separator     = \"/\"\n    action        = \"replace\"\n    replacement   = \"/var/log/pods/*$1/*.log\"\n    target_label  = \"__path__\"\n  }\n  rule {\n    action = \"replace\"\n    source_labels = [\"__meta_kubernetes_pod_container_id\"]\n    regex = \"^(\\\\w+):\\\\/\\\\/.+$\"\n    replacement = \"$1\"\n    target_label = \"tmp_container_runtime\"\n  }\n}\n\ndiscovery.relabel \"pod_metrics\" {\n  targets = discovery.kubernetes.pods.targets\n  \n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\"]\n        target_label  = \"namespace\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_name\"]\n        target_label  = \"pod\"\n      }\n      // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n      rule {\n        source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n        separator      = \"/\"\n        target_label   = \"__meta_app\"\n        action         = \"replace\"\n        regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n        replacement    = \"${1}\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n        separator     = \"/\"\n        target_label  = \"job\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_container_name\"]\n        target_label  = \"container\"\n      }\n      rule {\n        regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        regex   = \"pod_template_generation\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_phase\"]\n        regex = \"Pending|Succeeded|Failed|Completed\"\n        action = \"drop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_node_name\"]\n        action = \"replace\"\n        target_label = \"node\"\n      }\n      rule {\n        action = \"labelmap\"\n        regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n        replacement = \"__param_$1\"\n      }\n  // drop ports that do not expose Prometheus metrics, but might otherwise be exposed by a container which *also*\n  // exposes an HTTP port which exposes metrics\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_port_name\"]\n    regex         = \"grpc|http-(memberlist|console)\"\n    action        = \"drop\"\n  }\n  // adapted from the Prometheus helm chart\n  // https://github.com/prometheus-community/helm-charts/blob/862870fc3c847e32479b509e511584d5283126a3/charts/prometheus/values.yaml#L1070\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scrape\"]\n    action = \"keep\"\n    regex = \"true\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scheme\"]\n    action = \"replace\"\n    regex = \"(https?)\"\n    target_label = \"__scheme__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_path\"]\n    action = \"replace\"\n    target_label = \"__metrics_path__\"\n    regex = \"(.+)\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n    replacement = \"[$2]:$1\"\n    target_label = \"__address__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n    replacement = \"$2:$1\"\n    target_label = \"__address__\"\n  }\n}\n\ndiscovery.relabel \"pod_pprof\" {\n  targets = discovery.kubernetes.pods.targets\n  \n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\"]\n        target_label  = \"namespace\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_name\"]\n        target_label  = \"pod\"\n      }\n      // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n      rule {\n        source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n        separator      = \"/\"\n        target_label   = \"__meta_app\"\n        action         = \"replace\"\n        regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n        replacement    = \"${1}\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n        separator     = \"/\"\n        target_label  = \"job\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_container_name\"]\n        target_label  = \"container\"\n      }\n      rule {\n        regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        regex   = \"pod_template_generation\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_phase\"]\n        regex = \"Pending|Succeeded|Failed|Completed\"\n        action = \"drop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_node_name\"]\n        action = \"replace\"\n        target_label = \"node\"\n      }\n      rule {\n        action = \"labelmap\"\n        regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n        replacement = \"__param_$1\"\n      }\n  // The relabeling allows the actual pod scrape endpoint to be configured via the\n  // following annotations:\n  //\n  // * `pyroscope.io/scrape`: Only scrape pods that have a value of `true`.\n  // * `pyroscope.io/application-name`: Name of the application being profiled.\n  // * `pyroscope.io/scheme`: If the metrics endpoint is secured then you will need\n  // to set this to `https` & most likely set the `tls_config` of the scrape config.\n  // * `pyroscope.io/port`: Scrape the pod on the indicated port.\n  //\n  // Kubernetes labels will be added as Pyroscope labels on metrics via the\n  // `labelmap` relabeling action.\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_scrape\"]\n    action = \"keep\"\n    regex = \"true\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_application_name\"]\n    action = \"replace\"\n    target_label = \"__name__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_scheme\"]\n    action = \"replace\"\n    regex = \"(https?)\"\n    target_label = \"__scheme__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n    replacement = \"[$2]:$1\"\n    target_label = \"__address__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n    replacement = \"$2:$1\"\n    target_label = \"__address__\"\n  }\n}\n\nlocal.file_match \"pod_logs\" {\n  path_targets = discovery.relabel.pod_logs.output\n}\n\nloki.source.file \"pod_logs\" {\n  targets    = local.file_match.pod_logs.targets\n  forward_to = [loki.process.pod_logs.receiver]\n}\n\nloki.process \"pod_logs\" {\n  stage.match {\n    selector = \"{tmp_container_runtime=\\\"containerd\\\"}\"\n    // the cri processing stage extracts the following k/v pairs: log, stream, time, flags\n    stage.cri {}\n    // Set the extract flags and stream values as labels\n    stage.labels {\n      values = {\n        flags   = \"\",\n        stream  = \"\",\n      }\n    }\n  }\n\n  // if the label tmp_container_runtime from above is docker parse using docker\n  stage.match {\n    selector = \"{tmp_container_runtime=\\\"docker\\\"}\"\n    // the docker processing stage extracts the following k/v pairs: log, stream, time\n    stage.docker {}\n\n    // Set the extract stream value as a label\n    stage.labels {\n      values = {\n        stream  = \"\",\n      }\n    }\n  }\n\n  // drop the temporary container runtime label as it is no longer needed\n  stage.label_drop {\n    values = [\"tmp_container_runtime\"]\n  }\n\n  // parse Coder logs and extract level & logger for efficient filtering\n  stage.match {\n    selector = \"{pod=~\\\"coder.*\\\"}\" // TODO: make configurable\n\n    stage.multiline {\n      firstline     = \"^(?P<ts>\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\"\n      max_wait_time = \"10s\"\n    }\n\n    stage.regex {\n      expression = \"^(?P<ts>\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\\\\s\\\\[(?P<level>\\\\w+)\\\\]\\\\s\\\\s(?P<logger>[^:]+):\\\\s(?P<line>.+)\"\n    }\n\n    stage.timestamp {\n      source            = \"ts\"\n      format            = \"2006-01-02 15:04:05.000\"\n      action_on_failure = \"fudge\" // rather have inaccurate time than drop the log line\n    }\n\n    stage.labels {\n      values = {\n        level  = \"\",\n        logger = \"\",\n      }\n    }\n  }\n\n  forward_to = [loki.write.loki.receiver]\n}\n\nloki.write \"loki\" {\n  endpoint {\n    url = \"http://loki-gateway.coder-observability.svc/loki/api/v1/push\"\n  }\n}\n\npyroscope.scrape \"pods\" {\n  targets = discovery.relabel.pod_pprof.output\n  forward_to = [pyroscope.write.pods.receiver]\n\n  scrape_interval = \"60s\"\n  scrape_timeout = \"60s\"\n}\n\npyroscope.write \"pods\" {\n  endpoint {\n    url = \"http://pyroscope.coder-observability.svc:4040\"\n  }\n}\n\nprometheus.scrape \"pods\" {\n  targets = discovery.relabel.pod_metrics.output\n  forward_to = [prometheus.relabel.pods.receiver]\n\n  scrape_interval = \"15s\"\n  scrape_timeout = \"12s\"\n}\n\n// These are metric_relabel_configs while discovery.relabel are relabel_configs.\n// See https://github.com/grafana/agent/blob/main/internal/converter/internal/prometheusconvert/prometheusconvert.go#L95-L106\nprometheus.relabel \"pods\" {\n  forward_to = [prometheus.remote_write.default.receiver]\n\n  // Drop kube-state-metrics' labels which clash with ours\n  rule {\n    source_labels = [\"__name__\", \"container\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"container\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"pod\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"pod\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"namespace\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"namespace\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_container\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"container\"\n    replacement   = \"$1\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_pod\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"pod\"\n    replacement   = \"$1\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_namespace\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"namespace\"\n    replacement   = \"$1\"\n  }\n  rule {\n    regex         = \"^(exported_.*|image_.*|container_id|id|uid)$\"\n    action        = \"labeldrop\"\n  }\n}\n\ndiscovery.relabel \"cadvisor\" {\n  targets = discovery.kubernetes.nodes.targets\n  rule {\n    replacement   = \"/metrics/cadvisor\"\n    target_label  = \"__metrics_path__\"\n  }\n}\n\nprometheus.scrape \"cadvisor\" {\n  targets    = discovery.relabel.cadvisor.output\n  forward_to = [ prometheus.relabel.cadvisor.receiver ]\n  scheme     = \"https\"\n  tls_config {\n    insecure_skip_verify = true\n  }\n  bearer_token_file = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n  scrape_interval   = \"15s\"\n  scrape_timeout    = \"12s\"\n}\n\nprometheus.relabel \"cadvisor\" {\n  forward_to = [ prometheus.remote_write.default.receiver ]\n\n  // Drop empty container labels, addressing https://github.com/google/cadvisor/issues/2688\n  rule {\n    source_labels = [\"__name__\",\"container\"]\n    separator = \"@\"\n    regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*)@\"\n    action = \"drop\"\n  }\n  // Drop empty image labels, addressing https://github.com/google/cadvisor/issues/2688\n  rule {\n    source_labels = [\"__name__\",\"image\"]\n    separator = \"@\"\n    regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*|container_network_.*)@\"\n    action = \"drop\"\n  }\n  // Drop irrelevant series\n  rule {\n    source_labels = [\"container\"]\n    regex         = \"^POD$\"\n    action        = \"drop\"\n  }\n  // Drop unnecessary labels\n  rule {\n    source_labels = [\"id\"]\n    target_label  = \"id\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"job\"]\n    target_label  = \"job\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"name\"]\n    target_label  = \"name\"\n    replacement   = \"\"\n  }\n}\n\nprometheus.remote_write \"default\" {\n  endpoint {\n    url =\"http://prometheus.coder-observability.svc/api/v1/write\"\n\n    // drop instance label which unnecessarily adds new series when pods are restarted, since pod IPs are dynamically assigned\n    // NOTE: \"__address__\" is mapped to \"instance\", so will contain <hostname>:<port>\n    write_relabel_config {\n      regex  = \"instance\"\n      action = \"labeldrop\"\n    }\n  }\n}"
+>>>>>>> ad8b169 (chore: switch pyroscope from 80 -> 4040 (to unprivledged))
 ---
 # Source: coder-observability/templates/configmap-prometheus-alerts.yaml
 apiVersion: v1
@@ -10681,6 +10685,81 @@ spec:
   sessionAffinity: None
   type: "ClusterIP"
 ---
+<<<<<<< HEAD
+=======
+# Source: coder-observability/charts/pyroscope/templates/memberlist-service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: pyroscope-memberlist
+  namespace: coder-observability
+  labels:
+    app.kubernetes.io/name: pyroscope
+    app.kubernetes.io/instance: coder-observability
+    app.kubernetes.io/managed-by: Helm
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+    - name: memberlist
+      port: 7946
+      protocol: TCP
+      targetPort: 7946
+  publishNotReadyAddresses: true
+  selector:
+    app.kubernetes.io/name: pyroscope
+    app.kubernetes.io/instance: coder-observability
+    # TODO: Ensure only services that offer memberlist register
+    # pyroscope.grafana.com/memberlist: "true"
+---
+# Source: coder-observability/charts/pyroscope/templates/services.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: pyroscope
+  namespace: coder-observability
+  labels:
+    app.kubernetes.io/name: pyroscope
+    app.kubernetes.io/instance: coder-observability
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: "all"
+spec:
+  type: ClusterIP
+  ports:
+    - port: 4040
+      targetPort: http2
+      protocol: TCP
+      name: http2
+  selector:
+    app.kubernetes.io/name: pyroscope
+    app.kubernetes.io/instance: coder-observability
+    app.kubernetes.io/component: "all"
+---
+# Source: coder-observability/charts/pyroscope/templates/services.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: pyroscope-headless
+  namespace: coder-observability
+  labels:
+    app.kubernetes.io/name: pyroscope
+    app.kubernetes.io/instance: coder-observability
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: "all"
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+    - port: 4040
+      targetPort: http2
+      protocol: TCP
+      name: http2
+  selector:
+    app.kubernetes.io/name: pyroscope
+    app.kubernetes.io/instance: coder-observability
+    app.kubernetes.io/component: "all"
+---
+>>>>>>> ad8b169 (chore: switch pyroscope from 80 -> 4040 (to unprivledged))
 # Source: coder-observability/templates/service-runbook-viewer.yaml
 apiVersion: v1
 kind: Service
@@ -11255,7 +11334,7 @@ spec:
         app.kubernetes.io/name: grafana
         app.kubernetes.io/instance: coder-observability
       annotations:
-        checksum/config: 666fc9d25c33f33d23342eb8113dc2abcd9ba71cc245634e924f8397fb8d777b
+        checksum/config: 8143dd78a48b90e972a84c3078e9812a2536a4efff7ca13f404626d5eae08ab2
         checksum/dashboards-json-config: 010b57348b6dd1f09007330c03d22a0570022534712646511cad39a9e3cb4bb7
         checksum/sc-dashboard-provider-config: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
         kubectl.kubernetes.io/default-container: grafana
@@ -12194,6 +12273,113 @@ spec:
           requests:
             storage: "12Gi"
 ---
+<<<<<<< HEAD
+=======
+# Source: coder-observability/charts/pyroscope/templates/deployments-statefulsets.yaml
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: pyroscope
+  namespace: coder-observability
+  labels:
+    app.kubernetes.io/name: pyroscope
+    app.kubernetes.io/instance: coder-observability
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: "all"
+spec:
+  serviceName: pyroscope-headless
+  podManagementPolicy: Parallel
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: pyroscope
+      app.kubernetes.io/instance: coder-observability
+      app.kubernetes.io/component: "all"
+  template:
+    metadata:
+      annotations:
+        checksum/config: 1b47e0b4cfaba4c3fb572bf60178419eff351bf17d1a1d1a6b8e5a54f6ffcbea
+        profiles.grafana.com/service_repository: "https://github.com/grafana/pyroscope"
+        profiles.grafana.com/service_git_ref: "v1.14.0"
+        profiles.grafana.com/cpu.port_name: http2
+        profiles.grafana.com/cpu.scrape: "true"
+        profiles.grafana.com/goroutine.port_name: http2
+        profiles.grafana.com/goroutine.scrape: "true"
+        profiles.grafana.com/memory.port_name: http2
+        profiles.grafana.com/memory.scrape: "true"
+      labels:
+        app.kubernetes.io/name: pyroscope
+        app.kubernetes.io/instance: coder-observability
+        app.kubernetes.io/component: "all"
+        name: "pyroscope"
+    spec:
+      serviceAccountName: pyroscope
+      securityContext:
+        fsGroup: 10001
+        runAsNonRoot: true
+        runAsUser: 10001
+      dnsPolicy: ClusterFirst
+      containers:
+        - name: "pyroscope"
+          securityContext: {}
+          imagePullPolicy: IfNotPresent
+          args:
+            - "-target=all"
+            - "-self-profiling.disable-push=true"
+            - "-server.http-listen-port=4040"
+            - "-memberlist.cluster-label=coder-observability-pyroscope"
+            - "-memberlist.join=dns+pyroscope-memberlist.coder-observability.svc.cluster.local.:7946"
+            - "-config.file=/etc/pyroscope/config.yaml"
+            - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
+            - "-log.level=info"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "coder-observability.svc.cluster.local."
+          ports:
+            - name: http2
+              containerPort: 4040
+              protocol: TCP
+            - name: memberlist
+              containerPort: 7946
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: http2
+              scheme: HTTP
+          volumeMounts:
+            - name: config
+              mountPath: /etc/pyroscope/config.yaml
+              subPath: config.yaml
+            - name: overrides-config
+              mountPath: /etc/pyroscope/overrides/
+            - name: data
+              mountPath: /data
+          resources: {}
+      volumes:
+        - name: config
+          configMap:
+            name: pyroscope-config
+        - name: overrides-config
+          configMap:
+            name: pyroscope-overrides-config
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+        annotations: {}
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: "10Gi"
+        storageClassName:
+---
+>>>>>>> ad8b169 (chore: switch pyroscope from 80 -> 4040 (to unprivledged))
 # Source: coder-observability/templates/statefulset-postgres-exporter.yaml
 apiVersion: apps/v1
 kind: StatefulSet

From ca7f026fae14c4b575ad4489face3596b0c68cb8 Mon Sep 17 00:00:00 2001
From: Steven Masley <stevenmasley@gmail.com>
Date: Wed, 6 Aug 2025 14:44:23 -0500
Subject: [PATCH 2/3] compile.sh

---
 compiled/resources.yaml | 186 ----------------------------------------
 1 file changed, 186 deletions(-)

diff --git a/compiled/resources.yaml b/compiled/resources.yaml
index 3c8bde4..a2788a9 100644
--- a/compiled/resources.yaml
+++ b/compiled/resources.yaml
@@ -776,11 +776,7 @@ metadata:
   name: collector-config
   namespace: coder-observability
 data:
-<<<<<<< HEAD
   config.river: "\n// Discover k8s nodes\ndiscovery.kubernetes \"nodes\" {\n  role = \"node\"\n}\n\n// Discover k8s pods\ndiscovery.kubernetes \"pods\" {\n  role = \"pod\"\n  selectors {\n   role  = \"pod\"\n  }\n}\n\ndiscovery.relabel \"pod_logs\" {\n  targets = discovery.kubernetes.pods.targets\n  \n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\"]\n    target_label  = \"namespace\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_name\"]\n    target_label  = \"pod\"\n  }\n  // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n  rule {\n    source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n    separator      = \"/\"\n    target_label   = \"__meta_app\"\n    action         = \"replace\"\n    regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n    replacement    = \"${1}\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n    separator     = \"/\"\n    target_label  = \"job\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_name\"]\n    target_label  = \"container\"\n  }\n  rule {\n    regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n    action  = \"labeldrop\"\n  }\n  rule {\n    regex   = \"pod_template_generation\"\n    action  = \"labeldrop\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_phase\"]\n    regex = \"Pending|Succeeded|Failed|Completed\"\n    action = \"drop\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_node_name\"]\n    action = \"replace\"\n    target_label = \"node\"\n  }\n  rule {\n    action = \"labelmap\"\n    regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n    replacement = \"__param_$1\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_uid\", \"__meta_kubernetes_pod_container_name\"]\n    separator     = \"/\"\n    action        = \"replace\"\n    replacement   = \"/var/log/pods/*$1/*.log\"\n    target_label  = \"__path__\"\n  }\n  rule {\n    action = \"replace\"\n    source_labels = [\"__meta_kubernetes_pod_container_id\"]\n    regex = \"^(\\\\w+):\\\\/\\\\/.+$\"\n    replacement = \"$1\"\n    target_label = \"tmp_container_runtime\"\n  }\n}\n\ndiscovery.relabel \"pod_metrics\" {\n  targets = discovery.kubernetes.pods.targets\n  \n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\"]\n        target_label  = \"namespace\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_name\"]\n        target_label  = \"pod\"\n      }\n      // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n      rule {\n        source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n        separator      = \"/\"\n        target_label   = \"__meta_app\"\n        action         = \"replace\"\n        regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n        replacement    = \"${1}\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n        separator     = \"/\"\n        target_label  = \"job\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_container_name\"]\n        target_label  = \"container\"\n      }\n      rule {\n        regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        regex   = \"pod_template_generation\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_phase\"]\n        regex = \"Pending|Succeeded|Failed|Completed\"\n        action = \"drop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_node_name\"]\n        action = \"replace\"\n        target_label = \"node\"\n      }\n      rule {\n        action = \"labelmap\"\n        regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n        replacement = \"__param_$1\"\n      }\n  // drop ports that do not expose Prometheus metrics, but might otherwise be exposed by a container which *also*\n  // exposes an HTTP port which exposes metrics\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_port_name\"]\n    regex         = \"grpc|http-(memberlist|console)\"\n    action        = \"drop\"\n  }\n  // adapted from the Prometheus helm chart\n  // https://github.com/prometheus-community/helm-charts/blob/862870fc3c847e32479b509e511584d5283126a3/charts/prometheus/values.yaml#L1070\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scrape\"]\n    action = \"keep\"\n    regex = \"true\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scheme\"]\n    action = \"replace\"\n    regex = \"(https?)\"\n    target_label = \"__scheme__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_path\"]\n    action = \"replace\"\n    target_label = \"__metrics_path__\"\n    regex = \"(.+)\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n    replacement = \"[$2]:$1\"\n    target_label = \"__address__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n    replacement = \"$2:$1\"\n    target_label = \"__address__\"\n  }\n}\n\ndiscovery.relabel \"pod_pprof\" {\n  targets = discovery.kubernetes.pods.targets\n  \n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\"]\n        target_label  = \"namespace\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_name\"]\n        target_label  = \"pod\"\n      }\n      // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n      rule {\n        source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n        separator      = \"/\"\n        target_label   = \"__meta_app\"\n        action         = \"replace\"\n        regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n        replacement    = \"${1}\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n        separator     = \"/\"\n        target_label  = \"job\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_container_name\"]\n        target_label  = \"container\"\n      }\n      rule {\n        regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        regex   = \"pod_template_generation\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_phase\"]\n        regex = \"Pending|Succeeded|Failed|Completed\"\n        action = \"drop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_node_name\"]\n        action = \"replace\"\n        target_label = \"node\"\n      }\n      rule {\n        action = \"labelmap\"\n        regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n        replacement = \"__param_$1\"\n      }\n  // The relabeling allows the actual pod scrape endpoint to be configured via the\n  // following annotations:\n  //\n  // * `pyroscope.io/scrape`: Only scrape pods that have a value of `true`.\n  // * `pyroscope.io/application-name`: Name of the application being profiled.\n  // * `pyroscope.io/scheme`: If the metrics endpoint is secured then you will need\n  // to set this to `https` & most likely set the `tls_config` of the scrape config.\n  // * `pyroscope.io/port`: Scrape the pod on the indicated port.\n  //\n  // Kubernetes labels will be added as Pyroscope labels on metrics via the\n  // `labelmap` relabeling action.\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_scrape\"]\n    action = \"keep\"\n    regex = \"true\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_application_name\"]\n    action = \"replace\"\n    target_label = \"__name__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_scheme\"]\n    action = \"replace\"\n    regex = \"(https?)\"\n    target_label = \"__scheme__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n    replacement = \"[$2]:$1\"\n    target_label = \"__address__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n    replacement = \"$2:$1\"\n    target_label = \"__address__\"\n  }\n}\n\nlocal.file_match \"pod_logs\" {\n  path_targets = discovery.relabel.pod_logs.output\n}\n\nloki.source.file \"pod_logs\" {\n  targets    = local.file_match.pod_logs.targets\n  forward_to = [loki.process.pod_logs.receiver]\n}\n\nloki.process \"pod_logs\" {\n  stage.match {\n    selector = \"{tmp_container_runtime=\\\"containerd\\\"}\"\n    // the cri processing stage extracts the following k/v pairs: log, stream, time, flags\n    stage.cri {}\n    // Set the extract flags and stream values as labels\n    stage.labels {\n      values = {\n        flags   = \"\",\n        stream  = \"\",\n      }\n    }\n  }\n\n  // if the label tmp_container_runtime from above is docker parse using docker\n  stage.match {\n    selector = \"{tmp_container_runtime=\\\"docker\\\"}\"\n    // the docker processing stage extracts the following k/v pairs: log, stream, time\n    stage.docker {}\n\n    // Set the extract stream value as a label\n    stage.labels {\n      values = {\n        stream  = \"\",\n      }\n    }\n  }\n\n  // drop the temporary container runtime label as it is no longer needed\n  stage.label_drop {\n    values = [\"tmp_container_runtime\"]\n  }\n\n  // parse Coder logs and extract level & logger for efficient filtering\n  stage.match {\n    selector = \"{pod=~\\\"coder.*\\\"}\" // TODO: make configurable\n\n    stage.multiline {\n      firstline     = \"^(?P<ts>\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\"\n      max_wait_time = \"10s\"\n    }\n\n    stage.regex {\n      expression = \"^(?P<ts>\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\\\\s\\\\[(?P<level>\\\\w+)\\\\]\\\\s\\\\s(?P<logger>[^:]+):\\\\s(?P<line>.+)\"\n    }\n\n    stage.timestamp {\n      source            = \"ts\"\n      format            = \"2006-01-02 15:04:05.000\"\n      action_on_failure = \"fudge\" // rather have inaccurate time than drop the log line\n    }\n\n    stage.labels {\n      values = {\n        level  = \"\",\n        logger = \"\",\n      }\n    }\n  }\n\n  forward_to = [loki.write.loki.receiver]\n}\n\nloki.write \"loki\" {\n  endpoint {\n    url = \"http://loki-gateway.coder-observability.svc/loki/api/v1/push\"\n  }\n}\n\n\n\nprometheus.scrape \"pods\" {\n  targets = discovery.relabel.pod_metrics.output\n  forward_to = [prometheus.relabel.pods.receiver]\n\n  scrape_interval = \"15s\"\n  scrape_timeout = \"12s\"\n}\n\n// These are metric_relabel_configs while discovery.relabel are relabel_configs.\n// See https://github.com/grafana/agent/blob/main/internal/converter/internal/prometheusconvert/prometheusconvert.go#L95-L106\nprometheus.relabel \"pods\" {\n  forward_to = [prometheus.remote_write.default.receiver]\n\n  // Drop kube-state-metrics' labels which clash with ours\n  rule {\n    source_labels = [\"__name__\", \"container\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"container\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"pod\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"pod\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"namespace\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"namespace\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_container\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"container\"\n    replacement   = \"$1\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_pod\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"pod\"\n    replacement   = \"$1\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_namespace\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"namespace\"\n    replacement   = \"$1\"\n  }\n  rule {\n    regex         = \"^(exported_.*|image_.*|container_id|id|uid)$\"\n    action        = \"labeldrop\"\n  }\n}\n\ndiscovery.relabel \"cadvisor\" {\n  targets = discovery.kubernetes.nodes.targets\n  rule {\n    replacement   = \"/metrics/cadvisor\"\n    target_label  = \"__metrics_path__\"\n  }\n}\n\nprometheus.scrape \"cadvisor\" {\n  targets    = discovery.relabel.cadvisor.output\n  forward_to = [ prometheus.relabel.cadvisor.receiver ]\n  scheme     = \"https\"\n  tls_config {\n    insecure_skip_verify = true\n  }\n  bearer_token_file = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n  scrape_interval   = \"15s\"\n  scrape_timeout    = \"12s\"\n}\n\nprometheus.relabel \"cadvisor\" {\n  forward_to = [ prometheus.remote_write.default.receiver ]\n\n  // Drop empty container labels, addressing https://github.com/google/cadvisor/issues/2688\n  rule {\n    source_labels = [\"__name__\",\"container\"]\n    separator = \"@\"\n    regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*)@\"\n    action = \"drop\"\n  }\n  // Drop empty image labels, addressing https://github.com/google/cadvisor/issues/2688\n  rule {\n    source_labels = [\"__name__\",\"image\"]\n    separator = \"@\"\n    regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*|container_network_.*)@\"\n    action = \"drop\"\n  }\n  // Drop irrelevant series\n  rule {\n    source_labels = [\"container\"]\n    regex         = \"^POD$\"\n    action        = \"drop\"\n  }\n  // Drop unnecessary labels\n  rule {\n    source_labels = [\"id\"]\n    target_label  = \"id\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"job\"]\n    target_label  = \"job\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"name\"]\n    target_label  = \"name\"\n    replacement   = \"\"\n  }\n}\n\nprometheus.remote_write \"default\" {\n  endpoint {\n    url =\"http://prometheus.coder-observability.svc/api/v1/write\"\n\n    // drop instance label which unnecessarily adds new series when pods are restarted, since pod IPs are dynamically assigned\n    // NOTE: \"__address__\" is mapped to \"instance\", so will contain <hostname>:<port>\n    write_relabel_config {\n      regex  = \"instance\"\n      action = \"labeldrop\"\n    }\n  }\n}"
-=======
-  config.river: "\n// Discover k8s nodes\ndiscovery.kubernetes \"nodes\" {\n  role = \"node\"\n}\n\n// Discover k8s pods\ndiscovery.kubernetes \"pods\" {\n  role = \"pod\"\n  selectors {\n   role  = \"pod\"\n  }\n}\n\ndiscovery.relabel \"pod_logs\" {\n  targets = discovery.kubernetes.pods.targets\n  \n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\"]\n    target_label  = \"namespace\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_name\"]\n    target_label  = \"pod\"\n  }\n  // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n  rule {\n    source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n    separator      = \"/\"\n    target_label   = \"__meta_app\"\n    action         = \"replace\"\n    regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n    replacement    = \"${1}\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n    separator     = \"/\"\n    target_label  = \"job\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_name\"]\n    target_label  = \"container\"\n  }\n  rule {\n    regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n    action  = \"labeldrop\"\n  }\n  rule {\n    regex   = \"pod_template_generation\"\n    action  = \"labeldrop\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_phase\"]\n    regex = \"Pending|Succeeded|Failed|Completed\"\n    action = \"drop\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_node_name\"]\n    action = \"replace\"\n    target_label = \"node\"\n  }\n  rule {\n    action = \"labelmap\"\n    regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n    replacement = \"__param_$1\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_uid\", \"__meta_kubernetes_pod_container_name\"]\n    separator     = \"/\"\n    action        = \"replace\"\n    replacement   = \"/var/log/pods/*$1/*.log\"\n    target_label  = \"__path__\"\n  }\n  rule {\n    action = \"replace\"\n    source_labels = [\"__meta_kubernetes_pod_container_id\"]\n    regex = \"^(\\\\w+):\\\\/\\\\/.+$\"\n    replacement = \"$1\"\n    target_label = \"tmp_container_runtime\"\n  }\n}\n\ndiscovery.relabel \"pod_metrics\" {\n  targets = discovery.kubernetes.pods.targets\n  \n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\"]\n        target_label  = \"namespace\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_name\"]\n        target_label  = \"pod\"\n      }\n      // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n      rule {\n        source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n        separator      = \"/\"\n        target_label   = \"__meta_app\"\n        action         = \"replace\"\n        regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n        replacement    = \"${1}\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n        separator     = \"/\"\n        target_label  = \"job\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_container_name\"]\n        target_label  = \"container\"\n      }\n      rule {\n        regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        regex   = \"pod_template_generation\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_phase\"]\n        regex = \"Pending|Succeeded|Failed|Completed\"\n        action = \"drop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_node_name\"]\n        action = \"replace\"\n        target_label = \"node\"\n      }\n      rule {\n        action = \"labelmap\"\n        regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n        replacement = \"__param_$1\"\n      }\n  // drop ports that do not expose Prometheus metrics, but might otherwise be exposed by a container which *also*\n  // exposes an HTTP port which exposes metrics\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_container_port_name\"]\n    regex         = \"grpc|http-(memberlist|console)\"\n    action        = \"drop\"\n  }\n  // adapted from the Prometheus helm chart\n  // https://github.com/prometheus-community/helm-charts/blob/862870fc3c847e32479b509e511584d5283126a3/charts/prometheus/values.yaml#L1070\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scrape\"]\n    action = \"keep\"\n    regex = \"true\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scheme\"]\n    action = \"replace\"\n    regex = \"(https?)\"\n    target_label = \"__scheme__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_path\"]\n    action = \"replace\"\n    target_label = \"__metrics_path__\"\n    regex = \"(.+)\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n    replacement = \"[$2]:$1\"\n    target_label = \"__address__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n    replacement = \"$2:$1\"\n    target_label = \"__address__\"\n  }\n}\n\ndiscovery.relabel \"pod_pprof\" {\n  targets = discovery.kubernetes.pods.targets\n  \n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\"]\n        target_label  = \"namespace\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_name\"]\n        target_label  = \"pod\"\n      }\n      // coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\n      rule {\n        source_labels  = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n        separator      = \"/\"\n        target_label   = \"__meta_app\"\n        action         = \"replace\"\n        regex          = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n        replacement    = \"${1}\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n        separator     = \"/\"\n        target_label  = \"job\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_container_name\"]\n        target_label  = \"container\"\n      }\n      rule {\n        regex   = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        regex   = \"pod_template_generation\"\n        action  = \"labeldrop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_phase\"]\n        regex = \"Pending|Succeeded|Failed|Completed\"\n        action = \"drop\"\n      }\n      rule {\n        source_labels = [\"__meta_kubernetes_pod_node_name\"]\n        action = \"replace\"\n        target_label = \"node\"\n      }\n      rule {\n        action = \"labelmap\"\n        regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n        replacement = \"__param_$1\"\n      }\n  // The relabeling allows the actual pod scrape endpoint to be configured via the\n  // following annotations:\n  //\n  // * `pyroscope.io/scrape`: Only scrape pods that have a value of `true`.\n  // * `pyroscope.io/application-name`: Name of the application being profiled.\n  // * `pyroscope.io/scheme`: If the metrics endpoint is secured then you will need\n  // to set this to `https` & most likely set the `tls_config` of the scrape config.\n  // * `pyroscope.io/port`: Scrape the pod on the indicated port.\n  //\n  // Kubernetes labels will be added as Pyroscope labels on metrics via the\n  // `labelmap` relabeling action.\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_scrape\"]\n    action = \"keep\"\n    regex = \"true\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_application_name\"]\n    action = \"replace\"\n    target_label = \"__name__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_scheme\"]\n    action = \"replace\"\n    regex = \"(https?)\"\n    target_label = \"__scheme__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n    replacement = \"[$2]:$1\"\n    target_label = \"__address__\"\n  }\n  rule {\n    source_labels = [\"__meta_kubernetes_pod_annotation_pyroscope_io_port\", \"__meta_kubernetes_pod_ip\"]\n    action = \"replace\"\n    regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\"\n    replacement = \"$2:$1\"\n    target_label = \"__address__\"\n  }\n}\n\nlocal.file_match \"pod_logs\" {\n  path_targets = discovery.relabel.pod_logs.output\n}\n\nloki.source.file \"pod_logs\" {\n  targets    = local.file_match.pod_logs.targets\n  forward_to = [loki.process.pod_logs.receiver]\n}\n\nloki.process \"pod_logs\" {\n  stage.match {\n    selector = \"{tmp_container_runtime=\\\"containerd\\\"}\"\n    // the cri processing stage extracts the following k/v pairs: log, stream, time, flags\n    stage.cri {}\n    // Set the extract flags and stream values as labels\n    stage.labels {\n      values = {\n        flags   = \"\",\n        stream  = \"\",\n      }\n    }\n  }\n\n  // if the label tmp_container_runtime from above is docker parse using docker\n  stage.match {\n    selector = \"{tmp_container_runtime=\\\"docker\\\"}\"\n    // the docker processing stage extracts the following k/v pairs: log, stream, time\n    stage.docker {}\n\n    // Set the extract stream value as a label\n    stage.labels {\n      values = {\n        stream  = \"\",\n      }\n    }\n  }\n\n  // drop the temporary container runtime label as it is no longer needed\n  stage.label_drop {\n    values = [\"tmp_container_runtime\"]\n  }\n\n  // parse Coder logs and extract level & logger for efficient filtering\n  stage.match {\n    selector = \"{pod=~\\\"coder.*\\\"}\" // TODO: make configurable\n\n    stage.multiline {\n      firstline     = \"^(?P<ts>\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\"\n      max_wait_time = \"10s\"\n    }\n\n    stage.regex {\n      expression = \"^(?P<ts>\\\\d{4}-\\\\d{2}-\\\\d{2}\\\\s\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3})\\\\s\\\\[(?P<level>\\\\w+)\\\\]\\\\s\\\\s(?P<logger>[^:]+):\\\\s(?P<line>.+)\"\n    }\n\n    stage.timestamp {\n      source            = \"ts\"\n      format            = \"2006-01-02 15:04:05.000\"\n      action_on_failure = \"fudge\" // rather have inaccurate time than drop the log line\n    }\n\n    stage.labels {\n      values = {\n        level  = \"\",\n        logger = \"\",\n      }\n    }\n  }\n\n  forward_to = [loki.write.loki.receiver]\n}\n\nloki.write \"loki\" {\n  endpoint {\n    url = \"http://loki-gateway.coder-observability.svc/loki/api/v1/push\"\n  }\n}\n\npyroscope.scrape \"pods\" {\n  targets = discovery.relabel.pod_pprof.output\n  forward_to = [pyroscope.write.pods.receiver]\n\n  scrape_interval = \"60s\"\n  scrape_timeout = \"60s\"\n}\n\npyroscope.write \"pods\" {\n  endpoint {\n    url = \"http://pyroscope.coder-observability.svc:4040\"\n  }\n}\n\nprometheus.scrape \"pods\" {\n  targets = discovery.relabel.pod_metrics.output\n  forward_to = [prometheus.relabel.pods.receiver]\n\n  scrape_interval = \"15s\"\n  scrape_timeout = \"12s\"\n}\n\n// These are metric_relabel_configs while discovery.relabel are relabel_configs.\n// See https://github.com/grafana/agent/blob/main/internal/converter/internal/prometheusconvert/prometheusconvert.go#L95-L106\nprometheus.relabel \"pods\" {\n  forward_to = [prometheus.remote_write.default.receiver]\n\n  // Drop kube-state-metrics' labels which clash with ours\n  rule {\n    source_labels = [\"__name__\", \"container\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"container\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"pod\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"pod\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"namespace\"]\n    regex         = \"kube_pod.+;(.+)\"\n    target_label  = \"namespace\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_container\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"container\"\n    replacement   = \"$1\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_pod\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"pod\"\n    replacement   = \"$1\"\n  }\n  rule {\n    source_labels = [\"__name__\", \"exported_namespace\"]\n    // don't replace an empty label\n    regex         = \"^kube_pod.+;(.+)$\"\n    target_label  = \"namespace\"\n    replacement   = \"$1\"\n  }\n  rule {\n    regex         = \"^(exported_.*|image_.*|container_id|id|uid)$\"\n    action        = \"labeldrop\"\n  }\n}\n\ndiscovery.relabel \"cadvisor\" {\n  targets = discovery.kubernetes.nodes.targets\n  rule {\n    replacement   = \"/metrics/cadvisor\"\n    target_label  = \"__metrics_path__\"\n  }\n}\n\nprometheus.scrape \"cadvisor\" {\n  targets    = discovery.relabel.cadvisor.output\n  forward_to = [ prometheus.relabel.cadvisor.receiver ]\n  scheme     = \"https\"\n  tls_config {\n    insecure_skip_verify = true\n  }\n  bearer_token_file = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n  scrape_interval   = \"15s\"\n  scrape_timeout    = \"12s\"\n}\n\nprometheus.relabel \"cadvisor\" {\n  forward_to = [ prometheus.remote_write.default.receiver ]\n\n  // Drop empty container labels, addressing https://github.com/google/cadvisor/issues/2688\n  rule {\n    source_labels = [\"__name__\",\"container\"]\n    separator = \"@\"\n    regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*)@\"\n    action = \"drop\"\n  }\n  // Drop empty image labels, addressing https://github.com/google/cadvisor/issues/2688\n  rule {\n    source_labels = [\"__name__\",\"image\"]\n    separator = \"@\"\n    regex = \"(container_cpu_.*|container_fs_.*|container_memory_.*|container_network_.*)@\"\n    action = \"drop\"\n  }\n  // Drop irrelevant series\n  rule {\n    source_labels = [\"container\"]\n    regex         = \"^POD$\"\n    action        = \"drop\"\n  }\n  // Drop unnecessary labels\n  rule {\n    source_labels = [\"id\"]\n    target_label  = \"id\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"job\"]\n    target_label  = \"job\"\n    replacement   = \"\"\n  }\n  rule {\n    source_labels = [\"name\"]\n    target_label  = \"name\"\n    replacement   = \"\"\n  }\n}\n\nprometheus.remote_write \"default\" {\n  endpoint {\n    url =\"http://prometheus.coder-observability.svc/api/v1/write\"\n\n    // drop instance label which unnecessarily adds new series when pods are restarted, since pod IPs are dynamically assigned\n    // NOTE: \"__address__\" is mapped to \"instance\", so will contain <hostname>:<port>\n    write_relabel_config {\n      regex  = \"instance\"\n      action = \"labeldrop\"\n    }\n  }\n}"
->>>>>>> ad8b169 (chore: switch pyroscope from 80 -> 4040 (to unprivledged))
 ---
 # Source: coder-observability/templates/configmap-prometheus-alerts.yaml
 apiVersion: v1
@@ -10685,81 +10681,6 @@ spec:
   sessionAffinity: None
   type: "ClusterIP"
 ---
-<<<<<<< HEAD
-=======
-# Source: coder-observability/charts/pyroscope/templates/memberlist-service.yaml
-apiVersion: v1
-kind: Service
-metadata:
-  name: pyroscope-memberlist
-  namespace: coder-observability
-  labels:
-    app.kubernetes.io/name: pyroscope
-    app.kubernetes.io/instance: coder-observability
-    app.kubernetes.io/managed-by: Helm
-spec:
-  type: ClusterIP
-  clusterIP: None
-  ports:
-    - name: memberlist
-      port: 7946
-      protocol: TCP
-      targetPort: 7946
-  publishNotReadyAddresses: true
-  selector:
-    app.kubernetes.io/name: pyroscope
-    app.kubernetes.io/instance: coder-observability
-    # TODO: Ensure only services that offer memberlist register
-    # pyroscope.grafana.com/memberlist: "true"
----
-# Source: coder-observability/charts/pyroscope/templates/services.yaml
-apiVersion: v1
-kind: Service
-metadata:
-  name: pyroscope
-  namespace: coder-observability
-  labels:
-    app.kubernetes.io/name: pyroscope
-    app.kubernetes.io/instance: coder-observability
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: "all"
-spec:
-  type: ClusterIP
-  ports:
-    - port: 4040
-      targetPort: http2
-      protocol: TCP
-      name: http2
-  selector:
-    app.kubernetes.io/name: pyroscope
-    app.kubernetes.io/instance: coder-observability
-    app.kubernetes.io/component: "all"
----
-# Source: coder-observability/charts/pyroscope/templates/services.yaml
-apiVersion: v1
-kind: Service
-metadata:
-  name: pyroscope-headless
-  namespace: coder-observability
-  labels:
-    app.kubernetes.io/name: pyroscope
-    app.kubernetes.io/instance: coder-observability
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: "all"
-spec:
-  type: ClusterIP
-  clusterIP: None
-  ports:
-    - port: 4040
-      targetPort: http2
-      protocol: TCP
-      name: http2
-  selector:
-    app.kubernetes.io/name: pyroscope
-    app.kubernetes.io/instance: coder-observability
-    app.kubernetes.io/component: "all"
----
->>>>>>> ad8b169 (chore: switch pyroscope from 80 -> 4040 (to unprivledged))
 # Source: coder-observability/templates/service-runbook-viewer.yaml
 apiVersion: v1
 kind: Service
@@ -12273,113 +12194,6 @@ spec:
           requests:
             storage: "12Gi"
 ---
-<<<<<<< HEAD
-=======
-# Source: coder-observability/charts/pyroscope/templates/deployments-statefulsets.yaml
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: pyroscope
-  namespace: coder-observability
-  labels:
-    app.kubernetes.io/name: pyroscope
-    app.kubernetes.io/instance: coder-observability
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: "all"
-spec:
-  serviceName: pyroscope-headless
-  podManagementPolicy: Parallel
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: pyroscope
-      app.kubernetes.io/instance: coder-observability
-      app.kubernetes.io/component: "all"
-  template:
-    metadata:
-      annotations:
-        checksum/config: 1b47e0b4cfaba4c3fb572bf60178419eff351bf17d1a1d1a6b8e5a54f6ffcbea
-        profiles.grafana.com/service_repository: "https://github.com/grafana/pyroscope"
-        profiles.grafana.com/service_git_ref: "v1.14.0"
-        profiles.grafana.com/cpu.port_name: http2
-        profiles.grafana.com/cpu.scrape: "true"
-        profiles.grafana.com/goroutine.port_name: http2
-        profiles.grafana.com/goroutine.scrape: "true"
-        profiles.grafana.com/memory.port_name: http2
-        profiles.grafana.com/memory.scrape: "true"
-      labels:
-        app.kubernetes.io/name: pyroscope
-        app.kubernetes.io/instance: coder-observability
-        app.kubernetes.io/component: "all"
-        name: "pyroscope"
-    spec:
-      serviceAccountName: pyroscope
-      securityContext:
-        fsGroup: 10001
-        runAsNonRoot: true
-        runAsUser: 10001
-      dnsPolicy: ClusterFirst
-      containers:
-        - name: "pyroscope"
-          securityContext: {}
-          imagePullPolicy: IfNotPresent
-          args:
-            - "-target=all"
-            - "-self-profiling.disable-push=true"
-            - "-server.http-listen-port=4040"
-            - "-memberlist.cluster-label=coder-observability-pyroscope"
-            - "-memberlist.join=dns+pyroscope-memberlist.coder-observability.svc.cluster.local.:7946"
-            - "-config.file=/etc/pyroscope/config.yaml"
-            - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
-            - "-log.level=info"
-          env:
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: NAMESPACE_FQDN
-              value: "coder-observability.svc.cluster.local."
-          ports:
-            - name: http2
-              containerPort: 4040
-              protocol: TCP
-            - name: memberlist
-              containerPort: 7946
-              protocol: TCP
-          readinessProbe:
-            httpGet:
-              path: /ready
-              port: http2
-              scheme: HTTP
-          volumeMounts:
-            - name: config
-              mountPath: /etc/pyroscope/config.yaml
-              subPath: config.yaml
-            - name: overrides-config
-              mountPath: /etc/pyroscope/overrides/
-            - name: data
-              mountPath: /data
-          resources: {}
-      volumes:
-        - name: config
-          configMap:
-            name: pyroscope-config
-        - name: overrides-config
-          configMap:
-            name: pyroscope-overrides-config
-  volumeClaimTemplates:
-    - metadata:
-        name: data
-        annotations: {}
-      spec:
-        accessModes:
-          - ReadWriteOnce
-        resources:
-          requests:
-            storage: "10Gi"
-        storageClassName:
----
->>>>>>> ad8b169 (chore: switch pyroscope from 80 -> 4040 (to unprivledged))
 # Source: coder-observability/templates/statefulset-postgres-exporter.yaml
 apiVersion: apps/v1
 kind: StatefulSet

From 361b0a58cd393d7bd2ddeb2dc1360a7696eebdba Mon Sep 17 00:00:00 2001
From: Steven Masley <stevenmasley@gmail.com>
Date: Wed, 6 Aug 2025 14:47:44 -0500
Subject: [PATCH 3/3] make lint

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9770a71..7f7cd75 100644
--- a/README.md
+++ b/README.md
@@ -337,7 +337,7 @@ values which are defined [here](https://github.com/grafana/helm-charts/tree/main
 | grafana.datasources."datasources.yaml".datasources[0].timeout | string | `"{{ add $.Values.global.dashboards.queryTimeout 5 }}"` |  |
 | grafana.datasources."datasources.yaml".datasources[0].type | string | `"grafana-pyroscope-datasource"` |  |
 | grafana.datasources."datasources.yaml".datasources[0].uid | string | `"pyroscope"` |  |
-| grafana.datasources."datasources.yaml".datasources[0].url | string | `"http://pyroscope.{{ .Release.Namespace }}.{{ $.Values.global.zone }}"` |  |
+| grafana.datasources."datasources.yaml".datasources[0].url | string | `"http://pyroscope.{{ .Release.Namespace }}.{{ $.Values.global.zone }}:4040"` |  |
 | grafana.datasources."datasources.yaml".datasources[1].access | string | `"proxy"` |  |
 | grafana.datasources."datasources.yaml".datasources[1].editable | bool | `false` |  |
 | grafana.datasources."datasources.yaml".datasources[1].isDefault | bool | `true` |  |
@@ -505,7 +505,7 @@ values which are defined [here](https://github.com/grafana/helm-charts/tree/main
 | pyroscope.pyroscope.persistence.enabled | bool | `true` |  |
 | pyroscope.pyroscope.persistence.size | string | `"10Gi"` |  |
 | pyroscope.pyroscope.replicaCount | int | `1` |  |
-| pyroscope.pyroscope.service.port | int | `80` |  |
+| pyroscope.pyroscope.service.port | int | `4040` |  |
 | pyroscope.pyroscope.service.type | string | `"ClusterIP"` |  |
 | runbookViewer.image | string | `"dannyben/madness"` |  |
 | sqlExporter.image | string | `"burningalchemist/sql_exporter"` |  |
