From 814655cc5f01ee00ef25329ea48c7e7a9b395be8 Mon Sep 17 00:00:00 2001
From: Gavin Holahan <gholahan9@gmail.com>
Date: Fri, 28 Feb 2025 01:09:05 +0000
Subject: [PATCH] Removed Static, added better prom .yml

---
 jobs/monitoring/prometheus.hcl | 80 ++++++++++++++--------------------
 1 file changed, 32 insertions(+), 48 deletions(-)

diff --git a/jobs/monitoring/prometheus.hcl b/jobs/monitoring/prometheus.hcl
index 764673a..6ffa3af 100644
--- a/jobs/monitoring/prometheus.hcl
+++ b/jobs/monitoring/prometheus.hcl
@@ -1,87 +1,71 @@
 job "prometheus" {
   datacenters = ["aperture"]
+  type = "service"
 
   group "prometheus" {
+    count = 1
+
     network {
       port "http" {
-        static = 9090
+        to = 9090
       }
     }
 
-    service {
-      name = "prometheus"
-      port = "http"
-    }
-
     task "prometheus" {
       driver = "docker"
+
       config {
-        image = "quay.io/prometheus/prometheus"
+        image = "prom/prometheus:latest"
         ports = ["http"]
 
         volumes = [
-          "/storage/nomad/${NOMAD_JOB_NAME}/${NOMAD_TASK_NAME}:/prometheus"
-        ]
-
-        args = [
-          "--config.file=$${NOMAD_TASK_DIR}/prometheus.yml",
-          "--log.level=info",
-          "--storage.tsdb.retention.time=90d",
-          "--storage.tsdb.path=/prometheus",
-          "--web.console.libraries=/usr/share/prometheus/console_libraries",
-          "--web.console.templates=/usr/share/prometheus/consoles"
+          "local/prometheus.yml:/etc/prometheus/prometheus.yml"
         ]
       }
 
       template {
+        destination = "local/prometheus.yml"
         data = <<EOF
 global:
   scrape_interval: 10s
   evaluation_interval: 10s
 
 scrape_configs:
-
 - job_name: 'nomad_metrics'
   consul_sd_configs:
-  - server: '{{ env "attr.unique.network.ip-address" }}:8500'
-    services: ['nomad-client', 'nomad']
-
-  relabel_configs:
-  - source_labels: ['__meta_consul_tags']
-    regex: '(.*)http(.*)'
-    action: keep
-  - source_labels: ['__meta_consul_node']
-    target_label: 'node'
-    # If nomad is available on multiple IPs, drop the ones which are not scrapable
-  - source_labels: ['__address__']
-    regex: '172(.*)'
-    action: drop
+    - server: 'consul.service.consul:8500'
+      services: ['nomad-client', 'nomad'] # This allows for Client (Workload) and Server (Orchastration) metrics
+      tags: ['http']
 
   metrics_path: /v1/metrics
   params:
     format: ['prometheus']
 
-- job_name: 'application_metrics'
+- job_name: 'container-metrics'
   consul_sd_configs:
-  - server: '{{ env "attr.unique.network.ip-address" }}:8500'
-
+  - server: 'consul.service.consul:8500'
+    tags: ['prometheus.enable=true']
   relabel_configs:
-  - source_labels: ['__meta_consul_service']
-    regex: 'nomad|nomad-client|consul'
-    action: drop
-    # Drop services which do not want to be scraped.
-    # Typically used when a job does not expose prometheus metrics.
-  - source_labels: ['__meta_consul_tags']
-    regex: '.*prometheus.io/scrape=false.*'
-    action: 'drop'
-  - source_labels: ['__meta_consul_node']
-    target_label: 'node'
-  - source_labels: ['__meta_consul_service']
-    target_label: 'service'
-EOF
+    - source_labels: ['__meta_consul_service']
+      target_label: 'job'
+      replacement: 'consul-service'
+    - source_labels: ['__meta_consul_tags']
+      regex: '.*prometheus.path=([^,]+).*'  # Extract path from tag
+      target_label: '__metrics_path__'
 
-        destination = "local/prometheus.yml"
+  metrics_path: /v1/metrics
+  params:
+    format: ['prometheus']
+
+
+EOF
+      }
+
+      resources {
+        cpu    = 500
+        memory = 512
       }
     }
   }
 }
+