Updated Section 7 dashboards and content

06-collecting-k8s-infra-metrics.md

@@ -45,6 +45,13 @@ For our tutorial, we set up the OpenTelemetry collector to scrape these embedded
 4. **kubernetes-service-endpoints:** All service endpoints with annotations for scraping and port specifications are scraped.
 5. **kubernetes-cadvisor:** This job captures metrics from cAdvisor, providing container metrics.
 
+
+To check how targets are currently active for each job, visit Grafana Explore and use the provided query.
+
+To view the list of all scrape jobs and the count of active targets for each job, you can access [Grafana Explore](http://localhost:8080/grafana/explore?orgId=1&left=%7B%22datasource%22:%22PA58DA793C7250F1B%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22prometheus%22,%22uid%22:%22PA58DA793C7250F1B%22%7D,%22editorMode%22:%22code%22,%22expr%22:%22count%28up%29%20by%20%28job%29%22,%22legendFormat%22:%22__auto%22,%22range%22:true,%22instant%22:true%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D) 
+
+![](./images/grafana-metrics-k8s-scrape-jobs.jpg)
+
 To view Prometheus metrics for the Kubernetes API server, you can access the [k8s API Server Dashboard](http://localhost:8080/grafana/d/k8s_system_apisrv/kubernetes-system-api-server?orgId=1)
 
 ![](./images/grafana-metrics-k8s-api-server.jpg)

app/k8s-annotated.yaml

@@ -23,6 +23,7 @@ spec:
         app: backend1
       annotations:
         instrumentation.opentelemetry.io/inject-python: "true"
+        prometheus.io/scrape: "true"
     spec:
       containers:
       - name: backend1
@@ -37,6 +38,8 @@ metadata:
   namespace: tutorial-application
   labels:
     app: backend1
+  annotations:
+    prometheus.io/scrape: "true"
 spec:
   ports:
   - name: http
@@ -63,6 +66,7 @@ spec:
         app: backend2
       annotations:
         instrumentation.opentelemetry.io/inject-java: "true"
+        prometheus.io/scrape: "true"
     spec:
       containers:
       - name: backend2
@@ -75,6 +79,10 @@ kind: Service
 metadata:
   name: backend2-service
   namespace: tutorial-application
+  labels:
+    app: backend2
+  annotations:
+    prometheus.io/scrape: "true"
 spec:
   ports:
   - name: http
@@ -101,6 +109,7 @@ spec:
         app: frontend
       annotations:
         instrumentation.opentelemetry.io/inject-sdk: "true"
+        prometheus.io/scrape: "true"
     spec:
       containers:
       - name: frontend
@@ -120,6 +129,10 @@ kind: Service
 metadata:
   name: frontend-service
   namespace: tutorial-application
+  labels:
+    app: frontend
+  annotations:
+    prometheus.io/scrape: "true"
 spec:
   ports:
   - name: http

backend/06-collector-k8s-cluster-metrics.yaml

@@ -66,8 +66,107 @@ spec:
             tls_config:
               ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
               insecure_skip_verify: true
+          - job_name: kubernetes-nodes
+            honor_timestamps: true
+            scrape_interval: 20s
+            scrape_timeout: 10s
+            metrics_path: /metrics
+            scheme: https
+            authorization:
+              type: Bearer
+              credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+            tls_config:
+              ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+              insecure_skip_verify: true
+            follow_redirects: true
+            relabel_configs:
+            - separator: ;
+              regex: __meta_kubernetes_node_label_(.+)
+              replacement: $$1
+              action: labelmap
+            - separator: ;
+              regex: (.*)
+              target_label: __address__
+              replacement: kubernetes.default.svc:443
+              action: replace
+            kubernetes_sd_configs:
+            - role: node
+              kubeconfig_file: ""
+              follow_redirects: true
+          - job_name: kubernetes-nodes-cadvisor
+            honor_timestamps: true
+            scrape_interval: 20s
+            scrape_timeout: 10s
+            metrics_path: /metrics
+            scheme: https
+            authorization:
+              type: Bearer
+              credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+            tls_config:
+              ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+              insecure_skip_verify: true
+            follow_redirects: true
+            relabel_configs:
+            - separator: ;
+              regex: __meta_kubernetes_node_label_(.+)
+              replacement: $$1
+              action: labelmap
+            - separator: ;
+              regex: (.*)
+              target_label: __address__
+              replacement: kubernetes.default.svc:443
+              action: replace
+            kubernetes_sd_configs:
+            - role: node
+              kubeconfig_file: ""
+              follow_redirects: true
+          - job_name: kubernetes-service-endpoints
+            honor_labels: true
+            honor_timestamps: true
+            scrape_interval: 20s
+            scrape_timeout: 10s
+            metrics_path: /metrics
+            scheme: http
+            follow_redirects: true
+            relabel_configs:
+                - action: labelmap
+                  regex: __meta_kubernetes_service_label_(.+)
+                - source_labels: [__meta_kubernetes_namespace]
+                  action: replace
+                  target_label: namespace
+                - source_labels: [__meta_kubernetes_service_name]
+                  action: replace
+                  target_label: service
+            kubernetes_sd_configs:
+            - role: pod
+              kubeconfig_file: ""
+              follow_redirects: true
+          - job_name: kubernetes-pods
+            honor_labels: true
+            honor_timestamps: true
+            scrape_interval: 1m
+            scrape_timeout: 20s
+            metrics_path: /metrics
+            scheme: http
+            tls_config:
+              ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+              insecure_skip_verify: true
+            follow_redirects: true
+            relabel_configs:
+              - action: labelmap
+                regex: __meta_kubernetes_pod_label_(.+)
+              - source_labels: [__meta_kubernetes_namespace]
+                action: replace
+                target_label: namespace
+              - source_labels: [__meta_kubernetes_pod_name]
+                action: replace
+                target_label: pod
+            kubernetes_sd_configs:
+            - role: pod
+              kubeconfig_file: ""
+              follow_redirects: true
         target_allocator:
-          endpoint: http://otel-prom-k8s-metrics-targetallocator:80
+          endpoint: http://otel-k8s-cluster-metrics-targetallocator:80
           interval: 30s
           collector_id: ${POD_NAME}
           http_sd_config: