Merge pull request #143 from robusta-dev/prometheus-metric-fix

Aggregate in case of node change

2 files changed, 49 insertions, 35 deletions

diff --git a/robusta_krr/core/integrations/prometheus/metrics/cpu.py b/robusta_krr/core/integrations/prometheus/metrics/cpu.py
index 3b274d8..c7a2c73 100644
--- a/robusta_krr/core/integrations/prometheus/metrics/cpu.py
+++ b/robusta_krr/core/integrations/prometheus/metrics/cpu.py
@@ -14,14 +14,16 @@ class CPULoader(PrometheusMetric):
         pods_selector = "|".join(pod.name for pod in object.pods)
         cluster_label = self.get_prometheus_cluster_label()
         return f"""
-            rate(
-                container_cpu_usage_seconds_total{{
-                    namespace="{object.namespace}",
-                    pod=~"{pods_selector}",
-                    container="{object.container}"
-                    {cluster_label}
-                }}[{step}]
-            )
+            max(
+                rate(
+                    container_cpu_usage_seconds_total{{
+                        namespace="{object.namespace}",
+                        pod=~"{pods_selector}",
+                        container="{object.container}"
+                        {cluster_label}
+                    }}[{step}]
+                )
+            ) by (container, pod, job)
         """
 
 
@@ -40,14 +42,17 @@ def PercentileCPULoader(percentile: float) -> type[PrometheusMetric]:
             return f"""
                 quantile_over_time(
                     {round(percentile / 100, 2)},
-                    rate(
-                        container_cpu_usage_seconds_total{{
-                            namespace="{object.namespace}",
-                            pod=~"{pods_selector}",
-                            container="{object.container}"
-                            {cluster_label}
-                        }}[{step}]
-                    )[{duration}:{step}]
+                    max(
+                        rate(
+                            container_cpu_usage_seconds_total{{
+                                namespace="{object.namespace}",
+                                pod=~"{pods_selector}",
+                                container="{object.container}"
+                                {cluster_label}
+                            }}[{step}]
+                        )
+                    ) by (container, pod, job)
+                    [{duration}:{step}]
                 )
             """
 
@@ -64,11 +69,14 @@ class CPUAmountLoader(PrometheusMetric):
         cluster_label = self.get_prometheus_cluster_label()
         return f"""
             count_over_time(
-                container_cpu_usage_seconds_total{{
-                    namespace="{object.namespace}",
-                    pod=~"{pods_selector}",
-                    container="{object.container}"
-                    {cluster_label}
-                }}[{duration}]
+                max(
+                    container_cpu_usage_seconds_total{{
+                        namespace="{object.namespace}",
+                        pod=~"{pods_selector}",
+                        container="{object.container}"
+                        {cluster_label}
+                    }}
+                ) by (container, pod, job)
+                [{duration}:{step}]
             )
         """
diff --git a/robusta_krr/core/integrations/prometheus/metrics/memory.py b/robusta_krr/core/integrations/prometheus/metrics/memory.py
index 604756a..85031bc 100644
--- a/robusta_krr/core/integrations/prometheus/metrics/memory.py
+++ b/robusta_krr/core/integrations/prometheus/metrics/memory.py
@@ -14,7 +14,7 @@ class MemoryLoader(PrometheusMetric):
         pods_selector = "|".join(pod.name for pod in object.pods)
         cluster_label = self.get_prometheus_cluster_label()
         return f"""
-            sum(
+            max(
                 container_memory_working_set_bytes{{
                     namespace="{object.namespace}",
                     pod=~"{pods_selector}",
@@ -35,12 +35,15 @@ class MaxMemoryLoader(PrometheusMetric):
         cluster_label = self.get_prometheus_cluster_label()
         return f"""
             max_over_time(
-                container_memory_working_set_bytes{{
-                    namespace="{object.namespace}",
-                    pod=~"{pods_selector}",
-                    container="{object.container}"
-                    {cluster_label}
-                }}[{duration}:{step}]
+                max(
+                    container_memory_working_set_bytes{{
+                        namespace="{object.namespace}",
+                        pod=~"{pods_selector}",
+                        container="{object.container}"
+                        {cluster_label}
+                    }}
+                ) by (container, pod, job)
+                [{duration}:{step}]
             )
         """
 
@@ -55,11 +58,14 @@ class MemoryAmountLoader(PrometheusMetric):
         cluster_label = self.get_prometheus_cluster_label()
         return f"""
             count_over_time(
-                container_memory_working_set_bytes{{
-                    namespace="{object.namespace}",
-                    pod=~"{pods_selector}",
-                    container="{object.container}"
-                    {cluster_label}
-                }}[{duration}:{step}]
+                max(
+                    container_memory_working_set_bytes{{
+                        namespace="{object.namespace}",
+                        pod=~"{pods_selector}",
+                        container="{object.container}"
+                        {cluster_label}
+                    }}
+                ) by (container, pod, job)
+                [{duration}:{step}]
             )
         """