From 46b79409235a92f2480300397033d71fbbfe28b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB=20=D0=96=D1=83=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2?= <33721692+LeaveMyYard@users.noreply.github.com> Date: Tue, 23 May 2023 13:48:10 +0300 Subject: Change format for metric, add start_time, end_time and step --- robusta_krr/core/abstract/strategies.py | 4 ++-- .../integrations/prometheus/metrics/base_metric.py | 19 ++++++++++++++----- robusta_krr/core/models/result.py | 10 +++++++++- robusta_krr/core/runner.py | 4 ++-- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/robusta_krr/core/abstract/strategies.py b/robusta_krr/core/abstract/strategies.py index 5882493..6ca0d96 100644 --- a/robusta_krr/core/abstract/strategies.py +++ b/robusta_krr/core/abstract/strategies.py @@ -9,7 +9,7 @@ from textwrap import dedent import pydantic as pd -from robusta_krr.core.models.result import K8sObjectData, ResourceType +from robusta_krr.core.models.result import K8sObjectData, ResourceType, Metric from robusta_krr.utils.display_name import add_display_name @@ -46,7 +46,7 @@ ArrayNx2 = Annotated[NDArray[np.float64], Literal["N", 2]] class ResourceHistoryData(pd.BaseModel): - query: str # The query used to get the data + metric: Metric data: dict[str, ArrayNx2] # Mapping: pod -> (time, value) class Config: diff --git a/robusta_krr/core/integrations/prometheus/metrics/base_metric.py b/robusta_krr/core/integrations/prometheus/metrics/base_metric.py index 2547d16..e79e19a 100644 --- a/robusta_krr/core/integrations/prometheus/metrics/base_metric.py +++ b/robusta_krr/core/integrations/prometheus/metrics/base_metric.py @@ -5,7 +5,7 @@ import asyncio import datetime from typing import TYPE_CHECKING, Callable, TypeVar -from robusta_krr.core.abstract.strategies import ResourceHistoryData +from robusta_krr.core.abstract.strategies import ResourceHistoryData, Metric from robusta_krr.core.models.config import Config from robusta_krr.core.models.objects import K8sObjectData from robusta_krr.utils.configurable import Configurable @@ -27,6 +27,9 @@ class BaseMetricLoader(Configurable, abc.ABC): def get_query(self, object: K8sObjectData) -> str: ... + def _step_to_string(self, step: datetime.timedelta) -> str: + return f"{int(step.total_seconds()) // 60}m" + async def query_prometheus( self, query: str, start_time: datetime.datetime, end_time: datetime.datetime, step: datetime.timedelta ) -> list[dict]: @@ -35,17 +38,18 @@ class BaseMetricLoader(Configurable, abc.ABC): query=query, start_time=start_time, end_time=end_time, - step=f"{int(step.total_seconds()) // 60}m", + step=self._step_to_string(step), ) async def load_data( self, object: K8sObjectData, period: datetime.timedelta, step: datetime.timedelta ) -> ResourceHistoryData: query = self.get_query(object) + end_time = datetime.datetime.now() result = await self.query_prometheus( query=query, - start_time=datetime.datetime.now() - period, - end_time=datetime.datetime.now(), + start_time=end_time - period, + end_time=end_time, step=step, ) @@ -54,7 +58,12 @@ class BaseMetricLoader(Configurable, abc.ABC): return ResourceHistoryData(query=query, data={}) return ResourceHistoryData( - query=query, + metric=Metric( + query=query, + start_time=end_time - period, + end_time=end_time, + step=self._step_to_string(step), + ), data={ pod_result['metric']['pod']: np.array(pod_result["values"], dtype=np.float64) for pod_result in result diff --git a/robusta_krr/core/models/result.py b/robusta_krr/core/models/result.py index 5119de8..0bad398 100644 --- a/robusta_krr/core/models/result.py +++ b/robusta_krr/core/models/result.py @@ -3,6 +3,7 @@ from __future__ import annotations import enum import itertools from typing import Any, Union, Optional +from datetime import datetime import pydantic as pd @@ -60,7 +61,14 @@ class ResourceRecommendation(pd.BaseModel): limits: dict[ResourceType, RecommendationValue] -MetricsData = dict[ResourceType, str] +class Metric(pd.BaseModel): + query: str + start_time: datetime + end_time: datetime + step: str + + +MetricsData = dict[ResourceType, Metric] class ResourceScan(pd.BaseModel): diff --git a/robusta_krr/core/runner.py b/robusta_krr/core/runner.py index 51642fe..5543a06 100644 --- a/robusta_krr/core/runner.py +++ b/robusta_krr/core/runner.py @@ -109,12 +109,12 @@ class Runner(Configurable): ] ) data = dict(zip(ResourceType, data_tuple)) - queries = {resource: data[resource].query for resource in ResourceType} + metrics = {resource: data[resource].metric for resource in ResourceType} # NOTE: We run this in a threadpool as the strategy calculation might be CPU intensive # But keep in mind that numpy calcluations will not block the GIL result = await asyncio.to_thread(self._strategy.run, data, object) - return self._format_result(result), queries + return self._format_result(result), metrics async def _gather_objects_recommendations(self, objects: list[K8sObjectData]) -> list[tuple[ResourceAllocations, MetricsData]]: recommendations: list[tuple[RunResult, MetricsData]] = await asyncio.gather( -- cgit v1.2.3 From 4e75369726da3419fc25b276ca5a8b73de68a49c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB=20=D0=96=D1=83=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2?= <33721692+LeaveMyYard@users.noreply.github.com> Date: Tue, 23 May 2023 22:19:56 +0300 Subject: Fix metric data computation --- .../prometheus/metrics/base_filtered_metric.py | 12 ++++----- .../integrations/prometheus/metrics/base_metric.py | 29 ++++++++-------------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/robusta_krr/core/integrations/prometheus/metrics/base_filtered_metric.py b/robusta_krr/core/integrations/prometheus/metrics/base_filtered_metric.py index a409810..b0c7699 100644 --- a/robusta_krr/core/integrations/prometheus/metrics/base_filtered_metric.py +++ b/robusta_krr/core/integrations/prometheus/metrics/base_filtered_metric.py @@ -1,7 +1,7 @@ -import datetime from typing import Any, Optional from .base_metric import BaseMetricLoader +from robusta_krr.core.abstract.strategies import Metric PrometheusSeries = Any @@ -45,7 +45,9 @@ class BaseFilteredMetricLoader(BaseMetricLoader): # takes kubelet job if exists, return first job alphabetically if it doesn't for target_name in target_names: relevant_series = [ - series for series in series_list_result if BaseFilteredMetricLoader.get_target_name(series) == target_name + series + for series in series_list_result + if BaseFilteredMetricLoader.get_target_name(series) == target_name ] relevant_kubelet_metric = [series for series in relevant_series if series["metric"].get("job") == "kubelet"] if len(relevant_kubelet_metric) == 1: @@ -55,8 +57,6 @@ class BaseFilteredMetricLoader(BaseMetricLoader): return_list.append(sorted_relevant_series[0]) return return_list - async def query_prometheus( - self, query: str, start_time: datetime.datetime, end_time: datetime.datetime, step: datetime.timedelta - ) -> list[PrometheusSeries]: - result = await super().query_prometheus(query, start_time, end_time, step) + async def query_prometheus(self, metric: Metric) -> list[PrometheusSeries]: + result = await super().query_prometheus(metric) return self.filter_prom_jobs_results(result) diff --git a/robusta_krr/core/integrations/prometheus/metrics/base_metric.py b/robusta_krr/core/integrations/prometheus/metrics/base_metric.py index e79e19a..de4cf59 100644 --- a/robusta_krr/core/integrations/prometheus/metrics/base_metric.py +++ b/robusta_krr/core/integrations/prometheus/metrics/base_metric.py @@ -30,15 +30,13 @@ class BaseMetricLoader(Configurable, abc.ABC): def _step_to_string(self, step: datetime.timedelta) -> str: return f"{int(step.total_seconds()) // 60}m" - async def query_prometheus( - self, query: str, start_time: datetime.datetime, end_time: datetime.datetime, step: datetime.timedelta - ) -> list[dict]: + async def query_prometheus(self, metric: Metric) -> list[dict]: return await asyncio.to_thread( self.prometheus.custom_query_range, - query=query, - start_time=start_time, - end_time=end_time, - step=self._step_to_string(step), + query=metric.query, + start_time=metric.start_time, + end_time=metric.end_time, + step=metric.step, ) async def load_data( @@ -46,27 +44,22 @@ class BaseMetricLoader(Configurable, abc.ABC): ) -> ResourceHistoryData: query = self.get_query(object) end_time = datetime.datetime.now() - result = await self.query_prometheus( + metric = Metric( query=query, start_time=end_time - period, end_time=end_time, - step=step, + step=self._step_to_string(step), ) + result = await self.query_prometheus(metric) if result == []: self.warning(f"Prometheus returned no {self.__class__.__name__} metrics for {object}") - return ResourceHistoryData(query=query, data={}) + return ResourceHistoryData(metric=metric, data={}) return ResourceHistoryData( - metric=Metric( - query=query, - start_time=end_time - period, - end_time=end_time, - step=self._step_to_string(step), - ), + metric=metric, data={ - pod_result['metric']['pod']: np.array(pod_result["values"], dtype=np.float64) - for pod_result in result + pod_result["metric"]["pod"]: np.array(pod_result["values"], dtype=np.float64) for pod_result in result }, ) -- cgit v1.2.3