summaryrefslogtreecommitdiff
path: root/robusta_krr
diff options
context:
space:
mode:
authorLeaveMyYard <zhukovpavel2001@gmail.com>2023-10-05 16:21:56 +0300
committerLeaveMyYard <zhukovpavel2001@gmail.com>2023-10-05 16:21:56 +0300
commit248be94aaa84c7680622b3ecb57a6f934d9165ff (patch)
tree02efe754285557b722e4f415e3815a86b760a2fe /robusta_krr
parent31758d8170f4704a7d66414248f0743adc132760 (diff)
Add warnings to json result (for UI to use)
Diffstat (limited to 'robusta_krr')
-rw-r--r--robusta_krr/core/integrations/prometheus/loader.py6
-rw-r--r--robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py5
-rw-r--r--robusta_krr/core/models/objects.py11
-rw-r--r--robusta_krr/core/runner.py5
-rw-r--r--robusta_krr/utils/progress_bar.py5
5 files changed, 27 insertions, 5 deletions
diff --git a/robusta_krr/core/integrations/prometheus/loader.py b/robusta_krr/core/integrations/prometheus/loader.py
index 9129bb6..c099536 100644
--- a/robusta_krr/core/integrations/prometheus/loader.py
+++ b/robusta_krr/core/integrations/prometheus/loader.py
@@ -70,7 +70,11 @@ class PrometheusMetricsLoader:
return None
async def load_pods(self, object: K8sObjectData, period: datetime.timedelta) -> list[PodData]:
- return await self.loader.load_pods(object, period)
+ try:
+ return await self.loader.load_pods(object, period)
+ except Exception as e:
+ logger.exception(f"Failed to load pods for {object}: {e}")
+ return []
async def gather_data(
self,
diff --git a/robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py b/robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py
index 7894e53..091041e 100644
--- a/robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py
+++ b/robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py
@@ -142,6 +142,11 @@ class PrometheusMetricsService(MetricsService):
data = await metric_loader.load_data(object, period, step)
if len(data) == 0:
+ if "CPU" in LoaderClass.__name__:
+ object.add_warning("NoPrometheusCPUMetrics")
+ elif "Memory" in LoaderClass.__name__:
+ object.add_warning("NoPrometheusMemoryMetrics")
+
logger.warning(
f"{metric_loader.service_name} returned no {metric_loader.__class__.__name__} metrics for {object}"
)
diff --git a/robusta_krr/core/models/objects.py b/robusta_krr/core/models/objects.py
index c5f6a98..62149c3 100644
--- a/robusta_krr/core/models/objects.py
+++ b/robusta_krr/core/models/objects.py
@@ -27,6 +27,13 @@ class HPAData(pd.BaseModel):
target_memory_utilization_percentage: Optional[float]
+PodWarning = Literal[
+ "NoPrometheusPods",
+ "NoPrometheusCPUMetrics",
+ "NoPrometheusMemoryMetrics",
+]
+
+
class K8sObjectData(pd.BaseModel):
# NOTE: Here None means that we are running inside the cluster
cluster: Optional[str]
@@ -37,6 +44,7 @@ class K8sObjectData(pd.BaseModel):
namespace: str
kind: KindLiteral
allocations: ResourceAllocations
+ warnings: set[PodWarning] = set()
_api_resource = pd.PrivateAttr(None)
@@ -46,6 +54,9 @@ class K8sObjectData(pd.BaseModel):
def __hash__(self) -> int:
return hash(str(self))
+ def add_warning(self, warning: PodWarning) -> None:
+ self.warnings.add(warning)
+
@property
def current_pods_count(self) -> int:
return len([pod for pod in self.pods if not pod.deleted])
diff --git a/robusta_krr/core/runner.py b/robusta_krr/core/runner.py
index 7bf5b32..f50f55f 100644
--- a/robusta_krr/core/runner.py
+++ b/robusta_krr/core/runner.py
@@ -141,6 +141,7 @@ class Runner:
# NOTE: Kubernetes API returned pods, but Prometheus did not
if object.pods != []:
+ object.add_warning("NoPrometheusPods")
logger.warning(
f"Was not able to load any pods for {object} from Prometheus.\n\t"
"This could mean that Prometheus is missing some required metrics.\n\t"
@@ -154,12 +155,12 @@ class Runner:
step=self._strategy.settings.timeframe_timedelta,
)
- logger.debug(f"Calculating recommendations for {object} with {len(metrics)} metrics")
-
# NOTE: We run this in a threadpool as the strategy calculation might be CPU intensive
# But keep in mind that numpy calcluations will not block the GIL
loop = asyncio.get_running_loop()
result = await loop.run_in_executor(self._executor, self._strategy.run, metrics, object)
+
+ logger.info(f"Calculated recommendations for {object} (using {len(metrics)} metrics)")
return self._format_result(result)
async def _gather_object_allocations(self, k8s_object: K8sObjectData) -> ResourceScan:
diff --git a/robusta_krr/utils/progress_bar.py b/robusta_krr/utils/progress_bar.py
index 57dc113..b796b6a 100644
--- a/robusta_krr/utils/progress_bar.py
+++ b/robusta_krr/utils/progress_bar.py
@@ -1,6 +1,6 @@
from alive_progress import alive_bar
-from robusta_krr.core.models.config import settings
+# from robusta_krr.core.models.config import settings
class ProgressBar:
@@ -12,7 +12,8 @@ class ProgressBar:
"""
def __init__(self, **kwargs) -> None:
- self.show_bar = not settings.quiet and not settings.log_to_stderr
+ # self.show_bar = not settings.quiet and not settings.log_to_stderr
+ self.show_bar = False # FIXME: Progress bar is not working good with other logs
if self.show_bar:
self.alive_bar = alive_bar(**kwargs, enrich_print=False)