diff options
| author | LeaveMyYard <zhukovpavel2001@gmail.com> | 2023-10-05 16:21:56 +0300 |
|---|---|---|
| committer | LeaveMyYard <zhukovpavel2001@gmail.com> | 2023-10-05 16:21:56 +0300 |
| commit | 248be94aaa84c7680622b3ecb57a6f934d9165ff (patch) | |
| tree | 02efe754285557b722e4f415e3815a86b760a2fe | |
| parent | 31758d8170f4704a7d66414248f0743adc132760 (diff) | |
Add warnings to json result (for UI to use)
| -rw-r--r-- | robusta_krr/core/integrations/prometheus/loader.py | 6 | ||||
| -rw-r--r-- | robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py | 5 | ||||
| -rw-r--r-- | robusta_krr/core/models/objects.py | 11 | ||||
| -rw-r--r-- | robusta_krr/core/runner.py | 5 | ||||
| -rw-r--r-- | robusta_krr/utils/progress_bar.py | 5 |
5 files changed, 27 insertions, 5 deletions
diff --git a/robusta_krr/core/integrations/prometheus/loader.py b/robusta_krr/core/integrations/prometheus/loader.py index 9129bb6..c099536 100644 --- a/robusta_krr/core/integrations/prometheus/loader.py +++ b/robusta_krr/core/integrations/prometheus/loader.py @@ -70,7 +70,11 @@ class PrometheusMetricsLoader: return None async def load_pods(self, object: K8sObjectData, period: datetime.timedelta) -> list[PodData]: - return await self.loader.load_pods(object, period) + try: + return await self.loader.load_pods(object, period) + except Exception as e: + logger.exception(f"Failed to load pods for {object}: {e}") + return [] async def gather_data( self, diff --git a/robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py b/robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py index 7894e53..091041e 100644 --- a/robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py +++ b/robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py @@ -142,6 +142,11 @@ class PrometheusMetricsService(MetricsService): data = await metric_loader.load_data(object, period, step) if len(data) == 0: + if "CPU" in LoaderClass.__name__: + object.add_warning("NoPrometheusCPUMetrics") + elif "Memory" in LoaderClass.__name__: + object.add_warning("NoPrometheusMemoryMetrics") + logger.warning( f"{metric_loader.service_name} returned no {metric_loader.__class__.__name__} metrics for {object}" ) diff --git a/robusta_krr/core/models/objects.py b/robusta_krr/core/models/objects.py index c5f6a98..62149c3 100644 --- a/robusta_krr/core/models/objects.py +++ b/robusta_krr/core/models/objects.py @@ -27,6 +27,13 @@ class HPAData(pd.BaseModel): target_memory_utilization_percentage: Optional[float] +PodWarning = Literal[ + "NoPrometheusPods", + "NoPrometheusCPUMetrics", + "NoPrometheusMemoryMetrics", +] + + class K8sObjectData(pd.BaseModel): # NOTE: Here None means that we are running inside the cluster cluster: Optional[str] @@ -37,6 +44,7 @@ class K8sObjectData(pd.BaseModel): namespace: str kind: KindLiteral allocations: ResourceAllocations + warnings: set[PodWarning] = set() _api_resource = pd.PrivateAttr(None) @@ -46,6 +54,9 @@ class K8sObjectData(pd.BaseModel): def __hash__(self) -> int: return hash(str(self)) + def add_warning(self, warning: PodWarning) -> None: + self.warnings.add(warning) + @property def current_pods_count(self) -> int: return len([pod for pod in self.pods if not pod.deleted]) diff --git a/robusta_krr/core/runner.py b/robusta_krr/core/runner.py index 7bf5b32..f50f55f 100644 --- a/robusta_krr/core/runner.py +++ b/robusta_krr/core/runner.py @@ -141,6 +141,7 @@ class Runner: # NOTE: Kubernetes API returned pods, but Prometheus did not if object.pods != []: + object.add_warning("NoPrometheusPods") logger.warning( f"Was not able to load any pods for {object} from Prometheus.\n\t" "This could mean that Prometheus is missing some required metrics.\n\t" @@ -154,12 +155,12 @@ class Runner: step=self._strategy.settings.timeframe_timedelta, ) - logger.debug(f"Calculating recommendations for {object} with {len(metrics)} metrics") - # NOTE: We run this in a threadpool as the strategy calculation might be CPU intensive # But keep in mind that numpy calcluations will not block the GIL loop = asyncio.get_running_loop() result = await loop.run_in_executor(self._executor, self._strategy.run, metrics, object) + + logger.info(f"Calculated recommendations for {object} (using {len(metrics)} metrics)") return self._format_result(result) async def _gather_object_allocations(self, k8s_object: K8sObjectData) -> ResourceScan: diff --git a/robusta_krr/utils/progress_bar.py b/robusta_krr/utils/progress_bar.py index 57dc113..b796b6a 100644 --- a/robusta_krr/utils/progress_bar.py +++ b/robusta_krr/utils/progress_bar.py @@ -1,6 +1,6 @@ from alive_progress import alive_bar -from robusta_krr.core.models.config import settings +# from robusta_krr.core.models.config import settings class ProgressBar: @@ -12,7 +12,8 @@ class ProgressBar: """ def __init__(self, **kwargs) -> None: - self.show_bar = not settings.quiet and not settings.log_to_stderr + # self.show_bar = not settings.quiet and not settings.log_to_stderr + self.show_bar = False # FIXME: Progress bar is not working good with other logs if self.show_bar: self.alive_bar = alive_bar(**kwargs, enrich_print=False) |
