Merge pull request #51 from robusta-dev/ci-cd-pipelines

CI/CD pipelines

7 files changed, 368 insertions, 17 deletions

diff --git a/.github/workflows/build-on-release.yml b/.github/workflows/build-on-release.yml
new file mode 100644
index 0000000..498393f
--- /dev/null
+++ b/.github/workflows/build-on-release.yml
@@ -0,0 +1,200 @@
+name: Build and Release
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pyinstaller
+
+    - name: Install dependancies (Linux)
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        sudo apt-get install -y binutils
+
+    - name: Install the Apple certificate and provisioning profile
+      if: matrix.os == 'macos-latest'
+      env:
+        BUILD_CERTIFICATE_BASE64: ${{ secrets.BUILD_CERTIFICATE_BASE64 }}
+        P12_PASSWORD: ${{ secrets.P12_PASSWORD }}
+        BUILD_PROVISION_PROFILE_BASE64: ${{ secrets.BUILD_PROVISION_PROFILE_BASE64 }}
+        KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+      run: |
+        # create variables
+        CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12
+        PP_PATH=$RUNNER_TEMP/build_pp.mobileprovision
+        KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db
+
+        # import certificate and provisioning profile from secrets
+        echo -n "$BUILD_CERTIFICATE_BASE64" | base64 --decode -o $CERTIFICATE_PATH
+        echo -n "$BUILD_PROVISION_PROFILE_BASE64" | base64 --decode -o $PP_PATH
+
+        # create temporary keychain
+        security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
+        security set-keychain-settings -lut 21600 $KEYCHAIN_PATH
+        security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
+
+        # import certificate to keychain
+        security import $CERTIFICATE_PATH -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH
+        security list-keychain -d user -s $KEYCHAIN_PATH
+
+        # apply provisioning profile
+        mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles
+        cp $PP_PATH ~/Library/MobileDevice/Provisioning\ Profiles
+
+    - name: Set version in code (Unix)
+      if: matrix.os == 'macos-latest' || matrix.os == 'ubuntu-latest'
+      run: |
+        awk 'NR==3{$0="__version__ = \"'${{ github.ref_name }}'\""}1' ./robusta_krr/__init__.py > temp && mv temp ./robusta_krr/__init__.py
+        cat ./robusta_krr/__init__.py
+
+    - name: Set version in code (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        $content = Get-Content -Path .\robusta_krr\__init__.py
+        $content[2] = "__version__=`"$($env:GITHUB_REF_NAME)`""
+        $content | Out-File -FilePath .\robusta_krr\__init__.py -Encoding ascii
+        Get-Content .\robusta_krr\__init__.py
+      shell: pwsh
+      env:
+        GITHUB_REF_NAME: ${{ github.ref_name }}
+
+
+    - name: Build with PyInstaller
+      shell: bash
+      run: |
+        pyinstaller krr.py
+        mkdir -p ./dist/krr/grapheme/data
+        cp $(python -c "import grapheme; print(grapheme.__path__[0] + '/data/grapheme_break_property.json')") ./dist/krr/grapheme/data/grapheme_break_property.json
+
+    - name: Zip the application (Unix)
+      if: matrix.os == 'macos-latest' || matrix.os == 'ubuntu-latest'
+      run: |
+        cd dist
+        zip -r krr-${{ matrix.os }}-${{ github.ref_name }}.zip krr
+        mv krr-${{ matrix.os }}-${{ github.ref_name }}.zip ../
+        cd ..
+
+    - name: Zip the application (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        Set-Location -Path dist
+        Compress-Archive -Path krr -DestinationPath krr-${{ matrix.os }}-${{ github.ref_name }}.zip -Force
+        Move-Item -Path krr-${{ matrix.os }}-${{ github.ref_name }}.zip -Destination ..\
+        Set-Location -Path ..
+
+    - name: Upload Release Asset
+      uses: actions/upload-release-asset@v1.0.2
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }} 
+        asset_path: ./krr-${{ matrix.os }}-${{ github.ref_name }}.zip
+        asset_name: krr-${{ matrix.os }}-${{ github.ref_name }}.zip
+        asset_content_type: application/octet-stream
+
+    - name: Upload build as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: krr-${{ matrix.os }}-${{ github.ref_name }}
+        path: ./krr-${{ matrix.os }}-${{ github.ref_name }}.zip
+
+    - name: Clean up keychain and provisioning profile
+      if: (matrix.os == 'macos-latest') && always()
+      run: |
+        security delete-keychain $RUNNER_TEMP/app-signing.keychain-db
+        rm ~/Library/MobileDevice/Provisioning\ Profiles/build_pp.mobileprovision
+
+  check-latest:
+    needs: build
+    runs-on: ubuntu-latest
+    outputs: 
+      IS_LATEST: ${{ steps.check-latest.outputs.release == github.ref_name }}
+    steps:
+      - id: check-latest
+        uses: pozetroninc/github-action-get-latest-release@v0.7.0
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          repository: ${{ github.repository }}
+          excludes: prerelease, draft
+
+  # Define MacOS hash job
+  mac-hash:
+    needs: check-latest
+    runs-on: ubuntu-latest
+    if: needs.check-latest.outputs.IS_LATEST
+    outputs:
+      MAC_BUILD_HASH: ${{ steps.calc-hash.outputs.MAC_BUILD_HASH }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+      - name: Download MacOS artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: krr-macos-latest-${{ github.ref_name }}
+      - name: Calculate hash
+        id: calc-hash
+        run: echo "::set-output name=MAC_BUILD_HASH::$(sha256sum krr-macos-latest-${{ github.ref_name }}.zip | awk '{print $1}')"
+
+  # Define Linux hash job
+  linux-hash:
+    needs: check-latest
+    runs-on: ubuntu-latest
+    if: needs.check-latest.outputs.IS_LATEST
+    outputs:
+      LINUX_BUILD_HASH: ${{ steps.calc-hash.outputs.LINUX_BUILD_HASH }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+      - name: Download Linux artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: krr-ubuntu-latest-${{ github.ref_name }}
+      - name: Calculate hash
+        id: calc-hash
+        run: echo "::set-output name=LINUX_BUILD_HASH::$(sha256sum krr-ubuntu-latest-${{ github.ref_name }}.zip | awk '{print $1}')"
+
+  # Define job to update homebrew formula
+  update-formula:
+    needs: [mac-hash, linux-hash]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout homebrew-krr repository
+        uses: actions/checkout@v2
+        with:
+          repository: robusta-dev/homebrew-krr
+          token: ${{ secrets.MULTIREPO_GITHUB_TOKEN }}
+      - name: Update krr.rb formula
+        run: |
+          MAC_BUILD_HASH=${{ needs.mac-hash.outputs.MAC_BUILD_HASH }}
+          LINUX_BUILD_HASH=${{ needs.linux-hash.outputs.LINUX_BUILD_HASH }}
+          TAG_NAME=${{ github.ref_name }}
+          awk 'NR==6{$0="        url \"https://github.com/robusta-dev/krr/releases/download/'"$TAG_NAME"'/krr-macos-latest-'"$TAG_NAME"'.zip\""}1' ./Formula/krr.rb > temp && mv temp ./Formula/krr.rb
+          awk 'NR==7{$0="        sha256 \"'$MAC_BUILD_HASH'\""}1' ./Formula/krr.rb > temp && mv temp ./Formula/krr.rb
+          awk 'NR==9{$0="        url \"https://github.com/robusta-dev/krr/releases/download/'"$TAG_NAME"'/krr-linux-latest-'"$TAG_NAME"'.zip\""}1' ./Formula/krr.rb > temp && mv temp ./Formula/krr.rb
+          awk 'NR==10{$0="        sha256 \"'$LINUX_BUILD_HASH'\""}1' ./Formula/krr.rb > temp && mv temp ./Formula/krr.rb
+      - name: Commit and push changes
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+          git commit -am "Update formula for release ${TAG_NAME}"
+          git push
diff --git a/.github/workflows/docker-build-on-tag.yml b/.github/workflows/docker-build-on-tag.yml
new file mode 100644
index 0000000..97829f5
--- /dev/null
+++ b/.github/workflows/docker-build-on-tag.yml
@@ -0,0 +1,43 @@
+name: Docker Build and Push
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up gcloud CLI
+      uses: google-github-actions/setup-gcloud@v0.2.0
+      with:
+        service_account_key: ${{ secrets.GCP_SA_KEY }}
+        project_id: genuine-flight-317411
+        export_default_credentials: true
+
+    # Configure Docker to use the gcloud command-line tool as a credential helper for authentication
+    - name: Configure Docker
+      run: |-
+        gcloud auth configure-docker us-central1-docker.pkg.dev
+
+    - name: Verify gcloud configuration
+      run: |-
+        gcloud config get-value project
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Build and push Docker images
+      uses: docker/build-push-action@v2
+      with:
+        context: .
+        platforms: linux/arm64,linux/amd64
+        push: true
+        tags: us-central1-docker.pkg.dev/genuine-flight-317411/devel/krr:${{ github.ref_name }}
+        build-args: |
+          BUILDKIT_INLINE_CACHE=1
diff --git a/.github/workflows/pytest-on-push.yml b/.github/workflows/pytest-on-push.yml
new file mode 100644
index 0000000..a867168
--- /dev/null
+++ b/.github/workflows/pytest-on-push.yml
@@ -0,0 +1,27 @@
+name: Pytest
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -e .
+        pip install pytest
+
+    - name: Test with pytest
+      run: |
+        pytest
diff --git a/robusta_krr/api/models.py b/robusta_krr/api/models.py
index be2a3ad..a0c4ea9 100644
--- a/robusta_krr/api/models.py
+++ b/robusta_krr/api/models.py
@@ -1,6 +1,6 @@
-from robusta_krr.core.abstract.strategies import HistoryData, ResourceRecommendation, RunResult
+from robusta_krr.core.abstract.strategies import HistoryData, ResourceHistoryData, ResourceRecommendation, RunResult
 from robusta_krr.core.models.allocations import RecommendationValue, ResourceAllocations, ResourceType
-from robusta_krr.core.models.objects import K8sObjectData
+from robusta_krr.core.models.objects import K8sObjectData, PodData
 from robusta_krr.core.models.result import ResourceScan, Result, Severity
 
 __all__ = [
@@ -8,10 +8,12 @@ __all__ = [
     "ResourceAllocations",
     "RecommendationValue",
     "K8sObjectData",
+    "PodData",
     "Result",
     "Severity",
     "ResourceScan",
     "ResourceRecommendation",
     "HistoryData",
+    "ResourceHistoryData",
     "RunResult",
 ]
diff --git a/robusta_krr/core/models/objects.py b/robusta_krr/core/models/objects.py
index 21f5d61..84e468e 100644
--- a/robusta_krr/core/models/objects.py
+++ b/robusta_krr/core/models/objects.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
 import pydantic as pd
 
 from robusta_krr.core.models.allocations import ResourceAllocations
@@ -14,12 +12,12 @@ class PodData(pd.BaseModel):
 
 
 class K8sObjectData(pd.BaseModel):
-    cluster: Optional[str]
+    cluster: str
     name: str
     container: str
     pods: list[PodData]
     namespace: str
-    kind: Optional[str]
+    kind: str
     allocations: ResourceAllocations
 
     def __str__(self) -> str:
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..0617ddc
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,81 @@
+import random
+from datetime import datetime, timedelta
+from unittest.mock import AsyncMock, PropertyMock, patch
+
+import numpy as np
+import pytest
+
+from robusta_krr.api.models import K8sObjectData, PodData, ResourceAllocations, ResourceHistoryData
+
+TEST_OBJECT = K8sObjectData(
+    cluster="mock-cluster",
+    name="mock-object-1",
+    container="mock-container-1",
+    pods=[
+        PodData(name="mock-pod-1", deleted=False),
+        PodData(name="mock-pod-2", deleted=False),
+        PodData(name="mock-pod-3", deleted=True),
+    ],
+    namespace="default",
+    kind="Deployment",
+    allocations=ResourceAllocations(
+        requests={"cpu": 1, "memory": 1},  # type: ignore
+        limits={"cpu": 2, "memory": 2},  # type: ignore
+    ),
+)
+
+
+@pytest.fixture(autouse=True, scope="session")
+def mock_list_clusters():
+    with patch(
+        "robusta_krr.core.integrations.kubernetes.KubernetesLoader.list_clusters",
+        new=AsyncMock(return_value=[TEST_OBJECT.cluster]),
+    ):
+        yield
+
+
+@pytest.fixture(autouse=True, scope="session")
+def mock_list_scannable_objects():
+    with patch(
+        "robusta_krr.core.integrations.kubernetes.KubernetesLoader.list_scannable_objects",
+        new=AsyncMock(return_value=[TEST_OBJECT]),
+    ):
+        yield
+
+
+@pytest.fixture(autouse=True, scope="session")
+def mock_config_loaded():
+    with patch("robusta_krr.core.models.config.Config.config_loaded", new_callable=PropertyMock) as mock_config:
+        mock_config.return_value = True
+        yield
+
+
+@pytest.fixture(autouse=True, scope="session")
+def mock_prometheus_loader():
+    now = datetime.now()
+    start = now - timedelta(hours=1)
+    now_ts, start_ts = now.timestamp(), start.timestamp()
+    metric_points_data = np.array([(t, random.randrange(0, 100)) for t in np.linspace(start_ts, now_ts, 3600)])
+
+    with patch(
+        "robusta_krr.core.integrations.prometheus.loader.PrometheusLoader.gather_data",
+        new=AsyncMock(
+            return_value=ResourceHistoryData(
+                data={pod.name: metric_points_data for pod in TEST_OBJECT.pods},
+                metric={  # type: ignore
+                    "query": f"example_promql_metric{{pod_name=~\"{'|'.join(pod.name for pod in TEST_OBJECT.pods)}\"}}",
+                    "start_time": start,
+                    "end_time": now,
+                    "step": "30s",
+                },
+            )
+        ),
+    ) as mock_prometheus_loader:
+        mock_prometheus_loader
+        yield
+
+
+@pytest.fixture(autouse=True, scope="session")
+def mock_prometheus_init():
+    with patch("robusta_krr.core.integrations.prometheus.loader.PrometheusLoader.__init__", return_value=None):
+        yield
diff --git a/tests/test_krr.py b/tests/test_krr.py
index 23dab01..f013ee4 100644
--- a/tests/test_krr.py
+++ b/tests/test_krr.py
@@ -1,8 +1,3 @@
-"""
-    Test the krr command line interface and a general execution.
-    Requires a running kubernetes cluster with the kubectl command configured.
-"""
-
 import json
 
 import pytest
@@ -36,14 +31,19 @@ def test_run(log_flag: str):
 
 @pytest.mark.parametrize("format", ["json", "yaml", "table", "pprint"])
 def test_output_formats(format: str):
-    result = runner.invoke(app, [STRATEGY_NAME, "-q", "-f", format, "--namespace", "default"])
+    result = runner.invoke(app, [STRATEGY_NAME, "-q", "-f", format])
     try:
         assert result.exit_code == 0, result.exc_info
     except AssertionError as e:
         raise e from result.exception
 
-    if format == "json":
-        assert json.loads(result.stdout), result.stdout
-
-    if format == "yaml":
-        assert yaml.safe_load(result.stdout), result.stdout
+    try:
+        if format == "json":
+            json_output = json.loads(result.stdout)
+            assert json_output, result.stdout
+            assert len(json_output["scans"]) > 0, result.stdout
+
+        if format == "yaml":
+            assert yaml.safe_load(result.stdout), result.stdout
+    except Exception as e:
+        raise Exception(result.stdout) from e