feat: Export Prometheus compatible metrics (#123)

* feat: Export Prometheus compatible metrics

Signed-off-by: jannfis <jann@mistrust.net>

* adapt spelling action configuration

Signed-off-by: jannfis <jann@mistrust.net>

* adapt spelling action configuration

Signed-off-by: jannfis <jann@mistrust.net>

5 files changed, 213 insertions, 1 deletions

diff --git a/pkg/argocd/argocd.go b/pkg/argocd/argocd.go
index 4832c6f..b1dcf45 100644
--- a/pkg/argocd/argocd.go
+++ b/pkg/argocd/argocd.go
@@ -10,6 +10,7 @@ import (
 	"github.com/argoproj-labs/argocd-image-updater/pkg/common"
 	"github.com/argoproj-labs/argocd-image-updater/pkg/image"
 	"github.com/argoproj-labs/argocd-image-updater/pkg/log"
+	"github.com/argoproj-labs/argocd-image-updater/pkg/metrics"
 
 	argocdclient "github.com/argoproj/argo-cd/pkg/apiclient"
 	"github.com/argoproj/argo-cd/pkg/apiclient/application"
@@ -143,13 +144,17 @@ func FilterApplicationsForUpdate(apps []v1alpha1.Application, patterns []string)
 // GetApplication gets the application named appName from Argo CD API
 func (client *argoCD) GetApplication(ctx context.Context, appName string) (*v1alpha1.Application, error) {
 	conn, appClient, err := client.Client.NewApplicationClient()
+	metrics.Clients().IncreaseArgoCDClientRequest(client.Client.ClientOptions().ServerAddr, 1)
 	if err != nil {
+		metrics.Clients().IncreaseArgoCDClientError(client.Client.ClientOptions().ServerAddr, 1)
 		return nil, err
 	}
 	defer conn.Close()
 
+	metrics.Clients().IncreaseArgoCDClientRequest(client.Client.ClientOptions().ServerAddr, 1)
 	app, err := appClient.Get(ctx, &application.ApplicationQuery{Name: &appName})
 	if err != nil {
+		metrics.Clients().IncreaseArgoCDClientError(client.Client.ClientOptions().ServerAddr, 1)
 		return nil, err
 	}
 
@@ -160,13 +165,17 @@ func (client *argoCD) GetApplication(ctx context.Context, appName string) (*v1al
 // has access to.
 func (client *argoCD) ListApplications() ([]v1alpha1.Application, error) {
 	conn, appClient, err := client.Client.NewApplicationClient()
+	metrics.Clients().IncreaseArgoCDClientRequest(client.Client.ClientOptions().ServerAddr, 1)
 	if err != nil {
+		metrics.Clients().IncreaseArgoCDClientError(client.Client.ClientOptions().ServerAddr, 1)
 		return nil, err
 	}
 	defer conn.Close()
 
+	metrics.Clients().IncreaseArgoCDClientRequest(client.Client.ClientOptions().ServerAddr, 1)
 	apps, err := appClient.List(context.TODO(), &application.ApplicationQuery{})
 	if err != nil {
+		metrics.Clients().IncreaseArgoCDClientError(client.Client.ClientOptions().ServerAddr, 1)
 		return nil, err
 	}
 
@@ -176,13 +185,17 @@ func (client *argoCD) ListApplications() ([]v1alpha1.Application, error) {
 // UpdateSpec updates the spec for given application
 func (client *argoCD) UpdateSpec(ctx context.Context, in *application.ApplicationUpdateSpecRequest) (*v1alpha1.ApplicationSpec, error) {
 	conn, appClient, err := client.Client.NewApplicationClient()
+	metrics.Clients().IncreaseArgoCDClientRequest(client.Client.ClientOptions().ServerAddr, 1)
 	if err != nil {
+		metrics.Clients().IncreaseArgoCDClientError(client.Client.ClientOptions().ServerAddr, 1)
 		return nil, err
 	}
 	defer conn.Close()
 
+	metrics.Clients().IncreaseArgoCDClientRequest(client.Client.ClientOptions().ServerAddr, 1)
 	spec, err := appClient.UpdateSpec(ctx, in)
 	if err != nil {
+		metrics.Clients().IncreaseArgoCDClientError(client.Client.ClientOptions().ServerAddr, 1)
 		return nil, err
 	}
 
diff --git a/pkg/argocd/update.go b/pkg/argocd/update.go
index cf68011..0ed2e0c 100644
--- a/pkg/argocd/update.go
+++ b/pkg/argocd/update.go
@@ -15,6 +15,7 @@ import (
 // Stores some statistics about the results of a run
 type ImageUpdaterResult struct {
 	NumApplicationsProcessed int
+	NumImagesFound           int
 	NumImagesUpdated         int
 	NumImagesConsidered      int
 	NumSkipped               int
diff --git a/pkg/client/kubernetes.go b/pkg/client/kubernetes.go
index 99e6d8b..5f4f512 100644
--- a/pkg/client/kubernetes.go
+++ b/pkg/client/kubernetes.go
@@ -6,6 +6,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/argoproj-labs/argocd-image-updater/pkg/metrics"
+
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -46,7 +48,9 @@ func NewKubernetesClient(kubeconfig string) (*KubernetesClient, error) {
 // GetSecretData returns the raw data from named K8s secret in given namespace
 func (client *KubernetesClient) GetSecretData(namespace string, secretName string) (map[string][]byte, error) {
 	secret, err := client.Clientset.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, v1.GetOptions{})
+	metrics.Clients().IncreaseK8sClientRequest(1)
 	if err != nil {
+		metrics.Clients().IncreaseK8sClientRequest(1)
 		return nil, err
 	}
 	return secret.Data, nil
@@ -55,7 +59,9 @@ func (client *KubernetesClient) GetSecretData(namespace string, secretName strin
 // GetSecretField returns the value of a field from named K8s secret in given namespace
 func (client *KubernetesClient) GetSecretField(namespace string, secretName string, field string) (string, error) {
 	secret, err := client.GetSecretData(namespace, secretName)
+	metrics.Clients().IncreaseK8sClientRequest(1)
 	if err != nil {
+		metrics.Clients().IncreaseK8sClientRequest(1)
 		return "", err
 	}
 	if data, ok := secret[field]; !ok {
diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
new file mode 100644
index 0000000..08c8f4b
--- /dev/null
+++ b/pkg/metrics/metrics.go
@@ -0,0 +1,187 @@
+package metrics
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// TODO: These should not be global vars with this package
+var epm *EndpointMetrics
+var apm *ApplicationMetrics
+var cpm *ClientMetrics
+
+// EndpointMetrics stores metrics for registry endpoints
+type EndpointMetrics struct {
+	requestsTotal  *prometheus.CounterVec
+	requestsFailed *prometheus.CounterVec
+}
+
+// ApplicationMetrics stores metrics for applications
+type ApplicationMetrics struct {
+	applicationsTotal        prometheus.Gauge
+	imagesWatchedTotal       *prometheus.GaugeVec
+	imagesUpdatedTotal       *prometheus.CounterVec
+	imagesUpdatedErrorsTotal *prometheus.CounterVec
+}
+
+// ClientMetrics stores metrics for K8s and ArgoCD clients
+type ClientMetrics struct {
+	argoCDRequestsTotal        *prometheus.CounterVec
+	argoCDRequestsErrorsTotal  *prometheus.CounterVec
+	kubeAPIRequestsTotal       prometheus.Counter
+	kubeAPIRequestsErrorsTotal prometheus.Counter
+}
+
+// StartMetricsServer starts a new HTTP server for metrics on given port
+func StartMetricsServer(port int) chan error {
+	errCh := make(chan error)
+	go func() {
+		http.Handle("/metrics", promhttp.Handler())
+		errCh <- http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
+	}()
+	return errCh
+}
+
+// NewEndpointMetrics returns a new endpoint metrics object
+func NewEndpointMetrics() *EndpointMetrics {
+	metrics := &EndpointMetrics{}
+
+	metrics.requestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "argocd_image_updater_registry_requests_total",
+		Help: "The total number of requests to this endpoint",
+	}, []string{"registry"})
+	metrics.requestsFailed = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "argocd_image_updater_registry_requests_failed_total",
+		Help: "The number of failed requests to this endpoint",
+	}, []string{"registry"})
+
+	return metrics
+}
+
+// NewApplicationsMetrics returns a new application metrics object
+func NewApplicationsMetrics() *ApplicationMetrics {
+	metrics := &ApplicationMetrics{}
+
+	metrics.applicationsTotal = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "argocd_image_updater_applications_watched_total",
+		Help: "The total number of applications watched by Argo CD Image Updater",
+	})
+
+	metrics.imagesWatchedTotal = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "argocd_image_updater_images_watched_total",
+		Help: "Number of images watched by Argo CD Image Updater",
+	}, []string{"application"})
+
+	metrics.imagesUpdatedTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "argocd_image_updater_images_updated_total",
+		Help: "Number of images updates by Argo CD Image Updater",
+	}, []string{"application"})
+
+	metrics.imagesUpdatedErrorsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "argocd_image_updater_images_errors_total",
+		Help: "Number of errors reported by Argo CD Image Updater",
+	}, []string{"application"})
+
+	return metrics
+}
+
+// NewClientMetrics returns a new client metrics object
+func NewClientMetrics() *ClientMetrics {
+	metrics := &ClientMetrics{}
+
+	metrics.argoCDRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "argocd_image_updater_argocd_api_requests_total",
+		Help: "The total number of Argo CD API requests performed by the Argo CD Image Updater",
+	}, []string{"argocd_server"})
+
+	metrics.argoCDRequestsErrorsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "argocd_image_updater_argocd_api_errors_total",
+		Help: "The total number of Argo CD API requests resulting in error",
+	}, []string{"argocd_server"})
+
+	metrics.kubeAPIRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "argocd_image_updater_k8s_api_requests_total",
+		Help: "The total number of Argo CD API requests resulting in error",
+	})
+
+	metrics.kubeAPIRequestsErrorsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "argocd_image_updater_k8s_api_errors_total",
+		Help: "The total number of Argo CD API requests resulting in error",
+	})
+
+	return metrics
+}
+
+// Endpoint returns the global EndpointMetrics object
+func Endpoint() *EndpointMetrics {
+	return epm
+}
+
+// Applications returns the global ApplicationMetrics object
+func Applications() *ApplicationMetrics {
+	return apm
+}
+
+// Clients returns the global ClientMetrics object
+func Clients() *ClientMetrics {
+	return cpm
+}
+
+// IncreaseRequest increases the request counter of EndpointMetrics object
+func (epm *EndpointMetrics) IncreaseRequest(registryURL string, isFailed bool) {
+	epm.requestsTotal.WithLabelValues(registryURL).Inc()
+	if isFailed {
+		epm.requestsFailed.WithLabelValues(registryURL).Inc()
+	}
+}
+
+// SetNumberOfApplications sets the total number of currently watched applications
+func (apm *ApplicationMetrics) SetNumberOfApplications(num int) {
+	apm.applicationsTotal.Set(float64(num))
+}
+
+// SetNumberOfImagesWatched sets the total number of currently watched images for given application
+func (apm *ApplicationMetrics) SetNumberOfImagesWatched(application string, num int) {
+	apm.imagesWatchedTotal.WithLabelValues(application).Set(float64(num))
+}
+
+// IncreaseImageUpdate increases the number of image updates for given application
+func (apm *ApplicationMetrics) IncreaseImageUpdate(application string, by int) {
+	apm.imagesUpdatedTotal.WithLabelValues(application).Add(float64(by))
+}
+
+// IncreaseUpdateErrors increases the number of errors for given application occured during update process
+func (apm *ApplicationMetrics) IncreaseUpdateErrors(application string, by int) {
+	apm.imagesUpdatedErrorsTotal.WithLabelValues(application).Add(float64(by))
+}
+
+// IncreaseArgoCDClientRequest increases the number of Argo CD API requests for given server
+func (cpm *ClientMetrics) IncreaseArgoCDClientRequest(server string, by int) {
+	cpm.argoCDRequestsTotal.WithLabelValues(server).Add(float64(by))
+}
+
+// IncreaseArgoCDClientError increases the number of failed Argo CD API requests for given server
+func (cpm *ClientMetrics) IncreaseArgoCDClientError(server string, by int) {
+	cpm.argoCDRequestsErrorsTotal.WithLabelValues(server).Add(float64(by))
+}
+
+// IncreaseK8sClientRequest increases the number of K8s API requests
+func (cpm *ClientMetrics) IncreaseK8sClientRequest(by int) {
+	cpm.kubeAPIRequestsTotal.Add(float64(by))
+}
+
+// IncreaseK8sClientRequest increases the number of failed K8s API requests
+func (cpm *ClientMetrics) IncreaseK8sClientError(by int) {
+	cpm.kubeAPIRequestsErrorsTotal.Add(float64(by))
+}
+
+// TODO: This is a lazy workaround, better initialize it somehwere else
+func init() {
+	epm = NewEndpointMetrics()
+	apm = NewApplicationsMetrics()
+	cpm = NewClientMetrics()
+}
diff --git a/pkg/registry/client.go b/pkg/registry/client.go
index 7eac4ea..69992ee 100644
--- a/pkg/registry/client.go
+++ b/pkg/registry/client.go
@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/argoproj-labs/argocd-image-updater/pkg/log"
+	"github.com/argoproj-labs/argocd-image-updater/pkg/metrics"
 	"github.com/argoproj-labs/argocd-image-updater/pkg/tag"
 
 	"github.com/docker/distribution"
@@ -41,13 +42,16 @@ type registryClient struct {
 type rateLimitTransport struct {
 	limiter   ratelimit.Limiter
 	transport http.RoundTripper
+	endpoint  string
 }
 
 // RoundTrip is a custom RoundTrip method with rate-limiter
 func (rlt *rateLimitTransport) RoundTrip(r *http.Request) (*http.Response, error) {
 	rlt.limiter.Take()
 	log.Tracef("%s", r.URL)
-	return rlt.transport.RoundTrip(r)
+	resp, err := rlt.transport.RoundTrip(r)
+	metrics.Endpoint().IncreaseRequest(rlt.endpoint, err != nil)
+	return resp, err
 }
 
 // newRegistry is a wrapper for creating a registry client that is possibly
@@ -69,6 +73,7 @@ func newRegistry(ep *RegistryEndpoint, opts registry.Options) (*registry.Registr
 	rlt := &rateLimitTransport{
 		limiter:   ep.Limiter,
 		transport: transport,
+		endpoint:  ep.RegistryAPI,
 	}
 
 	logf := opts.Logf