OLS-3546 Dynamically reconcile metrics-reader ClusterRoleBinding namespace

config/prometheus/clusterrole_binding.yaml

@@ -14,4 +14,4 @@ subjects:
 
   - kind: ServiceAccount
     name: metrics-reader
-    namespace: openshift-lightspeed
+    namespace: system

internal/controller/appserver/assets.go

@@ -1004,6 +1004,33 @@ func GenerateMetricsReaderSecret(r reconciler.Reconciler, cr *olsv1alpha1.OLSCon
 	return secret, nil
 }
 
+func generateMetricsReaderClusterRoleBinding(r reconciler.Reconciler, cr *olsv1alpha1.OLSConfig) (*rbacv1.ClusterRoleBinding, error) {
+	rb := rbacv1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   utils.MetricsReaderClusterRoleBindingName,
+			Labels: utils.GenerateAppServerSelectorLabels(),
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      utils.MetricsReaderServiceAccountName,
+				Namespace: r.GetNamespace(),
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "ClusterRole",
+			Name:     utils.MetricsReaderClusterRoleName,
+		},
+	}
+
+	if err := controllerutil.SetControllerReference(cr, &rb, r.GetScheme()); err != nil {
+		return nil, fmt.Errorf("%s: %w", utils.ErrSetMetricsReaderCRBOwnerReference, err)
+	}
+
+	return &rb, nil
+}
+
 func serviceTLSProfileType(profileType configv1.TLSProfileType) string {
 	switch profileType {
 	case configv1.TLSProfileOldType:

internal/controller/appserver/assets_test.go

@@ -1661,6 +1661,32 @@ var _ = Describe("App server assets", func() {
 			Expect(secret.Type).To(Equal(corev1.SecretTypeServiceAccountToken))
 		})
 
+		It("should generate the metrics reader ClusterRoleBinding", func() {
+			rb, err := generateMetricsReaderClusterRoleBinding(testReconcilerInstance, cr)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(rb.Name).To(Equal(utils.MetricsReaderClusterRoleBindingName))
+			Expect(rb.Labels).To(Equal(utils.GenerateAppServerSelectorLabels()))
+			Expect(rb.RoleRef.Kind).To(Equal("ClusterRole"))
+			Expect(rb.RoleRef.Name).To(Equal(utils.MetricsReaderClusterRoleName))
+			Expect(rb.Subjects).To(HaveLen(1))
+			Expect(rb.Subjects[0].Kind).To(Equal("ServiceAccount"))
+			Expect(rb.Subjects[0].Name).To(Equal(utils.MetricsReaderServiceAccountName))
+			Expect(rb.Subjects[0].Namespace).To(Equal(utils.OLSNamespaceDefault))
+		})
+
+		It("should use the reconciler's actual namespace in the metrics reader ClusterRoleBinding subject", func() {
+			customNS := "custom-operator-namespace"
+			customReconciler := utils.NewTestReconciler(
+				k8sClient,
+				testReconcilerInstance.GetLogger(),
+				testReconcilerInstance.GetScheme(),
+				customNS,
+			)
+			rb, err := generateMetricsReaderClusterRoleBinding(customReconciler, cr)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(rb.Subjects[0].Namespace).To(Equal(customNS))
+		})
+
 		It("should generate the OLS prometheus rules", func() {
 			prometheusRule, err := GeneratePrometheusRule(testReconcilerInstance, cr)
 			Expect(err).NotTo(HaveOccurred())

internal/controller/appserver/reconciler.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"reflect"
 	"time"
 
 	"github.com/openshift/lightspeed-operator/internal/controller/reconciler"
@@ -71,6 +72,10 @@ func ReconcileAppServerResources(r reconciler.Reconciler, ctx context.Context, o
 			Name: "reconcile Metrics Reader Secret",
 			Task: reconcileMetricsReaderSecret,
 		},
+		{
+			Name: "reconcile Metrics Reader ClusterRoleBinding",
+			Task: reconcileMetricsReaderClusterRoleBinding,
+		},
 		{
 			Name: "reconcile App NetworkPolicy",
 			Task: reconcileAppServerNetworkPolicy,
@@ -405,6 +410,75 @@ func reconcileMetricsReaderSecret(r reconciler.Reconciler, ctx context.Context,
 	return nil
 }
 
+// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;create;update
+func reconcileMetricsReaderClusterRoleBinding(r reconciler.Reconciler, ctx context.Context, cr *olsv1alpha1.OLSConfig) error {
+	if os.Getenv("LOCAL_DEV_MODE") == "true" {
+		r.GetLogger().Info("Skipping metrics reader ClusterRoleBinding reconciliation in LOCAL_DEV_MODE")
+		return nil
+	}
+
+	desired, err := generateMetricsReaderClusterRoleBinding(r, cr)
+	if err != nil {
+		return fmt.Errorf("%s: %w", utils.ErrGenerateMetricsReaderCRB, err)
+	}
+
+	found := &rbacv1.ClusterRoleBinding{}
+	err = r.Get(ctx, client.ObjectKey{Name: desired.Name}, found)
+	if err != nil && errors.IsNotFound(err) {
+		r.GetLogger().Info("creating metrics reader ClusterRoleBinding", "ClusterRoleBinding", desired.Name)
+		err = r.Create(ctx, desired)
+		if err != nil {
+			return fmt.Errorf("%s: %w", utils.ErrCreateMetricsReaderCRB, err)
+		}
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("%s: %w", utils.ErrGetMetricsReaderCRB, err)
+	}
+
+	// RoleRef is immutable in Kubernetes, so a mismatch can only be fixed by
+	// deleting and recreating the ClusterRoleBinding.
+	if found.RoleRef != desired.RoleRef {
+		r.GetLogger().Info("recreating metrics reader ClusterRoleBinding due to RoleRef mismatch",
+			"ClusterRoleBinding", found.Name, "foundRoleRef", found.RoleRef, "desiredRoleRef", desired.RoleRef)
+		if err := r.Delete(ctx, found); err != nil {
+			return fmt.Errorf("%s: %w", utils.ErrDeleteMetricsReaderCRB, err)
+		}
+		if err := r.Create(ctx, desired); err != nil {
+			return fmt.Errorf("%s: %w", utils.ErrCreateMetricsReaderCRB, err)
+		}
+		return nil
+	}
+
+	needsUpdate := false
+	if !reflect.DeepEqual(found.Subjects, desired.Subjects) {
+		found.Subjects = desired.Subjects
+		needsUpdate = true
+	}
+
+	if !reflect.DeepEqual(found.OwnerReferences, desired.OwnerReferences) {
+		found.OwnerReferences = desired.OwnerReferences
+		needsUpdate = true
+	}
+
+	if !reflect.DeepEqual(found.Labels, desired.Labels) {
+		found.Labels = desired.Labels
+		needsUpdate = true
+	}
+
+	if needsUpdate {
+		r.GetLogger().Info("updating metrics reader ClusterRoleBinding",
+			"ClusterRoleBinding", found.Name, "namespace", r.GetNamespace())
+		err = r.Update(ctx, found)
+		if err != nil {
+			return fmt.Errorf("%s: %w", utils.ErrUpdateMetricsReaderCRB, err)
+		}
+	} else {
+		r.GetLogger().Info("metrics reader ClusterRoleBinding reconciled", "ClusterRoleBinding", found.Name)
+	}
+
+	return nil
+}
+
 func reconcileServiceMonitor(r reconciler.Reconciler, ctx context.Context, cr *olsv1alpha1.OLSConfig) error {
 	sm, err := GenerateServiceMonitor(r, cr)
 	if err != nil {

internal/controller/appserver/reconciler_test.go

@@ -452,6 +452,101 @@ var _ = Describe("App server reconciliator", Ordered, func() {
 			Expect(err).NotTo(HaveOccurred())
 		})
 
+		It("should create a metrics reader ClusterRoleBinding with correct namespace", func() {
+			By("Get the metrics reader ClusterRoleBinding")
+			rb := &rbacv1.ClusterRoleBinding{}
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: utils.MetricsReaderClusterRoleBindingName}, rb)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(rb.Subjects).To(HaveLen(1))
+			Expect(rb.Subjects[0].Name).To(Equal(utils.MetricsReaderServiceAccountName))
+			Expect(rb.Subjects[0].Namespace).To(Equal(utils.OLSNamespaceDefault))
+			Expect(rb.RoleRef.Name).To(Equal(utils.MetricsReaderClusterRoleName))
+		})
+
+		It("should fix a metrics reader ClusterRoleBinding with wrong namespace", func() {
+			By("Patch the ClusterRoleBinding to have a wrong namespace")
+			rb := &rbacv1.ClusterRoleBinding{}
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: utils.MetricsReaderClusterRoleBindingName}, rb)
+			Expect(err).NotTo(HaveOccurred())
+
+			rb.Subjects[0].Namespace = "wrong-namespace"
+			err = k8sClient.Update(ctx, rb)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Re-reconcile to fix the namespace")
+			err = ReconcileAppServer(testReconcilerInstance, ctx, cr)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Verify the namespace was corrected")
+			err = k8sClient.Get(ctx, client.ObjectKey{Name: utils.MetricsReaderClusterRoleBindingName}, rb)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(rb.Subjects[0].Namespace).To(Equal(utils.OLSNamespaceDefault))
+		})
+
+		It("should fix a metrics reader ClusterRoleBinding missing owner references and labels", func() {
+			By("Strip the owner references and labels from the ClusterRoleBinding")
+			rb := &rbacv1.ClusterRoleBinding{}
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: utils.MetricsReaderClusterRoleBindingName}, rb)
+			Expect(err).NotTo(HaveOccurred())
+
+			rb.OwnerReferences = nil
+			rb.Labels = nil
+			err = k8sClient.Update(ctx, rb)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Re-reconcile to restore owner references and labels")
+			err = ReconcileAppServer(testReconcilerInstance, ctx, cr)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Verify the owner references and labels were restored")
+			err = k8sClient.Get(ctx, client.ObjectKey{Name: utils.MetricsReaderClusterRoleBindingName}, rb)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(rb.OwnerReferences).NotTo(BeEmpty())
+			Expect(rb.Labels).To(Equal(utils.GenerateAppServerSelectorLabels()))
+		})
+
+		It("should recreate a metrics reader ClusterRoleBinding with a mismatched RoleRef", func() {
+			By("Delete the ClusterRoleBinding and recreate it with a wrong RoleRef")
+			rb := &rbacv1.ClusterRoleBinding{}
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: utils.MetricsReaderClusterRoleBindingName}, rb)
+			Expect(err).NotTo(HaveOccurred())
+			originalUID := rb.UID
+
+			err = k8sClient.Delete(ctx, rb)
+			Expect(err).NotTo(HaveOccurred())
+
+			wrongRB := &rbacv1.ClusterRoleBinding{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: utils.MetricsReaderClusterRoleBindingName,
+				},
+				Subjects: []rbacv1.Subject{
+					{
+						Kind:      "ServiceAccount",
+						Name:      utils.MetricsReaderServiceAccountName,
+						Namespace: utils.OLSNamespaceDefault,
+					},
+				},
+				RoleRef: rbacv1.RoleRef{
+					APIGroup: "rbac.authorization.k8s.io",
+					Kind:     "ClusterRole",
+					Name:     "wrong-cluster-role",
+				},
+			}
+			err = k8sClient.Create(ctx, wrongRB)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Re-reconcile to fix the RoleRef")
+			err = ReconcileAppServer(testReconcilerInstance, ctx, cr)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Verify the ClusterRoleBinding was recreated with the correct RoleRef")
+			fixedRB := &rbacv1.ClusterRoleBinding{}
+			err = k8sClient.Get(ctx, client.ObjectKey{Name: utils.MetricsReaderClusterRoleBindingName}, fixedRB)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(fixedRB.RoleRef.Name).To(Equal(utils.MetricsReaderClusterRoleName))
+			Expect(fixedRB.UID).NotTo(Equal(originalUID))
+		})
+
 		It("should create a prometheus rule", func() {
 			By("Get the prometheus rule")
 			pr := &monv1.PrometheusRule{}

internal/controller/utils/constants.go

@@ -454,6 +454,10 @@ ssl_ca_file = '/etc/certs/cm-olspostgresca/service-ca.crt'
 	MetricsReaderServiceAccountTokenSecretName = "metrics-reader-token" // #nosec G101
 	// MetricsReaderServiceAccountName is the name of the service account for the metrics reader
 	MetricsReaderServiceAccountName = "lightspeed-operator-metrics-reader"
+	// MetricsReaderClusterRoleName is the name of the ClusterRole granting metrics read access
+	MetricsReaderClusterRoleName = "lightspeed-operator-ols-metrics-reader"
+	// MetricsReaderClusterRoleBindingName is the name of the ClusterRoleBinding for the metrics reader
+	MetricsReaderClusterRoleBindingName = "lightspeed-operator-ols-metrics-reader"
 	// OpenShiftMCPServerHTTPSPort is the standalone Service/container HTTPS port (service-ca TLS).
 	OpenShiftMCPServerHTTPSPort = 8443
 	// OpenShiftMCPServerDeploymentName is the standalone openshift-mcp-server Deployment.

internal/controller/utils/errors.go

@@ -79,6 +79,12 @@ const (
 	ErrGetSARClusterRoleBinding            = "failed to get SAR cluster role binding"
 	ErrGetServiceMonitor                   = "failed to get ServiceMonitor"
 	ErrGetMetricsReaderSecret              = "failed to get metrics reader secret"
+	ErrGenerateMetricsReaderCRB            = "failed to generate metrics reader cluster role binding"
+	ErrCreateMetricsReaderCRB              = "failed to create metrics reader cluster role binding"
+	ErrGetMetricsReaderCRB                 = "failed to get metrics reader cluster role binding"
+	ErrUpdateMetricsReaderCRB              = "failed to update metrics reader cluster role binding"
+	ErrDeleteMetricsReaderCRB              = "failed to delete metrics reader cluster role binding"
+	ErrSetMetricsReaderCRBOwnerReference   = "failed to set metrics reader cluster role binding owner reference"
 	ErrGetPrometheusRule                   = "failed to get PrometheusRule"
 	ErrUpdateAPIConfigmap                  = "failed to update OLS configmap"
 	ErrUpdateAPIDeployment                 = "failed to update OLS deployment"