Skip to content

Commit

Permalink
Require Bearer token for /metrics endpoint
Browse files Browse the repository at this point in the history
Signed-off-by: João Vilaça <[email protected]>
  • Loading branch information
machadovilaca committed Feb 10, 2025
1 parent 8aa2052 commit 757aed9
Show file tree
Hide file tree
Showing 46 changed files with 2,968 additions and 8 deletions.
4 changes: 3 additions & 1 deletion cmd/hyperconverged-cluster-operator/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ import (
"github.com/kubevirt/hyperconverged-cluster-operator/controllers/nodes"
"github.com/kubevirt/hyperconverged-cluster-operator/controllers/observability"
"github.com/kubevirt/hyperconverged-cluster-operator/controllers/operands"
"github.com/kubevirt/hyperconverged-cluster-operator/pkg/jwt"
"github.com/kubevirt/hyperconverged-cluster-operator/pkg/monitoring/hyperconverged/metrics"
"github.com/kubevirt/hyperconverged-cluster-operator/pkg/upgradepatch"
hcoutil "github.com/kubevirt/hyperconverged-cluster-operator/pkg/util"
Expand Down Expand Up @@ -347,7 +348,8 @@ func getCacheOption(operatorNamespace string, ci hcoutil.ClusterInfo) cache.Opti
func getManagerOptions(operatorNamespace string, needLeaderElection bool, ci hcoutil.ClusterInfo, scheme *apiruntime.Scheme) manager.Options {
return manager.Options{
Metrics: server.Options{
BindAddress: fmt.Sprintf("%s:%d", hcoutil.MetricsHost, hcoutil.MetricsPort),
BindAddress: fmt.Sprintf("%s:%d", hcoutil.MetricsHost, hcoutil.MetricsPort),
FilterProvider: jwt.HttpWithBearerToken,
},
HealthProbeBindAddress: fmt.Sprintf("%s:%d", hcoutil.HealthProbeHost, hcoutil.HealthProbePort),
ReadinessEndpointName: hcoutil.ReadinessEndpointName,
Expand Down
4 changes: 3 additions & 1 deletion cmd/hyperconverged-cluster-webhook/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/webhook"

webhookscontrollers "github.com/kubevirt/hyperconverged-cluster-operator/controllers/webhooks"
"github.com/kubevirt/hyperconverged-cluster-operator/pkg/jwt"
"github.com/kubevirt/hyperconverged-cluster-operator/pkg/webhooks/validator"

csvv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
Expand Down Expand Up @@ -93,7 +94,8 @@ func main() {
// Create a new Cmd to provide shared dependencies and start components
mgr, err := manager.New(cfg, manager.Options{
Metrics: server.Options{
BindAddress: fmt.Sprintf("%s:%d", hcoutil.MetricsHost, hcoutil.MetricsPort),
BindAddress: fmt.Sprintf("%s:%d", hcoutil.MetricsHost, hcoutil.MetricsPort),
FilterProvider: jwt.HttpWithBearerToken,
},
HealthProbeBindAddress: fmt.Sprintf("%s:%d", hcoutil.HealthProbeHost, hcoutil.HealthProbePort),
ReadinessEndpointName: hcoutil.ReadinessEndpointName,
Expand Down
2 changes: 1 addition & 1 deletion controllers/alerts/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ var _ = Describe("alert tests", func() {
req = commontestutils.NewReq(hco)
Expect(r.UpdateRelatedObjects(req)).To(Succeed())
Expect(req.StatusDirty).To(BeTrue())
Expect(hco.Status.RelatedObjects).To(HaveLen(5))
Expect(hco.Status.RelatedObjects).To(HaveLen(6))

Expect(ee.CheckEvents(expectedEvents)).To(BeTrue())
})
Expand Down
1 change: 1 addition & 0 deletions controllers/alerts/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ func NewMonitoringReconciler(ci hcoutil.ClusterInfo, cl client.Client, ee hcouti
newRoleReconciler(namespace, owner),
newRoleBindingReconciler(namespace, owner, ci),
newMetricServiceReconciler(namespace, owner),
newSecretReconciler(namespace, owner),
newServiceMonitorReconciler(namespace, owner),
},
scheme: scheme,
Expand Down
104 changes: 104 additions & 0 deletions controllers/alerts/secret.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
package alerts

import (
"context"

"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/kubevirt/hyperconverged-cluster-operator/pkg/jwt"
hcoutil "github.com/kubevirt/hyperconverged-cluster-operator/pkg/util"
)

const (
secretName = "hco-bearer-auth"
)

type secretReconciler struct {
theSecret *corev1.Secret
}

func newSecretReconciler(namespace string, owner metav1.OwnerReference) *secretReconciler {
return &secretReconciler{
theSecret: NewSecret(namespace, owner),
}
}

func (r *secretReconciler) Kind() string {
return "Secret"
}

func (r *secretReconciler) ResourceName() string {
return secretName
}

func (r *secretReconciler) GetFullResource() client.Object {
return r.theSecret.DeepCopy()
}

func (r *secretReconciler) EmptyObject() client.Object {
return &corev1.Secret{}
}

func (r *secretReconciler) UpdateExistingResource(ctx context.Context, cl client.Client, resource client.Object, logger logr.Logger) (client.Object, bool, error) {
found := resource.(*corev1.Secret)
modified := false

logger.Info("creating bearer token")
token, err := jwt.CreateToken()
if err != nil {
logger.Info("failed to create bearer token", "message", err)
token = ""
}

logger.Info("bearer token created", "token", token)

if found.Data["token"] == nil || string(found.Data["token"]) != token {
logger.Info("updating bearer token")
found.StringData = map[string]string{
"token": token,
}
modified = true
}

logger.Info("updating common details")

modified = updateCommonDetails(&r.theSecret.ObjectMeta, &found.ObjectMeta) || modified

if modified {
err := cl.Update(ctx, found)
if err != nil {
logger.Error(err, "failed to update the Secret")
return nil, false, err
}
logger.Info("successfully updated the Secret")
}

return found, modified, nil
}

func NewSecret(namespace string, owner metav1.OwnerReference) *corev1.Secret {
token, err := jwt.CreateToken()
if err != nil {
logger.Info("failed to create bearer token", "message", err)
token = ""
}

return &corev1.Secret{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Secret",
},
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: namespace,
Labels: hcoutil.GetLabels(hcoutil.HyperConvergedName, hcoutil.AppComponentMonitoring),
OwnerReferences: []metav1.OwnerReference{owner},
},
StringData: map[string]string{
"token": token,
},
}
}
13 changes: 12 additions & 1 deletion controllers/alerts/serviceMonitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/go-logr/logr"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

Expand Down Expand Up @@ -63,7 +64,17 @@ func NewServiceMonitor(namespace string, owner metav1.OwnerReference) *monitorin
Selector: metav1.LabelSelector{
MatchLabels: labels,
},
Endpoints: []monitoringv1.Endpoint{{Port: operatorPortName}},
Endpoints: []monitoringv1.Endpoint{{
Port: operatorPortName,
Authorization: &monitoringv1.SafeAuthorization{
Credentials: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secretName,
},
Key: "token",
},
},
}},
}

return &monitoringv1.ServiceMonitor{
Expand Down
6 changes: 3 additions & 3 deletions controllers/hyperconverged/hyperconverged_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ var _ = Describe("HyperconvergedController", func() {
foundResource),
).ToNot(HaveOccurred())
// Check conditions
Expect(foundResource.Status.RelatedObjects).To(HaveLen(26))
Expect(foundResource.Status.RelatedObjects).To(HaveLen(27))
expectedRef := corev1.ObjectReference{
Kind: "PrometheusRule",
Namespace: namespace,
Expand Down Expand Up @@ -309,7 +309,7 @@ var _ = Describe("HyperconvergedController", func() {

verifySystemHealthStatusError(foundResource)

Expect(foundResource.Status.RelatedObjects).To(HaveLen(21))
Expect(foundResource.Status.RelatedObjects).To(HaveLen(22))
expectedRef := corev1.ObjectReference{
Kind: "PrometheusRule",
Namespace: namespace,
Expand Down Expand Up @@ -820,7 +820,7 @@ var _ = Describe("HyperconvergedController", func() {
).To(Succeed())

Expect(foundResource.Status.RelatedObjects).ToNot(BeNil())
Expect(foundResource.Status.RelatedObjects).To(HaveLen(21))
Expect(foundResource.Status.RelatedObjects).To(HaveLen(22))
Expect(foundResource.ObjectMeta.Finalizers).To(Equal([]string{FinalizerName}))

// Now, delete HCO
Expand Down
10 changes: 10 additions & 0 deletions deploy/cluster_role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1086,6 +1086,16 @@ rules:
- list
- create
- delete
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- create
- update
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -557,6 +557,16 @@ spec:
- list
- create
- delete
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- create
- update
- watch
serviceAccountName: hyperconverged-cluster-operator
- rules:
- apiGroups:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ metadata:
certified: "false"
console.openshift.io/disable-operand-delete: "true"
containerImage: quay.io/kubevirt/hyperconverged-cluster-operator:1.15.0-unstable
createdAt: "2025-02-09 09:42:58"
createdAt: "2025-02-10 17:18:01"
description: A unified operator deploying and controlling KubeVirt and its supporting
operators with opinionated defaults
features.operators.openshift.io/cnf: "false"
Expand Down Expand Up @@ -557,6 +557,16 @@ spec:
- list
- create
- delete
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- create
- update
- watch
serviceAccountName: hyperconverged-cluster-operator
- rules:
- apiGroups:
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require (
github.com/gertd/go-pluralize v0.2.1
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
github.com/go-logr/logr v1.4.2
github.com/golang-jwt/jwt/v5 v5.2.1
github.com/google/uuid v1.6.0
github.com/kubevirt/cluster-network-addons-operator v0.98.1
github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20240505100225-e29dee0bb12b
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
Expand Down
5 changes: 5 additions & 0 deletions pkg/components/components.go
Original file line number Diff line number Diff line change
Expand Up @@ -602,6 +602,11 @@ func GetClusterPermissions() []rbacv1.PolicyRule {
Resources: stringListToSlice("alertmanagers", "alertmanagers/api"),
Verbs: stringListToSlice("get", "list", "create", "delete"),
},
{
APIGroups: stringListToSlice(""),
Resources: stringListToSlice("secrets"),
Verbs: stringListToSlice("get", "list", "create", "update", "watch"),
},
}
}

Expand Down
44 changes: 44 additions & 0 deletions pkg/jwt/bearer_token.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package jwt

import (
"fmt"
"os"

"github.com/golang-jwt/jwt/v5"
)

func CreateToken() (string, error) {
token := jwt.New(jwt.SigningMethodHS256)

secretKey, err := getSecretKey()
if err != nil {
return "", fmt.Errorf("error getting secret key: %v", err)
}

tokenString, err := token.SignedString(secretKey)
if err != nil {
return "", fmt.Errorf("error signing token: %v", err)
}

return tokenString, nil
}

func ValidateToken(tokenString string) (bool, error) {
token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
return getSecretKey()
})
if err != nil {
return false, fmt.Errorf("error parsing token: %v", err)
}

return token.Valid, nil
}

func getSecretKey() ([]byte, error) {
token, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
if err != nil {
return nil, fmt.Errorf("error reading token: %v", err)
}

return token, nil
}
37 changes: 37 additions & 0 deletions pkg/jwt/http.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package jwt

import (
"net/http"
"strings"

"github.com/go-logr/logr"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
)

func HttpWithBearerToken(config *rest.Config, httpClient *http.Client) (server.Filter, error) {
return func(log logr.Logger, handler http.Handler) (http.Handler, error) {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
authValue := req.Header.Get("Authorization")
token := strings.TrimPrefix(authValue, "Bearer ")

if token == "" {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}

valid, err := ValidateToken(token)
if err != nil {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}

if !valid {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}

handler.ServeHTTP(w, req)
}), nil
}, nil
}
4 changes: 4 additions & 0 deletions vendor/github.com/golang-jwt/jwt/v5/.gitignore

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions vendor/github.com/golang-jwt/jwt/v5/LICENSE

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 757aed9

Please sign in to comment.