Skip to content

Commit

Permalink
Require Bearer token for /metrics endpoint
Browse files Browse the repository at this point in the history
Signed-off-by: João Vilaça <[email protected]>
  • Loading branch information
machadovilaca committed Feb 20, 2025
1 parent de3d79a commit 7332493
Show file tree
Hide file tree
Showing 49 changed files with 3,089 additions and 12 deletions.
4 changes: 3 additions & 1 deletion cmd/hyperconverged-cluster-operator/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ import (
"github.com/kubevirt/hyperconverged-cluster-operator/controllers/nodes"
"github.com/kubevirt/hyperconverged-cluster-operator/controllers/observability"
"github.com/kubevirt/hyperconverged-cluster-operator/controllers/operands"
"github.com/kubevirt/hyperconverged-cluster-operator/pkg/authorization"
"github.com/kubevirt/hyperconverged-cluster-operator/pkg/monitoring/hyperconverged/metrics"
"github.com/kubevirt/hyperconverged-cluster-operator/pkg/upgradepatch"
hcoutil "github.com/kubevirt/hyperconverged-cluster-operator/pkg/util"
Expand Down Expand Up @@ -360,7 +361,8 @@ func getCacheOption(operatorNamespace string, ci hcoutil.ClusterInfo) cache.Opti
func getManagerOptions(operatorNamespace string, needLeaderElection bool, ci hcoutil.ClusterInfo, scheme *apiruntime.Scheme) manager.Options {
return manager.Options{
Metrics: server.Options{
BindAddress: fmt.Sprintf("%s:%d", hcoutil.MetricsHost, hcoutil.MetricsPort),
BindAddress: fmt.Sprintf("%s:%d", hcoutil.MetricsHost, hcoutil.MetricsPort),
FilterProvider: authorization.HttpWithBearerToken,
},
HealthProbeBindAddress: fmt.Sprintf("%s:%d", hcoutil.HealthProbeHost, hcoutil.HealthProbePort),
ReadinessEndpointName: hcoutil.ReadinessEndpointName,
Expand Down
4 changes: 3 additions & 1 deletion cmd/hyperconverged-cluster-webhook/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/webhook"

webhookscontrollers "github.com/kubevirt/hyperconverged-cluster-operator/controllers/webhooks"
"github.com/kubevirt/hyperconverged-cluster-operator/pkg/authorization"
"github.com/kubevirt/hyperconverged-cluster-operator/pkg/webhooks/validator"

csvv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
Expand Down Expand Up @@ -92,7 +93,8 @@ func main() {
// Create a new Cmd to provide shared dependencies and start components
mgr, err := manager.New(cfg, manager.Options{
Metrics: server.Options{
BindAddress: fmt.Sprintf("%s:%d", hcoutil.MetricsHost, hcoutil.MetricsPort),
BindAddress: fmt.Sprintf("%s:%d", hcoutil.MetricsHost, hcoutil.MetricsPort),
FilterProvider: authorization.HttpWithBearerToken,
},
HealthProbeBindAddress: fmt.Sprintf("%s:%d", hcoutil.HealthProbeHost, hcoutil.HealthProbePort),
ReadinessEndpointName: hcoutil.ReadinessEndpointName,
Expand Down
2 changes: 1 addition & 1 deletion controllers/alerts/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ var _ = Describe("alert tests", func() {
req = commontestutils.NewReq(hco)
Expect(r.UpdateRelatedObjects(req)).To(Succeed())
Expect(req.StatusDirty).To(BeTrue())
Expect(hco.Status.RelatedObjects).To(HaveLen(5))
Expect(hco.Status.RelatedObjects).To(HaveLen(6))

Expect(ee.CheckEvents(expectedEvents)).To(BeTrue())
})
Expand Down
1 change: 1 addition & 0 deletions controllers/alerts/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ func NewMonitoringReconciler(ci hcoutil.ClusterInfo, cl client.Client, ee hcouti
newRoleReconciler(namespace, owner),
newRoleBindingReconciler(namespace, owner, ci),
newMetricServiceReconciler(namespace, owner),
newSecretReconciler(namespace, owner),
newServiceMonitorReconciler(namespace, owner),
},
scheme: scheme,
Expand Down
96 changes: 96 additions & 0 deletions controllers/alerts/secret.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package alerts

import (
"context"

"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/kubevirt/hyperconverged-cluster-operator/pkg/authorization"
hcoutil "github.com/kubevirt/hyperconverged-cluster-operator/pkg/util"
)

const (
secretName = "hco-bearer-auth"
)

type secretReconciler struct {
theSecret *corev1.Secret
}

func newSecretReconciler(namespace string, owner metav1.OwnerReference) *secretReconciler {
return &secretReconciler{
theSecret: NewSecret(namespace, owner),
}
}

func (r *secretReconciler) Kind() string {
return "Secret"
}

func (r *secretReconciler) ResourceName() string {
return secretName
}

func (r *secretReconciler) GetFullResource() client.Object {
return r.theSecret.DeepCopy()
}

func (r *secretReconciler) EmptyObject() client.Object {
return &corev1.Secret{}
}

func (r *secretReconciler) UpdateExistingResource(ctx context.Context, cl client.Client, resource client.Object, logger logr.Logger) (client.Object, bool, error) {
found := resource.(*corev1.Secret)
modified := false

token, err := authorization.CreateToken()
if err != nil {
return nil, false, err
}

if found.Data["token"] == nil || string(found.Data["token"]) != token {
found.StringData = map[string]string{
"token": token,
}
modified = true
}

modified = updateCommonDetails(&r.theSecret.ObjectMeta, &found.ObjectMeta) || modified

if modified {
err := cl.Update(ctx, found)
if err != nil {
logger.Error(err, "failed to update the Secret")
return nil, false, err
}
}

return found, modified, nil
}

func NewSecret(namespace string, owner metav1.OwnerReference) *corev1.Secret {
token, err := authorization.CreateToken()
if err != nil {
logger.Error(err, "failed to create bearer token")
return nil
}

return &corev1.Secret{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Secret",
},
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: namespace,
Labels: hcoutil.GetLabels(hcoutil.HyperConvergedName, hcoutil.AppComponentMonitoring),
OwnerReferences: []metav1.OwnerReference{owner},
},
StringData: map[string]string{
"token": token,
},
}
}
13 changes: 12 additions & 1 deletion controllers/alerts/serviceMonitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/go-logr/logr"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

Expand Down Expand Up @@ -63,7 +64,17 @@ func NewServiceMonitor(namespace string, owner metav1.OwnerReference) *monitorin
Selector: metav1.LabelSelector{
MatchLabels: labels,
},
Endpoints: []monitoringv1.Endpoint{{Port: operatorPortName}},
Endpoints: []monitoringv1.Endpoint{{
Port: operatorPortName,
Authorization: &monitoringv1.SafeAuthorization{
Credentials: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secretName,
},
Key: "token",
},
},
}},
}

return &monitoringv1.ServiceMonitor{
Expand Down
6 changes: 3 additions & 3 deletions controllers/hyperconverged/hyperconverged_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ var _ = Describe("HyperconvergedController", func() {
foundResource),
).ToNot(HaveOccurred())
// Check conditions
Expect(foundResource.Status.RelatedObjects).To(HaveLen(26))
Expect(foundResource.Status.RelatedObjects).To(HaveLen(27))
expectedRef := corev1.ObjectReference{
Kind: "PrometheusRule",
Namespace: namespace,
Expand Down Expand Up @@ -312,7 +312,7 @@ var _ = Describe("HyperconvergedController", func() {

verifySystemHealthStatusError(foundResource)

Expect(foundResource.Status.RelatedObjects).To(HaveLen(21))
Expect(foundResource.Status.RelatedObjects).To(HaveLen(22))
expectedRef := corev1.ObjectReference{
Kind: "PrometheusRule",
Namespace: namespace,
Expand Down Expand Up @@ -811,7 +811,7 @@ var _ = Describe("HyperconvergedController", func() {
).To(Succeed())

Expect(foundResource.Status.RelatedObjects).ToNot(BeNil())
Expect(foundResource.Status.RelatedObjects).To(HaveLen(21))
Expect(foundResource.Status.RelatedObjects).To(HaveLen(22))
Expect(foundResource.ObjectMeta.Finalizers).To(Equal([]string{FinalizerName}))

// Now, delete HCO
Expand Down
9 changes: 9 additions & 0 deletions deploy/cluster_role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -847,11 +847,20 @@ rules:
resources:
- pods
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- watch
- create
- update
- apiGroups:
- ""
resources:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -318,11 +318,20 @@ spec:
resources:
- pods
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- watch
- create
- update
- apiGroups:
- ""
resources:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -318,11 +318,20 @@ spec:
resources:
- pods
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- watch
- create
- update
- apiGroups:
- ""
resources:
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require (
github.com/gertd/go-pluralize v0.2.1
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
github.com/go-logr/logr v1.4.2
github.com/golang-jwt/jwt/v5 v5.2.1
github.com/google/uuid v1.6.0
github.com/kubevirt/cluster-network-addons-operator v0.98.1
github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20240505100225-e29dee0bb12b
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
Expand Down
13 changes: 13 additions & 0 deletions pkg/authorization/authorization_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package authorization_test

import (
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

func TestAuthorization(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Authorization Suite")
}
75 changes: 75 additions & 0 deletions pkg/authorization/bearer_token.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package authorization

import (
"crypto/rand"
"fmt"
"os"

"github.com/golang-jwt/jwt/v5"
)

const (
TokenPathEnvVar = "KUBERNETES_SERVICE_TOKEN_PATH"

defaultTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"
)

var inMemoryToken *[]byte

func CreateToken() (string, error) {
token := jwt.New(jwt.SigningMethodHS256)

secretKey, err := getSecretKey()
if err != nil {
return "", fmt.Errorf("error getting secret key: %v", err)
}

tokenString, err := token.SignedString(secretKey)
if err != nil {
return "", fmt.Errorf("error signing token: %v", err)
}

return tokenString, nil
}

func ValidateToken(tokenString string) (bool, error) {
token, err := jwt.Parse(tokenString, func(token *jwt.Token) (any, error) {
return getSecretKey()
})
if err != nil {
return false, fmt.Errorf("error parsing token: %v", err)
}

return token.Valid, nil
}

func getSecretKey() ([]byte, error) {
if inMemoryToken != nil {
return *inMemoryToken, nil
}

tokenPath := os.Getenv(TokenPathEnvVar)
if tokenPath == "" {
tokenPath = defaultTokenPath
}

token, err := os.ReadFile(tokenPath)
if err != nil {
// if ServiceAccount token is not available, generate an in-memory token
return generateInMemoryToken()
}

return token, nil
}

func generateInMemoryToken() ([]byte, error) {
tokenBytes := make([]byte, 32)

_, err := rand.Read(tokenBytes)
if err != nil {
return nil, fmt.Errorf("error generating in-memory token: %v", err)
}

inMemoryToken = &tokenBytes
return tokenBytes, nil
}
Loading

0 comments on commit 7332493

Please sign in to comment.