Allow for graceful delete

pkg/config/config.go

@@ -20,6 +20,10 @@ const (
 	PrepareDownscaleLabelKey   = "grafana.com/prepare-downscale"
 	PrepareDownscaleLabelValue = "true"
 
+	// GracefulShutdownAnnotationKey is the annotation to gracefully shutdown a pod of a statefulset.
+	GracefulShutdownAnnotationKey   = "grafana.com/graceful-shutdown"
+	GracefulShutdownAnnotationValue = "delete-pod"
+
 	// RolloutGroupLabelKey is the group to which multiple statefulsets belong and must be operated on together.
 	RolloutGroupLabelKey = "rollout-group"
 	// RolloutMaxUnavailableAnnotationKey is the max number of pods in each statefulset that may be stopped at

pkg/controller/controller.go

@@ -491,7 +491,7 @@ func (c *RolloutController) listPods(sel labels.Selector) ([]*corev1.Pod, error)
 func (c *RolloutController) updateStatefulSetPods(ctx context.Context, sts *v1.StatefulSet) (bool, error) {
 	level.Debug(c.logger).Log("msg", "reconciling StatefulSet", "statefulset", sts.Name)
 
-	podsToUpdate, err := c.podsNotMatchingUpdateRevision(sts)
+	podsToUpdate, err := c.getPodsToUpdate(sts)
 	if err != nil {
 		return false, errors.Wrap(err, "failed to get pods to update")
 	}
@@ -565,7 +565,7 @@ func (c *RolloutController) updateStatefulSetPods(ctx context.Context, sts *v1.S
 	return false, nil
 }
 
-func (c *RolloutController) podsNotMatchingUpdateRevision(sts *v1.StatefulSet) ([]*corev1.Pod, error) {
+func (c *RolloutController) getPodsToUpdate(sts *v1.StatefulSet) ([]*corev1.Pod, error) {
 	var (
 		currRev   = sts.Status.CurrentRevision
 		updateRev = sts.Status.UpdateRevision
@@ -592,6 +592,20 @@ func (c *RolloutController) podsNotMatchingUpdateRevision(sts *v1.StatefulSet) (
 		return nil, err
 	}
 
+	// If there are no pods to be found, check if there are pods that need to be gracefully terminated.
+	if len(pods) == 0 {
+		podsSelector = labels.NewSelector().Add(
+			util.MustNewLabelsRequirement(config.GracefulShutdownAnnotationKey, selection.Equals, []string{config.GracefulShutdownAnnotationValue}),
+			util.MustNewLabelsRequirement("name", selection.Equals, []string{sts.Spec.Template.Labels["name"]}),
+		)
+
+		if pods, err = c.listPods(podsSelector); err != nil {
+			return nil, err
+		} else if len(pods) > 0 {
+			level.Debug(c.logger).Log("msg", fmt.Sprintf("found %d pods that need to be gracefully terminated", len(pods)))
+		}
+	}
+
 	// Sort pods in order to provide a deterministic behaviour.
 	util.SortPods(pods)