Add cluster autoscaler annotation to allow safe eviction of collector…

…s and rule evaluators (#237)

pkg/operator/collection.go

@@ -229,6 +229,10 @@ func (r *collectionReconciler) makeCollectorDaemonSet(spec *monitoringv1.Collect
 
 	podAnnotations := map[string]string{
 		AnnotationMetricName: componentName,
+		// Allow cluster autoscaler to evict collector Pods even though the Pods
+		// have an emptyDir volume mounted. This is okay since the node where the
+		// Pod runs will be scaled down and therefore does not need metrics reporting.
+		ClusterAutoscalerSafeEvictionLabel: "true",
 	}
 
 	collectorArgs := []string{

pkg/operator/e2e/main_test.go

@@ -340,6 +340,15 @@ func testRuleEvaluatorDeployment(ctx context.Context, t *testContext) {
 			return false, nil
 		}
 
+		// Assert we have the expected annotations.
+		wantedAnnotations := map[string]string{
+			"components.gke.io/component-name":               "managed_prometheus",
+			"cluster-autoscaler.kubernetes.io/safe-to-evict": "true",
+		}
+		if diff := cmp.Diff(wantedAnnotations, deploy.Spec.Template.Annotations); diff != "" {
+			return false, errors.Errorf("unexpected annotations (-want, +got): %s", diff)
+		}
+
 		for _, c := range deploy.Spec.Template.Spec.Containers {
 			if c.Name != "evaluator" {
 				continue
@@ -549,10 +558,21 @@ func testCollectorDeployed(ctx context.Context, t *testContext) {
 				return false, nil
 			}
 		}
+
+		// Assert we have the expected annotations.
+		wantedAnnotations := map[string]string{
+			"components.gke.io/component-name":               "managed_prometheus",
+			"cluster-autoscaler.kubernetes.io/safe-to-evict": "true",
+		}
+		if diff := cmp.Diff(wantedAnnotations, ds.Spec.Template.Annotations); diff != "" {
+			return false, errors.Errorf("unexpected annotations (-want, +got): %s", diff)
+		}
+
 		for _, c := range ds.Spec.Template.Spec.Containers {
 			if c.Name != "prometheus" {
 				continue
 			}
+
 			// We're mainly interested in the dynamic flags but checking the entire set including
 			// the static ones is ultimately simpler.
 			wantArgs := []string{
@@ -583,7 +603,6 @@ func testCollectorDeployed(ctx context.Context, t *testContext) {
 			sort.Strings(c.Args)
 
 			if diff := cmp.Diff(wantArgs, c.Args); diff != "" {
-				t.Log(errors.Errorf("unexpected flags (-want, +got): %s", diff))
 				return false, errors.Errorf("unexpected flags (-want, +got): %s", diff)
 			}
 			return true, nil

pkg/operator/operator.go

@@ -72,6 +72,10 @@ const (
 	LabelAppName = "app.kubernetes.io/name"
 	// The component name, will be exposed as metric name.
 	AnnotationMetricName = "components.gke.io/component-name"
+	// ClusterAutoscalerSafeEvictionLabel is the annotation label that determines
+	// whether the cluster autoscaler can safely evict a Pod when the Pod doesn't
+	// satisfy certain eviction criteria.
+	ClusterAutoscalerSafeEvictionLabel = "cluster-autoscaler.kubernetes.io/safe-to-evict"
 
 	// The official images to be used with this version of the operator. For debugging
 	// and emergency use cases they may be overwritten through options.

pkg/operator/operator_config.go

@@ -79,6 +79,10 @@ func rulesLabels() map[string]string {
 func rulesAnnotations() map[string]string {
 	return map[string]string{
 		AnnotationMetricName: componentName,
+		// Allow cluster autoscaler to evict evaluator Pods even though the Pods
+		// have an emptyDir volume mounted. This is okay since the node where the
+		// Pod runs will be scaled down.
+		ClusterAutoscalerSafeEvictionLabel: "true",
 	}
 }