feat: make the probe timings configurable

docs/hugo/content/guide/diagnosing-problems/_index.md

@@ -26,6 +26,31 @@ Events:
   ...
 ```
 
+### Operator pod is restarted before it finishes applying CRDs
+
+On a cluster with a large number of CRDs, the operator can be killed by the kubelet while it is still starting.
+The pod restarts repeatedly and never reports ready, and `kubectl describe pod` shows the startup probe failing:
+
+```
+  Warning  Unhealthy  2m (x12 over 4m)  kubelet  Startup probe failed: Get "http://10.244.0.9:8081/healthz": context deadline exceeded
+```
+
+The operator applies its CRDs and starts a controller for each installed CRD before it serves `/healthz`, so on
+a large cluster that work can outlast the startup probe's budget of `periodSeconds` x `failureThreshold`.
+
+Give it more time by raising `failureThreshold`. In Helm:
+```yaml
+probes:
+  startup:
+    failureThreshold: 60
+```
+
+That allows ten minutes rather than the default two. Raising it costs nothing when the operator starts quickly,
+because the probe stops as soon as it first succeeds.
+
+Installing fewer CRDs also shortens startup, since the operator only applies and watches the CRDs it is asked
+for. See [CRD management]( {{< relref "crd-management" >}} ).
+
 ### Helm installation via Argo missing ClusterRole and other resources
 
 See reference issue [#4184](https://github.com/Azure/azure-service-operator/issues/4184).

v2/charts/azure-service-operator/templates/apps_v1_deployment_azureserviceoperator-controller-manager.yaml

@@ -220,6 +220,9 @@ spec:
           httpGet:
             path: /healthz
             port: 8081
+          periodSeconds: {{ .Values.probes.liveness.periodSeconds }}
+          failureThreshold: {{ .Values.probes.liveness.failureThreshold }}
+          timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }}
         name: manager
         ports:
         - containerPort: {{ .Values.webhook.port }}
@@ -237,6 +240,9 @@ spec:
           httpGet:
             path: /readyz
             port: 8081
+          periodSeconds: {{ .Values.probes.readiness.periodSeconds }}
+          failureThreshold: {{ .Values.probes.readiness.failureThreshold }}
+          timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }}
         {{- with .Values.resources }}
         resources:
           {{- toYaml . | nindent 10 }}
@@ -249,8 +255,9 @@ spec:
           httpGet:
             path: /healthz
             port: 8081
-          periodSeconds: 10
-          failureThreshold: 12
+          periodSeconds: {{ .Values.probes.startup.periodSeconds }}
+          failureThreshold: {{ .Values.probes.startup.failureThreshold }}
+          timeoutSeconds: {{ .Values.probes.startup.timeoutSeconds }}
         volumeMounts:
         - mountPath: /var/run/secrets/tokens
           name: azure-identity

v2/charts/azure-service-operator/values.yaml

@@ -236,6 +236,26 @@ resources:
 go:
   memLimit: 400MiB # This should be set to ~80-90% of the hard memory limit set above in resources
 
+# probes configures the timings of the operator's liveness, readiness and startup probes.
+# The startup probe is the one most likely to need adjusting. The operator applies its CRDs and starts a
+# controller per installed CRD before it serves /healthz, so on a cluster with a large number of CRDs the default
+# budget of periodSeconds x failureThreshold can expire while it is still working, and the kubelet restarts it
+# mid-apply. Raise failureThreshold in that case.
+# The probe paths and ports are not configurable; only the timings are.
+probes:
+  startup:
+    periodSeconds: 10
+    failureThreshold: 12
+    timeoutSeconds: 1
+  liveness:
+    periodSeconds: 10
+    failureThreshold: 3
+    timeoutSeconds: 1
+  readiness:
+    periodSeconds: 10
+    failureThreshold: 3
+    timeoutSeconds: 1
+
 # Number of old history to retain to allow rollback
 # Default Kubernetes value is set to 10
 revisionHistoryLimit: 10

v2/config/manager/manager.yaml

@@ -61,14 +61,21 @@ spec:
             port: 8081
           periodSeconds: 10
           failureThreshold: 12
+          timeoutSeconds: 1
         livenessProbe:
           httpGet:
             path: /healthz
             port: 8081
+          periodSeconds: 10
+          failureThreshold: 3
+          timeoutSeconds: 1
         readinessProbe:
           httpGet:
             path: /readyz
             port: 8081
+          periodSeconds: 10
+          failureThreshold: 3
+          timeoutSeconds: 1
         image: controller:latest
         name: manager
         resources: