Add ability to downscale HPA's minReplicas

Author: Vishal Tak

README.rst

@@ -18,9 +18,9 @@ Kubernetes Downscaler
  :target: http://calver.org/
  :alt: CalVer
 
-Scale down Kubernetes deployments and/or statefulsets during non-work hours.
+Scale down Kubernetes deployments and/or statefulsets and/or horizontalpodautoscalers during non-work hours.
 
-Deployments are interchangeable by statefulset for this whole guide.
+Deployments are interchangeable by statefulset/horizontalpodautoscalers for this whole guide unless explicitly stated otherwise.
 
 It will scale down the deployment's replicas if all of the following conditions are met:
 
@@ -39,6 +39,7 @@ It will scale down the deployment's replicas if all of the following conditions
 * there are no active pods that force the whole cluster into uptime (annotation ``downscaler/force-uptime: "true"``)
 
 The deployment by default will be scaled down to zero replicas. This can be configured with a deployment or its namespace's annotation of ``downscaler/downtime-replicas`` (e.g. ``downscaler/downtime-replicas: "1"``) or via CLI with ``--downtime-replicas``.
+In case of horizontalpodautoscalers, the `minReplicas` field cannot be set to zero and thus ``downscaler/downtime-replicas`` should be atleast ``1``.
 
 Example use cases:
 
@@ -81,6 +82,10 @@ The downscaler will eventually log something like:
 
  INFO: Scaling down Deployment default/nginx from 1 to 0 replicas (uptime: Mon-Fri 09:00-17:00 America/Buenos_Aires, downtime: never)
 
+Note that in cases where HPA is used along with Deployments, consider the following:
+
+* If downscale to 0 is desired, annotation is applied at Deployment. (Special case, since minReplicas of 0 at HPA is not allowed. Setting Deployment replica to 0, essentially makes the HPA behave as disabled. In such a case, the HPA will emit events like `` failed to get memory utilization: unable to get metrics for resource memory: no metrics returned from resource metrics API`` as there is no Pod to retrieve metric from.)
+* If downscale greater than 0 is desired, annotation is applied at HPA. This allows for dynamic scaling of the Pods even during downtime based upon the external traffic as well as maintain a lower minReplicas during downtime if there is no/low traffic. If the Deployment is annotated instead of the HPA, it leads to a race condition where kube-downscaler downscales the Deployment and HPA upscales it as its minReplica is higher.
 
 Configuration
 =============
@@ -134,7 +139,7 @@ Available command line options:
 ``--namespace``
  Restrict the downscaler to work only in a single namespace (default: all namespaces). This is mainly useful for deployment scenarios where the deployer of kube-downscaler only has access to a given namespace (instead of cluster access).
 ``--include-resources``
- Downscale resources of this kind as comma separated list. [deployments, statefulsets, stacks] (default: deployments)
+ Downscale resources of this kind as comma separated list. [deployments, statefulsets, stacks, horizontalpodautoscalers] (default: deployments)
 ``--grace-period``
  Grace period in seconds for new deployments before scaling them down (default: 15min). The grace period counts from time of creation of the deployment, i.e. updated deployments will immediately be scaled down regardless of the grace period.
 ``--upscale-period``

helm-chart/templates/clusterrole.yaml

@@ -30,6 +30,16 @@ rules:
  - list
  - update
  - patch
+- apiGroups:
+ - autoscaling
+ resources:
+ - horizontalpodautoscalers
+ verbs:
+ - get
+ - watch
+ - list
+ - update
+ - patch
 - apiGroups:
  - ""
  resources:

kube_downscaler/cmd.py

@@ -1,7 +1,9 @@
 import argparse
 import os
 
-VALID_RESOURCES = frozenset(["deployments", "statefulsets", "stacks", "cronjobs"])
+VALID_RESOURCES = frozenset(
+ ["deployments", "statefulsets", "stacks", "cronjobs", "horizontalpodautoscalers"]
+)
 
 
 def check_include_resources(value):
@@ -36,7 +38,7 @@ def get_parser():
  "--include-resources",
  type=check_include_resources,
  default="deployments",
- help="Downscale resources of this kind as comma separated list. [deployments, statefulsets, stacks] (default: deployments)",
+ help="Downscale resources of this kind as comma separated list. [deployments, statefulsets, stacks, horizontalpodautoscalers] (default: deployments)",
  )
  parser.add_argument(
  "--grace-period",

kube_downscaler/scaler.py

@@ -6,6 +6,7 @@
 import pykube
 from pykube import CronJob
 from pykube import Deployment
+from pykube import HorizontalPodAutoscaler
 from pykube import StatefulSet
 
 from kube_downscaler import helper
@@ -22,7 +23,7 @@
 DOWNTIME_ANNOTATION = "downscaler/downtime"
 DOWNTIME_REPLICAS_ANNOTATION = "downscaler/downtime-replicas"
 
-RESOURCE_CLASSES = [Deployment, StatefulSet, Stack, CronJob]
+RESOURCE_CLASSES = [Deployment, StatefulSet, Stack, CronJob, HorizontalPodAutoscaler]
 
 TIMESTAMP_FORMATS = [
  "%Y-%m-%dT%H:%M:%SZ",
@@ -200,6 +201,17 @@ def autoscale_resource(
  "suspended" if original_replicas == 0 else "not suspended",
  uptime,
  )
+ elif resource.kind == "HorizontalPodAutoscaler":
+ replicas = resource.obj["spec"]["minReplicas"]
+ logger.debug(
+ "%s %s/%s has %s minReplicas (original: %s, uptime: %s)",
+ resource.kind,
+ resource.namespace,
+ resource.name,
+ replicas,
+ original_replicas,
+ uptime,
+ )
  else:
  replicas = resource.replicas
  logger.debug(
@@ -232,6 +244,18 @@ def autoscale_resource(
  uptime,
  downtime,
  )
+ elif resource.kind == "HorizontalPodAutoscaler":
+ resource.obj["spec"]["minReplicas"] = int(original_replicas)
+ logger.info(
+ "Scaling up %s %s/%s from %s to %s minReplicas (uptime: %s, downtime: %s)",
+ resource.kind,
+ resource.namespace,
+ resource.name,
+ replicas,
+ original_replicas,
+ uptime,
+ downtime,
+ )
  else:
  resource.replicas = int(original_replicas)
  logger.info(
@@ -279,6 +303,18 @@ def autoscale_resource(
  uptime,
  downtime,
  )
+ elif resource.kind == "HorizontalPodAutoscaler":
+ resource.obj["spec"]["minReplicas"] = target_replicas
+ logger.info(
+ "Scaling down %s %s/%s from %s to %s minReplicas (uptime: %s, downtime: %s)",
+ resource.kind,
+ resource.namespace,
+ resource.name,
+ replicas,
+ target_replicas,
+ uptime,
+ downtime,
+ )
  else:
  resource.replicas = target_replicas
  logger.info(

tests/test_autoscale_resource.py

@@ -7,6 +7,7 @@
 import pykube
 import pytest
 from pykube import Deployment
+from pykube import HorizontalPodAutoscaler
 
 from kube_downscaler.resources.stack import Stack
 from kube_downscaler.scaler import autoscale_resource
@@ -613,3 +614,66 @@ def test_upscale_stack_with_autoscaling():
  assert stack.obj["spec"]["replicas"] is None
  assert stack.replicas == 4
  assert stack.annotations[ORIGINAL_REPLICAS_ANNOTATION] is None
+
+
+def test_downscale_hpa_with_autoscaling():
+ hpa = HorizontalPodAutoscaler(
+ None,
+ {
+ "metadata": {
+ "name": "my-hpa",
+ "namespace": "my-ns",
+ "creationTimestamp": "2018-10-23T21:55:00Z",
+ "annotations": {DOWNTIME_REPLICAS_ANNOTATION: str(1)},
+ },
+ "spec": {"minReplicas": 4},
+ },
+ )
+ now = datetime.strptime("2018-10-23T21:56:00Z", "%Y-%m-%dT%H:%M:%SZ").replace(
+ tzinfo=timezone.utc
+ )
+ autoscale_resource(
+ hpa,
+ upscale_period="never",
+ downscale_period="never",
+ default_uptime="never",
+ default_downtime="always",
+ forced_uptime=False,
+ dry_run=True,
+ now=now,
+ )
+ assert hpa.obj["spec"]["minReplicas"] == 1
+ assert hpa.obj["metadata"]["annotations"][ORIGINAL_REPLICAS_ANNOTATION] == str(4)
+
+
+def test_upscale_hpa_with_autoscaling():
+ hpa = HorizontalPodAutoscaler(
+ None,
+ {
+ "metadata": {
+ "name": "my-hpa",
+ "namespace": "my-ns",
+ "creationTimestamp": "2018-10-23T21:55:00Z",
+ "annotations": {
+ DOWNTIME_REPLICAS_ANNOTATION: str(1),
+ ORIGINAL_REPLICAS_ANNOTATION: str(4),
+ },
+ },
+ "spec": {"minReplicas": 1},
+ },
+ )
+ now = datetime.strptime("2018-10-23T22:15:00Z", "%Y-%m-%dT%H:%M:%SZ").replace(
+ tzinfo=timezone.utc
+ )
+ autoscale_resource(
+ hpa,
+ upscale_period="never",
+ downscale_period="never",
+ default_uptime="always",
+ default_downtime="never",
+ forced_uptime=False,
+ dry_run=True,
+ now=now,
+ )
+ assert hpa.obj["spec"]["minReplicas"] == 4
+ assert hpa.obj["metadata"]["annotations"][ORIGINAL_REPLICAS_ANNOTATION] is None