From f011648c963c1bfafe5db64befa2c238b3b8e22f Mon Sep 17 00:00:00 2001 From: Corneil du Plessis Date: Wed, 2 Nov 2022 13:45:35 +0200 Subject: [PATCH] [GitHub Actions] Update scaling --- config/defaults.json | 2 +- scripts/calculate-nodes.groovy | 23 +++++++++++++++-------- scripts/ceil.bc | 11 +++++++++++ scripts/scale-cluster-nodes.sh | 5 +++-- scripts/scale-cluster-pods.sh | 19 +++++++++++++++++-- 5 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 scripts/ceil.bc diff --git a/config/defaults.json b/config/defaults.json index 3977b345..cc0e04f3 100644 --- a/config/defaults.json +++ b/config/defaults.json @@ -12,7 +12,7 @@ }, "stream_apps_gh_runners": { "pods_per_job": 1, - "ram_per_pod": 3, + "ram_per_pod": 2, "cpu_per_pod": 1.5, "machine_type": "e2-highcpu-8", "runner_scaling": "manual", diff --git a/scripts/calculate-nodes.groovy b/scripts/calculate-nodes.groovy index 6cca0b1e..4c0ec813 100644 --- a/scripts/calculate-nodes.groovy +++ b/scripts/calculate-nodes.groovy @@ -146,10 +146,10 @@ int maxNodes = -1 int minNodes = -1 String outputFile String machineType -String podFile; -int ramUsed = 0; -int cpuUsed = 0; - +String podFile +int ramUsed = 0 +int cpuUsed = 0 +int additionalNodes = 0 int requiredRam = 0 int requiredCpu = 0 Integer requiredNodes = null @@ -179,11 +179,18 @@ for (int i = 0; i < args.length; i++) { outputFile = args[i + 1] i += 1 break + case '--add': + if (args.length <= i + 1) { + usageExit('Missing arguments after --add') + } + additionalNodes = args[i+1].toInteger() + i += 1 + break case '--cpu': if (args.length <= i + 1) { usageExit('Missing arguments after --cpu') } - requiredCpu = args[i + 1].toInteger() + requiredCpu = Math.ceil(Double.parseDouble(args[i + 1])).intValue() if (requiredCpu < 0) { shrink = true } @@ -193,7 +200,7 @@ for (int i = 0; i < args.length; i++) { if (args.length <= i + 1) { usageExit('Missing arguments after --ram') } - requiredRam = args[i + 1].toInteger() + requiredRam = Math.ceil(Double.parseDouble(args[i + 1])).intValue() if (requiredRam < 0) { shrink = true } @@ -403,7 +410,7 @@ if (cpuNodes != 0 && ramNodes != 0) { if (requiredNodes == null) { requiredNodes = 0 } -int targetNodes = currentNodes + requiredNodes +int targetNodes = currentNodes + requiredNodes + additionalNodes if (minNodes >= 0 && targetNodes < minNodes) { if (verbose || outputFile != null) { println "Min:$minNodes" @@ -422,7 +429,7 @@ if (!shrink && targetNodes < currentNodes) { } if (verbose || outputFile != null) { - println "Nodes: current=$currentNodes, required: $requiredNodes, target=$targetNodes, shrink=$shrink" + println "Nodes: current=$currentNodes, required=$requiredNodes, target=$targetNodes, shrink=$shrink" } def result = [nodes: targetNodes, shrink: shrink] String output = JsonOutput.toJson(result) diff --git a/scripts/ceil.bc b/scripts/ceil.bc new file mode 100644 index 00000000..3eaf93e5 --- /dev/null +++ b/scripts/ceil.bc @@ -0,0 +1,11 @@ +define ceil(x) { + auto s, r + s = scale + scale = 0 + r = x / 1 + if(r < x) { + r = r + 1 + } + scale = s + return (r) +} diff --git a/scripts/scale-cluster-nodes.sh b/scripts/scale-cluster-nodes.sh index 093a4888..c68424bc 100755 --- a/scripts/scale-cluster-nodes.sh +++ b/scripts/scale-cluster-nodes.sh @@ -83,7 +83,7 @@ function set_nodecount_machine_type() { } function usage() { - echo "Usage $0 [--ram ] | [--nodes ] [--cpu ] [--max fixed-nodes] [--shrink]" + echo "Usage $0 [--ram ] | [--nodes ] [--cpu ] [--max fixed-nodes] [--add ] [--shrink]" echo " If nodes and fixed-nodes is the same the node count will be set to that value" echo " If nodes is negative the number will be reduced by that many nodes" echo " If --ram then the machine type will be used to determine the number of nodes" @@ -98,6 +98,7 @@ fi export CLUSTER_NAME=$1 DETERMINE_TYPE=false ZONES=1 +ADDITIONAL_NODES=0 VERBOSE=false ARGS= while [ "$2" != "" ]; do @@ -166,7 +167,7 @@ fi if [ "$TARGET" != "" ]; then if [ "$PROVIDER" == "gke" ]; then if [ "$DRY_RUN" == "true" ]; then - echo "gcloud container clusters resize "$CLUSTER_NAME" --region "$REGION" "--num-nodes=$TARGET" --node-pool=$NODE_POOL --quiet" + echo "gcloud container clusters resize $CLUSTER_NAME --region $REGION --num-nodes=$TARGET --node-pool=$NODE_POOL --quiet" else gcloud container clusters resize "$CLUSTER_NAME" --region "$REGION" "--num-nodes=$TARGET" --node-pool=$NODE_POOL --quiet fi diff --git a/scripts/scale-cluster-pods.sh b/scripts/scale-cluster-pods.sh index 1352c9ee..8ca6ba75 100755 --- a/scripts/scale-cluster-pods.sh +++ b/scripts/scale-cluster-pods.sh @@ -30,8 +30,23 @@ PODS=$2 RAM_PER_POD=$($SCDIR/determine-default.sh $CLUSTER_NAME "ram-per-pod") CPU_PER_POD=$($SCDIR/determine-default.sh $CLUSTER_NAME "cpu-per-pod") PODS_PER_JOB=$($SCDIR/determine-default.sh $CLUSTER_NAME "pods_per_job") -TOTAL_RAM=$((RAM_PER_POD * PODS * PODS_PER_JOB)) -TOTAL_CPU=$((CPU_PER_POD * PODS * PODS_PER_JOB)) +# provide for fractions +set +e +bc -v > /dev/null +RC=$? +if ((RC > 0)); then + bc -v + echo "Error finding bc" + exit 1 +fi +cp $SCDIR/ceil.bc calc-ram.bc +echo "ceil($RAM_PER_POD * $PODS * $PODS_PER_JOB)" >> calc-ram.bc +echo "quit" >> calc-ram.bc +TOTAL_RAM=$(bc calc-ram.bc) +cp $SCDIR/ceil.bc calc-cpu.bc +echo "ceil($CPU_PER_POD * $PODS * $PODS_PER_JOB)" >> calc-cpu.bc +echo "quit" >> calc-cpu.bc +TOTAL_CPU=$(bc calc-cpu.bc) echo "Scaling $CLUSTER_NAME with $PODS pods at ${RAM_PER_POD}Gi and $CPU_PER_POD vCPUs per pod using $PODS_PER_JOB per job. Total RAM:$TOTAL_RAM and CPU:$TOTAL_CPU" ARGS= while [ "$3" != "" ]; do