[GitHub Actions] Update scaling

config/defaults.json

@@ -12,7 +12,7 @@
   },
   "stream_apps_gh_runners": {
     "pods_per_job": 1,
-    "ram_per_pod": 3,
+    "ram_per_pod": 2,
     "cpu_per_pod": 1.5,
     "machine_type": "e2-highcpu-8",
     "runner_scaling": "manual",

scripts/calculate-nodes.groovy

@@ -146,10 +146,10 @@ int maxNodes = -1
 int minNodes = -1
 String outputFile
 String machineType
-String podFile;
-int ramUsed = 0;
-int cpuUsed = 0;
-
+String podFile
+int ramUsed = 0
+int cpuUsed = 0
+int additionalNodes = 0
 int requiredRam = 0
 int requiredCpu = 0
 Integer requiredNodes = null
@@ -179,11 +179,18 @@ for (int i = 0; i < args.length; i++) {
             outputFile = args[i + 1]
             i += 1
             break
+        case '--add':
+            if (args.length <= i + 1) {
+                usageExit('Missing arguments after --add')
+            }
+            additionalNodes = args[i+1].toInteger()
+            i += 1
+            break
         case '--cpu':
             if (args.length <= i + 1) {
                 usageExit('Missing arguments after --cpu')
             }
-            requiredCpu = args[i + 1].toInteger()
+            requiredCpu = Math.ceil(Double.parseDouble(args[i + 1])).intValue()
             if (requiredCpu < 0) {
                 shrink = true
             }
@@ -193,7 +200,7 @@ for (int i = 0; i < args.length; i++) {
             if (args.length <= i + 1) {
                 usageExit('Missing arguments after --ram')
             }
-            requiredRam = args[i + 1].toInteger()
+            requiredRam = Math.ceil(Double.parseDouble(args[i + 1])).intValue()
             if (requiredRam < 0) {
                 shrink = true
             }
@@ -403,7 +410,7 @@ if (cpuNodes != 0 && ramNodes != 0) {
 if (requiredNodes == null) {
     requiredNodes = 0
 }
-int targetNodes = currentNodes + requiredNodes
+int targetNodes = currentNodes + requiredNodes + additionalNodes
 if (minNodes >= 0 && targetNodes < minNodes) {
     if (verbose || outputFile != null) {
         println "Min:$minNodes"
@@ -422,7 +429,7 @@ if (!shrink && targetNodes < currentNodes) {
 }
 
 if (verbose || outputFile != null) {
-    println "Nodes: current=$currentNodes, required: $requiredNodes, target=$targetNodes, shrink=$shrink"
+    println "Nodes: current=$currentNodes, required=$requiredNodes, target=$targetNodes, shrink=$shrink"
 }
 def result = [nodes: targetNodes, shrink: shrink]
 String output = JsonOutput.toJson(result)

scripts/ceil.bc

@@ -0,0 +1,11 @@
+define ceil(x) {
+  auto s, r
+  s = scale
+  scale = 0
+  r = x / 1
+  if(r < x) {
+    r = r + 1
+  }
+  scale = s
+  return (r)
+}

scripts/scale-cluster-nodes.sh

@@ -83,7 +83,7 @@ function set_nodecount_machine_type() {
 }
 
 function usage() {
-  echo "Usage $0 <provider> <cluster> [--ram <min-ram>] | [--nodes <min-nodes>] [--cpu <min-cpu>] [--max fixed-nodes] [--shrink]"
+  echo "Usage $0 <provider> <cluster> [--ram <min-ram>] | [--nodes <min-nodes>] [--cpu <min-cpu>] [--max fixed-nodes] [--add <additional>] [--shrink]"
   echo "  If nodes and fixed-nodes is the same the node count will be set to that value"
   echo "  If nodes is negative the number will be reduced by that many nodes"
   echo "  If --ram then the machine type will be used to determine the number of nodes"
@@ -98,6 +98,7 @@ fi
 export CLUSTER_NAME=$1
 DETERMINE_TYPE=false
 ZONES=1
+ADDITIONAL_NODES=0
 VERBOSE=false
 ARGS=
 while [ "$2" != "" ]; do
@@ -166,7 +167,7 @@ fi
 if [ "$TARGET" != "" ]; then
   if [ "$PROVIDER" == "gke" ]; then
     if [ "$DRY_RUN" == "true" ]; then
-      echo "gcloud container clusters resize "$CLUSTER_NAME" --region "$REGION" "--num-nodes=$TARGET" --node-pool=$NODE_POOL --quiet"
+      echo "gcloud container clusters resize $CLUSTER_NAME --region $REGION --num-nodes=$TARGET --node-pool=$NODE_POOL --quiet"
     else
       gcloud container clusters resize "$CLUSTER_NAME" --region "$REGION" "--num-nodes=$TARGET" --node-pool=$NODE_POOL --quiet
     fi

scripts/scale-cluster-pods.sh

@@ -30,8 +30,23 @@ PODS=$2
 RAM_PER_POD=$($SCDIR/determine-default.sh $CLUSTER_NAME "ram-per-pod")
 CPU_PER_POD=$($SCDIR/determine-default.sh $CLUSTER_NAME "cpu-per-pod")
 PODS_PER_JOB=$($SCDIR/determine-default.sh $CLUSTER_NAME "pods_per_job")
-TOTAL_RAM=$((RAM_PER_POD * PODS * PODS_PER_JOB))
-TOTAL_CPU=$((CPU_PER_POD * PODS * PODS_PER_JOB))
+# provide for fractions
+set +e
+bc -v > /dev/null
+RC=$?
+if ((RC > 0)); then
+  bc -v
+  echo "Error finding bc"
+  exit 1
+fi
+cp $SCDIR/ceil.bc calc-ram.bc
+echo "ceil($RAM_PER_POD * $PODS * $PODS_PER_JOB)" >> calc-ram.bc
+echo "quit" >> calc-ram.bc
+TOTAL_RAM=$(bc calc-ram.bc)
+cp $SCDIR/ceil.bc calc-cpu.bc
+echo "ceil($CPU_PER_POD * $PODS * $PODS_PER_JOB)" >> calc-cpu.bc
+echo "quit" >> calc-cpu.bc
+TOTAL_CPU=$(bc calc-cpu.bc)
 echo "Scaling $CLUSTER_NAME with $PODS pods at ${RAM_PER_POD}Gi and $CPU_PER_POD vCPUs per pod using $PODS_PER_JOB per job. Total RAM:$TOTAL_RAM and CPU:$TOTAL_CPU"
 ARGS=
 while [ "$3" != "" ]; do