SpringCloud/spring-cloud-kubernetes
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Simplifiy existing split of tests (#1352)

Browse Source
This commit is contained in:
erabii
2023-06-05 21:36:56 +03:00
committed by GitHub
parent b0b26c7b2a
commit a64758f35f
2 changed files with 84 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
.github/workflows/composites/test-bounds/action.yaml vendored Normal file
View File

@@ -0,0 +1,65 @@
name: test-bounds
description: test-bounds
runs:
using: "composite"
steps:
- name: test-bounds
shell: bash
run: |
PLAIN_TEST_CLASSNAMES=($(cat /tmp/tests.txt | grep 'spring.cloud.k8s.test.to.run' | awk '{print $3}'))
NUMBER_OF_TESTS=${#PLAIN_TEST_CLASSNAMES[@]}
echo "current index : ${CURRENT_INDEX}"
echo "total number of tests: ${NUMBER_OF_TESTS} to be run on ${NUMBER_OF_JOBS} instances"
##########################################################################################################
##########################################################################################################
# Split tests per instance. The simplest form is when (for example), there are 100 tests and 10 instances.
# In this case, we will run 10 tests per a single instance. This is the case when reminder == 0 (100 / 10).
# In this case we will compute start_from as = current index * per_instance, that is: 0, 10, 20...
# and so on. How many tests are supposed to be taken in each matrix is equal to per_instance, that is 10.
# Slightly more interesting is when the tests are not perfectly distributed, when reminder != 0.
# For example 323 tests on 32 instances. In this case, we logically split this problem in two:
# when current_index < reminder and the rest.
# The batches in the example above are going to be [0, 11) [11, 22) [22, 33) [33, 43) [43, 53) ...
# So when current_index < reminder, we need to start at : current_index * per_instance + current_index,
# i.e.: 0, 11, 22
# and how many elements to take is : per_instance + 1, i.e. : 11
# On the other hand when current_index is not < reminder, how_many is easy, it's equal to per_instance.
# start_from on the other hand is : per_instance * current_index + reminder, i.e. : 33, 43, 53 and so on
##########################################################################################################
##########################################################################################################
per_instance=$((${NUMBER_OF_TESTS} / ${NUMBER_OF_JOBS}))
reminder=$((${NUMBER_OF_TESTS} - ${NUMBER_OF_JOBS} * ${per_instance}))
start_from=0
how_many=0
if [[ $reminder -eq 0 ]]; then
start_from=$(( ${CURRENT_INDEX} * ${per_instance} ))
how_many=${per_instance}
else
if [[ $CURRENT_INDEX -eq 0 ]]; then
start_from=0
how_many=$(( $per_instance + 1 ))
else
if [[ $CURRENT_INDEX -lt $reminder ]]; then
start_from=$(( $per_instance * $CURRENT_INDEX + $CURRENT_INDEX ))
how_many=$(( $per_instance +1 ))
else
start_from=$(( $per_instance * $CURRENT_INDEX + $reminder ))
how_many=$per_instance
fi
fi
fi
start_from_name=start_from_${CURRENT_INDEX}
how_many_name=how_many_${CURRENT_INDEX}
echo "$start_from_name : $start_from"
echo "$how_many_name : $how_many"
echo "$start_from_name=$start_from" >> $GITHUB_ENV
echo "$how_many_name=$how_many" >> $GITHUB_ENV

73
.github/workflows/maven.yaml vendored
View File

@@ -139,71 +139,36 @@ jobs:
name: tests.txt
path: /tmp
- name: run tests
- name: compute single step test bounds
uses: ./.github/workflows/composites/test-bounds
env:
CURRENT_INDEX: ${{ matrix.current_index }}
NUMBER_OF_JOBS: ${{ matrix.number_of_jobs }}
- name: run tests
env:
CURRENT_INDEX: ${{ matrix.current_index }}
run: |
PLAIN_TEST_CLASSNAMES=($(cat /tmp/tests.txt | grep 'spring.cloud.k8s.test.to.run' | awk '{print $3}'))
IFS=$'\n'
SORTED_TEST_CLASSNAMES=( $(sort <<< "${PLAIN_TEST_CLASSNAMES[*]}") )
unset IFS
number_of_tests=${#SORTED_TEST_CLASSNAMES[@]}
number_of_jobs=${NUMBER_OF_JOBS}
current_index=${CURRENT_INDEX}
start_from_name=start_from_${CURRENT_INDEX}
how_many_name=how_many_${CURRENT_INDEX}
start_from=${!start_from_name}
how_many=${!how_many_name}
echo "${SORTED_TEST_CLASSNAMES[@]}"
echo "$start_from_name : ${start_from}"
echo "$how_many_name : ${how_many}"
sliced_array=(${SORTED_TEST_CLASSNAMES[@]:$start_from:$how_many})
per_instance=$((number_of_tests / number_of_jobs))
# we do not get an ideal distribution all the time, so this is needed to add one more test
# to the first "reminder" number of instances.
# consider the case when there are 10 tests, and 4 instances
# 10/4=2 (and this is "per_instance"), at the same time "reminder" = (10 - 4 * 2) = 2
# this means that the first instance will run (2 + 1) tests
# second instance will run (2 + 1) tests
# all subsequent instances will run 2 tests.
reminder=$((number_of_tests - number_of_jobs * per_instance))
elements_in_current_instance=$((per_instance + 1))
left_bound=0
right_bound=0
# we are in a range where we might need to add one more test to each instance
# notice the "less then" condition here, it is important and must not change
if [[ $current_index -lt $reminder ]]; then
# this one is easy, the range will be [0..3] (following our example above)
if [[ $current_index == 0 ]]; then
left_bound=0
right_bound=$elements_in_current_instance
# this one will be [3..6]
else
left_bound=$((current_index * elements_in_current_instance))
right_bound=$(((current_index + 1) * elements_in_current_instance))
fi
echo "total tests : $number_of_tests, jobs: $number_of_jobs, current index : $current_index. will run tests in range : [$left_bound..$right_bound]"
else
# reminder can be zero here (in case of a perfect distribution): in such a case, this is just "current_index * per_instance".
# if reminder is not zero, we have two regions here, logically. the one of the left is "reminder * elements_in_current_instance",
# basically [0..3] and [3..6]
# and the region on the right [6..8].
left_bound=$((reminder * elements_in_current_instance + ((current_index - reminder) * per_instance)))
right_bound=$((left_bound + per_instance))
echo "total tests : $number_of_tests, jobs: $number_of_jobs, current index : $current_index. will run tests in range : [$left_bound..$right_bound]"
fi
diff=$((right_bound - left_bound))
sliced_array=("${SORTED_TEST_CLASSNAMES[@]:$left_bound:$diff}")
TEST_ARG=$(echo ${sliced_array[@]} | sed 's/ /,/g')
echo "will run tests : ${TEST_ARG[@]}"
echo "$TEST_ARG"
if [[ $baseBranch == "2.1.x" ]]; then