From 104e9ce81e52eae93736bd5d120bfc2cd3b13707 Mon Sep 17 00:00:00 2001 From: erabii Date: Fri, 15 Sep 2023 14:59:42 +0300 Subject: [PATCH] fix pipeline issue that skips some integration tests (#1441) --- .../action.yaml | 68 ++++++++++++++-- .../action.yaml | 81 ++++++------------- .github/workflows/maven.yaml | 4 + 3 files changed, 92 insertions(+), 61 deletions(-) diff --git a/.github/workflows/composites/matrix-bounds-on-test-times-cache-hit/action.yaml b/.github/workflows/composites/matrix-bounds-on-test-times-cache-hit/action.yaml index 8be1192c..4badaf1b 100644 --- a/.github/workflows/composites/matrix-bounds-on-test-times-cache-hit/action.yaml +++ b/.github/workflows/composites/matrix-bounds-on-test-times-cache-hit/action.yaml @@ -18,15 +18,56 @@ runs: steps: + - name: restore test times cache + uses: actions/cache/restore@v3 + with: + path: /tmp/sorted.txt + key: ${{ runner.os }}-spring-cloud-k8s-existing-test-times-cache-${{ github.run_id }} + restore-keys: ${{ runner.os }}-spring-cloud-k8s-existing-test-times-cache- + + - name: show cached test times + shell: bash + run: cat /tmp/sorted.txt + - name: compute matrix steps shell: bash run: | - sum_of_all_tests=$(awk -F' ' '{sum+=$2;} END{print sum;}' /tmp/sorted.txt) + PLAIN_TEST_CLASSNAMES=($(cat /tmp/tests.txt | grep -o 'spring.cloud.k8s.test.to.run -> org.*' | awk '{print $3}')) + + ####################################################################################################### + ### split into tests that we know the running times for and in tests that we do not know the times for. + ####################################################################################################### + for test in "${PLAIN_TEST_CLASSNAMES[@]}"; do + + find_test_in_sorted=$(grep "$test " /tmp/sorted.txt || true) + if [[ -z "$find_test_in_sorted" ]]; then + echo $test >> /tmp/tests-without-times.txt + else + echo $find_test_in_sorted >> /tmp/tests-with-times.txt + fi + + done + + sort -t' ' -nk2 /tmp/tests-with-times.txt >> /tmp/tests-with-times-sorted.txt + + # this is a work-around for the fact that 'actions/download-artifact@v3' does not support + # something like: 'ignore if downloaded file is missing'. + # so unless we create a dummy file for upload, download will fail + if [[ ! -f /tmp/tests-without-times.txt ]]; then touch /tmp/tests-without-times.txt; fi + + echo '--------------------------------------------------------------' + cat /tmp/tests-with-times-sorted.txt + echo '--------------------------------------------------------------' + cat /tmp/tests-without-times.txt + echo '--------------------------------------------------------------' + + + sum_of_all_tests=$(awk -F' ' '{sum+=$2;} END{print sum;}' /tmp/tests-with-times-sorted.txt) sum_of_all_tests_as_int=$(printf "%.0f\n" "$sum_of_all_tests") echo "sum of all tests : $sum_of_all_tests_as_int" - max_test_time=$(tail -1 /tmp/sorted.txt | awk '{print $2}') + max_test_time=$(tail -1 /tmp/tests-with-times-sorted.txt | awk '{print $2}') max_test_time_as_int=$(printf "%.0f\n" "$max_test_time") echo "max test time : $max_test_time_as_int" @@ -38,7 +79,10 @@ runs: number_of_instances_as_array+=']' average_time_per_instance=$(( sum_of_all_tests_as_int / $number_of_instances )) - echo "average time per instance $average_time_per_instance" + average_time_per_instance_array=() + average_time_per_instance_array+='[' + average_time_per_instance_array+=$average_time_per_instance + average_time_per_instance_array+=']' matrix_array=() matrix_array+='[' @@ -52,8 +96,9 @@ runs: matrix_array+=']' + echo "********************************************************************************************************" echo "number of instances : $number_of_instances_as_array" - echo "average time per instance : $average_time_per_instance" + echo "average_time_per_instance_as_array: $average_time_per_instance_as_array" echo "matrix_array : $matrix_array" echo "********************************************************************************************************" @@ -62,10 +107,23 @@ runs: matrix_array_json=$(jq -r -c . <<< $matrix_array) echo "matrix_array_json : $matrix_array_json" + + average_time_per_instance_json=$(jq -r -c . <<< $average_time_per_instance_array) + echo "average_time_per_instance_json : $average_time_per_instance_json" echo "TEST_TIMES_CACHE_PRESENT=true" >> $GITHUB_ENV echo "NUMBER_OF_MATRIX_INSTANCES=$(echo $number_of_instances_json)" >> $GITHUB_ENV echo "MATRIX_ARRAY=$(echo $matrix_array_json)" >> $GITHUB_ENV - echo "AVERAGE_TIME_PER_INSTANCE=$(echo $average_time_per_instance)" >> $GITHUB_ENV + echo "AVERAGE_TIME_PER_INSTANCE=$(echo $average_time_per_instance_json)" >> $GITHUB_ENV + - name: upload test with times + uses: actions/upload-artifact@v3 + with: + name: tests-with-times-sorted.txt + path: /tmp/tests-with-times-sorted.txt + - name: upload test without times + uses: actions/upload-artifact@v3 + with: + name: tests-without-times.txt + path: /tmp/tests-without-times.txt diff --git a/.github/workflows/composites/run-and-save-test-times-when-cache-present/action.yaml b/.github/workflows/composites/run-and-save-test-times-when-cache-present/action.yaml index 147475f4..7b6af8a4 100644 --- a/.github/workflows/composites/run-and-save-test-times-when-cache-present/action.yaml +++ b/.github/workflows/composites/run-and-save-test-times-when-cache-present/action.yaml @@ -4,53 +4,30 @@ runs: using: "composite" steps: - - name: restore test times cache - uses: actions/cache/restore@v3 - with: - path: /tmp/sorted.txt - key: ${{ runner.os }}-spring-cloud-k8s-existing-test-times-cache-${{ github.run_id }} - restore-keys: ${{ runner.os }}-spring-cloud-k8s-existing-test-times-cache- - - name: show cached test times - shell: bash - run: cat /tmp/sorted.txt + - name: download tests with times + if: env.BASE_BRANCH != '2.1.x' + uses: actions/download-artifact@v3 + with: + name: tests-with-times-sorted.txt + path: /tmp/ + + - name: download tests without times + if: env.BASE_BRANCH != '2.1.x' + uses: actions/download-artifact@v3 + with: + name: tests-without-times.txt + path: /tmp/ - name: split tests into known times and un-known times shell: bash run: | - ############################################################################################################ - ############################################################################################################ - ############################################################################################################ - # 1. Get all existing tests and place them in PLAIN_TEST_CLASSNAMES - # 2. Get all test times from the existing cache : /tmp/sorted.txt - # 3. Split tests from PLAIN_TEST_CLASSNAMES into two files depending if we already know their running times - # or not : tests-without-times.txt and tests-with-times.txt - - ############################################################################################################ - ############################################################################################################ - ############################################################################################################ - PLAIN_TEST_CLASSNAMES=($(cat /tmp/tests.txt | grep -o 'spring.cloud.k8s.test.to.run -> org.*' | awk '{print $3}')) - - echo "${PLAIN_TEST_CLASSNAMES[@]}" - - temp_dir=$(mktemp -d) - - for test in "${PLAIN_TEST_CLASSNAMES[@]}"; do - - find_test_in_sorted=$(grep "$test " /tmp/sorted.txt || true) - if [[ -z "$find_test_in_sorted" ]]; then - echo $test >> $temp_dir/tests-without-times.txt - else - echo $find_test_in_sorted >> $temp_dir/tests-with-times.txt - fi - - done - - echo "tests with times:" - cat $temp_dir/tests-with-times.txt - - sort -t' ' -nk2 $temp_dir/tests-with-times.txt >> $temp_dir/tests-with-times-sorted.txt + echo "------------------------------------------------------------------------------" + cat /tmp/tests-with-times-sorted.txt + echo "------------------------------------------------------------------------------" + if [[ -f /tmp/tests-without-times.txt ]]; then cat /tmp/tests-without-times.txt; fi + echo "------------------------------------------------------------------------------" ############################################################################################################ ############################################################################################################ @@ -70,7 +47,7 @@ runs: # this test needs to be taken in 'tests_to_take_in_current_iteration', thus: # tests_to_take_in_current_iteration=testC; also next_sum becomes 3sec # we then drop this line from /tmp/sorted.txt because we have already processed it - # ('sed -i "" "${j}d" $temp_dir/tests-with-times-sorted.txt') + # ('sed -i "" "${j}d" /tmp/tests-with-times-sorted.txt') # we also decrement j, since we removed one line from the file # we then take testB, add its time to next_sum, thus next_sum = 5 sec, but now the time is NOT < 3.1 sec @@ -81,22 +58,14 @@ runs: # we repeat the process again and now tests_to_take_in_current_iteration=testB,testA because their cumulative # sum will be 3 sec and it's < 3.1 sec. - - sum_of_all_tests=$(awk -F' ' '{sum+=$2;} END{print sum;}' $temp_dir/tests-with-times-sorted.txt) - sum_of_all_tests_as_int=$(printf "%.0f\n" "$sum_of_all_tests") - echo "sum of all tests : $sum_of_all_tests_as_int" - - max_test_time=$(tail -1 $temp_dir/tests-with-times-sorted.txt | awk '{print $2}') - max_test_time_as_int=$(printf "%.0f\n" "$max_test_time") - echo "max test time : $max_test_time_as_int" number_of_instances=${NUMBER_OF_JOBS} echo "number of instances $number_of_instances" - average_time_per_instance=$(( sum_of_all_tests_as_int / number_of_instances )) + average_time_per_instance=${AVERAGE_TIME_PER_INSTANCE} echo "average time per instance $average_time_per_instance" - number_of_lines_in_file=$(grep -c ^ $temp_dir/tests-with-times-sorted.txt) + number_of_lines_in_file=$(grep -c ^ /tmp/tests-with-times-sorted.txt) echo "number of lines in fine : $number_of_lines_in_file" tests_to_run_in_current_index='' @@ -106,7 +75,7 @@ runs: tests_to_take_in_current_iteration='' for ((j=$number_of_lines_in_file; j>0; j--)) ; do - current_line_in_file=$(awk "NR == ${j}" $temp_dir/tests-with-times-sorted.txt) + current_line_in_file=$(awk "NR == ${j}" /tmp/tests-with-times-sorted.txt) current_test_time=$(echo $current_line_in_file | awk '{print $2}') current_test_time=$(printf "%.0f\n" "$current_test_time") current_test_name=$(echo $current_line_in_file | awk '{print $1}') @@ -122,7 +91,7 @@ runs: tests_to_take_in_current_iteration="$tests_to_take_in_current_iteration,$current_test_name" fi - sed -i "${j}d" $temp_dir/tests-with-times-sorted.txt + sed -i "${j}d" /tmp/tests-with-times-sorted.txt number_of_lines_in_file=$(( $number_of_lines_in_file-1 )) continue fi @@ -154,7 +123,7 @@ runs: if [[ ${CURRENT_INDEX} = ${NUMBER_OF_JOBS} ]]; then echo "last index spotted" - if [ ! -f $temp_dir/tests-without-times.txt ]; then + if [ ! -f /tmp/tests-without-times.txt ]; then echo "no tests outside cache found" if [ -z "$tests_to_run_in_current_index" ]; then @@ -163,7 +132,7 @@ runs: fi else - TESTS_WITHOUT_TIMES=($(cat $temp_dir/tests-without-times.txt)) + TESTS_WITHOUT_TIMES=($(cat /tmp/tests-without-times.txt)) for test in "${TESTS_WITHOUT_TIMES[@]}"; do if [[ -z "$tests_to_run_in_current_index" ]]; then diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml index 84f4dc35..6264d4c0 100644 --- a/.github/workflows/maven.yaml +++ b/.github/workflows/maven.yaml @@ -23,6 +23,7 @@ jobs: test_times_cache_present: ${{ steps.test_times_cache_present_init.outputs.test_times_cache_present }} number_of_matrix_instances: ${{ steps.test_times_cache_present_init.outputs.number_of_matrix_instances }} matrix_array: ${{ steps.test_times_cache_present_init.outputs.matrix_array }} + average_time_per_instance: ${{ steps.test_times_cache_present_init.outputs.average_time_per_instance }} steps: - name: checkout project @@ -91,6 +92,7 @@ jobs: echo "test_times_cache_present=${{ env.TEST_TIMES_CACHE_PRESENT }}" >> $GITHUB_OUTPUT echo "number_of_matrix_instances=${{ env.NUMBER_OF_MATRIX_INSTANCES }}" >> $GITHUB_OUTPUT echo "matrix_array=${{ env.MATRIX_ARRAY }}" >> $GITHUB_OUTPUT + echo "average_time_per_instance=${{ env.AVERAGE_TIME_PER_INSTANCE }}" >> $GITHUB_OUTPUT test_when_cache_present: needs: [ build ] @@ -106,6 +108,7 @@ jobs: matrix: current_index: [ "${{ fromJSON(needs.build.outputs.matrix_array) }}" ] number_of_jobs: [ "${{ fromJSON(needs.build.outputs.number_of_matrix_instances) }}" ] + average_time_per_instance: [ "${{ fromJSON(needs.build.outputs.average_time_per_instance) }}" ] steps: @@ -138,6 +141,7 @@ jobs: env: CURRENT_INDEX: ${{ matrix.current_index }} NUMBER_OF_JOBS: ${{ matrix.number_of_jobs }} + AVERAGE_TIME_PER_INSTANCE: ${{ matrix.average_time_per_instance }} test_when_cache_missing: needs: [ build ]