name: github-workflow on: push: branches: [ main ] pull_request: branches: [ main ] jobs: build: runs-on: ubuntu-latest env: TESTCONTAINERS_REUSE_ENABLE: false steps: - name: checkout project uses: actions/checkout@v2 - name: setup project uses: ./.github/workflows/composites/setup - name: set env variables uses: ./.github/workflows/composites/env-variables - name: cache local maven repository uses: ./.github/workflows/composites/cache - name: build fabric8 istio uses: ./.github/workflows/composites/fabric8-istio - name: build with skip tests and skip images run: ./mvnw -T 1C -s .settings.xml clean install -U -DskipTests -Dskip.build.image=true - name: build controllers project uses: ./.github/workflows/composites/build-controllers-project - name: build integration tests project uses: ./.github/workflows/composites/build-integration-tests-project - name: save controller docker images uses: ./.github/workflows/composites/save-controller-images - name: save integration tests docker images uses: ./.github/workflows/composites/save-integration-tests-images - name: echo saved images uses: ./.github/workflows/composites/echo-saved-images - name: upload docker images uses: ./.github/workflows/composites/upload-docker-images # we need to run some test, so that K3s container is started and then all other instances will re-use this one. # Otherwise, (since we use static ports) there might be two instances starting at the same time, and ports might conflict fabric8_istio_integration_test: needs: build runs-on: ubuntu-latest steps: - name: checkout project uses: actions/checkout@v2 - name: setup project uses: ./.github/workflows/composites/setup - name: set env variables uses: ./.github/workflows/composites/env-variables - name: cache local maven repository uses: ./.github/workflows/composites/cache - name: download docker images uses: ./.github/workflows/composites/download-docker-images - name: echo saved images uses: ./.github/workflows/composites/echo-saved-images - name: integration test fabric8 istio uses: ./.github/workflows/composites/fabric8-istio-integration-test test: needs: [ build, fabric8_istio_integration_test ] runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: true matrix: current_index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] number_of_jobs: [32] steps: - name: testcontainers reuse support run: echo "testcontainers.reuse.enable=true" > ~/.testcontainers.properties - name: checkout project uses: actions/checkout@v2 - name: setup project uses: ./.github/workflows/composites/setup - name: set env variables uses: ./.github/workflows/composites/env-variables - name: cache local maven repository uses: ./.github/workflows/composites/cache - name: download docker images uses: ./.github/workflows/composites/download-docker-images - name: load docker images into local repo uses: ./.github/workflows/composites/load-docker-images - name: run tests env: CURRENT_INDEX: ${{ matrix.current_index }} NUMBER_OF_JOBS: ${{ matrix.number_of_jobs }} run: | # - find all tests # - exclude Fabric8IstionIT # - only take classes that have @Test inside them # - ignore the ones that have 'abstract class'. we do this because otherwise we would pass # to -DtestsToRun an abstract class, and it will not run anything. # - drop the "begining" xxx/src/test/java # - replace / with . # - drop last ".java" # - replace newline with space # - replace '\n' with ' ' PLAIN_TEST_CLASSNAMES=($(find . -name '*.java' \ | grep 'src/test/java' \ | grep -v 'Fabric8IstioIT' \ | xargs grep -l '@Test' \ | xargs grep -L 'abstract class' \ | sed 's/.*src.test.java.//g' \ | sed 's@/@.@g' \ | sed 's/.\{5\}$//' \ | tr '\n' ' ')) # classes that have @Test and are abstract, for example: "LabeledSecretWithPrefixTests" # - exclude Fabric8IstionIT # - only take classes that have @Test inside them # - only take classes that are abstract # - drop everything up until the last "/" # - drop ".java" ABSTRACT_TEST_CLASSNAMES_COMMAND="find . -name '*.java' \ | grep 'src/test/java' \ | grep -v 'Fabric8IstioIT' \ | xargs grep -l '@Test' \ | xargs grep -l 'abstract class' \ | sed 's/.*\///g' \ | sed 's/.java//g'" # find classes that extend abstract test classes DERIVED_FROM_ABSTRACT_CLASSES_COMMAND="find . -name '*.java' \ | grep 'src/test/java' \ | grep -v 'Fabric8IstioIT' \ | xargs grep -l 'extends replace_me ' \ | sed 's/.*src.test.java.//g' \ | sed 's@/@.@g' \ | sed 's/.\{5\}$//' \ | tr '\n' ' '" while read class_name; do replaced=$(echo ${DERIVED_FROM_ABSTRACT_CLASSES_COMMAND/replace_me/"$class_name"}) result=($(eval $replaced)) PLAIN_TEST_CLASSNAMES+=(${result[@]}) done < <(eval $ABSTRACT_TEST_CLASSNAMES_COMMAND) IFS=$'\n' SORTED_TEST_CLASSNAMES=( $(sort <<< "${PLAIN_TEST_CLASSNAMES[*]} | uniq -u") ) unset IFS number_of_tests=${#SORTED_TEST_CLASSNAMES[@]} number_of_jobs=${NUMBER_OF_JOBS} current_index=${CURRENT_INDEX} per_instance=$((number_of_tests / number_of_jobs)) # we do not get an ideal distribution all the time, so this is needed to add one more test # to the first "reminder" number of instances. # consider the case when there are 10 tests, and 4 instances # 10/4=2 (and this is "per_instance"), at the same time "reminder" = (10 - 4 * 2) = 2 # this means that the first instance will run (2 + 1) tests # second instance will run (2 + 1) tests # all subsequent instances will run 2 tests. reminder=$((number_of_tests - number_of_jobs * per_instance)) elements_in_current_instance=$((per_instance + 1)) left_bound=0 right_bound=0 # we are in a range where we might need to add one more test to each instance # notice the "less then" condition here, it is important and must not change if [[ $current_index -lt $reminder ]]; then # this one is easy, the range will be [0..3] (following our example above) if [[ $current_index == 0 ]]; then left_bound=0 right_bound=$elements_in_current_instance # this one will be [3..6] else left_bound=$((current_index * elements_in_current_instance)) right_bound=$(((current_index + 1) * elements_in_current_instance)) fi echo "total tests : $number_of_tests, jobs: $number_of_jobs, current index : $current_index. will run tests in range : [$left_bound..$right_bound]" else # reminder can be zero here (in case of a perfect distribution): in such a case, this is just "current_index * per_instance". # if reminder is not zero, we have two regions here, logically. the one of the left is "reminder * elements_in_current_instance", # basically [0..3] and [3..6] # and the region on the right [6..8]. left_bound=$((reminder * elements_in_current_instance + ((current_index - reminder) * per_instance))) right_bound=$((left_bound + per_instance)) echo "total tests : $number_of_tests, jobs: $number_of_jobs, current index : $current_index. will run tests in range : [$left_bound..$right_bound]" fi diff=$((right_bound - left_bound)) sliced_array=("${SORTED_TEST_CLASSNAMES[@]:$left_bound:$diff}") TEST_ARG=$(echo ${sliced_array[@]} | sed 's/ /,/g') echo "will run tests : ${TEST_ARG[@]}" ./mvnw -s .settings.xml \ -DtestsToRun=${TEST_ARG[@]} \ -e clean install \ -U -P sonar -nsu --batch-mode \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ -Dhttp.keepAlive=false \ -Dmaven.wagon.http.pool=false \ -Dmaven.wagon.http.retryHandler.class=standard \ -Dmaven.wagon.http.retryHandler.count=3 \ -Dskip.build.image=true - name: Publish Test Report uses: mikepenz/action-junit-report@v2 if: always() # always run even if the previous step fails with: report_paths: '**/surefire-reports/TEST-*.xml' - name: Archive code coverage results uses: actions/upload-artifact@v2 with: name: surefire-reports path: '**/surefire-reports/*'