SpringCloud/spring-cloud-kubernetes
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
spencergibb
2023-09-19 13:10:02 -04:00
parent 2e4d529f30
commit 8a57c69c88
1303 changed files with 8661 additions and 95772 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/composites/build-controllers-project/action.yaml vendored
View File

@@ -1,11 +0,0 @@
name: build controllers project
description: build controllers project
runs:
using: "composite"
steps:
- name: build controllers project
shell: bash
run: |
cd spring-cloud-kubernetes-controllers
.././mvnw -T 1C -U clean install
cd ..

17
.github/workflows/composites/build-integration-tests-project/action.yaml vendored
View File

@@ -1,17 +0,0 @@
name: build integration tests project without tests
description: build integration tests project without tests
runs:
using: "composite"
steps:
- name: build integration tests project without tests
shell: bash
run: |
cd spring-cloud-kubernetes-test-support
.././mvnw clean install -U
cd ..
cd spring-cloud-kubernetes-integration-tests
# build the images, but dont run the tests
.././mvnw -T 1C clean install -DskipTests
cd ..

15
.github/workflows/composites/cache/action.yaml vendored
View File

@@ -1,15 +0,0 @@
name: cache
description: cache
runs:
using: "composite"
steps:
- uses: Wandalen/wretry.action@master
with:
attempt_limit: 3
action: buildjet/cache@v3
with: |
path: ~/.m2/repository
key: ${{ runner.os }}-cache-${{ env.BRANCH_NAME }}-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-cache-${{ env.BRANCH_NAME }}-${{ hashFiles('**/pom.xml') }}

19
.github/workflows/composites/clean-space/action.yaml vendored
View File

@@ -1,19 +0,0 @@
name: clean space
description: clean space
runs:
using: "composite"
steps:
- name: apt-update
shell: bash
run: |
sudo apt-get update
- name: Free Disk Space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: true
swap-storage: true

24
.github/workflows/composites/env-variables/action.yaml vendored
View File

@@ -1,24 +0,0 @@
name: sets environment variables
description: sets environment variables
runs:
using: "composite"
steps:
- name: set env variables
shell: bash
run: |
baseRef=$GITHUB_BASE_REF
if [[ ! -z $baseRef ]]; then
echo "This is a PR to branch : $baseRef"
echo "BASE_BRANCH=$(echo $baseRef)" >> $GITHUB_ENV
else
raw=${{ github.ref }}
echo "${raw}"
branch=${raw##*/}
echo "This is a merge to branch : ${branch}"
echo "BASE_BRANCH=$(echo ${branch})" >> $GITHUB_ENV
fi
echo "BRANCH_NAME=$(echo $GITHUB_HEAD_REF)" >> $GITHUB_ENV
echo "BASE_BRANCH_NAME=$(echo $GITHUB_BASE_REF)" >> $GITHUB_ENV
echo "DOCKER_IMAGES_KEY=$(echo $GITHUB_RUN_ID)" >> $GITHUB_ENV

129
.github/workflows/composites/matrix-bounds-on-test-times-cache-hit/action.yaml vendored
View File

@@ -1,129 +0,0 @@
########################################### find test times #######################################
# when cache of test times is present, compute matrix steps
# we do this because the number of steps required in a matrix is not stable
# it is computed as sum_of_times_of_all_tests / max_time_of_a_single_test
# For example if there is no test times cache, we will be using 32 steps in the matrix
# on the other hand if cache is present, we might need less steps:
# sum_of_times_of_all_tests = 8000 seconds
# max_time_of_a_single_test = 400
# we would require 20 steps in the matrix, as having 32 would make no sense
# because one step will run around 400 seconds and others will wait for this one to finish
# as they finish earlier
name: matrix bounds on test times cache hit
description: matrix bounds on test times cache hit
runs:
using: "composite"
steps:
- name: restore test times cache
uses: actions/cache/restore@v3
with:
path: /tmp/sorted.txt
key: ${{ runner.os }}-spring-cloud-k8s-existing-test-times-cache-${{ github.run_id }}
restore-keys: ${{ runner.os }}-spring-cloud-k8s-existing-test-times-cache-
- name: show cached test times
shell: bash
run: cat /tmp/sorted.txt
- name: compute matrix steps
shell: bash
run: |
PLAIN_TEST_CLASSNAMES=($(cat /tmp/tests.txt | grep -o 'spring.cloud.k8s.test.to.run -> org.*' | awk '{print $3}'))
#######################################################################################################
### split into tests that we know the running times for and in tests that we do not know the times for.
#######################################################################################################
for test in "${PLAIN_TEST_CLASSNAMES[@]}"; do
find_test_in_sorted=$(grep "$test " /tmp/sorted.txt || true)
if [[ -z "$find_test_in_sorted" ]]; then
echo $test >> /tmp/tests-without-times.txt
else
echo $find_test_in_sorted >> /tmp/tests-with-times.txt
fi
done
sort -t' ' -nk2 /tmp/tests-with-times.txt >> /tmp/tests-with-times-sorted.txt
# this is a work-around for the fact that 'actions/download-artifact@v3' does not support
# something like: 'ignore if downloaded file is missing'.
# so unless we create a dummy file for upload, download will fail
if [[ ! -f /tmp/tests-without-times.txt ]]; then touch /tmp/tests-without-times.txt; fi
echo '--------------------------------------------------------------'
cat /tmp/tests-with-times-sorted.txt
echo '--------------------------------------------------------------'
cat /tmp/tests-without-times.txt
echo '--------------------------------------------------------------'
sum_of_all_tests=$(awk -F' ' '{sum+=$2;} END{print sum;}' /tmp/tests-with-times-sorted.txt)
sum_of_all_tests_as_int=$(printf "%.0f\n" "$sum_of_all_tests")
echo "sum of all tests : $sum_of_all_tests_as_int"
max_test_time=$(tail -1 /tmp/tests-with-times-sorted.txt | awk '{print $2}')
max_test_time_as_int=$(printf "%.0f\n" "$max_test_time")
echo "max test time : $max_test_time_as_int"
number_of_instances=$(( $sum_of_all_tests_as_int / $max_test_time_as_int ))
number_of_instances_as_array=()
number_of_instances_as_array+='['
number_of_instances_as_array+=$number_of_instances
number_of_instances_as_array+=']'
average_time_per_instance=$(( sum_of_all_tests_as_int / $number_of_instances ))
average_time_per_instance_array=()
average_time_per_instance_array+='['
average_time_per_instance_array+=$average_time_per_instance
average_time_per_instance_array+=']'
matrix_array=()
matrix_array+='['
for ((i=0; i<=${number_of_instances}; i++)); do
if (( $i == $(( $number_of_instances )) )); then
matrix_array+=$i
else
matrix_array+=$i,
fi
done
matrix_array+=']'
echo "********************************************************************************************************"
echo "number of instances : $number_of_instances_as_array"
echo "average_time_per_instance_as_array: $average_time_per_instance_as_array"
echo "matrix_array : $matrix_array"
echo "********************************************************************************************************"
number_of_instances_json=$(jq -r -c . <<< $number_of_instances_as_array)
echo "number_of_instances_json : $number_of_instances_json"
matrix_array_json=$(jq -r -c . <<< $matrix_array)
echo "matrix_array_json : $matrix_array_json"
average_time_per_instance_json=$(jq -r -c . <<< $average_time_per_instance_array)
echo "average_time_per_instance_json : $average_time_per_instance_json"
echo "TEST_TIMES_CACHE_PRESENT=true" >> $GITHUB_ENV
echo "NUMBER_OF_MATRIX_INSTANCES=$(echo $number_of_instances_json)" >> $GITHUB_ENV
echo "MATRIX_ARRAY=$(echo $matrix_array_json)" >> $GITHUB_ENV
echo "AVERAGE_TIME_PER_INSTANCE=$(echo $average_time_per_instance_json)" >> $GITHUB_ENV
- name: upload test with times
uses: actions/upload-artifact@v3
with:
name: tests-with-times-sorted.txt
path: /tmp/tests-with-times-sorted.txt
- name: upload test without times
uses: actions/upload-artifact@v3
with:
name: tests-without-times.txt
path: /tmp/tests-without-times.txt

37
.github/workflows/composites/maven-build-with-dry-run-for-tests/action.yaml vendored
View File

@@ -1,37 +0,0 @@
name: maven-build-with-dry-run-for-tests
description: maven-build-with-dry-run-for-tests
runs:
using: "composite"
steps:
- name: run 'package' on the project
shell: bash
run: |
./mvnw install -B \
-Dskip.build.image=true \
-DskipTests -DskipITs \
-T 1C -q
- name: find all classpath entries
shell: bash
run: |
./mvnw -q exec:exec -Dexec.classpathScope="test" -Dexec.executable="echo" -Dexec.args="%classpath" | tr : "\n" > /tmp/deps.txt
- name: find all tests
shell: bash
run: |
cd spring-cloud-kubernetes-test-support
.././mvnw -q exec:java -Prun-on-github-actions -Dexec.mainClass="org.springframework.cloud.kubernetes.tests.discovery.TestsDiscovery" > /tmp/tests.txt
cd ..
- name: show result
if: always()
shell: bash
run: cat /tmp/tests.txt
- name: upload test
uses: actions/upload-artifact@v3
with:
name: tests.txt
path: /tmp/tests.txt

39
.github/workflows/composites/pre-test-actions/action.yaml vendored
View File

@@ -1,39 +0,0 @@
# common steps before running actula tests for both
# when a cache of test times is presen and when it is not present
name: pre-test-actions
description: pre-test-actions
runs:
using: "composite"
steps:
- name: set env variables
uses: ./.github/workflows/composites/env-variables
- name: setup project jdk-17
uses: ./.github/workflows/composites/setup-jdk17
if: env.BASE_BRANCH == 'main'
- name: setup project jdk-8
uses: ./.github/workflows/composites/setup-jdk1.8
if: env.BASE_BRANCH == '2.1.x'
- name: cache local maven repository
uses: ./.github/workflows/composites/cache
- name: build controllers project
uses: ./.github/workflows/composites/build-controllers-project
- name: build integration tests project
uses: ./.github/workflows/composites/build-integration-tests-project
- name: download tests
uses: actions/download-artifact@v3
with:
name: tests.txt
path: /tmp

97
.github/workflows/composites/run-and-save-test-times-when-cache-missing/action.yaml vendored
View File

@@ -1,97 +0,0 @@
########################################### find test times #######################################
# 1. find files that look like this: 'org.springframework.xxx.txt', these files are generated
# by surefire-plugin when tests run.
# 2. extract the time it took to run a certain test (from the above .txt file)
# 3. create a file that contains test times from this current matrix
name: run and save test times when cache missing
description: run and save test times when cache missing
runs:
using: "composite"
steps:
- name: run and save test times
shell: bash
run: |
# take only likes that look like :
# 'spring.cloud.k8s.test.to.run -> org.springframework.cloud.kubernetes.fabric8.discovery.ServicePortSecureResolverTest'
PLAIN_TEST_CLASSNAMES=($(cat /tmp/tests.txt | grep -o 'spring.cloud.k8s.test.to.run -> org.*' | awk '{print $3}'))
IFS=$'\n'
SORTED_TEST_CLASSNAMES=( $(sort <<< "${PLAIN_TEST_CLASSNAMES[*]}") )
unset IFS
start_from_name=start_from_${CURRENT_INDEX}
how_many_name=how_many_${CURRENT_INDEX}
start_from=${!start_from_name}
how_many=${!how_many_name}
echo "${SORTED_TEST_CLASSNAMES[@]}"
echo "$start_from_name : ${start_from}"
echo "$how_many_name : ${how_many}"
sliced_array=(${SORTED_TEST_CLASSNAMES[@]:$start_from:$how_many})
TEST_ARG=$(echo ${sliced_array[@]} | sed 's/ /,/g')
echo "$TEST_ARG"
if [[ $baseBranch == "2.1.x" ]]; then
./mvnw -s .settings.xml -pl '-:kubernetes-leader-election-example' -pl '-:kubernetes-hello-world-example' \
-pl '-:kubernetes-reload-example' -pl '-:kubernetes-loadbalancer-example' \
-pl '-:spring-cloud-kubernetes-configserver' -pl '-:spring-cloud-kubernetes-configuration-watcher' \
-pl '-:spring-cloud-kubernetes-discoveryserver' \
-DtestsToRun=${TEST_ARG[@]} \
-e clean install \
-U -P sonar -nsu --batch-mode \
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
-Dhttp.keepAlive=false \
-Dmaven.wagon.http.pool=false \
-Dmaven.wagon.http.retryHandler.class=standard \
-Dmaven.wagon.http.retryHandler.count=3 \
-Dskip.build.image=true
else
version=$(java -version)
echo "version of java: $version"
./mvnw -s .settings.xml \
-DtestsToRun=${TEST_ARG[@]} \
-e clean install \
-U -P sonar -nsu --batch-mode \
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
-Dhttp.keepAlive=false \
-Dmaven.wagon.http.pool=false \
-Dmaven.wagon.http.retryHandler.class=standard \
-Dmaven.wagon.http.retryHandler.count=3 \
-Dskip.build.image=true
fi
touch /tmp/test_times_${{ env.CURRENT_INDEX }}.txt
for i in "${sliced_array[@]}"; do
filename="${i}.txt"
echo "searching for filename: ${filename}"
file=$(find . -name "${filename}")
echo "found file: ${file}"
result=$(cat "${file}" | grep 'elapsed' | awk '{print $12, $13}')
echo "run test: ${i} in : ${result}" >> /tmp/test_times_${{ env.CURRENT_INDEX }}.txt
done
- name: show individual test times
shell: bash
if: env.BASE_BRANCH != '2.1.x'
run: cat /tmp/test_times_${{ env.CURRENT_INDEX }}.txt
- name: upload individual tests
if: env.BASE_BRANCH != '2.1.x'
uses: actions/upload-artifact@v3
with:
name: test_times_${{ env.CURRENT_INDEX }}.txt
path: /tmp/test_times_${{ env.CURRENT_INDEX }}.txt

196
.github/workflows/composites/run-and-save-test-times-when-cache-present/action.yaml vendored
View File

@@ -1,196 +0,0 @@
name: run and save test times when cache present
description: run and save test times when cache present
runs:
using: "composite"
steps:
- name: download tests with times
if: env.BASE_BRANCH != '2.1.x'
uses: actions/download-artifact@v3
with:
name: tests-with-times-sorted.txt
path: /tmp/
- name: download tests without times
if: env.BASE_BRANCH != '2.1.x'
uses: actions/download-artifact@v3
with:
name: tests-without-times.txt
path: /tmp/
- name: split tests into known times and un-known times
shell: bash
run: |
echo "------------------------------------------------------------------------------"
cat /tmp/tests-with-times-sorted.txt
echo "------------------------------------------------------------------------------"
if [[ -f /tmp/tests-without-times.txt ]]; then cat /tmp/tests-without-times.txt; fi
echo "------------------------------------------------------------------------------"
############################################################################################################
############################################################################################################
############################################################################################################
# For each number_of_instances, iterate from bottom to top ('for ((j=$number_of_lines_in_file; j>0; j--))')
# (starting from the test that has the max time). Easier to understand is via an example.
# Suppose our /tmp/sorted.txt looks like this (very artificial):
# testA 1 sec
# testB 2 sec
# testC 3 sec
# average_time_per_instance = 3.1 sec
# number_of_lines_in_file = 3
# we start iterating from bottom to top and first get 'testC' that has a running time of 3 sec
# since 3 sec < 3.1 sec ('if [[ $next_sum -lt $average_time_per_instance ]];')
# this test needs to be taken in 'tests_to_take_in_current_iteration', thus:
# tests_to_take_in_current_iteration=testC; also next_sum becomes 3sec
# we then drop this line from /tmp/sorted.txt because we have already processed it
# ('sed -i "" "${j}d" /tmp/tests-with-times-sorted.txt')
# we also decrement j, since we removed one line from the file
# we then take testB, add its time to next_sum, thus next_sum = 5 sec, but now the time is NOT < 3.1 sec
# This means we do not take testB in current iteration and skip it.
# Same goes for testA, next_sum = 4 sec, and it is NOT < 3.1 sec
# There are no more lines in file to iterate, thus tests_to_take_in_current_iteration=testC
# we repeat the process again and now tests_to_take_in_current_iteration=testB,testA because their cumulative
# sum will be 3 sec and it's < 3.1 sec.
number_of_instances=${NUMBER_OF_JOBS}
echo "number of instances $number_of_instances"
average_time_per_instance=${AVERAGE_TIME_PER_INSTANCE}
echo "average time per instance $average_time_per_instance"
number_of_lines_in_file=$(grep -c ^ /tmp/tests-with-times-sorted.txt)
echo "number of lines in fine : $number_of_lines_in_file"
tests_to_run_in_current_index=''
for ((i=0; i<=${number_of_instances}; i++)) ; do
sum=0
tests_to_take_in_current_iteration=''
for ((j=$number_of_lines_in_file; j>0; j--)) ; do
current_line_in_file=$(awk "NR == ${j}" /tmp/tests-with-times-sorted.txt)
current_test_time=$(echo $current_line_in_file | awk '{print $2}')
current_test_time=$(printf "%.0f\n" "$current_test_time")
current_test_name=$(echo $current_line_in_file | awk '{print $1}')
next_sum=$(( $sum + $current_test_time ))
if [[ $next_sum -lt $average_time_per_instance ]]; then
sum=$(( $sum + $current_test_time ))
if [[ -z $tests_to_take_in_current_iteration ]]; then
tests_to_take_in_current_iteration="$current_test_name"
else
tests_to_take_in_current_iteration="$tests_to_take_in_current_iteration,$current_test_name"
fi
sed -i "${j}d" /tmp/tests-with-times-sorted.txt
number_of_lines_in_file=$(( $number_of_lines_in_file-1 ))
continue
fi
done
if [[ $i = ${CURRENT_INDEX} ]]; then
echo "current index : ${CURRENT_INDEX}"
tests_to_run_in_current_index=$tests_to_take_in_current_iteration
echo "time of tests in current index : $sum"
# this can be the case when we delete some tests in some PR for example
# the previous cache will contain more tests then we currently have, so some
# matrix instances will have no tests to run
if [[ "$sum" -eq "0" ]]; then
echo "no tests (with known times) to run in current index"
tests_to_run_in_current_index='none'
fi
fi
done
# here we compute tests that are "new", these are tests that our cache is not aware of
# for example these can come from a new PR where new tests are added
# This last step can have already existing tests to run, so we need to be careful to append
# new tests here.
if [[ ${CURRENT_INDEX} = ${NUMBER_OF_JOBS} ]]; then
echo "last index spotted"
if [ ! -f /tmp/tests-without-times.txt ]; then
echo "no tests outside cache found"
if [ -z "$tests_to_run_in_current_index" ]; then
echo "no tests to run in the last index"
tests_to_run_in_current_index='none'
fi
else
TESTS_WITHOUT_TIMES=($(cat /tmp/tests-without-times.txt))
for test in "${TESTS_WITHOUT_TIMES[@]}"; do
if [[ -z "$tests_to_run_in_current_index" ]]; then
tests_to_run_in_current_index="$test"
else
tests_to_run_in_current_index="$tests_to_run_in_current_index,$test"
fi
done
fi
fi
echo "will run tests : $tests_to_run_in_current_index"
./mvnw -s .settings.xml \
-DtestsToRun=${tests_to_run_in_current_index} \
-e clean install \
-U -P sonar -nsu --batch-mode \
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
-Dhttp.keepAlive=false \
-Dmaven.wagon.http.pool=false \
-Dmaven.wagon.http.retryHandler.class=standard \
-Dmaven.wagon.http.retryHandler.count=3 \
-Dskip.build.image=true
touch /tmp/test_times_${{ env.CURRENT_INDEX }}.txt
# if there are no tests to run in the last index, don't parse anything
if [ $tests_to_run_in_current_index != "none" ]; then
IFS=',' read -r -a sliced_array <<< "$tests_to_run_in_current_index"
unset IFS
for i in "${sliced_array[@]}"; do
# can be present in the last index as 'none,testA,testB'
if [[ "$i" == "none" ]]; then
echo "skipping 'none'"
else
filename="${i}.txt"
echo "searching for filename: ${filename}"
file=$(find . -name "${filename}")
echo "found file: ${file}"
result=$(cat "${file}" | grep 'elapsed' | awk '{print $12, $13}')
echo "run test: ${i} in : ${result}" >> /tmp/test_times_${{ env.CURRENT_INDEX }}.txt
fi
done
fi
- name: show individual test times
shell: bash
if: env.BASE_BRANCH != '2.1.x'
run: cat /tmp/test_times_${{ env.CURRENT_INDEX }}.txt
- name: upload individual tests
if: env.BASE_BRANCH != '2.1.x'
uses: actions/upload-artifact@v3
with:
name: test_times_${{ env.CURRENT_INDEX }}.txt
path: /tmp/test_times_${{ env.CURRENT_INDEX }}.txt

15
.github/workflows/composites/save-controller-images/action.yaml vendored
View File

@@ -1,15 +0,0 @@
name: save controller docker images
description: save controller docker images
runs:
using: "composite"
steps:
- name: save controller docker images
shell: bash
run: |
mkdir -p /tmp/docker/images
TAG=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout)
cd spring-cloud-kubernetes-controllers
while read controller_image; do
docker save -o /tmp/docker/images/${controller_image}.tar docker.io/springcloud/${controller_image}:$TAG
done < <(mvn -Dexec.executable='echo' -Dexec.args='${project.artifactId}' exec:exec -q | grep -v 'spring-cloud-kubernetes-controllers')
cd ..

20
.github/workflows/composites/save-integration-tests-images/action.yaml vendored
View File

@@ -1,20 +0,0 @@
name: save integration tests docker images
description: save integration tests docker images
runs:
using: "composite"
steps:
- name: save integration tests docker images
shell: bash
run: |
mkdir -p /tmp/docker/images
TAG=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout)
cd spring-cloud-kubernetes-integration-tests
while read integ_test; do
docker save -o /tmp/docker/images/${integ_test}.tar docker.io/springcloud/${integ_test}:$TAG
done < <(mvn -Dexec.executable='echo' -Dexec.args='${project.artifactId}' exec:exec -q \
| grep -v 'spring-cloud-kubernetes-integration-tests' \
| grep -v 'spring-cloud-kubernetes-client-configmap-event-reload-multiple-apps' \
| grep -v 'spring-cloud-kubernetes-client-configuration-watcher-configmap-test-app' \
| grep -v 'spring-cloud-kubernetes-client-secrets-event-reload-multiple-apps' \
| grep -v 'spring-cloud-kubernetes-client-configuration-watcher-secrets-test-app' )
cd ..

9
.github/workflows/composites/setup-jdk1.8/action.yaml vendored
View File

@@ -1,9 +0,0 @@
name: setup project with jdk-8
description: setup project with jdk-8
runs:
using: "composite"
steps:
- uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '8'

9
.github/workflows/composites/setup-jdk17/action.yaml vendored
View File

@@ -1,9 +0,0 @@
name: setup project with jdk-17
description: setup project with jdk-17
runs:
using: "composite"
steps:
- uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '17'

65
.github/workflows/composites/test-bounds/action.yaml vendored
View File

@@ -1,65 +0,0 @@
name: test-bounds
description: test-bounds
runs:
using: "composite"
steps:
- name: test-bounds
shell: bash
run: |
PLAIN_TEST_CLASSNAMES=($(cat /tmp/tests.txt | grep -o 'spring.cloud.k8s.test.to.run -> org.*' | awk '{print $3}'))
NUMBER_OF_TESTS=${#PLAIN_TEST_CLASSNAMES[@]}
echo "current index : ${CURRENT_INDEX}"
echo "total number of tests: ${NUMBER_OF_TESTS} to be run on ${NUMBER_OF_JOBS} instances"
##########################################################################################################
##########################################################################################################
# Split tests per instance. The simplest form is when (for example), there are 100 tests and 10 instances.
# In this case, we will run 10 tests per a single instance. This is the case when reminder == 0 (100 / 10).
# In this case we will compute start_from as = current index * per_instance, that is: 0, 10, 20...
# and so on. How many tests are supposed to be taken in each matrix is equal to per_instance, that is 10.
# Slightly more interesting is when the tests are not perfectly distributed, when reminder != 0.
# For example 323 tests on 32 instances. In this case, we logically split this problem in two:
# when current_index < reminder and the rest.
# The batches in the example above are going to be [0, 11) [11, 22) [22, 33) [33, 43) [43, 53) ...
# So when current_index < reminder, we need to start at : current_index * per_instance + current_index,
# i.e.: 0, 11, 22
# and how many elements to take is : per_instance + 1, i.e. : 11
# On the other hand when current_index is not < reminder, how_many is easy, it's equal to per_instance.
# start_from on the other hand is : per_instance * current_index + reminder, i.e. : 33, 43, 53 and so on
##########################################################################################################
##########################################################################################################
per_instance=$((${NUMBER_OF_TESTS} / ${NUMBER_OF_JOBS}))
reminder=$((${NUMBER_OF_TESTS} - ${NUMBER_OF_JOBS} * ${per_instance}))
start_from=0
how_many=0
if [[ $reminder -eq 0 ]]; then
start_from=$(( ${CURRENT_INDEX} * ${per_instance} ))
how_many=${per_instance}
else
if [[ $CURRENT_INDEX -eq 0 ]]; then
start_from=0
how_many=$(( $per_instance + 1 ))
else
if [[ $CURRENT_INDEX -lt $reminder ]]; then
start_from=$(( $per_instance * $CURRENT_INDEX + $CURRENT_INDEX ))
how_many=$(( $per_instance +1 ))
else
start_from=$(( $per_instance * $CURRENT_INDEX + $reminder ))
how_many=$per_instance
fi
fi
fi
start_from_name=start_from_${CURRENT_INDEX}
how_many_name=how_many_${CURRENT_INDEX}
echo "$start_from_name : $start_from"
echo "$how_many_name : $how_many"
echo "$start_from_name=$start_from" >> $GITHUB_ENV
echo "$how_many_name=$how_many" >> $GITHUB_ENV

58
.github/workflows/composites/test-times/action.yaml vendored
View File

@@ -1,58 +0,0 @@
# this stage sort all tests by their timings. For that we rely on the surefire-plugin reports
# that have the name as <TEST_NAME>.txt and inside it there is a field that holds the time it took
# We get a hold of that field via awk '{print $12, $13} in the previous step.
# That previous step stores only tests for one matrix step, which we uploaded.
# In the steps below we download all artifacts (thus all test times), put them into a single file
# called /tmp/times-of-all-tests.txt, then sort this file, thus get a histogram of all test names
# and the time it took to run them.
name: test-times
description: test-times
runs:
using: "composite"
steps:
# we omit the name, as such all artifacts are downloaded
- name: download all artifacts
if: env.BASE_BRANCH != '2.1.x'
uses: actions/download-artifact@v3
with:
path: /tmp/all-artifacts
- name: show all artifacts
if: env.BASE_BRANCH != '2.1.x'
shell: bash
run: ls -l /tmp/all-artifacts
- name: merge all artifacts into a single file and sort by time
if: env.BASE_BRANCH != '2.1.x'
shell: bash
run: |
arr=($(find /tmp/all-artifacts/ -type f -name "test_times*"))
for i in "${arr[@]}"; do
echo "parsing -> ${i}"
cat "${i}" | awk '{print $3" "$6}' >> /tmp/times-of-all-tests.txt
done
sort -t' ' -nk2 /tmp/times-of-all-tests.txt >> /tmp/sorted.txt
cat /tmp/sorted.txt
- name: show all tests in a sorted manner
if: env.BASE_BRANCH != '2.1.x'
shell: bash
run: cat /tmp/sorted.txt
# save with the current run_id, but restore it without it. This means two things:
# 1) if we re-run, cache will be available
# 2) if there is a new run, we restore as '${{ runner.os }}-spring-cloud-k8s-existing-test-times-cache-'
# meaning there could be many of them already present and this is not an exact match
# github in this case will pick up the latest one, exactly what we want.
- name: save test times in cache
uses: actions/cache/save@v3
if: env.BASE_BRANCH != '2.1.x'
with:
path: /tmp/sorted.txt
key: ${{ runner.os }}-spring-cloud-k8s-existing-test-times-cache-${{ github.run_id }}

19
.github/workflows/composites/upload-docker-images/action.yaml vendored
View File

@@ -1,19 +0,0 @@
# we are not going to use upload/download actions, since it is slower than cache usage
# as key to the cache we are going to use DOCKER_IMAGES_KEY, which is the same as GITHUB_RUN_ID
name: upload docker images
description: upload docker images
runs:
using: "composite"
steps:
- uses: Wandalen/wretry.action@master
with:
attempt_limit: 3
action: buildjet/cache@v3
with: |
path: /tmp/docker/images
key: docker-images-cache-${{ env.DOCKER_IMAGES_KEY }}
restore-keys: docker-images-cache-${{ env.DOCKER_IMAGES_KEY }}

53
.github/workflows/deploy-docs.yml vendored Normal file
View File

@@ -0,0 +1,53 @@
name: Deploy Docs
run-name: ${{ format('{0} ({1})', github.workflow, github.event.inputs.build-refname || 'all') }}
on:
workflow_dispatch:
inputs:
build-refname:
description: Enter git refname to build (e.g., 5.7.x).
required: false
push:
branches: docs-build
env:
GRADLE_ENTERPRISE_SECRET_ACCESS_KEY: ${{ secrets.GRADLE_ENTERPRISE_SECRET_ACCESS_KEY }}
permissions:
contents: write
jobs:
build:
if: github.repository_owner == 'spring-cloud'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 5
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
- name: Set up refname build
if: github.event.inputs.build-refname
run: |
git fetch --depth 1 https://github.com/$GITHUB_REPOSITORY ${{ github.event.inputs.build-refname }}
export BUILD_REFNAME=${{ github.event.inputs.build-refname }}
echo "BUILD_REFNAME=$BUILD_REFNAME" >> $GITHUB_ENV
export BUILD_VERSION=$(git cat-file --textconv FETCH_HEAD:pom.xml | python3 -c "import xml.etree.ElementTree as xml; from sys import stdin; print(xml.parse(stdin).getroot().find('{http://maven.apache.org/POM/4.0.0}version').text)")
echo BUILD_VERSION=$BUILD_VERSION >> $GITHUB_ENV
- name: Run Antora
run: |
./mvnw --no-transfer-progress -B antora
- name: Publish Docs
uses: spring-io/spring-doc-actions/rsync-antora-reference@v0.0.11
with:
docs-username: ${{ secrets.DOCS_USERNAME }}
docs-host: ${{ secrets.DOCS_HOST }}
docs-ssh-key: ${{ secrets.DOCS_SSH_KEY }}
docs-ssh-host-key: ${{ secrets.DOCS_SSH_HOST_KEY }}
site-path: target/antora/site
- name: Bust Cloudflare Cache
uses: spring-io/spring-doc-actions/bust-cloudflare-antora-cache@v0.0.11
with:
context-root: spring-cloud-kubernetes
cloudflare-zone-id: ${{ secrets.CLOUDFLARE_ZONE_ID }}
cloudflare-cache-token: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }}

223
.github/workflows/maven.yaml vendored
View File

@@ -1,223 +0,0 @@
name: github-workflow
on:
push:
branches: [ main, 2.1.x, 3.0.x ]
pull_request:
branches: [ main, 2.1.x, 3.0.x ]
jobs:
build:
runs-on: ubuntu-latest
env:
# this might get set to true if there is an existing cache of test times
# this happens in 'matrix-bounds-on-test-times-cache-hit'
TEST_TIMES_CACHE_PRESENT: false
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 30
# this job ('build') outputs a value from step 'test_times_cache_present_init', that has the name of:
# 'test_times_cache_present'. This can later be used by other jobs. For example, we use this one
# to skip the job responsible for running the tests, if a previous cache that has the test times is not present.
# Same for other two variables 'number_of_matrix_instances' and 'matrix_array'.
outputs:
test_times_cache_present: ${{ steps.test_times_cache_present_init.outputs.test_times_cache_present }}
number_of_matrix_instances: ${{ steps.test_times_cache_present_init.outputs.number_of_matrix_instances }}
matrix_array: ${{ steps.test_times_cache_present_init.outputs.matrix_array }}
average_time_per_instance: ${{ steps.test_times_cache_present_init.outputs.average_time_per_instance }}
steps:
- name: checkout project
uses: actions/checkout@v2
- name: clean space
uses: ./.github/workflows/composites/clean-space
- name: set env variables
uses: ./.github/workflows/composites/env-variables
- name: setup project jdk-17
id: jdk_17
uses: ./.github/workflows/composites/setup-jdk17
if: env.BASE_BRANCH == 'main' || env.BASE_BRANCH == '3.0.x'
- name: setup project jdk-8
uses: ./.github/workflows/composites/setup-jdk1.8
if: env.BASE_BRANCH == '2.1.x'
- name: cache local maven repository
uses: ./.github/workflows/composites/cache
- name: Show caches
uses: actions/github-script@v6
with:
script: |
const caches = await github.rest.actions.getActionsCacheList({
owner: context.repo.owner,
repo: context.repo.repo,
})
for (const cache of caches.data.actions_caches) {
console.log(cache)
}
- name: maven build with dry-run for tests
uses: ./.github/workflows/composites/maven-build-with-dry-run-for-tests
- name: restore test times cache if it exists
id: restore_test_times_cache
if: env.BASE_BRANCH != '2.1.x'
uses: actions/cache/restore@v3
with:
path: /tmp/sorted.txt
key: ${{ runner.os }}-spring-cloud-k8s-existing-test-times-cache-${{ github.run_id }}
restore-keys: ${{ runner.os }}-spring-cloud-k8s-existing-test-times-cache-
- name: check test times cache exists
id: check_files
uses: andstor/file-existence-action@v2
with:
files: /tmp/sorted.txt
- name: show existing cache of test times
if: steps.check_files.outputs.files_exists == 'true'
shell: bash
run: cat /tmp/sorted.txt
- name: compute matrix related fields when cache is present
if: steps.check_files.outputs.files_exists == 'true'
uses: ./.github/workflows/composites/matrix-bounds-on-test-times-cache-hit
- name: matrix related variables when cache is present
id: test_times_cache_present_init
run: |
echo "test_times_cache_present=${{ env.TEST_TIMES_CACHE_PRESENT }}" >> $GITHUB_OUTPUT
echo "number_of_matrix_instances=${{ env.NUMBER_OF_MATRIX_INSTANCES }}" >> $GITHUB_OUTPUT
echo "matrix_array=${{ env.MATRIX_ARRAY }}" >> $GITHUB_OUTPUT
echo "average_time_per_instance=${{ env.AVERAGE_TIME_PER_INSTANCE }}" >> $GITHUB_OUTPUT
test_when_cache_present:
needs: [ build ]
runs-on: ubuntu-latest
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 30
# only run this one if there is a previous cache of test times
if: needs.build.outputs.test_times_cache_present == 'true'
timeout-minutes: 60
strategy:
fail-fast: true
matrix:
current_index: [ "${{ fromJSON(needs.build.outputs.matrix_array) }}" ]
number_of_jobs: [ "${{ fromJSON(needs.build.outputs.number_of_matrix_instances) }}" ]
average_time_per_instance: [ "${{ fromJSON(needs.build.outputs.average_time_per_instance) }}" ]
steps:
- name: checkout project
uses: actions/checkout@v2
- name: clean space
uses: ./.github/workflows/composites/clean-space
- name: set env variables
uses: ./.github/workflows/composites/env-variables
- name: setup project jdk-17
uses: ./.github/workflows/composites/setup-jdk17
if: env.BASE_BRANCH == 'main' || env.BASE_BRANCH == '3.0.x'
- name: setup project jdk-8
uses: ./.github/workflows/composites/setup-jdk1.8
if: env.BASE_BRANCH == '2.1.x'
- name: pre-test-actions
uses: ./.github/workflows/composites/pre-test-actions
- name: testcontainers reuse support
shell: bash
run: echo "testcontainers.reuse.enable=true" > ~/.testcontainers.properties
- name: run and save test times when cache is present
uses: ./.github/workflows/composites/run-and-save-test-times-when-cache-present
env:
CURRENT_INDEX: ${{ matrix.current_index }}
NUMBER_OF_JOBS: ${{ matrix.number_of_jobs }}
AVERAGE_TIME_PER_INSTANCE: ${{ matrix.average_time_per_instance }}
test_when_cache_missing:
needs: [ build ]
runs-on: ubuntu-latest
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 30
timeout-minutes: 60
# only run this one if there is no previous cache of test times
if: needs.build.outputs.test_times_cache_present == 'false'
strategy:
fail-fast: true
matrix:
current_index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
number_of_jobs: [32]
steps:
- name: checkout project
uses: actions/checkout@v2
- name: clean space
uses: ./.github/workflows/composites/clean-space
- name: set env variables
uses: ./.github/workflows/composites/env-variables
- name: setup project jdk-17
uses: ./.github/workflows/composites/setup-jdk17
if: env.BASE_BRANCH == 'main' || env.BASE_BRANCH == '3.0.x'
- name: setup project jdk-8
uses: ./.github/workflows/composites/setup-jdk1.8
if: env.BASE_BRANCH == '2.1.x'
- name: pre-test-actions
uses: ./.github/workflows/composites/pre-test-actions
- name: compute single step test bounds
uses: ./.github/workflows/composites/test-bounds
env:
CURRENT_INDEX: ${{ matrix.current_index }}
NUMBER_OF_JOBS: ${{ matrix.number_of_jobs }}
- name: testcontainers reuse support
shell: bash
run: echo "testcontainers.reuse.enable=true" > ~/.testcontainers.properties
- name: run and save individual test times
env:
CURRENT_INDEX: ${{ matrix.current_index }}
if: env.BASE_BRANCH != '2.1.x'
uses: ./.github/workflows/composites/run-and-save-test-times-when-cache-missing
save_test_times_when_cache_missing:
runs-on: ubuntu-latest
needs: [build, test_when_cache_missing ]
steps:
- name: checkout project
uses: actions/checkout@v2
- name: compute and save running time of tests
if: env.BASE_BRANCH != '2.1.x'
uses: ./.github/workflows/composites/test-times
save_test_times_when_cache_present:
runs-on: ubuntu-latest
needs: [ build, test_when_cache_present ]
steps:
- name: checkout project
uses: actions/checkout@v2
- name: compute and save running time of tests
if: env.BASE_BRANCH != '2.1.x'
uses: ./.github/workflows/composites/test-times