SpringCloud/spring-cloud-kubernetes
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add common pipeline for main and 2.1.x branches (#1256)

Browse Source
This commit is contained in:
erabii
2023-03-15 17:24:24 +02:00
committed by GitHub
parent 4883ea6ff7
commit 0fb6b2d52f
19 changed files with 525 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/composites/build-controllers-project/action.yaml vendored Executable file
View File

@@ -0,0 +1,11 @@
name: build controllers project
description: build controllers project
runs:
using: "composite"
steps:
- name: build controllers project
shell: bash
run: |
cd spring-cloud-kubernetes-controllers
.././mvnw -T 1C -U clean install
cd ..

12
.github/workflows/composites/build-integration-tests-project/action.yaml vendored Executable file
View File

@@ -0,0 +1,12 @@
name: build integration tests project without tests
description: build integration tests project without tests
runs:
using: "composite"
steps:
- name: build integration tests project without tests
shell: bash
run: |
cd spring-cloud-kubernetes-integration-tests
# build the images, but dont run the tests
.././mvnw -T 1C clean install -DskipTests
cd ..

11
.github/workflows/composites/cache/action.yaml vendored Executable file
View File

@@ -0,0 +1,11 @@
name: cache
description: cache
runs:
using: "composite"
steps:
- uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-cache-${{ env.BRANCH_NAME }}-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-cache-${{ env.BRANCH_NAME }}-${{ hashFiles('**/pom.xml') }}

15
.github/workflows/composites/download-docker-images/action.yaml vendored Executable file
View File

@@ -0,0 +1,15 @@
# we are not going to use upload/download actions, since it is slower than cache usage
# as key to the cache we are going to use DOCKER_IMAGES_KEY, which is the same as GITHUB_RUN_ID
name: download docker images
description: download docker images
runs:
using: "composite"
steps:
- uses: actions/cache@v2
with:
path: /tmp/docker/images
key: docker-images-cache-${{ env.DOCKER_IMAGES_KEY }}
restore-keys: docker-images-cache-${{ env.DOCKER_IMAGES_KEY }}

10
.github/workflows/composites/echo-saved-images/action.yaml vendored Executable file
View File

@@ -0,0 +1,10 @@
name: echo saved images
description: echo saved images
runs:
using: "composite"
steps:
- name: echo saved images
shell: bash
run: |
VIEW=$(ls -l /tmp/docker/images)
echo "${VIEW}"

24
.github/workflows/composites/env-variables/action.yaml vendored Executable file
View File

@@ -0,0 +1,24 @@
name: sets environment variables
description: sets environment variables
runs:
using: "composite"
steps:
- name: set env variables
shell: bash
run: |
baseRef=$GITHUB_BASE_REF
if [[ ! -z $baseRef ]]; then
echo "This is a PR to branch : $baseRef"
echo "BASE_BRANCH=$(echo $baseRef)" >> $GITHUB_ENV
else
raw=${{ github.ref }}
echo "${raw}"
branch=${raw##*/}
echo "This is a merge to branch : ${branch}"
echo "BASE_BRANCH=$(echo ${branch})" >> $GITHUB_ENV
fi
echo "BRANCH_NAME=$(echo $GITHUB_HEAD_REF)" >> $GITHUB_ENV
echo "BASE_BRANCH_NAME=$(echo $GITHUB_BASE_REF)" >> $GITHUB_ENV
echo "DOCKER_IMAGES_KEY=$(echo $GITHUB_RUN_ID)" >> $GITHUB_ENV

17
.github/workflows/composites/fabric8-istio-integration-test/action.yaml vendored Executable file
View File

@@ -0,0 +1,17 @@
name: run fabric8 istio integration test
description: run fabric8 istio integration test
runs:
using: "composite"
steps:
- name: run fabric8 istio integration test
shell: bash
run: |
cd spring-cloud-kubernetes-test-support
.././mvnw clean install
cd ..
docker load -i /tmp/docker/images/spring-cloud-kubernetes-fabric8-istio-it.tar
cd spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-istio-it/
../.././mvnw clean install -Dskip.build.image=true
cd ../..

14
.github/workflows/composites/fabric8-istio/action.yaml vendored Executable file
View File

@@ -0,0 +1,14 @@
# build a lightweight project so that dependencies for the
# maven-surefire-plugin are downloaded and thus cached also
name: build fabric8 istio project
description: build fabric8 istio projects
runs:
using: "composite"
steps:
- name: build fabric8 istio project
shell: bash
run: |
cd spring-cloud-kubernetes-fabric8-istio
.././mvnw -s ../.settings.xml clean install -U
cd ..

27
.github/workflows/composites/load-docker-images/action.yaml vendored Executable file
View File

@@ -0,0 +1,27 @@
name: install docker images
description: load docker images from tar
runs:
using: "composite"
steps:
- name: load docker images
shell: bash
run: |
# get the name of the images to load from children of spring-cloud-kubernetes-controllers
cd spring-cloud-kubernetes-controllers
while read controller_image; do
docker load -i /tmp/docker/images/${controller_image}.tar
done < <(mvn -Dexec.executable='echo' -Dexec.args='${project.artifactId}' exec:exec -q | grep -v 'spring-cloud-kubernetes-controllers')
cd ..
# get the name of the images to load from children of spring-cloud-kubernetes-integration-tests
cd spring-cloud-kubernetes-integration-tests
while read integration_test_image; do
docker load -i /tmp/docker/images/${integration_test_image}.tar
done < <(mvn -Dexec.executable='echo' -Dexec.args='${project.artifactId}' exec:exec -q \
| grep -v 'spring-cloud-kubernetes-integration-tests' \
| grep -v 'spring-cloud-kubernetes-client-configmap-event-reload-multiple-apps' \
| grep -v 'spring-cloud-kubernetes-client-configuration-watcher-configmap-test-app' \
| grep -v 'spring-cloud-kubernetes-client-secrets-event-reload-multiple-apps' \
| grep -v 'spring-cloud-kubernetes-client-configuration-watcher-secrets-test-app' )
cd ..

15
.github/workflows/composites/save-controller-images/action.yaml vendored Executable file
View File

@@ -0,0 +1,15 @@
name: save controller docker images
description: save controller docker images
runs:
using: "composite"
steps:
- name: save controller docker images
shell: bash
run: |
mkdir -p /tmp/docker/images
TAG=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout)
cd spring-cloud-kubernetes-controllers
while read controller_image; do
docker save -o /tmp/docker/images/${controller_image}.tar docker.io/springcloud/${controller_image}:$TAG
done < <(mvn -Dexec.executable='echo' -Dexec.args='${project.artifactId}' exec:exec -q | grep -v 'spring-cloud-kubernetes-controllers')
cd ..

20
.github/workflows/composites/save-integration-tests-images/action.yaml vendored Executable file
View File

@@ -0,0 +1,20 @@
name: save integration tests docker images
description: save integration tests docker images
runs:
using: "composite"
steps:
- name: save integration tests docker images
shell: bash
run: |
mkdir -p /tmp/docker/images
TAG=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout)
cd spring-cloud-kubernetes-integration-tests
while read integ_test; do
docker save -o /tmp/docker/images/${integ_test}.tar docker.io/springcloud/${integ_test}:$TAG
done < <(mvn -Dexec.executable='echo' -Dexec.args='${project.artifactId}' exec:exec -q \
| grep -v 'spring-cloud-kubernetes-integration-tests' \
| grep -v 'spring-cloud-kubernetes-client-configmap-event-reload-multiple-apps' \
| grep -v 'spring-cloud-kubernetes-client-configuration-watcher-configmap-test-app' \
| grep -v 'spring-cloud-kubernetes-client-secrets-event-reload-multiple-apps' \
| grep -v 'spring-cloud-kubernetes-client-configuration-watcher-secrets-test-app' )
cd ..

9
.github/workflows/composites/setup-jdk1.8/action.yaml vendored Executable file
View File

@@ -0,0 +1,9 @@
name: setup project with jdk-8
description: setup project with jdk-8
runs:
using: "composite"
steps:
- uses: actions/setup-java@v2
with:
distribution: 'temurin'
java-version: '8'

9
.github/workflows/composites/setup-jdk17/action.yaml vendored Executable file
View File

@@ -0,0 +1,9 @@
name: setup project with jdk-17
description: setup project with jdk-17
runs:
using: "composite"
steps:
- uses: actions/setup-java@v2
with:
distribution: 'temurin'
java-version: '17'

15
.github/workflows/composites/upload-docker-images/action.yaml vendored Executable file
View File

@@ -0,0 +1,15 @@
# we are not going to use upload/download actions, since it is slower than cache usage
# as key to the cache we are going to use DOCKER_IMAGES_KEY, which is the same as GITHUB_RUN_ID
name: upload docker images
description: upload docker images
runs:
using: "composite"
steps:
- uses: actions/cache@v2
with:
path: /tmp/docker/images
key: docker-images-cache-${{ env.DOCKER_IMAGES_KEY }}
restore-keys: docker-images-cache-${{ env.DOCKER_IMAGES_KEY }}

314
.github/workflows/maven.yaml vendored Executable file
View File

@@ -0,0 +1,314 @@
name: github-workflow
on:
push:
branches: [ main, 2.1.x ]
pull_request:
branches: [ main, 2.1.x ]
jobs:
build:
runs-on: ubuntu-latest
env:
TESTCONTAINERS_REUSE_ENABLE: false
steps:
- name: checkout project
uses: actions/checkout@v2
- name: set env variables
uses: ./.github/workflows/composites/env-variables
- name: setup project jdk-17
id: jdk_17
uses: ./.github/workflows/composites/setup-jdk17
if: env.BASE_BRANCH == 'main'
- name: setup project jdk-8
uses: ./.github/workflows/composites/setup-jdk1.8
if: env.BASE_BRANCH == '2.1.x'
- name: cache local maven repository
uses: ./.github/workflows/composites/cache
- name: build fabric8 istio
uses: ./.github/workflows/composites/fabric8-istio
- name: build with skip tests and skip images
run: ./mvnw -T 1C -s .settings.xml clean install -U -DskipTests -Dskip.build.image=true
- name: build controllers project
uses: ./.github/workflows/composites/build-controllers-project
if: env.BASE_BRANCH != '2.1.x'
- name: build integration tests project
uses: ./.github/workflows/composites/build-integration-tests-project
if: env.BASE_BRANCH != '2.1.x'
- name: save controller docker images
uses: ./.github/workflows/composites/save-controller-images
if: env.BASE_BRANCH != '2.1.x'
- name: save integration tests docker images
uses: ./.github/workflows/composites/save-integration-tests-images
if: env.BASE_BRANCH != '2.1.x'
- name: echo saved images
uses: ./.github/workflows/composites/echo-saved-images
if: env.BASE_BRANCH != '2.1.x'
- name: upload docker images
uses: ./.github/workflows/composites/upload-docker-images
if: env.BASE_BRANCH != '2.1.x'
# we need to run some test, so that K3s container is started and then all other instances will re-use this one.
# Otherwise, (since we use static ports) there might be two instances starting at the same time, and ports might conflict
# This does not run on '2.1.x' branch.
fabric8_istio_integration_test:
needs: build
runs-on: ubuntu-latest
steps:
- name: checkout project
uses: actions/checkout@v2
- name: set env variables
uses: ./.github/workflows/composites/env-variables
- name: setup project
uses: ./.github/workflows/composites/setup-jdk17
if: env.BASE_BRANCH == 'main'
- name: cache local maven repository
uses: ./.github/workflows/composites/cache
if: env.BASE_BRANCH != '2.1.x'
- name: download docker images
uses: ./.github/workflows/composites/download-docker-images
if: env.BASE_BRANCH != '2.1.x'
- name: echo saved images
uses: ./.github/workflows/composites/echo-saved-images
if: env.BASE_BRANCH != '2.1.x'
- name: integration test fabric8 istio
uses: ./.github/workflows/composites/fabric8-istio-integration-test
if: env.BASE_BRANCH != '2.1.x'
test:
needs: [ build, fabric8_istio_integration_test ]
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: true
matrix:
current_index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
number_of_jobs: [32]
steps:
- name: testcontainers reuse support
run: echo "testcontainers.reuse.enable=true" > ~/.testcontainers.properties
- name: checkout project
uses: actions/checkout@v2
- name: set env variables
uses: ./.github/workflows/composites/env-variables
- name: setup project jdk-17
uses: ./.github/workflows/composites/setup-jdk17
if: env.BASE_BRANCH == 'main'
- name: setup project jdk-8
uses: ./.github/workflows/composites/setup-jdk1.8
if: env.BASE_BRANCH == '2.1.x'
- name: cache local maven repository
uses: ./.github/workflows/composites/cache
- name: download docker images
uses: ./.github/workflows/composites/download-docker-images
if: env.BASE_BRANCH != '2.1.x'
- name: load docker images into local repo
uses: ./.github/workflows/composites/load-docker-images
if: env.BASE_BRANCH != '2.1.x'
- name: run tests
env:
CURRENT_INDEX: ${{ matrix.current_index }}
NUMBER_OF_JOBS: ${{ matrix.number_of_jobs }}
run: |
# - find all tests
# - exclude Fabric8IstionIT
# - only take classes that have @Test inside them
# - ignore the ones that have 'abstract class'. we do this because otherwise we would pass
# to -DtestsToRun an abstract class, and it will not run anything.
# - drop the "begining" xxx/src/test/java
# - replace / with .
# - drop last ".java"
# - replace newline with space
# - replace '\n' with ' '
# exclude all integration tests for branch '2.1.x' via 'grep -v '.*IT.java''
echo "base_branch : $BASE_BRANCH"
declare -a PLAIN_TEST_CLASSNAMES
baseBranch=$BASE_BRANCH
if [[ $baseBranch == "2.1.x" ]]; then
PLAIN_TEST_CLASSNAMES=($(find . -name '*.java' \
| grep 'src/test/java' \
| grep -v 'Fabric8IstioIT' \
| grep -v '.*IT.java' \
| xargs grep -l '@Test' \
| xargs grep -L 'abstract class' \
| sed 's/.*src.test.java.//g' \
| sed 's@/@.@g' \
| sed 's/.\{5\}$//' \
| tr '\n' ' '))
else
PLAIN_TEST_CLASSNAMES=($(find . -name '*.java' \
| grep 'src/test/java' \
| grep -v 'Fabric8IstioIT' \
| xargs grep -l '@Test' \
| xargs grep -L 'abstract class' \
| sed 's/.*src.test.java.//g' \
| sed 's@/@.@g' \
| sed 's/.\{5\}$//' \
| tr '\n' ' '))
fi
# classes that have @Test and are abstract, for example: "LabeledSecretWithPrefixTests"
# - exclude Fabric8IstionIT
# - only take classes that have @Test inside them
# - only take classes that are abstract
# - drop everything up until the last "/"
# - drop ".java"
ABSTRACT_TEST_CLASSNAMES_COMMAND="find . -name '*.java' \
| grep 'src/test/java' \
| grep -v 'Fabric8IstioIT' \
| xargs grep -l '@Test' \
| xargs grep -l 'abstract class' \
| sed 's/.*\///g' \
| sed 's/.java//g'"
# find classes that extend abstract test classes
DERIVED_FROM_ABSTRACT_CLASSES_COMMAND="find . -name '*.java' \
| grep 'src/test/java' \
| grep -v 'Fabric8IstioIT' \
| xargs grep -l 'extends replace_me ' \
| sed 's/.*src.test.java.//g' \
| sed 's@/@.@g' \
| sed 's/.\{5\}$//' \
| tr '\n' ' '"
while read class_name; do
replaced=$(echo ${DERIVED_FROM_ABSTRACT_CLASSES_COMMAND/replace_me/"$class_name"})
result=($(eval $replaced))
PLAIN_TEST_CLASSNAMES+=(${result[@]})
done < <(eval $ABSTRACT_TEST_CLASSNAMES_COMMAND)
IFS=$'\n'
SORTED_TEST_CLASSNAMES=( $(sort <<< "${PLAIN_TEST_CLASSNAMES[*]} | uniq -u") )
unset IFS
number_of_tests=${#SORTED_TEST_CLASSNAMES[@]}
number_of_jobs=${NUMBER_OF_JOBS}
current_index=${CURRENT_INDEX}
per_instance=$((number_of_tests / number_of_jobs))
# we do not get an ideal distribution all the time, so this is needed to add one more test
# to the first "reminder" number of instances.
# consider the case when there are 10 tests, and 4 instances
# 10/4=2 (and this is "per_instance"), at the same time "reminder" = (10 - 4 * 2) = 2
# this means that the first instance will run (2 + 1) tests
# second instance will run (2 + 1) tests
# all subsequent instances will run 2 tests.
reminder=$((number_of_tests - number_of_jobs * per_instance))
elements_in_current_instance=$((per_instance + 1))
left_bound=0
right_bound=0
# we are in a range where we might need to add one more test to each instance
# notice the "less then" condition here, it is important and must not change
if [[ $current_index -lt $reminder ]]; then
# this one is easy, the range will be [0..3] (following our example above)
if [[ $current_index == 0 ]]; then
left_bound=0
right_bound=$elements_in_current_instance
# this one will be [3..6]
else
left_bound=$((current_index * elements_in_current_instance))
right_bound=$(((current_index + 1) * elements_in_current_instance))
fi
echo "total tests : $number_of_tests, jobs: $number_of_jobs, current index : $current_index. will run tests in range : [$left_bound..$right_bound]"
else
# reminder can be zero here (in case of a perfect distribution): in such a case, this is just "current_index * per_instance".
# if reminder is not zero, we have two regions here, logically. the one of the left is "reminder * elements_in_current_instance",
# basically [0..3] and [3..6]
# and the region on the right [6..8].
left_bound=$((reminder * elements_in_current_instance + ((current_index - reminder) * per_instance)))
right_bound=$((left_bound + per_instance))
echo "total tests : $number_of_tests, jobs: $number_of_jobs, current index : $current_index. will run tests in range : [$left_bound..$right_bound]"
fi
diff=$((right_bound - left_bound))
sliced_array=("${SORTED_TEST_CLASSNAMES[@]:$left_bound:$diff}")
TEST_ARG=$(echo ${sliced_array[@]} | sed 's/ /,/g')
echo "will run tests : ${TEST_ARG[@]}"
if [[ $baseBranch == "2.1.x" ]]; then
./mvnw -s .settings.xml -pl '-:kubernetes-leader-election-example' -pl '-:kubernetes-hello-world-example' \
-pl '-:kubernetes-reload-example' -pl '-:kubernetes-loadbalancer-example' \
-pl '-:spring-cloud-kubernetes-configserver' -pl '-:spring-cloud-kubernetes-configuration-watcher' \
-pl '-:spring-cloud-kubernetes-discoveryserver' \
-DtestsToRun=${TEST_ARG[@]} \
-e clean install \
-U -P sonar -nsu --batch-mode \
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
-Dhttp.keepAlive=false \
-Dmaven.wagon.http.pool=false \
-Dmaven.wagon.http.retryHandler.class=standard \
-Dmaven.wagon.http.retryHandler.count=3 \
-Dskip.build.image=true
else
./mvnw -s .settings.xml \
-DtestsToRun=${TEST_ARG[@]} \
-e clean install \
-U -P sonar -nsu --batch-mode \
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
-Dhttp.keepAlive=false \
-Dmaven.wagon.http.pool=false \
-Dmaven.wagon.http.retryHandler.class=standard \
-Dmaven.wagon.http.retryHandler.count=3 \
-Dskip.build.image=true
fi
- name: Publish Test Report
uses: mikepenz/action-junit-report@v2
if: always() # always run even if the previous step fails
with:
report_paths: '**/surefire-reports/TEST-*.xml'
- name: Archive code coverage results
uses: actions/upload-artifact@v2
with:
name: surefire-reports
path: '**/surefire-reports/*'