[CI] Fix test support

This commit is contained in:
Corneil du Plessis
2023-06-08 23:05:50 +02:00
parent 4abd19757e
commit ae0fcbe09d
26 changed files with 176 additions and 79 deletions

View File

@@ -26,4 +26,4 @@ jobs:
with:
branch: 'main'
verbose: ${{ inputs.verbose == 'true' }}
useMavenThreads: false
maven_threads: '0.5C'

31
.github/workflows/ci-manual.yml vendored Normal file
View File

@@ -0,0 +1,31 @@
name: 'CI - Manual'
on:
workflow_dispatch:
inputs:
verbose:
default: 'false'
description: 'Verbose output'
required: false
branch:
description: 'Enter branch name'
required: true
maven_threads:
description: 'Maven Threads'
default: '1'
required: true
jobs:
build:
uses: ./.github/workflows/common.yml
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TMC_API_TOKEN: ${{ secrets.TMC_API_TOKEN }}
GCP_CRED_JSON: ${{ secrets.GCP_CRED_JSON }}
ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
with:
branch: ${{ inputs.branch }}
verbose: ${{ inputs.verbose == 'true' }}
mavenThreads: ${{ inputs.maven_threads }}

View File

@@ -13,11 +13,11 @@ on:
required: false
default: false
description: 'Verbose output'
useMavenThreads:
type: boolean
mavenThreads:
type: string
required: false
default: false
description: 'Enable Maven Threads'
default: '1'
description: 'Maven Thread Option. Examples 0.5C, 2'
secrets:
DOCKERHUB_USERNAME:
DOCKERHUB_TOKEN:
@@ -33,7 +33,7 @@ on:
env:
VERBOSE: ${{ (github.debug || inputs.verbose) && 'true' || '' }}
BRANCH: ${{ inputs.branch }}
MAVEN_THREADS: ${{ inputs.useMavenThreads }}
MAVEN_THREADS: ${{ inputs.mavenThreads }}
jobs:
scan:
@@ -57,6 +57,7 @@ jobs:
shell: bash
run: echo "::info ::Scanned"
parameters:
name: 'parameters - ${{ inputs.branch }}'
if: ${{ github.repository == 'spring-cloud/stream-applications' }}
runs-on: ubuntu-latest
steps:
@@ -108,12 +109,12 @@ jobs:
echo "SOURCES=$SOURCES" >> $GITHUB_ENV
popd
if [ "${{ inputs.branch }}" == "main" ]; then
if [[ "${{ inputs.branch }}" == *"2021"* ]]; then
echo "JDK_BUILD=8" >> $GITHUB_ENV
echo "JRE_DEFAULT=11" >> $GITHUB_ENV
else
echo "JDK_BUILD=17" >> $GITHUB_ENV
echo "JRE_DEFAULT=17" >> $GITHUB_ENV
else
echo "JDK_BUILD=8" >> $GITHUB_ENV
echo "JRE_DEFAULT=11" >> $GITHUB_ENV
fi
if [ "${{ inputs.verbose }}" == "true" ]; then
echo "::debug ::MAX_PARALLEL=$MAX_PARALLEL"
@@ -142,6 +143,7 @@ jobs:
jdk_build: ${{ env.JDK_BUILD }}
jre_version: ${{ env.JRE_DEFAULT }}
core:
name: 'core - ${{ inputs.branch }}'
if: ${{ github.repository == 'spring-cloud/stream-applications' }}
runs-on: ubuntu-latest
needs:
@@ -213,13 +215,13 @@ jobs:
ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
VERBOSE: ${{ (github.debug || inputs.verbose) && 'true' || '' }}
MAVEN_THREADS: ${{ inputs.useMavenThreads }}
MAVEN_THREADS: ${{ inputs.mavenThreads }}
run: |
ROOT_DIR=$(realpath $PWD)
pushd stream-applications > /dev/null
echo "::notice ::building - stream-applications-build,functions,applications/stream-applications-core"
set -e
$ROOT_DIR/build-folder.sh stream-applications-build,functions,applications/stream-applications-core "install verify deploy"
$ROOT_DIR/build-core.sh "install verify deploy"
set +e
echo "::notice ::core build completed"
popd > /dev/null
@@ -343,7 +345,7 @@ jobs:
ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }}
VERBOSE: ${{ (github.debug || inputs.verbose) && 'true' || '' }}
MAVEN_THREADS: ${{ inputs.useMavenThreads }}
MAVEN_THREADS: ${{ inputs.mavenThreads }}
run: ./build-app.sh "stream-applications" "applications/processor/${{ matrix.app }}"
- name: 'Upload: Test Reports'
if: ${{ always() }}
@@ -444,7 +446,7 @@ jobs:
ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }}
VERBOSE: ${{ (github.debug || inputs.verbose) && 'true' || '' }}
MAVEN_THREADS: ${{ inputs.useMavenThreads }}
MAVEN_THREADS: ${{ inputs.mavenThreads }}
run: ./build-app.sh "stream-applications" "applications/sink/${{ matrix.app }}"
- name: 'Upload: Test Reports'
if: ${{ always() }}
@@ -545,7 +547,7 @@ jobs:
ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }}
VERBOSE: ${{ (github.debug || inputs.verbose) && 'true' || '' }}
MAVEN_THREADS: ${{ inputs.useMavenThreads }}
MAVEN_THREADS: ${{ inputs.mavenThreads }}
run: ./build-app.sh "stream-applications" "applications/source/${{ matrix.app }}"
- name: 'Upload: Test Reports'
if: ${{ always() }}
@@ -637,7 +639,7 @@ jobs:
ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }}
VERBOSE: ${{ (github.debug || inputs.verbose) && 'true' || '' }}
MAVEN_THREADS: ${{ inputs.useMavenThreads }}
MAVEN_THREADS: ${{ inputs.mavenThreads }}
run: |
ROOT_DIR=$(realpath $PWD)
pushd stream-applications > /dev/null

View File

@@ -33,19 +33,19 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>${test-containers.version}</version>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>${test-containers.version}</version>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>elasticsearch</artifactId>
<version>${test-containers.version}</version>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

View File

@@ -22,7 +22,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>${test-containers.version}</version>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

View File

@@ -37,16 +37,19 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>${testcontainers.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>rabbitmq</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

View File

@@ -23,6 +23,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>

View File

@@ -65,6 +65,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
@@ -77,12 +78,14 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mysql</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -123,7 +126,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mssqlserver</artifactId>
<version>1.18.2</version>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

View File

@@ -22,7 +22,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>${test-containers.version}</version>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

View File

@@ -27,13 +27,13 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>${test-containers.version}</version>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>rabbitmq</artifactId>
<version>${test-containers.version}</version>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>

View File

@@ -65,6 +65,20 @@
<artifactId>mockserver-client-java</artifactId>
<version>${mockserver.version}</version>
</dependency>
<dependency>
<groupId>org.junit</groupId>
<artifactId>junit-bom</artifactId>
<version>5.9.3</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-bom</artifactId>
<version>${testcontainers.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

View File

@@ -49,28 +49,38 @@
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<optional>true</optional>
<version>${testcontainers.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>kafka</artifactId>
<optional>true</optional>
<version>${testcontainers.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>rabbitmq</artifactId>
<optional>true</optional>
<version>${testcontainers.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
@@ -82,6 +92,7 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${apache-httpclient.version}</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>

View File

@@ -31,30 +31,36 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>kafka</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>rabbitmq</artifactId>
<version>${testcontainers.version}</version>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mongodb</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mysql</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -101,19 +107,9 @@
<artifactId>spring-boot-starter-jdbc</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-bom</artifactId>
<version>${test-containers.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>

View File

@@ -89,34 +89,36 @@ if [[ "$MAVEN_GOAL" == *"deploy"* ]]; then
check_env ARTIFACTORY_USERNAME
check_env ARTIFACTORY_PASSWORD
fi
MAVEN_THREADS_OPT=1
if [ "$MAVEN_THREADS" != "" ]; then
case $MAVEN_THREADS in
"true")
MAVEN_THREADS_OPT="0.5C"
;;
"false")
MAVEN_THREADS_OPT="1"
case $MAVEN_THREADS in
"true")
MVN_THR="-T 0.75C"
;;
"false")
MVN_THR=
;;
*)
MVN_THR="-T 0.5C"
;;
esac
;;
*)
MAVEN_THREADS_OPT="$MAVEN_THREADS"
;;
esac
fi
MVN_THR="-T $MAVEN_THREADS_OPT"
if [ "$SKIP_RESOLVE" = "" ]; then
SKIP_RESOLVE=true
fi
if [ "$SKIP_RESOLVE" = "true" ]; then
RETRIES=-1
RETRIES=0
RESULT=0
else
RETRIES=3
fi
while ((RETRIES >= 0)); do
while ((RETRIES > 0)); do
echo -e "Resolving dependencies for ${bold}$FOLDER_FOLDER_NAMES${end}"
set +e
$SCDIR/mvnw -U -pl "$FOLDER_NAMES" $MAVEN_OPTS $MVN_THR dependency:resolve
$SCDIR/mvnw -U -pl "$FOLDER_NAMES" $MAVEN_OPTS -T 0.5C dependency:resolve
RESULT=$?
set -e
if ((RESULT == 0)); then
@@ -139,12 +141,12 @@ if ((RESULT == 0)); then
source "$FOLDER/set-env.sh"
fi
set +e
echo -e "Maven goals:${bold}-f $FOLDER $MAVEN_OPTS $MVN_THR $MAVEN_GOAL${end}"
echo -e "Maven goals:${bold}-f $FOLDER $MAVEN_OPTS $MVN_THR "-DmavenThreads=$MAVEN_THREADS_OPT" $MAVEN_GOAL${end}"
MVNW="./mvnw"
if [ ! -f $MVNW ]; then
MVNW="$SCDIR/mvnw"
fi
$MVNW -f "$FOLDER" $MAVEN_OPTS $MVN_THR $MAVEN_GOAL
$MVNW -f "$FOLDER" $MAVEN_OPTS $MVN_THR "-DmavenThreads=$MAVEN_THREADS_OPT" $MAVEN_GOAL
RESULT=$?
set -e
if ((RESULT != 0)); then

View File

@@ -43,18 +43,29 @@
<groupId>org.apache.sshd</groupId>
<artifactId>sshd-sftp</artifactId>
<version>${sshd-sftp.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>${testcontainers.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>${testcontainers.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>

View File

@@ -18,6 +18,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mongodb</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

View File

@@ -26,6 +26,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>cassandra</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

View File

@@ -41,7 +41,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>elasticsearch</artifactId>
<version>${test-containers.version}</version>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

View File

@@ -26,6 +26,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mongodb</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>

View File

@@ -37,6 +37,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mongodb</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>

View File

@@ -56,6 +56,11 @@
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.integration</groupId>
<artifactId>spring-integration-test</artifactId>
@@ -68,12 +73,14 @@
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<artifactId>testcontainers</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<artifactId>junit-jupiter</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>

View File

@@ -36,6 +36,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mongodb</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>

View File

@@ -35,22 +35,23 @@
<nohttp-checkstyle.version>0.0.10</nohttp-checkstyle.version>
<disable.nohttp.checks>true</disable.nohttp.checks>
<spring-javaformat-checkstyle.version>0.0.35</spring-javaformat-checkstyle.version>
<spring-boot.version>3.0.7</spring-boot.version>
<spring-kafka.version>3.0.7</spring-kafka.version>
<spring-rabbit.version>3.0.4</spring-rabbit.version>
<spring.version>6.0.9</spring.version>
<spring-integration.version>6.0.5</spring-integration.version>
<spring-cloud.version>2022.0.2</spring-cloud.version>
<spring-cloud-starters.version>4.0.2</spring-cloud-starters.version>
<spring-cloud-function.version>4.0.2</spring-cloud-function.version>
<spring-cloud-stream-dependencies.version>4.0.2</spring-cloud-stream-dependencies.version>
<test-containers.version>1.17.6</test-containers.version>
<spring-cloud.version>2022.0.3</spring-cloud.version>
<spring-cloud-starters.version>4.0.3</spring-cloud-starters.version>
<spring-cloud-function.version>4.0.3</spring-cloud-function.version>
<spring-cloud-stream-dependencies.version>4.0.3</spring-cloud-stream-dependencies.version>
<testcontainers.version>1.18.3</testcontainers.version>
<mockserver.version>5.13.2</mockserver.version>
<groovy.version>4.0.5</groovy.version>
<jakarta-jms.version>3.1.0</jakarta-jms.version>
<mavenThreads>1</mavenThreads>
</properties>
<url>https://spring.io/projects/spring-cloud-stream-applications</url>
@@ -86,7 +87,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-bom</artifactId>
<version>${test-containers.version}</version>
<version>${testcontainers.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
@@ -150,7 +151,7 @@
<version>${maven-surefire-plugin.version}</version>
<configuration>
<parallel>classes</parallel>
<forkCount>0.25C</forkCount>
<forkCount>${mavenThreads}</forkCount>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
</configuration>
</plugin>
@@ -328,7 +329,7 @@
<version>${maven-surefire-plugin.version}</version>
<configuration>
<parallel>classes</parallel>
<forkCount>0.25C</forkCount>
<forkCount>${mavenThreads}</forkCount>
<excludedGroups>integration</excludedGroups>
</configuration>
</plugin>
@@ -345,7 +346,7 @@
<version>${maven-surefire-plugin.version}</version>
<configuration>
<parallel>classes</parallel>
<forkCount>0.25C</forkCount>
<forkCount>${mavenThreads}</forkCount>
<groups>integration</groups>
</configuration>
</plugin>

View File

@@ -33,11 +33,6 @@
<enabled>true</enabled>
</snapshots>
</repository>
<repository>
<id>spring-releases</id>
<name>Spring Releases</name>
<url>https://repo.spring.io/release</url>
</repository>
<repository>
<id>spring-milestones</id>
<name>Spring Milestones</name>

View File

@@ -41,9 +41,8 @@
<dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId>
<version>3.0.9</version>
<version>3.0.17</version>
<type>pom</type>
<scope>runtime</scope>
</dependency>
</dependencies>
<executions>

View File

@@ -16,9 +16,25 @@
</properties>
<repositories>
<repository>
<id>spring-release</id>
<name>spring-release</name>
<url>https://repo.spring.io/release</url>
<id>maven-central</id>
<name>Maven Central</name>
<url>https://repo.maven.apache.org/maven2</url>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
<repository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
</repository>
<repository>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/snapshot</url>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
<dependencies>