From e46b43b23f4d131ae6026424c6b5fd8ae4c13af1 Mon Sep 17 00:00:00 2001 From: Corneil du Plessis Date: Thu, 14 Nov 2024 13:23:32 +0200 Subject: [PATCH] Simplify CI-PR (#570) Removed checks for changed modules and ensure the build scripts works in the absence of credentials. This changes the distribution to use liberica since it properly detects the latest patch version of the distribution. See #550 --- .github/workflows/ci-pr.yml | 69 +++++++++---------------------------- build-app.sh | 8 +++-- build-apps.sh | 16 ++++++++- build-core.sh | 2 +- build-folder.sh | 12 ++++--- local/pack-containers.sh | 3 ++ 6 files changed, 50 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 15f2bc1d..f2549102 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -7,10 +7,16 @@ env: MAIN_PATH: 'build-dir' GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} MAVEN_THREADS: 'false' + BUILD_NAME: 'stream-applications-${{ github.ref_name }}' + BUILD_NUMBER: ${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} + ARTIFACTORY_USERNAME: 'anonymous' + ARTIFACTORY_PASSWORD: 'anonymous' + SKIP_DEPLOY: 'true' + jobs: build: if: github.repository == 'spring-cloud/stream-applications' - runs-on: ubuntu-latest + runs-on: ubuntu22-8-32-OSS steps: - name: 'Configure: checkout stream-applications@${{ github.ref_name }}' uses: actions/checkout@v3 @@ -26,18 +32,10 @@ jobs: shell: bash run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; - name: 'Configure: Install Java' - if: ${{ github.base_ref != 'main' }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: - java-version: ${{ needs.parameters.outputs.jdk_build }} - - name: 'Configure: Install GraalVM' - if: ${{ github.base_ref == 'main' }} - uses: graalvm/setup-graalvm@v1 - with: - version: 'latest' java-version: '17' - components: 'js' - github-token: ${{ secrets.GITHUB_TOKEN }} + distribution: 'liberica' - name: 'Configure: cache for maven dependencies' uses: actions/cache@v3 with: @@ -49,51 +47,18 @@ jobs: maven-repo- - name: 'Configure: Install xsltproc' uses: ./.github/actions/install-xsltproc - - name: 'Action: verify changed modules' + - name: 'Action: Build Core' shell: bash env: VERBOSE: ${{ github.debug && 'true' || '' }} run: | - PR=$(echo "${{ github.ref_name }}" | grep -o '[[:digit:]]*') - echo "Checking out pull request #$PR" - gh pr checkout $PR - BUILD_DIR=$(realpath $MAIN_PATH) - echo "Changed files:" - gh pr diff --name-only - CHANGED=$(gh pr diff --name-only) - $BUILD_DIR/scripts/determine-modules-changed.sh "$CHANGED" - MODULES=$(jq '.[]' modules.json | sed 's/\"//g') - FOLDERS= - ROOT_MODULES= - if [ "$MAVEN_THREADS" = "true" ]; then - MVN_THR="-T 0.5C" - else - MVN_THR= - fi - MAVEN_OPTS="-s $BUILD_DIR/.settings.xml -B -U $MVN_THR" - echo "VERBOSE=$VERBOSE" - if [ "$VERBOSE" = "true" ]; then - MAVEN_OPTS="-X $MAVEN_OPTS" - fi - for module in $MODULES; do - if [ "$FOLDERS" = "" ]; then - FOLDERS="$module" - else - FOLDERS="$FOLDERS,$module" - fi - done - set +e - if [ "$FOLDERS" != "" ]; then - echo "::notice::install $FOLDERS and dependents" - echo "::debug::MAVEN:./mvnw $MAVEN_OPTS -amd -pl $FOLDERS install" - ./mvnw $MAVEN_OPTS -amd -pl $FOLDERS install - RC=$? - if ((RC!=0)); then - exit $RC - fi - fi - - name: 'Configure: Install Trivy' - uses: ./.github/actions/install-trivy + ./build-core.sh "-T 1C package -Psnapshot" + - name: 'Action: Build Apps' + shell: bash + env: + VERBOSE: ${{ github.debug && 'true' || '' }} + run: | + ./build-apps.sh "-T 1C package -Psnapshot" - name: 'Upload: Error logs' if: ${{ failure() }} uses: actions/upload-artifact@v3 diff --git a/build-app.sh b/build-app.sh index 46aca6dd..ca73dd78 100755 --- a/build-app.sh +++ b/build-app.sh @@ -23,7 +23,9 @@ if [ "$2" == "" ]; then fi PROJECT_FOLDER=$(realpath "$1") APP_FOLDER=$2 -SKIP_DEPLOY=$3 +if [ "$3" != "" ]; then + SKIP_DEPLOY=$3 +fi set -e @@ -40,7 +42,7 @@ pushd "$PROJECT_FOLDER" >/dev/null else MAVEN_GOAL="install verify deploy" fi - if [[ "$MAVEN_GOAL" == *"deploy"* ]]; then + if [[ "$MAVEN_GOAL" == *"deploy"* ]] && [ "$SKIP_DEPLOY" != "true" ]; then check_env ARTIFACTORY_USERNAME check_env ARTIFACTORY_PASSWORD fi @@ -90,7 +92,9 @@ pushd "$PROJECT_FOLDER" >/dev/null if [ "$SKIP_DEPLOY" == "" ] || [ "$SKIP_DEPLOY" == "false" ]; then for v in $JDKS; do echo "Pack:$app:$VERSION-jdk$v" + set -e pack build \ + --pull-policy if-not-present \ --path "target/$app-$VERSION.jar" \ --builder paketobuildpacks/builder-jammy-base:latest \ --env BP_JVM_VERSION=$v \ diff --git a/build-apps.sh b/build-apps.sh index 620d0ba0..2544a112 100755 --- a/build-apps.sh +++ b/build-apps.sh @@ -19,17 +19,31 @@ if [ "$1" = "" ]; then else MAVEN_GOAL="$*" fi - +RC=0 +set +e $SCDIR/create-matrices.sh PROCESSORS=$(jq -c '.processors | .[]' matrix.json | sed 's/\"//g') SINKS=$(jq -c '.sinks | .[]' matrix.json | sed 's/\"//g') SOURCES=$(jq -c '.sources | .[]' matrix.json | sed 's/\"//g') for app in $PROCESSORS; do $SCDIR/build-app.sh . "applications/processor/$app" + RCC=$? + if ((RCC!=0)); then + RC=$RCC + fi done for app in $SOURCES; do $SCDIR/build-app.sh . "applications/source/$app" + RCC=$? + if ((RCC!=0)); then + RC=$RCC + fi done for app in $SINKS; do $SCDIR/build-app.sh . "applications/sink/$app" + RCC=$? + if ((RCC!=0)); then + RC=$RCC + fi done +exit $RC diff --git a/build-core.sh b/build-core.sh index a9bc42ed..95947c0d 100755 --- a/build-core.sh +++ b/build-core.sh @@ -19,5 +19,5 @@ if [ "$1" = "" ]; then else MAVEN_GOAL="$*" fi - +set -e $SCDIR/build-folder.sh stream-applications-build,applications/stream-applications-core "$MAVEN_GOAL" diff --git a/build-folder.sh b/build-folder.sh index ca3ff8d6..80ef356e 100755 --- a/build-folder.sh +++ b/build-folder.sh @@ -20,14 +20,18 @@ function check_env() { fi } +if [ "$1" == "" ]; then + echo -e "Options: ${bold} []*${end}" + exit 1 +fi if [ "$VERBOSE" = "true" ]; then MAVEN_OPTS="-X" elif [ "$VERBOSE" = "false" ]; then MAVEN_OPTS="-q" fi -MAVEN_OPTS="$MAVEN_OPTS -s $SCDIR/.settings.xml -B" -if [ "$1" == "" ]; then - echo -e "Options: ${bold} []*${end}" +MAVEN_OPTS="$MAVEN_OPTS -B" +if [ "$SKIP_DEPLOY" != "true" ]; then + MAVEN_OPTS="$MAVEN_OPTS -s $SCDIR/.settings.xml" fi FOLDER_NAMES="$1" shift @@ -83,7 +87,7 @@ for FOLDER in $FOLDER_NAMES; do done # IFS will affect mvnw ability to split arguments. IFS=$SAVED_IFS -if [[ "$MAVEN_GOAL" == *"deploy"* ]]; then +if [[ "$MAVEN_GOAL" == *"deploy"* ]] && [ "$SKIP_DEPLOY" != "true" ]; then check_env ARTIFACTORY_USERNAME check_env ARTIFACTORY_PASSWORD fi diff --git a/local/pack-containers.sh b/local/pack-containers.sh index 15add090..dc6ac986 100755 --- a/local/pack-containers.sh +++ b/local/pack-containers.sh @@ -76,6 +76,7 @@ for app in ${PROCESSORS[@]}; do TARGET_FILE="$APP_PATH/$app-$BROKER-$TAG.jar" if [ -f "$TARGET_FILE" ]; then pack build \ + --pull-policy if-not-present \ --path "$TARGET_FILE" \ --builder paketobuildpacks/builder-jammy-base:latest \ --env BP_JVM_VERSION=$v \ @@ -105,6 +106,7 @@ for app in ${SINKS[@]}; do TARGET_FILE="$APP_PATH/$app-$BROKER-$TAG.jar" if [ -f "$TARGET_FILE" ]; then pack build \ + --pull-policy if-not-present \ --path "$TARGET_FILE" \ --builder paketobuildpacks/builder-jammy-base:latest \ --env BP_JVM_VERSION=$v \ @@ -134,6 +136,7 @@ for app in ${SOURCES[@]}; do TARGET_FILE="$APP_PATH/$app-$BROKER-$TAG.jar" if [ -f "$TARGET_FILE" ]; then pack build \ + --pull-policy if-not-present \ --path "$TARGET_FILE" \ --builder paketobuildpacks/builder-jammy-base:latest \ --env BP_JVM_VERSION=$v \