diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 16c7bafb..2230cc9b 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -10,15 +10,12 @@ jobs: check: runs-on: 'stream-ci-large' steps: - - name: 'Configure: checkout' + - name: 'Configure: checkout stream-applications@${{ github.ref }}' + uses: actions/checkout@v3 + - name: 'Configure: checkout stream-applications@main' uses: actions/checkout@v3 with: - ref: 'main' - - name: 'Configure: checkout stream-applications@${{ inputs.branch }}' - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - path: 'stream-applications' + ref: 'build-dir' - name: 'Configure: Ensure scripts are executable' shell: bash run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; @@ -39,43 +36,55 @@ jobs: uses: actions/cache@v3 with: path: ~/.m2/repository - key: maven-repo-${{ hashFiles('**/pom.xml') }}-pr + key: maven-repo-${{ hashFiles('**/pom.xml') }}-pr-${{ github.base_ref }} restore-keys: | - maven-repo-${{ hashFiles('**/pom.xml') }}-pr + maven-repo-${{ hashFiles('**/pom.xml') }}-pr-${{ github.base_ref }} maven-repo-${{ hashFiles('**/pom.xml') }} maven-repo- - name: 'Configure: Install xsltproc' uses: ./.github/actions/install-xsltproc - name: 'Action: build initial dependencies' shell: bash + env: + VERBOSE: ${{ github.debug && 'true' || '' }} run: | - ROOT_DIR=$(realpath $PWD) + BUILD_DIR=$(realpath build-dir) MAVEN_OPT="-U -B -T 1C -s $ROOT_DIR/.settings.xml ${{ inputs.verbose && '--debug' || '' }}" echo "::notice ::building - stream-applications-build" set -e - ./mvnw $MAVEN_OPT install -DskipTests -f stream-applications-build + $BUILD_DIR/build-folder.sh stream-applications-build set +e echo "::notice ::building - functions" set -e - ./mvnw $MAVEN_OPT install -DskipTests -f functions + $BUILD_DIR/build-folder.sh functions set +e echo "::notice ::building - stream-applications-core" set -e - ./mvnw $MAVEN_OPT deploy -DskipTests -f applications/stream-applications-core + $BUILD_DIR/build-folder.sh applications/stream-applications-core set +e echo "::notice ::core build completed" - name: 'Configure: get changed files' id: changed-files uses: tj-actions/changed-files@v34 - name: 'Action: verify changed modules' + shell: bash + env: + VERBOSE: ${{ github.debug && 'true' || '' }} run: | - $ROOT_DIR=$(realpath $PWD) - pushd stream-applications > /dev/null - $ROOT_DIR/create-matrices.sh - $ROOT_DIR/scripts/determine-modules-changed.sh "${{ steps.changed-files.outputs.all_changed_files }}" - MODULES=$(jq '.[]' modules.json | sed 's/\"//g') - for module in $MODULES; do - echo "::info ::verify module $module" - ./mvnw $MAVEN_OPT verify -f $module -amd - done - popd > /dev/null + $BUILD_DIR=$(realpath build-dir) + $BUILD_DIR/create-matrices.sh + $BUILD_DIR/scripts/determine-modules-changed.sh "${{ steps.changed-files.outputs.all_changed_files }}" + MODULES=$(jq '.[]' modules.json | sed 's/\"//g') + FOLDERS= + for module in $MODULES; do + if [ "$FOLDERS" = "" ]; then + FOLDERS="$module" + else + FOLDERS="$FOLDERS,$module" + fi + if [[ "$module" == *"application" ]]; then + $BUILD_DIR/scripts/remove-jib-plugin.sh $module/pom.xml + fi + done + echo "::info ::verify $FOLDERS" + $BUILD_DIR/build-folder.sh $FOLDERS install verify -amd diff --git a/build-core.sh b/build-core.sh index 2801114d..5e5f11ab 100755 --- a/build-core.sh +++ b/build-core.sh @@ -1,4 +1,6 @@ #!/bin/bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath "$SCDIR") (return 0 2>/dev/null) && sourced=1 || sourced=0 function check_env() { eval ev='$'$1 @@ -18,19 +20,15 @@ if [ "$VERBOSE" == "true" ]; then MAVEN_OPT=--debug fi if [ "$MAVEN_OPT" == "" ]; then - MAVEN_OPT=-B + MAVEN_OPT="-B -U -T 1C" else - MAVEN_OPT="$MAVEN_OPT -B" + MAVEN_OPT="$MAVEN_OPT -B -U -T 1C" fi - -if [ "$LOCAL" == "true" ]; then +if [ "$1" = "" ]; then MAVEN_GOAL="install" else - MAVEN_GOAL="install deploy" + MAVEN_GOAL="$*" fi -# -T 2C -./mvnw $MAVEN_OPT -s ./.settings.xml $MAVEN_GOAL -T 1C -f stream-applications-build -./mvnw $MAVEN_OPT -s ./.settings.xml $MAVEN_GOAL -T 1C -f functions -./mvnw $MAVEN_OPT -s ./.settings.xml $MAVEN_GOAL -T 1C -f applications/stream-applications-core - - +$SCDIR/build-folder.sh stream-applications-build "$MAVEN_GOAL" +$SCDIR/build-folder.sh functions "$MAVEN_GOAL" +$SCDIR/build-folder.sh applications/stream-applications-core "$MAVEN_GOAL" diff --git a/build-folder.sh b/build-folder.sh new file mode 100755 index 00000000..545b6dfd --- /dev/null +++ b/build-folder.sh @@ -0,0 +1,82 @@ +#!/bin/bash +bold="\033[1m" +red="\033[31m" +dim="\033[2m" +end="\033[0m" + +(return 0 2>/dev/null) && sourced=1 || sourced=0 +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if ((sourced != 0)); then + return 1 + else + exit 1 + fi + fi +} + +if [ "$VERBOSE" == "true" ]; then + MAVEN_OPTS="-s ./.settings.xml --debug -B -T 1C" +else + MAVEN_OPTS="-s ./.settings.xml -B -T 1C" +fi +if [ "$1" == "" ]; then + echo -e "Options: ${bold} []*${end}" +fi +FOLDER_NAMES="$1" +shift +MAVEN_GOAL=$* +if [ "$MAVEN_GOAL" == "" ]; then + echo -e "Using default goal ${bold}install${end}" + MAVEN_GOAL="install" +fi +SAVED_IFS=$IFS +IFS=, +FOLDERS= +for FOLDER in $FOLDER_NAMES; do + if [ ! -d "$FOLDER" ]; then + echo -e "${bold}Folder not found ${red}$FOLDER${end}" + exit 2 + fi + if [ "$FOLDERS" == "" ]; then + FOLDERS="$FOLDER" + else + FOLDERS="$FOLDERS $FOLDER" + fi +done +# IFS will affect mvnw ability to split arguments. +IFS=$SAVED_IFS +if [[ "$MAVEN_GOAL" == *"deploy"* ]]; then + check_env CI_DEPLOY_USERNAME + check_env CI_DEPLOY_PASSWORD +fi +for FOLDER in $FOLDERS; do + RETRIES=3 + while ((RETRIES > 0)); do + set +e + ./mvnw -U -f "$FOLDER" $MAVEN_OPTS dependency:tree + set -e + RESULT=$? + if ((RESULT == 0)); then + break + fi + RETRIES=$((RETRIES - 1)) + done +done + +MODULE_ARGS="" +for FOLDER in $FOLDERS; do + if [ "$MAVEN_GOAL" != "install" ]; then + echo -e "Maven goals:${bold}-f $FOLDER -DskipTests install${end}" + ./mvnw -f "$FOLDER" -DskipTests $MAVEN_OPTS install + fi + if [ "$MODULE_ARGS" == "" ]; then + MODULE_ARGS="$FOLDER" + else + MODULE_ARGS="$MODULE_ARGS,$FOLDER" + fi +done +echo -e "Maven goals:${bold}-pl $MODULE_ARGS $MAVEN_GOAL${end}" +./mvnw -pl "$MODULE_ARGS" $MAVEN_OPTS $MAVEN_GOAL diff --git a/functions/function/aggregator-function/pom.xml b/functions/function/aggregator-function/pom.xml index 70d5dd99..9403407a 100644 --- a/functions/function/aggregator-function/pom.xml +++ b/functions/function/aggregator-function/pom.xml @@ -15,6 +15,10 @@ Spring Native Function for Aggregator + + org.springframework.integration + spring-integration-core + org.springframework.integration