diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 38f4a125..69af536e 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -50,7 +50,7 @@ jobs: maven-repo- - name: 'Configure: Install xsltproc' uses: ./.github/actions/install-xsltproc - - name: 'Action: build initial dependencies' + - name: 'Action: verify changed modules' shell: bash env: VERBOSE: ${{ github.debug && 'true' || '' }} @@ -58,39 +58,43 @@ jobs: PR=$(echo "${{ github.ref_name }}" | grep -o '[[:digit:]]*') echo "Checking out pull request #$PR" gh pr checkout $PR - BUILD_DIR=$(realpath $MAIN_PATH) - echo "::notice ::building - stream-applications - core" - set +e - $BUILD_DIR/build-folder.sh stream-applications-build,functions/common,functions/spring-functions-parent,applications/stream-applications-core install -DskipTests - RC=$? - if ((RC!=0)); then - exit $RC - fi - echo "::notice ::core build completed" - - name: 'Action: verify changed modules' - shell: bash - env: - VERBOSE: ${{ github.debug && 'true' || '' }} - run: | BUILD_DIR=$(realpath $MAIN_PATH) - $BUILD_DIR/create-matrices.sh echo "Changed files:" gh pr diff --name-only CHANGED=$(gh pr diff --name-only) $BUILD_DIR/scripts/determine-modules-changed.sh "$CHANGED" MODULES=$(jq '.[]' modules.json | sed 's/\"//g') FOLDERS= - for module in $MODULES; do - if [ "$FOLDERS" = "" ]; then - FOLDERS="$module" + ROOT_MODULES= + MAVEN_OPTS="-s .settings.xml -B -T 0.3C" + for module in $MODULES; do + if [[ "$module" == *"/"* ]]; then + if [ "$FOLDERS" = "" ]; then + FOLDERS="$module" + else + FOLDERS="$FOLDERS,$module" + fi else - FOLDERS="$FOLDERS,$module" + if [ "$ROOT_MODULES" == "" ]; then + ROOT_MODULES=$module + else + ROOT_MODULES="$ROOT_MODULES,$module" + fi fi done - echo "::info ::verify $FOLDERS" set +e + if [ "$ROOT_MODULES" != "" ]; then + # Module with no / are probable a root module that contains other modules and will trigger a lot of dependant builds without real value + echo "::info ::building $ROOT_MODULES" + ./mvnw $MAVEN_OPTS -pl $ROOT_MODULES install + RC=$? + if ((RC!=0)); then + exit $RC + fi + fi if [ "$FOLDERS" != "" ]; then - $BUILD_DIR/build-folder.sh $FOLDERS install verify -amd + echo "::info ::verify $FOLDERS and dependents" + ./mvnw $MAVEN_OPTS -pl $FOLDERS install -amd RC=$? if ((RC!=0)); then exit $RC diff --git a/scripts/build-modified-modules.sh b/scripts/build-modified-modules.sh new file mode 100755 index 00000000..b2483573 --- /dev/null +++ b/scripts/build-modified-modules.sh @@ -0,0 +1,54 @@ +#!/bin/bash +function itemInModules() { + local e + for e in ${MODULES}; do + if [[ "$e" == "$ITEM" ]]; then + echo "1" + return 0 + fi + done + echo "0" +} +function addItem() { + if [ "$MODULES" == "" ]; then + echo "$1" + else + echo "$MODULES $1" + fi +} +MODIFIED="$*" +ALL_MODULES=$(find . -name "pom.xml" -type f -exec dirname '{}' \; | sed 's/\.\///' | sort -r) +MODULES= +for file in $MODIFIED; do + FILE=$(realpath $file) + echo "$file was changed" + for ITEM in $ALL_MODULES; do + if [[ "$ITEM" != "." ]] && [[ "$file" == *"$ITEM"* ]]; then + echo "Matched $ITEM" + HAS_ITEM=$(itemInModules) + if ((HAS_ITEM == 0)); then + echo "Add:$ITEM" + MODULES=$(addItem "$ITEM") + fi + break + fi + done +done + +MVN_MODULES= + +for module in $MODULES; do + if [[ "$module" == *"/"* ]]; then + if [ "$MVN_MODULES" == "" ]; then + MVN_MODULES=$module + else + MVN_MODULES="$MVN_MODULES,$module" + fi + else + ./mvnw install -pl $module + fi +done +if [ "$MVN_MODULES" != "" ]; then + echo "Building $MVN_MODULES" + ./mvnw install -pl "$MVN_MODULES" -amd +fi diff --git a/scripts/determine-modules-changed.sh b/scripts/determine-modules-changed.sh index bd5ff1a5..ae6305f8 100755 --- a/scripts/determine-modules-changed.sh +++ b/scripts/determine-modules-changed.sh @@ -17,80 +17,20 @@ function addItem() { fi } MODIFIED="$*" -FUNCTIONS=$(jq -c '.functions | .[]' matrix.json | sed 's/\"//g') -CONSUMERS=$(jq -c '.consumers | .[]' matrix.json | sed 's/\"//g') -SUPPLIERS=$(jq -c '.suppliers | .[]' matrix.json | sed 's/\"//g') -PROCESSORS=$(jq -c '.processors | .[]' matrix.json | sed 's/\"//g') -SINKS=$(jq -c '.sinks | .[]' matrix.json | sed 's/\"//g') -SOURCES=$(jq -c '.sources | .[]' matrix.json | sed 's/\"//g') +ALL_MODULES=$(find . -name "pom.xml" -type f -exec dirname '{}' \; | sed 's/\.\///' | sort -r) MODULES= for file in $MODIFIED; do FILE=$(realpath $file) echo "$file was changed" - for app in $FUNCTIONS; do - ITEM="functions/function/$app" - if [[ "$file" == *"$ITEM"* ]]; then - echo "Matched $ITEM" - HAS_ITEM=$(itemInModules) - if ((HAS_ITEM == 0)); then - echo "Add:$ITEM" - MODULES=$(addItem "$ITEM") - fi - fi - done - for app in $CONSUMERS; do - ITEM="functions/consumer/$app" - if [[ "$file" == *"$ITEM"* ]]; then - echo "Matched $ITEM" - HAS_ITEM=$(itemInModules) - if ((HAS_ITEM == 0)); then - echo "Add:$ITEM" - MODULES=$(addItem "$ITEM") - fi - fi - done - for app in $SUPPLIERS; do - ITEM="functions/supplier/$app" - if [[ "$file" == *"$ITEM"* ]]; then - echo "Matched $ITEM" - HAS_ITEM=$(itemInModules) - if ((HAS_ITEM == 0)); then - echo "Add:$ITEM" - MODULES=$(addItem "$ITEM") - fi - fi - done - for app in $PROCESSORS; do - ITEM="applications/processor/$app" - if [[ "$file" == *"$ITEM"* ]]; then - echo "Matched $ITEM" - HAS_ITEM=$(itemInModules) - if ((HAS_ITEM == 0)); then - echo "Add:$ITEM" - MODULES=$(addItem "$ITEM") - fi - fi - done - for app in $SINKS; do - ITEM="applications/sink/$app" - if [[ "$file" == *"$ITEM"* ]]; then - echo "Matched $ITEM" - HAS_ITEM=$(itemInModules) - if ((HAS_ITEM == 0)); then - echo "Add:$ITEM" - MODULES=$(addItem "$ITEM") - fi - fi - done - for app in $SOURCES; do - ITEM="applications/source/$app" - if [[ "$file" == *"$ITEM"* ]]; then + for ITEM in $ALL_MODULES; do + if [[ "$ITEM" != "." ]] && [[ "$file" == *"$ITEM"* ]]; then echo "Matched $ITEM" HAS_ITEM=$(itemInModules) if ((HAS_ITEM == 0)); then echo "Add:$ITEM" MODULES=$(addItem "$ITEM") fi + break fi done done @@ -104,4 +44,3 @@ for module in $MODULES; do echo "\"$module\"" >> modules.json done echo "]" >>modules.json -