Refine CI-PR to only build modified modules.

.github/workflows/ci-pr.yml

@@ -50,7 +50,7 @@ jobs:
             maven-repo-
       - name: 'Configure: Install xsltproc'
         uses: ./.github/actions/install-xsltproc
-      - name: 'Action: build initial dependencies'
+      - name: 'Action: verify changed modules'
         shell: bash
         env:
           VERBOSE: ${{ github.debug && 'true' || '' }}
@@ -58,39 +58,43 @@ jobs:
           PR=$(echo "${{ github.ref_name }}" | grep -o '[[:digit:]]*')
           echo "Checking out pull request #$PR"
           gh pr checkout $PR
-          BUILD_DIR=$(realpath $MAIN_PATH)                    
-          echo "::notice ::building - stream-applications - core"
-          set +e
-          $BUILD_DIR/build-folder.sh stream-applications-build,functions/common,functions/spring-functions-parent,applications/stream-applications-core install -DskipTests
-          RC=$?
-          if ((RC!=0)); then
-            exit $RC
-          fi
-          echo "::notice ::core build completed"
-      - name: 'Action: verify changed modules'
-        shell: bash
-        env:
-          VERBOSE: ${{ github.debug && 'true' || '' }}
-        run: |
           BUILD_DIR=$(realpath $MAIN_PATH)
-          $BUILD_DIR/create-matrices.sh
           echo "Changed files:"
           gh pr diff --name-only
           CHANGED=$(gh pr diff --name-only)
           $BUILD_DIR/scripts/determine-modules-changed.sh "$CHANGED"
           MODULES=$(jq '.[]' modules.json | sed 's/\"//g')
           FOLDERS=
-          for module in $MODULES; do            
-             if [ "$FOLDERS" = "" ]; then
-              FOLDERS="$module"
+          ROOT_MODULES=
+          MAVEN_OPTS="-s .settings.xml -B -T 0.3C"
+          for module in $MODULES; do
+            if [[ "$module" == *"/"* ]]; then            
+              if [ "$FOLDERS" = "" ]; then
+                FOLDERS="$module"
+              else
+                FOLDERS="$FOLDERS,$module"
+              fi
             else
-              FOLDERS="$FOLDERS,$module"
+              if [ "$ROOT_MODULES" == "" ]; then
+                ROOT_MODULES=$module
+              else
+                ROOT_MODULES="$ROOT_MODULES,$module"
+              fi
             fi
           done
-          echo "::info ::verify $FOLDERS"
           set +e
+          if [ "$ROOT_MODULES" != "" ]; then
+              # Module with no / are probable a root module that contains other modules and will trigger a lot of dependant builds without real value
+              echo "::info ::building $ROOT_MODULES"
+              ./mvnw $MAVEN_OPTS -pl $ROOT_MODULES install
+              RC=$?
+              if ((RC!=0)); then
+                exit $RC
+              fi
+          fi
           if [ "$FOLDERS" != "" ]; then
-            $BUILD_DIR/build-folder.sh $FOLDERS install verify -amd
+            echo "::info ::verify $FOLDERS and dependents"
+            ./mvnw $MAVEN_OPTS -pl $FOLDERS install -amd
             RC=$?
             if ((RC!=0)); then
               exit $RC

scripts/build-modified-modules.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+function itemInModules() {
+  local e
+  for e in ${MODULES}; do
+    if [[ "$e" == "$ITEM" ]]; then
+      echo "1"
+      return 0
+    fi
+  done
+  echo "0"
+}
+function addItem() {
+    if [ "$MODULES" == "" ]; then
+      echo "$1"
+    else
+      echo "$MODULES $1"
+    fi
+}
+MODIFIED="$*"
+ALL_MODULES=$(find . -name "pom.xml" -type f -exec dirname '{}' \; | sed 's/\.\///' | sort -r)
+MODULES=
+for file in $MODIFIED; do
+  FILE=$(realpath $file)
+  echo "$file was changed"
+  for ITEM in $ALL_MODULES; do
+    if [[ "$ITEM" != "." ]] && [[ "$file" == *"$ITEM"* ]]; then
+      echo "Matched $ITEM"
+      HAS_ITEM=$(itemInModules)
+      if ((HAS_ITEM == 0)); then
+        echo "Add:$ITEM"
+        MODULES=$(addItem "$ITEM")
+      fi
+      break
+    fi
+  done
+done
+
+MVN_MODULES=
+
+for module in $MODULES; do
+  if [[ "$module" == *"/"* ]]; then
+    if [ "$MVN_MODULES" == "" ]; then
+      MVN_MODULES=$module
+    else
+      MVN_MODULES="$MVN_MODULES,$module"
+    fi
+  else
+    ./mvnw install -pl $module
+  fi
+done
+if [ "$MVN_MODULES" != "" ]; then
+  echo "Building $MVN_MODULES"
+  ./mvnw install -pl "$MVN_MODULES" -amd
+fi

scripts/determine-modules-changed.sh

@@ -17,80 +17,20 @@ function addItem() {
     fi
 }
 MODIFIED="$*"
-FUNCTIONS=$(jq -c '.functions | .[]' matrix.json | sed 's/\"//g')
-CONSUMERS=$(jq -c '.consumers | .[]' matrix.json | sed 's/\"//g')
-SUPPLIERS=$(jq -c '.suppliers | .[]' matrix.json | sed 's/\"//g')
-PROCESSORS=$(jq -c '.processors | .[]' matrix.json | sed 's/\"//g')
-SINKS=$(jq -c '.sinks | .[]' matrix.json | sed 's/\"//g')
-SOURCES=$(jq -c '.sources | .[]' matrix.json | sed 's/\"//g')
+ALL_MODULES=$(find . -name "pom.xml" -type f -exec dirname '{}' \; | sed 's/\.\///' | sort -r)
 MODULES=
 for file in $MODIFIED; do
   FILE=$(realpath $file)
   echo "$file was changed"
-  for app in $FUNCTIONS; do
-    ITEM="functions/function/$app"
-    if [[ "$file" == *"$ITEM"* ]]; then
-      echo "Matched $ITEM"
-      HAS_ITEM=$(itemInModules)
-      if ((HAS_ITEM == 0)); then
-        echo "Add:$ITEM"
-        MODULES=$(addItem "$ITEM")
-      fi
-    fi
-  done
-  for app in $CONSUMERS; do
-    ITEM="functions/consumer/$app"
-    if [[ "$file" == *"$ITEM"* ]]; then
-      echo "Matched $ITEM"
-      HAS_ITEM=$(itemInModules)
-      if ((HAS_ITEM == 0)); then
-        echo "Add:$ITEM"
-        MODULES=$(addItem "$ITEM")
-      fi
-    fi
-  done
-  for app in $SUPPLIERS; do
-    ITEM="functions/supplier/$app"
-    if [[ "$file" == *"$ITEM"* ]]; then
-      echo "Matched $ITEM"
-      HAS_ITEM=$(itemInModules)
-      if ((HAS_ITEM == 0)); then
-        echo "Add:$ITEM"
-        MODULES=$(addItem "$ITEM")
-      fi
-    fi
-  done
-  for app in $PROCESSORS; do
-    ITEM="applications/processor/$app"
-    if [[ "$file" == *"$ITEM"* ]]; then
-      echo "Matched $ITEM"
-      HAS_ITEM=$(itemInModules)
-      if ((HAS_ITEM == 0)); then
-        echo "Add:$ITEM"
-        MODULES=$(addItem "$ITEM")
-      fi
-    fi
-  done
-  for app in $SINKS; do
-    ITEM="applications/sink/$app"
-    if [[ "$file" == *"$ITEM"* ]]; then
-      echo "Matched $ITEM"
-      HAS_ITEM=$(itemInModules)
-      if ((HAS_ITEM == 0)); then
-        echo "Add:$ITEM"
-        MODULES=$(addItem "$ITEM")
-      fi
-    fi
-  done
-  for app in $SOURCES; do
-    ITEM="applications/source/$app"
-    if [[ "$file" == *"$ITEM"* ]]; then
+  for ITEM in $ALL_MODULES; do
+    if [[ "$ITEM" != "." ]] && [[ "$file" == *"$ITEM"* ]]; then
       echo "Matched $ITEM"
       HAS_ITEM=$(itemInModules)
       if ((HAS_ITEM == 0)); then
         echo "Add:$ITEM"
         MODULES=$(addItem "$ITEM")
       fi
+      break
     fi
   done
 done
@@ -104,4 +44,3 @@ for module in $MODULES; do
   echo "\"$module\"" >> modules.json
 done
 echo "]" >>modules.json
-