Updated build to use packeto buildpacks to create a container. (#384)

.github/actions/install-pack/action.yml

@@ -0,0 +1,9 @@
+name: 'Install Packeto Buildpacks'
+description: 'Install Packeto Buildpacks'
+
+runs:
+  using: composite
+  steps:
+    - name: 'Install Pack'
+      shell: bash
+      run: (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack)

.github/workflows/ci-2021-0-x.yml

@@ -14,5 +14,4 @@ jobs:
       CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }}
       CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }}
     with:
-      version: '2021.0.x'
       branch: '2021.0.x'

.github/workflows/ci-2021-1-x.yml

@@ -14,5 +14,4 @@ jobs:
       CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }}
       CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }}
     with:
-      version: '2021.1.x'
       branch: '2021.1.x'

.github/workflows/ci-main.yml

@@ -19,6 +19,5 @@ jobs:
       CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }}
       CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }}
     with:
-      version: 'main'
       branch: 'main'
       verbose: ${{ inputs.verbose == 'true' }}

.github/workflows/common.yml

@@ -3,11 +3,6 @@ name: 'Stream Applications - Common'
 on:
   workflow_call:
     inputs:
-      version:
-        type: string
-        required: false
-        default: 'main'
-        description: 'Version Tag'
       branch:
         type: string
         required: false
@@ -72,12 +67,21 @@ jobs:
               echo "SOURCES=$SOURCES"
             fi
           popd
+          if [ "${{ inputs.branch" == "main" ]; then
+            echo "JDK_BUILD=17" >> $GITHUB_ENV
+            echo "JRE_DEFAULT=17" >> $GITHUB_ENV
+          else
+            echo "JDK_BUILD=8" >> $GITHUB_ENV
+            echo "JRE_DEFAULT=11" >> $GITHUB_ENV
+          fi
     outputs:
       max_parallel: ${{ env.MAX_PARALLEL }}
       sources: ${{ env.SOURCES }}
       sinks: ${{ env.SINKS }}
       processors: ${{ env.PROCESSORS }}
       matrix: ${{ env.matrix }}
+      jdk_build: ${{ env.JDK_BUILD }}
+      jre_version: ${{ env.JRE_DEFAULT }}
   #  scale-runners:
   #    runs-on: ubuntu-latest
   #    needs:
@@ -184,6 +188,8 @@ jobs:
   #      runner-type: '${{ env.RUNNER_TYPE }}'
   core:
     runs-on: ubuntu-latest
+    needs:
+      - parameters
     steps:
       - name: 'Configure: checkout stream-applications@${{ inputs.branch }}'
         uses: actions/checkout@v3
@@ -195,7 +201,7 @@ jobs:
       - name: 'Configure: Install Java'
         uses: actions/setup-java@v1
         with:
-          java-version: 17
+          java-version: ${{ needs.parameters.outputs.jdk_build }}
       - name: 'Action: build initial dependencies'
         shell: bash
         env:
@@ -240,16 +246,23 @@ jobs:
       - name: 'Configure: Install Java'
         uses: actions/setup-java@v1
         with:
-          java-version: 17
+          java-version: ${{ needs.parameters.outputs.jdk_build }}
+      - name: 'Configure: Install Pack'
+        uses: ./.github/actions/install-pack
       - name: 'Build: ${{ matrix.app }}'
         shell: bash
         env:
-          VERSION: ${{ inputs.version }}
-          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
-          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }}
           CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }}
+          DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }}
         run: ./build-app.sh "stream-applications/applications/processor/${{ matrix.app }}"
+      - name: 'Publish: ${{ matrix.app }}'
+        shell: bash
+        env:
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }}
+        run: ./publish-app.sh "stream-applications/applications/sink/${{ matrix.app }}"
   sinks:
     if: ${{ needs.parameters.outputs.sinks != '' && needs.parameters.outputs.sinks != null }}
     needs:
@@ -277,16 +290,23 @@ jobs:
       - name: 'Configure: Install Java'
         uses: actions/setup-java@v1
         with:
-          java-version: 17
+          java-version: ${{ needs.parameters.outputs.jdk_build }}
+      - name: 'Configure: Install Pack'
+        uses: ./.github/actions/install-pack
       - name: 'Build: ${{ matrix.app }}'
         shell: bash
         env:
-          VERSION: ${{ inputs.version }}
-          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
-          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }}
           CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }}
+          DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }}
         run: ./build-app.sh "stream-applications/applications/sink/${{ matrix.app }}"
+      - name: 'Publish: ${{ matrix.app }}'
+        shell: bash
+        env:
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }}
+        run: ./publish-app.sh "stream-applications/applications/sink/${{ matrix.app }}"
   sources:
     if: ${{ needs.parameters.outputs.sources != '' && needs.parameters.outputs.sources != null }}
     needs:
@@ -315,16 +335,23 @@ jobs:
       - name: 'Configure: Install Java'
         uses: actions/setup-java@v1
         with:
-          java-version: 17
+          java-version: ${{ needs.parameters.outputs.jdk_build }}
+      - name: 'Configure: Install Pack'
+        uses: ./.github/actions/install-pack
       - name: 'Build: ${{ matrix.app }}'
         shell: bash
         env:
-          VERSION: ${{ inputs.version }}
-          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
-          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }}
           CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }}
+          DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }}
         run: ./build-app.sh "stream-applications/applications/source/${{ matrix.app }}"
+      - name: 'Publish: ${{ matrix.app }}'
+        shell: bash
+        env:
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }}
+        run: ./publish-app.sh "stream-applications/applications/sink/${{ matrix.app }}"
 #  scale-down-runners:
 #    if: ${{ success() }}
 #    runs-on: ubuntu-latest

.mvn/wrapper/maven-wrapper.properties

@@ -1,2 +1,2 @@
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.2/apache-maven-3.6.2-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar

README.adoc

@@ -221,13 +221,14 @@ cd applications/sink/log-sink/apps/log-sink-rabbit
 ./mvnw clean package
 ----
 
-Rebuild all container images.
+Rebuild all applications and container images.
 
 [source,shell]
 ----
 ./local/build-images.sh
 ----
 
+
 === Additional Resources
 
 Here is a list of resources where you can find out more about using and developing functions and stream applications:

build-app.sh

@@ -25,7 +25,7 @@ check_env DOCKER_HUB_USERNAME
 check_env DOCKER_HUB_PASSWORD
 check_env CI_DEPLOY_USERNAME
 check_env CI_DEPLOY_PASSWORD
-check_env VERSION
+
 
 ROOT_DIR=$(realpath $PWD)
 if [ "$VERBOSE" == "true" ]; then
@@ -33,13 +33,31 @@ if [ "$VERBOSE" == "true" ]; then
 else
   MAVEN_OPT=-q
 fi
-
+if [ "$VERSION" == "" ]; then
+  VERSIONS=$($ROOT_DIR/mvnw exec:exec -Dexec.executable='echo' -Dexec.args='${project.version}' --non-recursive -q)
+  for v in $VERSIONS; do
+    VERSION=$v
+  done
+fi
+echo "Project Version:$VERSION"
 if [ "$LOCAL" == "true" ]; then
   MAVEN_GOAL="install"
 else
   MAVEN_GOAL="install deploy"
 fi
 
+if [[ "$VERSION" == "4."* ]]; then
+  JDKS="17"
+  if [ "$DEFAULT_JDK" == "" ]; then
+    DEFAULT_JDK=17
+  fi
+else
+  JDKS="8 11 17"
+  if [ "$DEFAULT_JDK" == "" ]; then
+      DEFAULT_JDK=11
+    fi
+fi
+
 pushd $APP_FOLDER > /dev/null
   rm -rf apps
   if [ -d "src/main/java" ]; then
@@ -56,10 +74,23 @@ pushd $APP_FOLDER > /dev/null
   pushd apps > /dev/null
     echo "Building:$APP_FOLDER/apps"
     ./mvnw $MAVEN_OPT $MAVEN_GOAL -U -Pintegration
-    ./mvnw $MAVEN_OPT package jib:build -DskipTests \
-                  -Djib.to.tags="$VERSION" \
-                  -Djib.httpTimeout=1800000 \
-                  -Djib.to.auth.username="$DOCKER_HUB_USERNAME" \
-                  -Djib.to.auth.password="$DOCKER_HUB_PASSWORD"
+#    ./mvnw $MAVEN_OPT package jib:build -DskipTests \
+#                  -Djib.to.tags="$VERSION" \
+#                  -Djib.httpTimeout=1800000 \
+#                  -Djib.to.auth.username="$DOCKER_HUB_USERNAME" \
+#                  -Djib.to.auth.password="$DOCKER_HUB_PASSWORD"
+    APPS=$(find * -maxdepth 0 -type d)
+    for app in $APPS; do
+      for v in $JDKS; do
+        echo "Pack:springcloudstream/$app:$VERSION-jdk$v"
+        pack build \
+          --path "springcloudstream/$app-$VERSION.jar" \
+          --builder gcr.io/paketo-buildpacks/builder:base \
+          --env BP_JVM_VERSION=$v "springcloudstream/$app:$VERSION-jdk$v"
+      done
+      if [ "$DEFAULT_JDK" == "" ]; then
+        docker tag "springcloudstream/$app:$VERSION-jdk$DEFAULT_JDK" "springcloudstream/$app:$VERSION"
+      fi
+    done
   popd > /dev/null
 popd > /dev/null

local/build-images.sh

@@ -11,50 +11,25 @@ then
     exit 2
   fi
 fi
-version="3.2.1-SNAPSHOT"
-rootdir="$(pwd)"
-processors="aggregator bridge filter groovy header-enricher http-request image-recognition object-detection script semantic-segmentation splitter transform twitter-trend"
-echo "Build processors:$processor"
-pushd applications/processor
-  for app in $processors
-  do
-    pushd "${app}-processor"
-      echo "$Building $(pwd)"
-      rm -rf apps
-      $rootdir/mvnw clean package -Pintegration
-      pushd apps
-        $rootdir/mvnw package jib:dockerBuild -DskipTests -Djib.to.tags=${version}
-      popd
-    popd
-  done
-popd
-sinks="analytics cassandra elasticsearch file ftp geode jdbc log mongodb mqtt pgcopy rabbit redis router rsocket s3 sftp tcp throughput twitter-message twitter-update wavefront websocket zeromq"
-echo "Build sinks:$sinks"
-pushd applications/sink
-  for app in $sinks
-  do
-      pushd "${app}-sink"
-        echo "$Building $(pwd)"
-        rm -rf apps
-        $rootdir/mvnw clean package -Pintegration
-        pushd apps
-          $rootdir/mvnw package jib:dockerBuild -DskipTests -Djib.to.tags=${version}
-        popd
-      popd
-  done
-popd
-sources="cdc-debezium file ftp geode http jdbc jms load-generator mail mongodb mqtt rabbit s3 sftp syslog tcp time twitter-message twitter-search twitter-stream websocket zeromq"
-echo "Build sources:sources"
-pushd applications/source
-  for app in $sources
-  do
-      pushd "${app}-source"
-        echo "$Building $(pwd)"
-        rm -rf apps
-        $rootdir/mvnw clean package -Pintegration
-        pushd apps
-          $rootdir/mvnw package jib:dockerBuild -DskipTests -Djib.to.tags=${version}
-        popd
-      popd
-  done
-popd
+rootdir="$(realpath $PWD)"
+if [ "$VERSION" == "" ]; then
+  export VERSION=$($ROOT_DIR/mvnw exec:exec -Dexec.executable='echo' -Dexec.args='${project.version}' --non-recursive -q)
+fi
+pushd applications/processor > /dev/null
+processors=$(find * -maxdepth 0 -type d)
+popd  > /dev/null
+for app in $processors; do
+  $rootdir/build-app.sh "applications/processor/$app"
+done
+pushd applications/sink > /dev/null
+sinks=$(find * -maxdepth 0 -type d)
+popd  > /dev/null
+for app in $sinks; do
+  $rootdir/build-app.sh "applications/sink/$app"
+done
+pushd applications/source > /dev/null
+sources=$(find * -maxdepth 0 -type d)
+popd  > /dev/null
+for app in $sinks; do
+  $rootdir/build-app.sh "applications/source/$app"
+done

publish-app.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+(return 0 2>/dev/null) && sourced=1 || sourced=0
+function check_env() {
+  eval ev='$'$1
+  if [ "$ev" == "" ]; then
+    echo "$1 not defined"
+    if (( sourced != 0 )); then
+      return 1
+    else
+      exit 1
+    fi
+  fi
+}
+if [ "$1" == "" ]; then
+  echo "Argument: application-folder required"
+  if((sourced > 0)); then
+    exit 0
+  else
+    exit 1
+  fi
+fi
+APP_FOLDER=$1
+
+VERSION=$($ROOT_DIR/mvnw exec:exec -Dexec.executable='echo' -Dexec.args='${project.version}' --non-recursive -q)
+echo "Project Version:$VERSION"
+
+if [[ "$VERSION" == "4."* ]]; then
+  JDKS=17
+  if [ "$DEFAULT_JDK" == "" ];then
+      DEFAULT_JDK=17
+    fi
+else
+  if [ "$DEFAULT_JDK" == "" ];then
+    DEFAULT_JDK=11
+  fi
+  JDKS=8 11 17
+fi
+pushd $APP_FOLDER > /dev/null
+  pushd apps > /dev/null
+    echo "Pushing:$APP_FOLDER/apps"
+    BROKERS=$(find * -maxdepth 0 -type d)
+    for broker in $BROKERS; do
+      project="$APP_FOLDER-$broker"
+      docker tag "springcloudstream/$project:$VERSION-jdk$DEFAULT_JDK" "springcloudstream/$project:$VERSION"
+      docker push "springcloudstream/$project:$VERSION-jdk$DEFAULT_JDK"
+      for v in $JDKS; do
+        docker push "springcloudstream/$project:$VERSION-jdk$v"
+      done
+    done
+  popd > /dev/null
+popd > /dev/null