diff --git a/.github/actions/install-pack/action.yml b/.github/actions/install-pack/action.yml new file mode 100644 index 00000000..a9f82f96 --- /dev/null +++ b/.github/actions/install-pack/action.yml @@ -0,0 +1,9 @@ +name: 'Install Packeto Buildpacks' +description: 'Install Packeto Buildpacks' + +runs: + using: composite + steps: + - name: 'Install Pack' + shell: bash + run: (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack) diff --git a/.github/workflows/ci-2021-0-x.yml b/.github/workflows/ci-2021-0-x.yml index 5174a0fc..c0fc7bdc 100644 --- a/.github/workflows/ci-2021-0-x.yml +++ b/.github/workflows/ci-2021-0-x.yml @@ -14,5 +14,4 @@ jobs: CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} with: - version: '2021.0.x' branch: '2021.0.x' diff --git a/.github/workflows/ci-2021-1-x.yml b/.github/workflows/ci-2021-1-x.yml index 75397f92..8268b2f5 100644 --- a/.github/workflows/ci-2021-1-x.yml +++ b/.github/workflows/ci-2021-1-x.yml @@ -14,5 +14,4 @@ jobs: CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} with: - version: '2021.1.x' branch: '2021.1.x' diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index cd02d9e4..ef49a930 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -19,6 +19,5 @@ jobs: CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} with: - version: 'main' branch: 'main' verbose: ${{ inputs.verbose == 'true' }} diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml index d39d59a2..069b10f9 100644 --- a/.github/workflows/common.yml +++ b/.github/workflows/common.yml @@ -3,11 +3,6 @@ name: 'Stream Applications - Common' on: workflow_call: inputs: - version: - type: string - required: false - default: 'main' - description: 'Version Tag' branch: type: string required: false @@ -72,12 +67,21 @@ jobs: echo "SOURCES=$SOURCES" fi popd + if [ "${{ inputs.branch" == "main" ]; then + echo "JDK_BUILD=17" >> $GITHUB_ENV + echo "JRE_DEFAULT=17" >> $GITHUB_ENV + else + echo "JDK_BUILD=8" >> $GITHUB_ENV + echo "JRE_DEFAULT=11" >> $GITHUB_ENV + fi outputs: max_parallel: ${{ env.MAX_PARALLEL }} sources: ${{ env.SOURCES }} sinks: ${{ env.SINKS }} processors: ${{ env.PROCESSORS }} matrix: ${{ env.matrix }} + jdk_build: ${{ env.JDK_BUILD }} + jre_version: ${{ env.JRE_DEFAULT }} # scale-runners: # runs-on: ubuntu-latest # needs: @@ -184,6 +188,8 @@ jobs: # runner-type: '${{ env.RUNNER_TYPE }}' core: runs-on: ubuntu-latest + needs: + - parameters steps: - name: 'Configure: checkout stream-applications@${{ inputs.branch }}' uses: actions/checkout@v3 @@ -195,7 +201,7 @@ jobs: - name: 'Configure: Install Java' uses: actions/setup-java@v1 with: - java-version: 17 + java-version: ${{ needs.parameters.outputs.jdk_build }} - name: 'Action: build initial dependencies' shell: bash env: @@ -240,16 +246,23 @@ jobs: - name: 'Configure: Install Java' uses: actions/setup-java@v1 with: - java-version: 17 + java-version: ${{ needs.parameters.outputs.jdk_build }} + - name: 'Configure: Install Pack' + uses: ./.github/actions/install-pack - name: 'Build: ${{ matrix.app }}' shell: bash env: - VERSION: ${{ inputs.version }} - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} + DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }} run: ./build-app.sh "stream-applications/applications/processor/${{ matrix.app }}" + - name: 'Publish: ${{ matrix.app }}' + shell: bash + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }} + run: ./publish-app.sh "stream-applications/applications/sink/${{ matrix.app }}" sinks: if: ${{ needs.parameters.outputs.sinks != '' && needs.parameters.outputs.sinks != null }} needs: @@ -277,16 +290,23 @@ jobs: - name: 'Configure: Install Java' uses: actions/setup-java@v1 with: - java-version: 17 + java-version: ${{ needs.parameters.outputs.jdk_build }} + - name: 'Configure: Install Pack' + uses: ./.github/actions/install-pack - name: 'Build: ${{ matrix.app }}' shell: bash env: - VERSION: ${{ inputs.version }} - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} + DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }} run: ./build-app.sh "stream-applications/applications/sink/${{ matrix.app }}" + - name: 'Publish: ${{ matrix.app }}' + shell: bash + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }} + run: ./publish-app.sh "stream-applications/applications/sink/${{ matrix.app }}" sources: if: ${{ needs.parameters.outputs.sources != '' && needs.parameters.outputs.sources != null }} needs: @@ -315,16 +335,23 @@ jobs: - name: 'Configure: Install Java' uses: actions/setup-java@v1 with: - java-version: 17 + java-version: ${{ needs.parameters.outputs.jdk_build }} + - name: 'Configure: Install Pack' + uses: ./.github/actions/install-pack - name: 'Build: ${{ matrix.app }}' shell: bash env: - VERSION: ${{ inputs.version }} - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} + DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }} run: ./build-app.sh "stream-applications/applications/source/${{ matrix.app }}" + - name: 'Publish: ${{ matrix.app }}' + shell: bash + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + DEFAULT_JDK: ${{ needs.parameters.outputs.jre_version }} + run: ./publish-app.sh "stream-applications/applications/sink/${{ matrix.app }}" # scale-down-runners: # if: ${{ success() }} # runs-on: ubuntu-latest diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index 7d59a01f..598fb341 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -1,2 +1,2 @@ -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.2/apache-maven-3.6.2-bin.zip -wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar diff --git a/README.adoc b/README.adoc index bd52422e..089a43b1 100644 --- a/README.adoc +++ b/README.adoc @@ -221,13 +221,14 @@ cd applications/sink/log-sink/apps/log-sink-rabbit ./mvnw clean package ---- -Rebuild all container images. +Rebuild all applications and container images. [source,shell] ---- ./local/build-images.sh ---- + === Additional Resources Here is a list of resources where you can find out more about using and developing functions and stream applications: diff --git a/build-app.sh b/build-app.sh index ea85068d..971f19e1 100755 --- a/build-app.sh +++ b/build-app.sh @@ -25,7 +25,7 @@ check_env DOCKER_HUB_USERNAME check_env DOCKER_HUB_PASSWORD check_env CI_DEPLOY_USERNAME check_env CI_DEPLOY_PASSWORD -check_env VERSION + ROOT_DIR=$(realpath $PWD) if [ "$VERBOSE" == "true" ]; then @@ -33,13 +33,31 @@ if [ "$VERBOSE" == "true" ]; then else MAVEN_OPT=-q fi - +if [ "$VERSION" == "" ]; then + VERSIONS=$($ROOT_DIR/mvnw exec:exec -Dexec.executable='echo' -Dexec.args='${project.version}' --non-recursive -q) + for v in $VERSIONS; do + VERSION=$v + done +fi +echo "Project Version:$VERSION" if [ "$LOCAL" == "true" ]; then MAVEN_GOAL="install" else MAVEN_GOAL="install deploy" fi +if [[ "$VERSION" == "4."* ]]; then + JDKS="17" + if [ "$DEFAULT_JDK" == "" ]; then + DEFAULT_JDK=17 + fi +else + JDKS="8 11 17" + if [ "$DEFAULT_JDK" == "" ]; then + DEFAULT_JDK=11 + fi +fi + pushd $APP_FOLDER > /dev/null rm -rf apps if [ -d "src/main/java" ]; then @@ -56,10 +74,23 @@ pushd $APP_FOLDER > /dev/null pushd apps > /dev/null echo "Building:$APP_FOLDER/apps" ./mvnw $MAVEN_OPT $MAVEN_GOAL -U -Pintegration - ./mvnw $MAVEN_OPT package jib:build -DskipTests \ - -Djib.to.tags="$VERSION" \ - -Djib.httpTimeout=1800000 \ - -Djib.to.auth.username="$DOCKER_HUB_USERNAME" \ - -Djib.to.auth.password="$DOCKER_HUB_PASSWORD" +# ./mvnw $MAVEN_OPT package jib:build -DskipTests \ +# -Djib.to.tags="$VERSION" \ +# -Djib.httpTimeout=1800000 \ +# -Djib.to.auth.username="$DOCKER_HUB_USERNAME" \ +# -Djib.to.auth.password="$DOCKER_HUB_PASSWORD" + APPS=$(find * -maxdepth 0 -type d) + for app in $APPS; do + for v in $JDKS; do + echo "Pack:springcloudstream/$app:$VERSION-jdk$v" + pack build \ + --path "springcloudstream/$app-$VERSION.jar" \ + --builder gcr.io/paketo-buildpacks/builder:base \ + --env BP_JVM_VERSION=$v "springcloudstream/$app:$VERSION-jdk$v" + done + if [ "$DEFAULT_JDK" == "" ]; then + docker tag "springcloudstream/$app:$VERSION-jdk$DEFAULT_JDK" "springcloudstream/$app:$VERSION" + fi + done popd > /dev/null popd > /dev/null diff --git a/local/build-images.sh b/local/build-images.sh index 7718631a..597a5fd4 100755 --- a/local/build-images.sh +++ b/local/build-images.sh @@ -11,50 +11,25 @@ then exit 2 fi fi -version="3.2.1-SNAPSHOT" -rootdir="$(pwd)" -processors="aggregator bridge filter groovy header-enricher http-request image-recognition object-detection script semantic-segmentation splitter transform twitter-trend" -echo "Build processors:$processor" -pushd applications/processor - for app in $processors - do - pushd "${app}-processor" - echo "$Building $(pwd)" - rm -rf apps - $rootdir/mvnw clean package -Pintegration - pushd apps - $rootdir/mvnw package jib:dockerBuild -DskipTests -Djib.to.tags=${version} - popd - popd - done -popd -sinks="analytics cassandra elasticsearch file ftp geode jdbc log mongodb mqtt pgcopy rabbit redis router rsocket s3 sftp tcp throughput twitter-message twitter-update wavefront websocket zeromq" -echo "Build sinks:$sinks" -pushd applications/sink - for app in $sinks - do - pushd "${app}-sink" - echo "$Building $(pwd)" - rm -rf apps - $rootdir/mvnw clean package -Pintegration - pushd apps - $rootdir/mvnw package jib:dockerBuild -DskipTests -Djib.to.tags=${version} - popd - popd - done -popd -sources="cdc-debezium file ftp geode http jdbc jms load-generator mail mongodb mqtt rabbit s3 sftp syslog tcp time twitter-message twitter-search twitter-stream websocket zeromq" -echo "Build sources:sources" -pushd applications/source - for app in $sources - do - pushd "${app}-source" - echo "$Building $(pwd)" - rm -rf apps - $rootdir/mvnw clean package -Pintegration - pushd apps - $rootdir/mvnw package jib:dockerBuild -DskipTests -Djib.to.tags=${version} - popd - popd - done -popd +rootdir="$(realpath $PWD)" +if [ "$VERSION" == "" ]; then + export VERSION=$($ROOT_DIR/mvnw exec:exec -Dexec.executable='echo' -Dexec.args='${project.version}' --non-recursive -q) +fi +pushd applications/processor > /dev/null +processors=$(find * -maxdepth 0 -type d) +popd > /dev/null +for app in $processors; do + $rootdir/build-app.sh "applications/processor/$app" +done +pushd applications/sink > /dev/null +sinks=$(find * -maxdepth 0 -type d) +popd > /dev/null +for app in $sinks; do + $rootdir/build-app.sh "applications/sink/$app" +done +pushd applications/source > /dev/null +sources=$(find * -maxdepth 0 -type d) +popd > /dev/null +for app in $sinks; do + $rootdir/build-app.sh "applications/source/$app" +done diff --git a/publish-app.sh b/publish-app.sh new file mode 100644 index 00000000..6703f119 --- /dev/null +++ b/publish-app.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +(return 0 2>/dev/null) && sourced=1 || sourced=0 +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if (( sourced != 0 )); then + return 1 + else + exit 1 + fi + fi +} +if [ "$1" == "" ]; then + echo "Argument: application-folder required" + if((sourced > 0)); then + exit 0 + else + exit 1 + fi +fi +APP_FOLDER=$1 + +VERSION=$($ROOT_DIR/mvnw exec:exec -Dexec.executable='echo' -Dexec.args='${project.version}' --non-recursive -q) +echo "Project Version:$VERSION" + +if [[ "$VERSION" == "4."* ]]; then + JDKS=17 + if [ "$DEFAULT_JDK" == "" ];then + DEFAULT_JDK=17 + fi +else + if [ "$DEFAULT_JDK" == "" ];then + DEFAULT_JDK=11 + fi + JDKS=8 11 17 +fi +pushd $APP_FOLDER > /dev/null + pushd apps > /dev/null + echo "Pushing:$APP_FOLDER/apps" + BROKERS=$(find * -maxdepth 0 -type d) + for broker in $BROKERS; do + project="$APP_FOLDER-$broker" + docker tag "springcloudstream/$project:$VERSION-jdk$DEFAULT_JDK" "springcloudstream/$project:$VERSION" + docker push "springcloudstream/$project:$VERSION-jdk$DEFAULT_JDK" + for v in $JDKS; do + docker push "springcloudstream/$project:$VERSION-jdk$v" + done + done + popd > /dev/null +popd > /dev/null