diff --git a/.gitignore b/.gitignore index 137c178..7cbc838 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ spring-*/src/main/java/META-INF/MANIFEST.MF **/.idea/* rebel.xml **/.vscode/ + +dataflow-migrate-schedules/manifest.yml + diff --git a/dataflow-migrate-schedules/.mvn/wrapper/MavenWrapperDownloader.java b/dataflow-migrate-schedules/.mvn/wrapper/MavenWrapperDownloader.java new file mode 100644 index 0000000..da7c339 --- /dev/null +++ b/dataflow-migrate-schedules/.mvn/wrapper/MavenWrapperDownloader.java @@ -0,0 +1,118 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.util.Properties; + +public class MavenWrapperDownloader { + + /** + * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. + */ + private static final String DEFAULT_DOWNLOAD_URL = + "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar"; + + /** + * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to + * use instead of the default one. + */ + private static final String MAVEN_WRAPPER_PROPERTIES_PATH = + ".mvn/wrapper/maven-wrapper.properties"; + + /** + * Path where the maven-wrapper.jar will be saved to. + */ + private static final String MAVEN_WRAPPER_JAR_PATH = + ".mvn/wrapper/maven-wrapper.jar"; + + /** + * Name of the property which should be used to override the default download url for the wrapper. + */ + private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; + + public static void main(String args[]) { + System.out.println("- Downloader started"); + File baseDirectory = new File(args[0]); + System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); + + // If the maven-wrapper.properties exists, read it and check if it contains a custom + // wrapperUrl parameter. + File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); + String url = DEFAULT_DOWNLOAD_URL; + if (mavenWrapperPropertyFile.exists()) { + FileInputStream mavenWrapperPropertyFileInputStream = null; + try { + mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); + Properties mavenWrapperProperties = new Properties(); + mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); + url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); + } + catch (IOException e) { + System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); + } + finally { + try { + if (mavenWrapperPropertyFileInputStream != null) { + mavenWrapperPropertyFileInputStream.close(); + } + } + catch (IOException e) { + // Ignore ... + } + } + } + System.out.println("- Downloading from: : " + url); + + File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); + if (!outputFile.getParentFile().exists()) { + if (!outputFile.getParentFile().mkdirs()) { + System.out.println( + "- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'"); + } + } + System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); + try { + downloadFileFromURL(url, outputFile); + System.out.println("Done"); + System.exit(0); + } + catch (Throwable e) { + System.out.println("- Error downloading"); + e.printStackTrace(); + System.exit(1); + } + } + + private static void downloadFileFromURL(String urlString, File destination) throws Exception { + URL website = new URL(urlString); + ReadableByteChannel rbc; + rbc = Channels.newChannel(website.openStream()); + FileOutputStream fos = new FileOutputStream(destination); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + fos.close(); + rbc.close(); + } + +} diff --git a/dataflow-migrate-schedules/.mvn/wrapper/maven-wrapper.jar b/dataflow-migrate-schedules/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000..01e6799 Binary files /dev/null and b/dataflow-migrate-schedules/.mvn/wrapper/maven-wrapper.jar differ diff --git a/dataflow-migrate-schedules/.mvn/wrapper/maven-wrapper.properties b/dataflow-migrate-schedules/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..cd0d451 --- /dev/null +++ b/dataflow-migrate-schedules/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1 @@ +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.0/apache-maven-3.6.0-bin.zip diff --git a/dataflow-migrate-schedules/README.adoc b/dataflow-migrate-schedules/README.adoc new file mode 100644 index 0000000..8e3f3a2 --- /dev/null +++ b/dataflow-migrate-schedules/README.adoc @@ -0,0 +1,125 @@ += Cloud Scheduler Migration + +The purpose of this project is to migrate existing schedules created with Spring +Cloud Data Flow 2.2.x and before to the new 2.3.0 format and stage the +SchedulerTaskLauncher. This is a Spring Boot application that utilizes Spring Batch to create a workflow +to migrate the schedules. This is a single step Spring Batch Job that does the following: + +* Read - Retrieves all schedules from scheduler. + +* Process - Enriches the Schedule request with App and Deployer properties from the scheduler (or deployed app) +as well as data from the TaskDefinition. + +* Write - Deploys artifacts if required and creates the new schedule. Once the migrated +schedule has been created, the old schedule is destroyed. + +In the case that the application fails with an exception. You can re-execute the +application and it will pick up where it left off. Thanks Spring Batch! :-) + +== Build the project + +=== Services Required +In order to migrate the existing schedules to the 2.3.x Spring Cloud Data Flow Scheduling format, the Schedule Migrator requires the following services: + +1. Access to the database where Spring Cloud Data Flow stores its Task Definitions. For Cloud Foundry we need to bind the database to the Schedule Migrator. +2. Access to the Scheduling Agent. For Cloud Foundry we need to bind the PCF Scheduler to the Scheduler Migrator. + +=== Running the maven command + +``` +./mvnw clean package +``` + +== Running The Project For Cloud Foundry + +=== Prerequisites + +Spring Cloud Data Flow 2.3+ must be installed and running prior to launching the Cloud Scheduler Migration app. + +=== Launching your migration +1) Create a manifest.yml file in a work directory. +``` +--- +applications: +- name: schedulemigrator + host: schedulemigrator + memory: 1G + disk_quota: 1G + instances: 0 + path: + env: + SPRING_APPLICATION_NAME: schedulemigrator + spring_cloud_deployer_cloudfoundry_url: + spring_cloud_deployer_cloudfoundry_org: + spring_cloud_deployer_cloudfoundry_space: + spring_cloud_deployer_cloudfoundry_username: + spring_cloud_deployer_cloudfoundry_password: + spring_cloud_deployer_cloudfoundry_skipSslValidation: + spring_cloud_deployer_cloudfoundry_services: + spring_cloud_scheduler_cloudfoundry_schedulerUrl: + spring_profiles_active: cf + spring.cloud.deployer.cloudfoundry.healthCheckTimeout: 300 + spring.cloud.deployer.cloudfoundry.apiTimeout: 300 + dataflowServerUri: + spring_cloud_task_closecontextEnabled: true + remoteRepositories_repo1_url: https://repo.spring.io/libs-snapshot + services: + - + - +``` +2) From the command line use the cf cli to log into your org and space for which you will migrate your schedules +``` +cf login -a +``` +-or if you need to skip ssl validation- +``` +cf login -a --skip-ssl-validation +``` + +3) Now push the schedulemigrator from the directory where the manifest.yml is present: +``` +cf push +``` + +3) To start the migration: +From the `dataflow-migrate-schedules` directory launch the `runMigration.sh` using the commands below: +``` +chmod +x scripts/runMigration.sh +./scripts/runMigration.sh +``` +=== Picking which schedules to migrate +Use the `scheduleNamesToMigrate` property to specify a comma delimited list of +the schedules you wish to migrate. If you don't specify this property +all schedules will be migrated. For example: +``` +./scripts/runMigration.sh --scheduleNamesToMigrate=task-job3,task-job1 +``` + +=== Limiting one Scheduler to run at a time +If there is a requirement that only one `schedulemigrator` should run at a time you can set the `spring.cloud.task.single-instance-enabled` property to true. This will stop other executions of the schedulemigrator till the currently running instance completes. +To enable this feature use the `runMigration.sh` script as follows. +``` +./scripts/runMigration.sh --spring.cloud.task.single-instance-enabled=true +``` + +=== Configuring Your Deployer Properties +The following deployer properties will affect all schedules to be migrated. +If a property is not set then the default will be used. + +==== Deployer properties to be applied to all migrated schedules: +* healthCheckTimeout +* apiTimeout +* statusTimeout +* stagingTimeout +* startupTimeout +* maximumConcurrentTasks +* javaOpts + +NOTE: Descriptions of these properties can be found : https://github.com/cppwfs/spring-cloud-dataflow-samples/blob/SCDF-121/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/MigrateProperties.java[here] + +=== Supported Databases +The database supported are enumerated https://docs.spring.io/spring-cloud-dataflow/docs/current/reference/htmlsingle/#configuration-local-rdbms[here]. + +=== Previously Pushed Apps +The Cloud Schedule Migration app does not delete previously scheduled applications. +If these apps are no longer needed it is up to the user to delete them. diff --git a/dataflow-migrate-schedules/mvnw b/dataflow-migrate-schedules/mvnw new file mode 100755 index 0000000..8b9da3b --- /dev/null +++ b/dataflow-migrate-schedules/mvnw @@ -0,0 +1,286 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + wget "$jarUrl" -O "$wrapperJarPath" + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + curl -o "$wrapperJarPath" "$jarUrl" + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/dataflow-migrate-schedules/mvnw.cmd b/dataflow-migrate-schedules/mvnw.cmd new file mode 100644 index 0000000..fef5a8f --- /dev/null +++ b/dataflow-migrate-schedules/mvnw.cmd @@ -0,0 +1,161 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" +FOR /F "tokens=1,2 delims==" %%A IN (%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties) DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + echo Found %WRAPPER_JAR% +) else ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + powershell -Command "(New-Object Net.WebClient).DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')" + echo Finished downloading %WRAPPER_JAR% +) +@REM End of extension + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/dataflow-migrate-schedules/pom.xml b/dataflow-migrate-schedules/pom.xml new file mode 100644 index 0000000..7af61b8 --- /dev/null +++ b/dataflow-migrate-schedules/pom.xml @@ -0,0 +1,133 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.2.1.RELEASE + + + io.spring + migrateschedule + 1.0.0.BUILD-SNAPSHOT + Schedule Migrator + Migrates SCDF Schedules to the 2.3 format + + + 1.8 + 2.3.0.RC1 + 2.1.0.RELEASE + + + + + + org.springframework.cloud + spring-cloud-task-dependencies + 2.2.1.RELEASE + pom + import + + + + + + + org.springframework.boot + spring-boot-starter + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + org.springframework.cloud + spring-cloud-deployer-spi + ${deployer.version} + + + org.springframework.cloud + spring-cloud-deployer-resource-maven + ${deployer.version} + + + org.springframework.boot + spring-boot-starter-batch + + + org.springframework.batch + spring-batch-test + test + + + org.springframework.cloud + spring-cloud-starter-task + + + + com.h2database + h2 + + + org.mariadb.jdbc + mariadb-java-client + + + org.springframework.cloud + spring-cloud-deployer-cloudfoundry + ${deployer.version} + + + org.springframework.cloud + spring-cloud-deployer-resource-docker + ${deployer.version} + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.cloud + spring-cloud-dataflow-core + ${spring-cloud-data-flow.version} + + + org.springframework.cloud + spring-cloud-dataflow-registry + ${spring-cloud-data-flow.version} + + + org.assertj + assertj-core + test + + + org.springframework.integration + spring-integration-core + + + org.springframework.integration + spring-integration-jdbc + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + diff --git a/dataflow-migrate-schedules/scripts/runMigration.sh b/dataflow-migrate-schedules/scripts/runMigration.sh new file mode 100755 index 0000000..b4901cd --- /dev/null +++ b/dataflow-migrate-schedules/scripts/runMigration.sh @@ -0,0 +1,2 @@ +#!/bin/bash +cf rt schedulemigrator "JAVA_OPTS=\"-agentpath:\$PWD/.java-buildpack/open_jdk_jre/bin/jvmkill-1.16.0_RELEASE=printHeapHistogram=1 -Djava.io.tmpdir=\$TMPDIR -XX:ActiveProcessorCount=\$(nproc) -Djava.ext.dirs=\$PWD/.java-buildpack/container_security_provider:\$PWD/.java-buildpack/open_jdk_jre/lib/ext -Djava.security.properties=\$PWD/.java-buildpack/java_security/java.security \$JAVA_OPTS\" && CALCULATED_MEMORY=\$(\$PWD/.java-buildpack/open_jdk_jre/bin/java-buildpack-memory-calculator-3.13.0_RELEASE -totMemory=\$MEMORY_LIMIT -loadedClasses=26092 -poolType=metaspace -stackThreads=250 -vmOptions=\"\$JAVA_OPTS\") && echo JVM Memory Configuration: \$CALCULATED_MEMORY && JAVA_OPTS=\"\$JAVA_OPTS \$CALCULATED_MEMORY\" && MALLOC_ARENA_MAX=2 SERVER_PORT=\$PORT eval exec \$PWD/.java-buildpack/open_jdk_jre/bin/java \$JAVA_OPTS -cp \$PWD/. org.springframework.boot.loader.JarLauncher $*" diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/MigrateScheduleApplication.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/MigrateScheduleApplication.java new file mode 100644 index 0000000..6818f37 --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/MigrateScheduleApplication.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class MigrateScheduleApplication { + + public static void main(String[] args) { + SpringApplication.run(MigrateScheduleApplication.class, args); + } + +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/batch/SchedulerProcessor.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/batch/SchedulerProcessor.java new file mode 100644 index 0000000..2cb6e62 --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/batch/SchedulerProcessor.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.batch; + +import io.spring.migrateschedule.service.ConvertScheduleInfo; +import io.spring.migrateschedule.service.MigrateProperties; +import io.spring.migrateschedule.service.MigrateScheduleService; +import io.spring.migrateschedule.service.ScheduleProcessedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.batch.item.ItemProcessor; +import org.springframework.cloud.deployer.spi.scheduler.ScheduleInfo; + +/** + * Enriches the {@link ConvertScheduleInfo} with information obtained from the platform. + * The new name for the schedule is established and the properties as well as commandline args + * so that the SchedulerTaskLauncher can process the entries. + * + * @author Glenn Renfro + */ +public class SchedulerProcessor implements ItemProcessor{ + + private static final Logger logger = LoggerFactory.getLogger(SchedulerProcessor.class); + + private MigrateScheduleService migrateScheduleService; + + private MigrateProperties migrateProperties; + + public SchedulerProcessor(MigrateScheduleService migrateScheduleService, MigrateProperties migrateProperties) { + this.migrateScheduleService = migrateScheduleService; + this.migrateProperties = migrateProperties; + } + + @Override + public ConvertScheduleInfo process(ConvertScheduleInfo scheduleInfo){ + if(scheduleInfo.getScheduleName().contains(migrateProperties.getSchedulerToken())) { + throw new ScheduleProcessedException(scheduleInfo.getScheduleName()); + } + logger.info(String.format("Processing Schedule %s", scheduleInfo.getScheduleName())); + return this.migrateScheduleService.enrichScheduleMetadata(scheduleInfo); + } +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/batch/SchedulerReader.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/batch/SchedulerReader.java new file mode 100644 index 0000000..b673799 --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/batch/SchedulerReader.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.batch; + +import java.util.List; + +import io.spring.migrateschedule.service.ConvertScheduleInfo; +import io.spring.migrateschedule.service.MigrateScheduleService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.batch.item.support.AbstractItemCountingItemStreamItemReader; +import org.springframework.cloud.deployer.spi.scheduler.ScheduleInfo; +import org.springframework.util.Assert; + +/** + * Retrieves all of the available schedules. + * + * @author Glenn Renfro + */ +public class SchedulerReader extends AbstractItemCountingItemStreamItemReader { + + private static final Logger logger = LoggerFactory.getLogger(SchedulerReader.class); + + private List scheduleInfoList; + + private MigrateScheduleService migrateScheduleService; + + public SchedulerReader(MigrateScheduleService migrateScheduleService) { + Assert.notNull(migrateScheduleService, "convertScheduleService must not be null"); + logger.info("Retrieving schedules from PCF Scheduler"); + this.migrateScheduleService = migrateScheduleService; + } + + @Override + protected ConvertScheduleInfo doRead(){ + return this.scheduleInfoList.get(this.getCurrentItemCount()-1); + } + + @Override + protected void doOpen() { + this.scheduleInfoList = migrateScheduleService.scheduleInfoList(); + this.setMaxItemCount(this.scheduleInfoList.size()); + this.setName("scheduler-reader"); + } + + @Override + protected void doClose(){ + + } +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/batch/SchedulerWriter.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/batch/SchedulerWriter.java new file mode 100644 index 0000000..e480552 --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/batch/SchedulerWriter.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.batch; + +import java.util.List; + +import io.spring.migrateschedule.service.ConvertScheduleInfo; +import io.spring.migrateschedule.service.MigrateScheduleService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.batch.item.ItemWriter; +import org.springframework.cloud.deployer.spi.scheduler.ScheduleInfo; +import org.springframework.cloud.deployer.spi.scheduler.Scheduler; + +/** + * Migrates the original schedule to the new scheduler format required for SCDF + * and stages the SchedulerTaskLauncher. + * + * @author Glenn Renfro + */ +public class SchedulerWriter implements ItemWriter { + + private static final Logger logger = LoggerFactory.getLogger(SchedulerWriter.class); + + private Scheduler scheduler; + + private MigrateScheduleService scheduleService; + + public SchedulerWriter (MigrateScheduleService scheduleService, Scheduler scheduler) { + this.scheduleService = scheduleService; + this.scheduler = scheduler; + } + + @Override + public void write(List list) { + for(ConvertScheduleInfo scheduleInfo : list) { + logger.info(String.format("Migrating Schedule %s ", scheduleInfo.getScheduleName())); + this.scheduleService.migrateSchedule(this.scheduler, scheduleInfo); + logger.info(String.format("Migrated Schedule %s ", scheduleInfo.getScheduleName())); + }; + } +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/configuration/BatchConfiguration.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/configuration/BatchConfiguration.java new file mode 100644 index 0000000..8c7754f --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/configuration/BatchConfiguration.java @@ -0,0 +1,100 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.configuration; + +import io.spring.migrateschedule.service.ConvertScheduleInfo; +import io.spring.migrateschedule.service.MigrateScheduleService; +import io.spring.migrateschedule.service.MigrateProperties; +import io.spring.migrateschedule.batch.SchedulerProcessor; +import io.spring.migrateschedule.batch.SchedulerReader; +import io.spring.migrateschedule.batch.SchedulerWriter; +import io.spring.migrateschedule.service.ScheduleProcessedException; +import io.spring.migrateschedule.service.SchedulerSkipPolicy; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.deployer.spi.scheduler.ScheduleInfo; +import org.springframework.cloud.deployer.spi.scheduler.Scheduler; +import org.springframework.cloud.task.configuration.EnableTask; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +/** + * @author Glenn Renfro + */ +@Configuration +@EnableBatchProcessing +@EnableTask +public class BatchConfiguration { + + + @Autowired + public JobBuilderFactory jobBuilderFactory; + + @Autowired + public StepBuilderFactory stepBuilderFactory; + + @Bean + public Job importUserJob(Step migrationStep) { + return this.jobBuilderFactory.get("migrationJob") + .incrementer(new RunIdIncrementer()) + .start(migrationStep) + .build(); + } + + @Bean + public Step migrationStep(SchedulerReader itemReader, + SchedulerProcessor schedulerProcessor, SchedulerWriter writer) { + return this.stepBuilderFactory.get("migrationStep") + . chunk(1) + .reader(itemReader) + .processor(schedulerProcessor) + .writer(writer) + .faultTolerant() + .skip(ScheduleProcessedException.class) + .skipPolicy(new SchedulerSkipPolicy()) + .build(); + } + + @Bean + public SchedulerReader itemReader(MigrateScheduleService scheduler) { + return new SchedulerReader(scheduler); + } + + @Bean + public SchedulerWriter itemWriter(Scheduler scheduler, MigrateScheduleService scheduleService) { + return new SchedulerWriter(scheduleService, scheduler); + } + + @Bean + public SchedulerProcessor itemProcessor(MigrateScheduleService migrateScheduleService, MigrateProperties migrateProperties) { + return new SchedulerProcessor(migrateScheduleService, migrateProperties); + } + + @Bean + @ConfigurationProperties + public MigrateProperties converterProperties() { + return new MigrateProperties(); + } +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/configuration/CFMigrateScheduleConfiguration.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/configuration/CFMigrateScheduleConfiguration.java new file mode 100644 index 0000000..9cc5780 --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/configuration/CFMigrateScheduleConfiguration.java @@ -0,0 +1,90 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.configuration; + +import io.pivotal.reactor.scheduler.ReactorSchedulerClient; +import io.pivotal.scheduler.SchedulerClient; +import io.spring.migrateschedule.service.CFMigrateSchedulerService; +import io.spring.migrateschedule.service.MigrateProperties; +import io.spring.migrateschedule.service.MigrateScheduleService; +import io.spring.migrateschedule.service.TaskDefinitionRepository; +import org.cloudfoundry.operations.CloudFoundryOperations; +import org.cloudfoundry.reactor.ConnectionContext; +import org.cloudfoundry.reactor.TokenProvider; +import reactor.core.publisher.Mono; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.deployer.resource.maven.MavenProperties; +import org.springframework.cloud.deployer.spi.cloudfoundry.CloudFoundryConnectionProperties; +import org.springframework.cloud.deployer.spi.cloudfoundry.CloudFoundryTaskLauncher; +import org.springframework.cloud.deployer.spi.scheduler.cloudfoundry.CloudFoundryAppScheduler; +import org.springframework.cloud.deployer.spi.scheduler.cloudfoundry.CloudFoundrySchedulerProperties; +import org.springframework.cloud.deployer.spi.task.TaskLauncher; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; + +/** + * @author Glenn Renfro + */ +@Configuration +@EntityScan({ + "org.springframework.cloud.dataflow.core" +}) +public class CFMigrateScheduleConfiguration { + + @Bean + public ReactorSchedulerClient reactorSchedulerClient(ConnectionContext context, + TokenProvider passwordGrantTokenProvider, + CloudFoundrySchedulerProperties properties) { + return ReactorSchedulerClient.builder() + .connectionContext(context) + .tokenProvider(passwordGrantTokenProvider) + .root(Mono.just(properties.getSchedulerUrl())) + .build(); + } + + @Bean + public CloudFoundrySchedulerProperties cloudFoundrySchedulerProperties() { + return new CloudFoundrySchedulerProperties(); + } + + @Bean + public CFMigrateSchedulerService scheduleService(CloudFoundryOperations cloudFoundryOperations, + SchedulerClient schedulerClient, + CloudFoundryConnectionProperties properties, MigrateProperties migrateProperties, + TaskDefinitionRepository taskDefinitionRepository, MavenProperties mavenProperties) { + return new CFMigrateSchedulerService(cloudFoundryOperations, + schedulerClient, properties, migrateProperties, taskDefinitionRepository, mavenProperties); + } + + @Bean + public CloudFoundryAppScheduler scheduler(SchedulerClient client, CloudFoundryOperations operations, + CloudFoundryConnectionProperties properties, TaskLauncher taskLauncher, + CloudFoundrySchedulerProperties schedulerProperties) { + return new CloudFoundryAppScheduler(client, operations, properties, (CloudFoundryTaskLauncher) taskLauncher, schedulerProperties); + } + + @Bean + @ConfigurationProperties + public MavenProperties mavenProperties() { + return new MavenProperties(); + } + +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/AbstractMigrateService.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/AbstractMigrateService.java new file mode 100644 index 0000000..7077f79 --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/AbstractMigrateService.java @@ -0,0 +1,170 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.service; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import org.springframework.cloud.dataflow.core.TaskDefinition; +import org.springframework.cloud.dataflow.registry.support.AppResourceCommon; +import org.springframework.cloud.deployer.resource.maven.MavenProperties; +import org.springframework.core.io.DefaultResourceLoader; +import org.springframework.core.io.Resource; +import org.springframework.util.StringUtils; + +/** + * Abstract class containing methods that will be required for both Cloud Foundry + * and the Kubernetes Impls. + * + * @author Glenn Renfro + */ +public abstract class AbstractMigrateService implements MigrateScheduleService { + + private final static String DATA_FLOW_URI_KEY = "spring.cloud.dataflow.client.serverUri"; + + private final static String COMMAND_ARGUMENT_PREFIX = "cmdarg."; + + protected final static String APP_PREFIX = "app."; + + protected final static String DEPLOYER_PREFIX = "deployer."; + + protected MigrateProperties migrateProperties; + + private TaskDefinitionRepository taskDefinitionRepository; + + private MavenProperties mavenProperties; + + public AbstractMigrateService(MigrateProperties migrateProperties, TaskDefinitionRepository taskDefinitionRepository, MavenProperties mavenProperties) { + this.migrateProperties = migrateProperties; + this.taskDefinitionRepository = taskDefinitionRepository; + this.mavenProperties = mavenProperties; + } + + public TaskDefinition findTaskDefinitionByName(String taskDefinitionName) { + return this.taskDefinitionRepository.findByTaskName(taskDefinitionName); + } + + protected String getSchedulePrefixDefinitionName(String taskDefinitionName) { + return this.migrateProperties.getSchedulerToken() + taskDefinitionName; + } + + /** + * Retain only properties that are meant for the scheduler of a given task(those + * that start with {@code scheduler.}and qualify all + * property values with the {@code spring.cloud.scheduler.} prefix. + * + * @param input the scheduler properties + * @return scheduler properties for the task + */ + protected static Map extractAndQualifySchedulerProperties(Map input) { + final String prefix = "spring.cloud.scheduler."; + + Map result = new TreeMap<>(input).entrySet().stream() + .filter(kv -> kv.getKey().startsWith(prefix)) + .collect(Collectors.toMap(kv -> kv.getKey(), kv -> kv.getValue(), + (fromWildcard, fromApp) -> fromApp)); + + return result; + } + + /** + * Retrieve the resource for the SchedulerTaskLauncher and verify the URI. + * @return {@link Resource} for the SchedulerTaskLauncher. + */ + protected Resource getTaskLauncherResource() { + final URI url; + try { + new URI(this.migrateProperties.getSchedulerTaskLauncherUrl()); //verify url + } + catch (URISyntaxException uriSyntaxException) { + throw new IllegalArgumentException(uriSyntaxException); + } + AppResourceCommon appResourceCommon = new AppResourceCommon(this.mavenProperties, new DefaultResourceLoader()); + return appResourceCommon.getResource(this.migrateProperties.getSchedulerTaskLauncherUrl()); + } + + /** + * Add the appropriate tags to the command line args so that the SchedulerTaskLauncher can + * extract them. + * @param args the command line args to be tagged. + * @return the tagged command line args. + */ + protected List tagCommandLineArgs(List args) { + List taggedArgs = new ArrayList<>(); + + for(String arg : args) { + if(arg.contains("spring.cloud.task.name")) { + continue; + } + String updatedArg = arg; + if (!arg.startsWith(DATA_FLOW_URI_KEY) && !"--".concat(arg).startsWith(DATA_FLOW_URI_KEY)) { + updatedArg = COMMAND_ARGUMENT_PREFIX + + this.migrateProperties.getTaskLauncherPrefix() + "." + arg; + } + taggedArgs.add(updatedArg); + } + return taggedArgs; + } + + /** + * Add the appropriate tags to the command line args so that the SchedulerTaskLauncher can + * extract them. + * @param appName the name of the application to be associated with the property + * @param appProperties the properties to be tagged + * @param prefix the prefix to mark the property as to be used by the SchedulerTaskLauncher. + * @return the tagged command line args. + */ + protected Map tagProperties(String appName, Map appProperties, String prefix) { + Map taggedAppProperties = new HashMap<>(appProperties.size()); + + for(String key : appProperties.keySet()) { + if(key.contains("spring.cloud.task.name")) { + continue; + } + String updatedKey = key; + if (!key.startsWith(DATA_FLOW_URI_KEY)) { + if (StringUtils.hasText(appName)) { + updatedKey = this.migrateProperties.getTaskLauncherPrefix() + "." + + prefix + appName + "." + key; + } + else { + updatedKey = this.migrateProperties.getTaskLauncherPrefix() + "." + + prefix + key; + } + } + taggedAppProperties.put(updatedKey, appProperties.get(key)); + } + return taggedAppProperties; + } + + /** + * Add the required SchedulerTaskLauncher properties. + * @param properties the map of properties in which to add the SchedulerTaskLauncher properties. + * @return updated properties. + */ + protected Map addSchedulerAppProps(Map properties) { + Map appProperties = new HashMap<>(properties); + appProperties.put("spring.cloud.dataflow.client.serverUri", this.migrateProperties.getDataflowServerUri()); + return appProperties; + } +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/CFMigrateSchedulerService.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/CFMigrateSchedulerService.java new file mode 100644 index 0000000..0495ceb --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/CFMigrateSchedulerService.java @@ -0,0 +1,402 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.pivotal.scheduler.SchedulerClient; +import io.pivotal.scheduler.v1.jobs.ListJobsRequest; +import io.pivotal.scheduler.v1.jobs.ListJobsResponse; +import org.cloudfoundry.operations.CloudFoundryOperations; +import org.cloudfoundry.operations.applications.ApplicationEnvironments; +import org.cloudfoundry.operations.applications.ApplicationManifest; +import org.cloudfoundry.operations.applications.ApplicationSummary; +import org.cloudfoundry.operations.applications.GetApplicationEnvironmentsRequest; +import org.cloudfoundry.operations.applications.GetApplicationManifestRequest; +import org.cloudfoundry.operations.applications.Route; +import org.cloudfoundry.operations.spaces.SpaceSummary; +import org.codehaus.plexus.util.cli.CommandLineUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import org.springframework.cloud.dataflow.core.TaskDefinition; +import org.springframework.cloud.deployer.resource.maven.MavenProperties; +import org.springframework.cloud.deployer.spi.cloudfoundry.CloudFoundryConnectionProperties; +import org.springframework.cloud.deployer.spi.core.AppDefinition; +import org.springframework.cloud.deployer.spi.scheduler.ScheduleRequest; +import org.springframework.cloud.deployer.spi.scheduler.Scheduler; +import org.springframework.cloud.deployer.spi.scheduler.SchedulerException; +import org.springframework.cloud.deployer.spi.scheduler.SchedulerPropertyKeys; +import org.springframework.util.StringUtils; + +/** + * Services required to migrate schedules to the 2.3.0 format in Cloud Foundry + * and stage the SchedulerTaskLauncher. + * + * @author Glenn Renfro + */ +public class CFMigrateSchedulerService extends AbstractMigrateService { + + public static final String JAR_LAUNCHER = "org.springframework.boot.loader.JarLauncher"; + + private static final int JAR_LAUNCHER_LENGTH = JAR_LAUNCHER.length(); + + private static final Logger logger = LoggerFactory.getLogger(CFMigrateSchedulerService.class); + + private static final String CLOUD_FOUNDRY_PREFIX = "cloudfoundry"; + + private final static int PCF_PAGE_START_NUM = 1; //First PageNum for PCFScheduler starts at 1. + + private final static String SCHEDULER_SERVICE_ERROR_MESSAGE = "Scheduler Service returned a null response."; + + private CloudFoundryOperations cloudFoundryOperations; + + private SchedulerClient schedulerClient; + + private CloudFoundryConnectionProperties properties; + + + public CFMigrateSchedulerService(CloudFoundryOperations cloudFoundryOperations, + SchedulerClient schedulerClient, + CloudFoundryConnectionProperties properties, MigrateProperties migrateProperties, + TaskDefinitionRepository taskDefinitionRepository, MavenProperties mavenProperties) { + super(migrateProperties, taskDefinitionRepository, mavenProperties); + this.cloudFoundryOperations = cloudFoundryOperations; + this.schedulerClient = schedulerClient; + this.properties = properties; + } + + @Override + public List scheduleInfoList() { + List result = new ArrayList<>(); + int pageCount = getJobPageCount(); + for (int i = PCF_PAGE_START_NUM; i <= pageCount; i++) { + logger.info(String.format("Reading Schedules Page %s of %s ", i, pageCount )); + List scheduleInfoPage = getSchedules(i); + if(scheduleInfoPage == null) { + throw new SchedulerException(SCHEDULER_SERVICE_ERROR_MESSAGE); + } + result.addAll(scheduleInfoPage); + } + Collections.sort(result); + return result; + } + + public List getSchedules(int page) { + Flux applicationSummaries = cacheAppSummaries(); + return this.getSpace(this.properties.getSpace()).flatMap(requestSummary -> { + return this.schedulerClient.jobs().list(ListJobsRequest.builder() + .spaceId(requestSummary.getId()) + .page(page) + .detailed(true).build()); + }) + .flatMapIterable(jobs -> jobs.getResources())// iterate over the resources returned. + .flatMap(job -> { + return getApplication(applicationSummaries, + job.getApplicationId()) // get the application name for each job. + .map(optionalApp -> { + ConvertScheduleInfo scheduleInfo = new ConvertScheduleInfo(); + scheduleInfo.setScheduleProperties(new HashMap<>()); + scheduleInfo.setScheduleName(job.getName()); + scheduleInfo.setTaskDefinitionName(optionalApp.getName()); + String jobCommandLine = job.getCommand(); + String commandArgs = ""; + if (jobCommandLine != null && jobCommandLine.length() > JAR_LAUNCHER_LENGTH) { + int locationOfArgs = job.getCommand().indexOf(JAR_LAUNCHER) + JAR_LAUNCHER_LENGTH; + commandArgs = job.getCommand().substring(locationOfArgs); + } + else { + logger.warn(String.format("Job %s does not have commandArgs associated with it.", job.getName())); + } + if (StringUtils.hasText(commandArgs)) { + try { + scheduleInfo.setCommandLineArgs(Arrays.asList(CommandLineUtils.translateCommandline(commandArgs))); + } + catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + if (job.getJobSchedules() != null) { + scheduleInfo.getScheduleProperties().put(SchedulerPropertyKeys.CRON_EXPRESSION, + job.getJobSchedules().get(0).getExpression()); + } + else { + logger.warn(String.format("Job %s does not have an associated schedule", job.getName())); + } + return scheduleInfo; + }); + }) + .filter(job -> isScheduleMigratable(job.getScheduleName())) + .collectList().block(); + } + + private boolean isScheduleMigratable(String scheduleName) { + boolean result; + if(migrateProperties.getScheduleNamesToMigrate().size() > 0) { + result = migrateProperties.getScheduleNamesToMigrate().contains(scheduleName); + } + else { + result = true; + } + return result; + } + + @Override + public ConvertScheduleInfo enrichScheduleMetadata(ConvertScheduleInfo scheduleInfo) { + logger.info(String.format("Retrieving Properties from application %s for schedule %s", scheduleInfo.getTaskDefinitionName(), scheduleInfo.getScheduleName())); + ApplicationEnvironments environment = this.cloudFoundryOperations.applications(). + getEnvironments(GetApplicationEnvironmentsRequest.builder(). + name(scheduleInfo.getTaskDefinitionName()). + build()). + block(); + + logger.info(String.format("Retrieving ApplicationManifest for application %s for schedule %s", scheduleInfo.getTaskDefinitionName(), scheduleInfo.getScheduleName())); + ApplicationManifest applicationManifest = getApplicationManifest(scheduleInfo.getTaskDefinitionName()); + if(applicationManifest != null) { + addApplicationManifestPropsToConvertScheduleInfo(scheduleInfo, applicationManifest); + } + if (environment != null) { + for (Map.Entry var : environment.getUserProvided().entrySet()) { + scheduleInfo.getScheduleProperties().put(var.getKey(), (String) var.getValue()); + } + } + logger.info(String.format("Tagging command line args for application %s for schedule %s", scheduleInfo.getTaskDefinitionName(), scheduleInfo.getScheduleName())); + List revisedCommandLineArgs = tagCommandLineArgs(scheduleInfo.getCommandLineArgs()); + revisedCommandLineArgs.add("--spring.cloud.scheduler.task.launcher.taskName=" + scheduleInfo.getTaskDefinitionName()); + scheduleInfo.setCommandLineArgs(revisedCommandLineArgs); + Map appProperties = null; + try { + logger.info(String.format("Extracting Spring App Properties for application %s for schedule %s", scheduleInfo.getTaskDefinitionName(), scheduleInfo.getScheduleName())); + appProperties = getSpringAppProperties(scheduleInfo.getScheduleProperties()); + if(appProperties.size() > 0) { + scheduleInfo.setUseSpringApplicationJson(true); + } + } + catch (Exception exception) { + throw new IllegalArgumentException("Unable to parse SPRING_APPLICATION_JSON from USER VARIABLES", exception); + } + logger.info(String.format("Retrieving Task Definition for application %s for schedule %s", scheduleInfo.getTaskDefinitionName(), scheduleInfo.getScheduleName())); + TaskDefinition taskDefinition = findTaskDefinitionByName(appProperties.get("spring.cloud.task.name")); + if (appProperties.size() > 0 && taskDefinition == null) { + throw new IllegalStateException(String.format("The schedule %s contains " + + "properties but the task definition %s does not exist and thus can't be migrated", + scheduleInfo.getScheduleName(), scheduleInfo.getTaskDefinitionName())); + } + logger.info(String.format("Tagging app properties for application %s for schedule %s", scheduleInfo.getTaskDefinitionName(), scheduleInfo.getScheduleName())); + appProperties = tagProperties(taskDefinition.getRegisteredAppName(), appProperties, APP_PREFIX); + Map deployerProperties = tagProperties(taskDefinition.getRegisteredAppName(), + getDeployerProperties(scheduleInfo), DEPLOYER_PREFIX); + appProperties = addSchedulerAppProps(appProperties); + appProperties.putAll(deployerProperties); + scheduleInfo.setAppProperties(appProperties); + return scheduleInfo; + } + + @Override + public void migrateSchedule(Scheduler scheduler, ConvertScheduleInfo scheduleInfo) { + String scheduleName = scheduleInfo.getScheduleName() + "-" + getSchedulePrefixDefinitionName(scheduleInfo.getTaskDefinitionName()); + AppDefinition appDefinition = new AppDefinition(scheduleName, scheduleInfo.getAppProperties()); + logger.info(String.format("Extracting schedule specific properties for schedule %s", scheduleInfo.getScheduleName())); + Map schedulerProperties = extractAndQualifySchedulerProperties(scheduleInfo.getScheduleProperties()); + ScheduleRequest scheduleRequest = new ScheduleRequest(appDefinition, schedulerProperties, new HashMap<>(), scheduleInfo.getCommandLineArgs(), scheduleName, getTaskLauncherResource()); + logger.info(String.format("Staging ScheduleTaskLauncher and scheduling %s", scheduleInfo.getScheduleName())); + scheduler.schedule(scheduleRequest); + logger.info(String.format("Unscheduling original %s", scheduleInfo.getScheduleName())); + scheduler.unschedule(scheduleInfo.getScheduleName()); + } + + /** + * Retrieves the number of pages that can be returned when retrieving a list of jobs. + * @return an int containing the number of available pages. + */ + private int getJobPageCount() { + ListJobsResponse response = this.getSpace(this.properties.getSpace()).flatMap(requestSummary -> { + return this.schedulerClient.jobs().list(ListJobsRequest.builder() + .spaceId(requestSummary.getId()) + .detailed(false).build()); + }).block(); + if(response == null) { + throw new SchedulerException(SCHEDULER_SERVICE_ERROR_MESSAGE); + } + return response.getPagination().getTotalPages(); + } + + private Map getSpringAppProperties(Map properties) throws Exception { + Map result; + if(properties.containsKey("SPRING_APPLICATION_JSON")) { + result = new ObjectMapper() + .readValue(properties.get("SPRING_APPLICATION_JSON"), Map.class); + } + else { + result = new HashMap<>(); + } + return result; + } + + /** + * Retrieve a {@link Mono} containing a {@link SpaceSummary} for the specified name. + * + * @param spaceName the name of space to search. + * @return the {@link SpaceSummary} associated with the spaceName. + */ + private Mono getSpace(String spaceName) { + return requestSpaces() + .cache() //cache results from first call. + .filter(space -> spaceName.equals(space.getName())) + .singleOrEmpty() + .cast(SpaceSummary.class); + } + + /** + * Retrieve a {@link Flux} containing the available {@link SpaceSummary}s. + * + * @return {@link Flux} of {@link SpaceSummary}s. + */ + private Flux requestSpaces() { + return this.cloudFoundryOperations.spaces() + .list(); + } + + /** + * Retrieve a cached {@link Flux} of {@link ApplicationSummary}s. + */ + private Flux cacheAppSummaries() { + return requestListApplications() + .cache(); //cache results from first call. No need to re-retrieve each time. + } + + /** + * Retrieve a {@link Flux} of {@link ApplicationSummary}s. + */ + private Flux requestListApplications() { + return this.cloudFoundryOperations.applications() + .list(); + } + + /** + * Retrieve a {@link Mono} containing the {@link ApplicationSummary} associated with the appId. + * + * @param applicationSummaries {@link Flux} of {@link ApplicationSummary}s to filter. + * @param appId the id of the {@link ApplicationSummary} to search. + */ + private Mono getApplication(Flux applicationSummaries, + String appId) { + return applicationSummaries + .filter(application -> appId.equals(application.getId())) + .singleOrEmpty(); + } + private ApplicationManifest getApplicationManifest(String appName) { + return this.cloudFoundryOperations.applications() + .getApplicationManifest(GetApplicationManifestRequest + .builder().name(appName).build()) + .block(); + } + + private Map getDeployerProperties(ConvertScheduleInfo scheduleInfo) { + Map result = new HashMap<>(); + if (scheduleInfo.getJavaBuildPack() != null) { + result.put(CLOUD_FOUNDRY_PREFIX + ".buildpack", scheduleInfo.getJavaBuildPack()); + } + if (scheduleInfo.getMemoryInMB() != null) { + result.put(CLOUD_FOUNDRY_PREFIX + ".memory", scheduleInfo.getMemoryInMB() + "m"); + } + if (scheduleInfo.getDiskInMB() != null) { + result.put(CLOUD_FOUNDRY_PREFIX + ".disk", scheduleInfo.getDiskInMB() + "m"); + } + if (scheduleInfo.getApplicationHealthCheck() != null) { + result.put(CLOUD_FOUNDRY_PREFIX + ".health-check", scheduleInfo.getApplicationHealthCheck().getValue()); + } + if (scheduleInfo.getHealthCheckEndPoint() != null) { + result.put(CLOUD_FOUNDRY_PREFIX + ".health-check-http-endpoint", scheduleInfo.getHealthCheckEndPoint()); + } + if (scheduleInfo.getServices() != null && scheduleInfo.getServices().size() > 0) { + result.put(CLOUD_FOUNDRY_PREFIX + ".services", StringUtils.arrayToCommaDelimitedString(scheduleInfo.getServices().toArray())); + } + if (scheduleInfo.getDomains() != null && scheduleInfo.getDomains().size() > 0) { + result.put(CLOUD_FOUNDRY_PREFIX + ".domain", StringUtils.arrayToCommaDelimitedString(scheduleInfo.getDomains().toArray())); + } + if (scheduleInfo.getRoutes() != null && scheduleInfo.getRoutes().size() > 0) { + result.put(CLOUD_FOUNDRY_PREFIX + ".route-path", StringUtils.arrayToCommaDelimitedString(scheduleInfo.getRoutes().toArray())); + } + if (scheduleInfo.getHosts() != null && scheduleInfo.getHosts().size() > 0) { + result.put(CLOUD_FOUNDRY_PREFIX + ".host", StringUtils.arrayToCommaDelimitedString(scheduleInfo.getHosts().toArray())); + } + + // Global deployer properties; + if (this.migrateProperties.getHealthCheckTimeout() != null) { + result.put(CLOUD_FOUNDRY_PREFIX + ".health-check-timeout", this.migrateProperties.getHealthCheckTimeout()); + } + if (this.migrateProperties.getJavaOptions() != null) { + result.put(CLOUD_FOUNDRY_PREFIX + ".javaOpts", this.migrateProperties.getJavaOptions()); + } + if (this.migrateProperties.getApiTimeout() != null) { + result.put(CLOUD_FOUNDRY_PREFIX + ".api-timeout", String.valueOf(this.migrateProperties.getApiTimeout())); + } + if (this.migrateProperties.getStatusTimeout() != null) { + result.put(CLOUD_FOUNDRY_PREFIX + ".status-timeout", String.valueOf(this.migrateProperties.getStatusTimeout())); + } + if (this.migrateProperties.getStagingTimeout() != null) { + result.put(CLOUD_FOUNDRY_PREFIX + ".staging-timeout", String.valueOf(this.migrateProperties.getStagingTimeout())); + } + if (this.migrateProperties.getStartupTimeout() != null) { + result.put(CLOUD_FOUNDRY_PREFIX + ".startup-timeout", String.valueOf(this.migrateProperties.getStartupTimeout())); + } + if (this.migrateProperties.getMaximumConcurrentTasks() != null) { + result.put(CLOUD_FOUNDRY_PREFIX + ".maximum-concurrent-tasks", String.valueOf(this.migrateProperties.getMaximumConcurrentTasks())); + } + + return result; + } + + private ConvertScheduleInfo addApplicationManifestPropsToConvertScheduleInfo(ConvertScheduleInfo scheduleInfo, ApplicationManifest applicationManifest) { + scheduleInfo.setDiskInMB(applicationManifest.getDisk()); + scheduleInfo.setMemoryInMB(applicationManifest.getMemory()); + scheduleInfo.setApplicationHealthCheck(applicationManifest.getHealthCheckType()); + scheduleInfo.setJavaBuildPack(applicationManifest.getBuildpack()); + scheduleInfo.setHealthCheckEndPoint(applicationManifest.getHealthCheckHttpEndpoint()); + if(applicationManifest.getServices() != null && applicationManifest.getServices().size() > 0) { + scheduleInfo.setServices(applicationManifest.getServices()); + } + if(applicationManifest.getDomains() != null && applicationManifest.getDomains().size() > 0) { + scheduleInfo.setDomains(applicationManifest.getDomains()); + } + if(applicationManifest.getRoutes() != null && applicationManifest.getRoutes().size() > 0) { + List routes = new ArrayList<>(); + for (Route route : applicationManifest.getRoutes()) { + routes.add(route.getRoute()); + } + scheduleInfo.setRoutes(routes); + } + if (applicationManifest.getHosts() != null && applicationManifest.getHosts().size() > 0) { + List hosts = new ArrayList<>(); + for (String host : applicationManifest.getHosts()) { + hosts.add(host); + } + scheduleInfo.setHosts(hosts); + } + + return scheduleInfo; + } +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/ConvertScheduleInfo.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/ConvertScheduleInfo.java new file mode 100644 index 0000000..b1c13a2 --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/ConvertScheduleInfo.java @@ -0,0 +1,183 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.service; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.cloudfoundry.operations.applications.ApplicationHealthCheck; + +import org.springframework.cloud.deployer.spi.scheduler.ScheduleInfo; + +/** + * A child implementation of {@link ScheduleInfo} that adds additional attributes + * required to migrate to the new schedule format. + * + * @author Glenn Renfro + */ +public class ConvertScheduleInfo extends ScheduleInfo implements Comparable{ + + private List commandLineArgs = new ArrayList<>(); + + private String registeredAppName; + + private Map appProperties = new HashMap<>(); + + private Integer diskInMB; + + private Integer memoryInMB; + + private String javaBuildPack; + + private ApplicationHealthCheck applicationHealthCheck; + + private String healthCheckEndPoint; + + private Integer maximumConcurrentTasks = 20; + + private boolean useSpringApplicationJson; + + private List services; + + private List domains; + + private List routes; + + private List hosts; + + public List getCommandLineArgs() { + return commandLineArgs; + } + + public void setCommandLineArgs(List commandLineArgs) { + this.commandLineArgs = commandLineArgs; + } + + public String getRegisteredAppName() { + return registeredAppName; + } + + public void setRegisteredAppName(String registeredAppName) { + this.registeredAppName = registeredAppName; + } + + public Map getAppProperties() { + return appProperties; + } + + public void setAppProperties(Map appProperties) { + this.appProperties = appProperties; + } + + @Override + public int compareTo(Object o) { + if(o instanceof ConvertScheduleInfo) { + return this.getScheduleName().compareTo(((ConvertScheduleInfo) o).getScheduleName()); + } + throw new IllegalArgumentException("Can only compare Objects of type ConvertScheduleInfo"); + } + + public Integer getDiskInMB() { + return diskInMB; + } + + public void setDiskInMB(Integer diskInMB) { + this.diskInMB = diskInMB; + } + + public Integer getMemoryInMB() { + return memoryInMB; + } + + public void setMemoryInMB(Integer memoryInMB) { + this.memoryInMB = memoryInMB; + } + + public String getJavaBuildPack() { + return javaBuildPack; + } + + public void setJavaBuildPack(String javaBuildPack) { + this.javaBuildPack = javaBuildPack; + } + + public ApplicationHealthCheck getApplicationHealthCheck() { + return applicationHealthCheck; + } + + public void setApplicationHealthCheck(ApplicationHealthCheck applicationHealthCheck) { + this.applicationHealthCheck = applicationHealthCheck; + } + + public boolean isUseSpringApplicationJson() { + return useSpringApplicationJson; + } + + public void setUseSpringApplicationJson(boolean useSpringApplicationJson) { + this.useSpringApplicationJson = useSpringApplicationJson; + } + + public String getHealthCheckEndPoint() { + return healthCheckEndPoint; + } + + public void setHealthCheckEndPoint(String healthCheckEndPoint) { + this.healthCheckEndPoint = healthCheckEndPoint; + } + + public Integer getMaximumConcurrentTasks() { + return maximumConcurrentTasks; + } + + public void setMaximumConcurrentTasks(Integer maximumConcurrentTasks) { + this.maximumConcurrentTasks = maximumConcurrentTasks; + } + + public List getServices() { + return services; + } + + public void setServices(List services) { + this.services = services; + } + + public List getDomains() { + return domains; + } + + public void setDomains(List domains) { + this.domains = domains; + } + + public List getRoutes() { + return routes; + } + + public void setRoutes(List routes) { + this.routes = routes; + } + + public List getHosts() { + return hosts; + } + + public void setHosts(List hosts) { + this.hosts = hosts; + } +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/MigrateProperties.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/MigrateProperties.java new file mode 100644 index 0000000..8469169 --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/MigrateProperties.java @@ -0,0 +1,191 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.service; + +import java.util.ArrayList; +import java.util.List; + +/** + * Allows user to configure the migration application. + * + * @author Glenn Renfro + */ +public class MigrateProperties { + private String schedulerTaskLauncherUrl = "maven://org.springframework.cloud:spring-cloud-dataflow-scheduler-task-launcher:2.3.0.BUILD-SNAPSHOT"; + + /** + * The token for the updated schedules. + */ + private String schedulerToken = "scdf-"; + + /** + * The prefix to attach to the application properties to be sent to the SchedulerTaskLauncher. + */ + private String taskLauncherPrefix = "tasklauncher"; + + private String dataflowServerUri = "http://localhost:9393"; + + /** + * The global Java Options required for the applications to be launched by the schedulerTaskLauncher. + */ + private String javaOptions; + + /** + * The global timeout to be assigned to applications to be launched by the schedulerTaskLauncher. + */ + private String healthCheckTimeout; + + /** + * The global api timeout to be assigned to applications to be launched by scheduleTaskLauncher. + */ + private Long apiTimeout; + + /** + * Timeout for status API operations in milliseconds to be assigned to applications to be launched by scheduleTaskLauncher + */ + private Long statusTimeout; + + /** + * If set, the global override the timeout allocated for staging an app launched by the schedulefTaskLauncher. + */ + private Long stagingTimeout; + + /** + * If set, the global override the timeout allocated for starting an app launched by scheduleTaskLauncher. + */ + private Long startupTimeout; + + /** + * If set, the global override for the maximum number of concurrently running tasks. + */ + private Integer maximumConcurrentTasks; + + /** + * The number of seconds to wait for a schedule to complete. + * This excludes the time it takes to stage the application on Cloud Foundry. + */ + private int scheduleTimeoutInSeconds = 30; + + /** + * Comma delimited list of schedules to migrate. If empty then all schedules will be migrated. + */ + private List scheduleNamesToMigrate = new ArrayList<>(); + + public String getSchedulerTaskLauncherUrl() { + return schedulerTaskLauncherUrl; + } + + public void setSchedulerTaskLauncherUrl(String schedulerTaskLauncherUrl) { + this.schedulerTaskLauncherUrl = schedulerTaskLauncherUrl; + } + + public String getSchedulerToken() { + return schedulerToken; + } + + public void setSchedulerToken(String schedulerToken) { + this.schedulerToken = schedulerToken; + } + + public String getTaskLauncherPrefix() { + return taskLauncherPrefix; + } + + public void setTaskLauncherPrefix(String taskLauncherPrefix) { + this.taskLauncherPrefix = taskLauncherPrefix; + } + + public String getDataflowServerUri() { + return dataflowServerUri; + } + + public void setDataflowServerUri(String dataflowServerUri) { + this.dataflowServerUri = dataflowServerUri; + } + + public int getScheduleTimeoutInSeconds() { + return scheduleTimeoutInSeconds; + } + + public void setScheduleTimeoutInSeconds(int scheduleTimeoutInSeconds) { + this.scheduleTimeoutInSeconds = scheduleTimeoutInSeconds; + } + + public String getJavaOptions() { + return javaOptions; + } + + public void setJavaOptions(String javaOptions) { + this.javaOptions = javaOptions; + } + + public String getHealthCheckTimeout() { + return healthCheckTimeout; + } + + public void setHealthCheckTimeout(String healthCheckTimeout) { + this.healthCheckTimeout = healthCheckTimeout; + } + + public Long getApiTimeout() { + return apiTimeout; + } + + public void setApiTimeout(Long apiTimeout) { + this.apiTimeout = apiTimeout; + } + + public Long getStatusTimeout() { + return statusTimeout; + } + + public void setStatusTimeout(Long statusTimeout) { + this.statusTimeout = statusTimeout; + } + + public Long getStagingTimeout() { + return stagingTimeout; + } + + public void setStagingTimeout(Long stagingTimeout) { + this.stagingTimeout = stagingTimeout; + } + + public Long getStartupTimeout() { + return startupTimeout; + } + + public void setStartupTimeout(Long startupTimeout) { + this.startupTimeout = startupTimeout; + } + + public Integer getMaximumConcurrentTasks() { + return maximumConcurrentTasks; + } + + public void setMaximumConcurrentTasks(Integer maximumConcurrentTasks) { + this.maximumConcurrentTasks = maximumConcurrentTasks; + } + + public List getScheduleNamesToMigrate() { + return scheduleNamesToMigrate; + } + + public void setScheduleNamesToMigrate(List scheduleNamesToMigrate) { + this.scheduleNamesToMigrate = scheduleNamesToMigrate; + } +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/MigrateScheduleService.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/MigrateScheduleService.java new file mode 100644 index 0000000..1da6923 --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/MigrateScheduleService.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.service; + +import java.util.List; + +import org.springframework.cloud.dataflow.core.TaskDefinition; +import org.springframework.cloud.deployer.spi.scheduler.ScheduleInfo; +import org.springframework.cloud.deployer.spi.scheduler.Scheduler; + +/** + * Interface that establishes the method signatures required to migrate + * schedules to the 2.3.0 format as well as stage the application. + */ +public interface MigrateScheduleService { + + /** + * Retrieve all available {@link ScheduleInfo}s. + * @return list of available ScheduleInfos + */ + List scheduleInfoList(); + + /** + * Add properties and commandLine args to the {@link ScheduleInfo} + * @return enriched {@link ConvertScheduleInfo} + */ + ConvertScheduleInfo enrichScheduleMetadata(ConvertScheduleInfo scheduleInfo); + + /** + * Migrates existing schedule to new SCDF schedule. + * @param scheduler the deployer scheduler to build the new schedule. + * @param scheduleInfo the schedule info containing the existing schedule. + */ + void migrateSchedule(Scheduler scheduler, ConvertScheduleInfo scheduleInfo); + + /** + * Retrieve {@link TaskDefinition} for the name provided + * @param taskDefinitionName the name of the {@link TaskDefinition}. + * @return a TaskDefinition + */ + TaskDefinition findTaskDefinitionByName(String taskDefinitionName); +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/ScheduleProcessedException.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/ScheduleProcessedException.java new file mode 100644 index 0000000..669a747 --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/ScheduleProcessedException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.service; + +/** + * Thrown if a schedule has already been processed. + * + * @author Glenn Renfro + */ +public class ScheduleProcessedException extends RuntimeException { + public ScheduleProcessedException(String scheduleName) { + super(String.format("Schedule %s has already been migrated", scheduleName)); + } +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/SchedulerSkipPolicy.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/SchedulerSkipPolicy.java new file mode 100644 index 0000000..3905566 --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/SchedulerSkipPolicy.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.service; + +import org.springframework.batch.core.step.skip.SkipLimitExceededException; +import org.springframework.batch.core.step.skip.SkipPolicy; + +/** + * Establish that there is no max maximum skip count if {@link ScheduleProcessedException} is thrown. + * + * @author Glenn Renfro + * + */ +public class SchedulerSkipPolicy implements SkipPolicy { + @Override + public boolean shouldSkip(Throwable t, int skipCount) throws SkipLimitExceededException { + boolean result = false; + if(t instanceof ScheduleProcessedException) { + result = true; + } + return result; + } +} diff --git a/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/TaskDefinitionRepository.java b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/TaskDefinitionRepository.java new file mode 100644 index 0000000..878951a --- /dev/null +++ b/dataflow-migrate-schedules/src/main/java/io/spring/migrateschedule/service/TaskDefinitionRepository.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule.service; + +import org.springframework.cloud.dataflow.core.TaskDefinition; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.PagingAndSortingRepository; +import org.springframework.transaction.annotation.Transactional; + +/** + * Repository to access {@link org.springframework.cloud.dataflow.core.TaskDefinition}s. + * + * @author Michael Minella + * @author Gunnar Hillert + */ +@Transactional +public interface TaskDefinitionRepository extends PagingAndSortingRepository { + + Page findByTaskNameContains(String taskName, Pageable pageable); + + /** + * Performs a findByName query and throws an exception if the name is not found. + * @param name the name of the task definition + * @return The task definition instance or NoSuchTaskDefinitionException if not found. + */ + TaskDefinition findByTaskName(String name); +} diff --git a/dataflow-migrate-schedules/src/main/resources/application.properties b/dataflow-migrate-schedules/src/main/resources/application.properties new file mode 100644 index 0000000..398a1dc --- /dev/null +++ b/dataflow-migrate-schedules/src/main/resources/application.properties @@ -0,0 +1 @@ +logging.level.org.springframework.cloud.task=debug diff --git a/dataflow-migrate-schedules/src/main/resources/banner.txt b/dataflow-migrate-schedules/src/main/resources/banner.txt new file mode 100644 index 0000000..da98c5f --- /dev/null +++ b/dataflow-migrate-schedules/src/main/resources/banner.txt @@ -0,0 +1,6 @@ + ____ ____ ____ _____ ____ _ _ _ __ __ _ _ + / ___| / ___| | _ \ | ___| / ___| ___ | |__ ___ __| | _ _ | | ___ ___ | \/ | (_) __ _ _ __ __ _ | |_ ___ _ __ + \___ \ | | | | | | | |_ \___ \ / __| | '_ \ / _ \ / _` | | | | | | | / _ \ / __| | |\/| | | | / _` | | '__| / _` | | __| / _ \ | '__| + ___) | | |___ | |_| | | _| ___) | | (__ | | | | | __/ | (_| | | |_| | | | | __/ \__ \ | | | | | | | (_| | | | | (_| | | |_ | (_) | | | + |____/ \____| |____/ |_| |____/ \___| |_| |_| \___| \__,_| \__,_| |_| \___| |___/ |_| |_| |_| \__, | |_| \__,_| \__| \___/ |_| + |___/ diff --git a/dataflow-migrate-schedules/src/test/java/io/spring/migrateschedule/AbstractApplications.java b/dataflow-migrate-schedules/src/test/java/io/spring/migrateschedule/AbstractApplications.java new file mode 100644 index 0000000..10d0517 --- /dev/null +++ b/dataflow-migrate-schedules/src/test/java/io/spring/migrateschedule/AbstractApplications.java @@ -0,0 +1,166 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule; + +import org.cloudfoundry.doppler.LogMessage; +import org.cloudfoundry.operations.applications.ApplicationEvent; +import org.cloudfoundry.operations.applications.ApplicationManifest; +import org.cloudfoundry.operations.applications.ApplicationSshEnabledRequest; +import org.cloudfoundry.operations.applications.ApplicationSummary; +import org.cloudfoundry.operations.applications.Applications; +import org.cloudfoundry.operations.applications.CopySourceApplicationRequest; +import org.cloudfoundry.operations.applications.DeleteApplicationRequest; +import org.cloudfoundry.operations.applications.DisableApplicationSshRequest; +import org.cloudfoundry.operations.applications.EnableApplicationSshRequest; +import org.cloudfoundry.operations.applications.GetApplicationEventsRequest; +import org.cloudfoundry.operations.applications.GetApplicationManifestRequest; +import org.cloudfoundry.operations.applications.ListApplicationTasksRequest; +import org.cloudfoundry.operations.applications.LogsRequest; +import org.cloudfoundry.operations.applications.PushApplicationManifestRequest; +import org.cloudfoundry.operations.applications.PushApplicationRequest; +import org.cloudfoundry.operations.applications.RenameApplicationRequest; +import org.cloudfoundry.operations.applications.RestageApplicationRequest; +import org.cloudfoundry.operations.applications.RestartApplicationInstanceRequest; +import org.cloudfoundry.operations.applications.RestartApplicationRequest; +import org.cloudfoundry.operations.applications.RunApplicationTaskRequest; +import org.cloudfoundry.operations.applications.ScaleApplicationRequest; +import org.cloudfoundry.operations.applications.SetApplicationHealthCheckRequest; +import org.cloudfoundry.operations.applications.SetEnvironmentVariableApplicationRequest; +import org.cloudfoundry.operations.applications.StartApplicationRequest; +import org.cloudfoundry.operations.applications.StopApplicationRequest; +import org.cloudfoundry.operations.applications.Task; +import org.cloudfoundry.operations.applications.TerminateApplicationTaskRequest; +import org.cloudfoundry.operations.applications.UnsetEnvironmentVariableApplicationRequest; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public abstract class AbstractApplications implements Applications { + @Override + public Mono copySource(CopySourceApplicationRequest copySourceApplicationRequest) { + return null; + } + + @Override + public Mono delete(DeleteApplicationRequest deleteApplicationRequest) { + return null; + } + + @Override + public Mono disableSsh(DisableApplicationSshRequest disableApplicationSshRequest) { + return null; + } + + @Override + public Mono enableSsh(EnableApplicationSshRequest enableApplicationSshRequest) { + return null; + } + + @Override + public Flux getEvents(GetApplicationEventsRequest getApplicationEventsRequest) { + return null; + } + + @Override + public Flux list() { + return null; + } + + @Override + public Flux listTasks(ListApplicationTasksRequest listApplicationTasksRequest) { + return null; + } + + @Override + public Flux logs(LogsRequest logsRequest) { + return null; + } + + @Override + public Mono push(PushApplicationRequest pushApplicationRequest) { + return null; + } + + @Override + public Mono pushManifest(PushApplicationManifestRequest pushApplicationManifestRequest) { + return null; + } + + @Override + public Mono rename(RenameApplicationRequest renameApplicationRequest) { + return null; + } + + @Override + public Mono restage(RestageApplicationRequest restageApplicationRequest) { + return null; + } + + @Override + public Mono restart(RestartApplicationRequest restartApplicationRequest) { + return null; + } + + @Override + public Mono restartInstance(RestartApplicationInstanceRequest restartApplicationInstanceRequest) { + return null; + } + + @Override + public Mono runTask(RunApplicationTaskRequest runApplicationTaskRequest) { + return null; + } + + @Override + public Mono terminateTask(TerminateApplicationTaskRequest terminateApplicationTaskRequest) { + return null; + } + + @Override + public Mono scale(ScaleApplicationRequest scaleApplicationRequest) { + return null; + } + + @Override + public Mono setEnvironmentVariable(SetEnvironmentVariableApplicationRequest setEnvironmentVariableApplicationRequest) { + return null; + } + + @Override + public Mono setHealthCheck(SetApplicationHealthCheckRequest setApplicationHealthCheckRequest) { + return null; + } + + @Override + public Mono sshEnabled(ApplicationSshEnabledRequest applicationSshEnabledRequest) { + return null; + } + + @Override + public Mono start(StartApplicationRequest startApplicationRequest) { + return null; + } + + @Override + public Mono stop(StopApplicationRequest stopApplicationRequest) { + return null; + } + + @Override + public Mono unsetEnvironmentVariable(UnsetEnvironmentVariableApplicationRequest unsetEnvironmentVariableApplicationRequest) { + return null; + } +} diff --git a/dataflow-migrate-schedules/src/test/java/io/spring/migrateschedule/BatchIntegrationTests.java b/dataflow-migrate-schedules/src/test/java/io/spring/migrateschedule/BatchIntegrationTests.java new file mode 100644 index 0000000..3c4ac1c --- /dev/null +++ b/dataflow-migrate-schedules/src/test/java/io/spring/migrateschedule/BatchIntegrationTests.java @@ -0,0 +1,128 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +import io.spring.migrateschedule.service.ConvertScheduleInfo; +import io.spring.migrateschedule.configuration.BatchConfiguration; +import io.spring.migrateschedule.service.MigrateScheduleService; +import io.spring.migrateschedule.service.TaskDefinitionRepository; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; + +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobInstance; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.explore.JobExplorer; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.batch.test.context.SpringBatchTest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.cloud.deployer.spi.cloudfoundry.CloudFoundryDeployerAutoConfiguration; +import org.springframework.cloud.deployer.spi.scheduler.Scheduler; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestExecutionListeners; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; +import org.springframework.test.context.support.DirtiesContextTestExecutionListener; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(SpringRunner.class) +@SpringBatchTest +@SpringBootTest +@EnableAutoConfiguration(exclude = {CloudFoundryDeployerAutoConfiguration.class}) +@ContextConfiguration(classes = { BatchIntegrationTests.BatchTestConfiguration.class, BatchConfiguration.class}) +@TestExecutionListeners({ DependencyInjectionTestExecutionListener.class, + DirtiesContextTestExecutionListener.class}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +public class BatchIntegrationTests { + + public static final String DEFAULT_SCHEDULE_NAME = "defaultScheduleName"; + public static final String DEFAULT_TASK_DEFINITION_NAME = "defaultTaskDefinitionName"; + public static final String DEFAULT_APP_NAME = "defaultAppName"; + + @Autowired + private JobLauncherTestUtils jobLauncherTestUtils; + + @Autowired + private MigrateScheduleService migrateScheduleService; + + @Autowired + private JobExplorer jobExplorer; + + @MockBean + private TaskDefinitionRepository taskDefinitionRepository; + + @MockBean + private Scheduler scheduler; + + @Test + public void baseTest() throws Exception{ + final ArgumentCaptor schedulerArgumentCaptor = ArgumentCaptor.forClass(Scheduler.class); + final ArgumentCaptor scheduleInfoArgumentCaptor = ArgumentCaptor.forClass(ConvertScheduleInfo.class); + verify(this.migrateScheduleService, times(2)).enrichScheduleMetadata(scheduleInfoArgumentCaptor.capture()); + verify(this.migrateScheduleService, times(2)).migrateSchedule(schedulerArgumentCaptor.capture(), scheduleInfoArgumentCaptor.capture()); + assertThat(this.jobExplorer.getJobInstanceCount("migrationJob")).isEqualTo(1); + JobInstance jobInstance = this.jobExplorer.getJobInstances("migrationJob",0, 1).get(0); + List jobExecutions = this.jobExplorer.getJobExecutions(jobInstance); + assertThat(jobExecutions.size()).isEqualTo(1); + assertThat(jobExecutions.get(0).getExitStatus().getExitCode()).isEqualTo("COMPLETED"); + } + + @Configuration + public static class BatchTestConfiguration { + @Bean + public MigrateScheduleService convertScheduleService() { + MigrateScheduleService migrateScheduleService = mock(MigrateScheduleService.class); + ConvertScheduleInfo scheduleInfo = new ConvertScheduleInfo(); + scheduleInfo.setScheduleName(DEFAULT_SCHEDULE_NAME); + scheduleInfo.setTaskDefinitionName(DEFAULT_TASK_DEFINITION_NAME); + scheduleInfo.setScheduleProperties(new HashMap<>()); + scheduleInfo.setRegisteredAppName(DEFAULT_APP_NAME); + List schedules = new ArrayList<>(); + schedules.add(scheduleInfo); + scheduleInfo = new ConvertScheduleInfo(); + scheduleInfo.setScheduleName(DEFAULT_SCHEDULE_NAME + 1); + scheduleInfo.setTaskDefinitionName(DEFAULT_TASK_DEFINITION_NAME + 1); + scheduleInfo.setScheduleProperties(new HashMap<>()); + scheduleInfo.setRegisteredAppName(DEFAULT_APP_NAME + 1); + schedules.add(scheduleInfo); + when(migrateScheduleService.scheduleInfoList()).thenReturn(schedules); + when(migrateScheduleService.enrichScheduleMetadata(any())).thenReturn(scheduleInfo); + return migrateScheduleService; + } + } +} diff --git a/dataflow-migrate-schedules/src/test/java/io/spring/migrateschedule/CFGetSchedulesTests.java b/dataflow-migrate-schedules/src/test/java/io/spring/migrateschedule/CFGetSchedulesTests.java new file mode 100644 index 0000000..17b5ec7 --- /dev/null +++ b/dataflow-migrate-schedules/src/test/java/io/spring/migrateschedule/CFGetSchedulesTests.java @@ -0,0 +1,290 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule; + + +import java.util.ArrayList; +import java.util.List; + +import io.pivotal.scheduler.SchedulerClient; +import io.pivotal.scheduler.v1.jobs.CreateJobRequest; +import io.pivotal.scheduler.v1.jobs.CreateJobResponse; +import io.pivotal.scheduler.v1.jobs.DeleteJobRequest; +import io.pivotal.scheduler.v1.jobs.DeleteJobScheduleRequest; +import io.pivotal.scheduler.v1.jobs.ExecuteJobRequest; +import io.pivotal.scheduler.v1.jobs.ExecuteJobResponse; +import io.pivotal.scheduler.v1.jobs.GetJobRequest; +import io.pivotal.scheduler.v1.jobs.GetJobResponse; +import io.pivotal.scheduler.v1.jobs.Job; +import io.pivotal.scheduler.v1.jobs.JobSchedule; +import io.pivotal.scheduler.v1.jobs.Jobs; +import io.pivotal.scheduler.v1.jobs.ListJobHistoriesRequest; +import io.pivotal.scheduler.v1.jobs.ListJobHistoriesResponse; +import io.pivotal.scheduler.v1.jobs.ListJobScheduleHistoriesRequest; +import io.pivotal.scheduler.v1.jobs.ListJobScheduleHistoriesResponse; +import io.pivotal.scheduler.v1.jobs.ListJobSchedulesRequest; +import io.pivotal.scheduler.v1.jobs.ListJobSchedulesResponse; +import io.pivotal.scheduler.v1.jobs.ListJobsRequest; +import io.pivotal.scheduler.v1.jobs.ListJobsResponse; +import io.pivotal.scheduler.v1.jobs.ScheduleJobRequest; +import io.pivotal.scheduler.v1.jobs.ScheduleJobResponse; +import io.spring.migrateschedule.service.CFMigrateSchedulerService; +import io.spring.migrateschedule.service.ConvertScheduleInfo; +import io.spring.migrateschedule.service.MigrateProperties; +import io.spring.migrateschedule.service.TaskDefinitionRepository; +import org.cloudfoundry.operations.CloudFoundryOperations; +import org.cloudfoundry.operations.applications.ApplicationDetail; +import org.cloudfoundry.operations.applications.ApplicationEnvironments; +import org.cloudfoundry.operations.applications.ApplicationHealthCheck; +import org.cloudfoundry.operations.applications.ApplicationManifest; +import org.cloudfoundry.operations.applications.ApplicationSummary; +import org.cloudfoundry.operations.applications.GetApplicationEnvironmentsRequest; +import org.cloudfoundry.operations.applications.GetApplicationHealthCheckRequest; +import org.cloudfoundry.operations.applications.GetApplicationManifestRequest; +import org.cloudfoundry.operations.applications.GetApplicationRequest; +import org.cloudfoundry.operations.spaces.AllowSpaceSshRequest; +import org.cloudfoundry.operations.spaces.CreateSpaceRequest; +import org.cloudfoundry.operations.spaces.DeleteSpaceRequest; +import org.cloudfoundry.operations.spaces.DisallowSpaceSshRequest; +import org.cloudfoundry.operations.spaces.GetSpaceRequest; +import org.cloudfoundry.operations.spaces.RenameSpaceRequest; +import org.cloudfoundry.operations.spaces.SpaceDetail; +import org.cloudfoundry.operations.spaces.SpaceSshAllowedRequest; +import org.cloudfoundry.operations.spaces.SpaceSummary; +import org.cloudfoundry.operations.spaces.Spaces; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import org.springframework.cloud.deployer.resource.maven.MavenProperties; +import org.springframework.cloud.deployer.spi.cloudfoundry.CloudFoundryConnectionProperties; +import org.springframework.cloud.deployer.spi.scheduler.Scheduler; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CFGetSchedulesTests { + + private static final String DEFAULT_SPACE = "TESTSPACE"; + private static final String DEFAULT_APPLICATION_ID = "TEST_APPLICATION_ID"; + private static final String DEFAULT_COMMAND_ARG_PROP = "foo=bar"; + private static final String DEFAULT_SCHEDULE_EXPRESSION = "*/1 * ? * *"; + private static final String DEFAULT_COMMAND_ARG = CFMigrateSchedulerService.JAR_LAUNCHER + " " + DEFAULT_COMMAND_ARG_PROP; + + private CFMigrateSchedulerService cfConvertSchedulerService; + private CloudFoundryOperations cloudFoundryOperations; + private CloudFoundryConnectionProperties cloudFoundryConnectionProperties; + private MigrateProperties migrateProperties; + private TaskDefinitionRepository taskDefinitionRepository; + private SchedulerClient schedulerClient; + private Scheduler scheduler; + + @BeforeEach + public void setup() { + this.cloudFoundryOperations = Mockito.mock(CloudFoundryOperations.class); + this.schedulerClient = Mockito.mock(SchedulerClient.class); + this.cloudFoundryConnectionProperties = new CloudFoundryConnectionProperties(); + this.cloudFoundryConnectionProperties.setSpace(DEFAULT_SPACE); + this.migrateProperties = new MigrateProperties(); + this.taskDefinitionRepository = Mockito.mock(TaskDefinitionRepository.class); + this.scheduler = Mockito.mock(Scheduler.class); + this.cfConvertSchedulerService = new CFMigrateSchedulerService(this.cloudFoundryOperations, + this.schedulerClient, + this.cloudFoundryConnectionProperties, this.migrateProperties, + this.taskDefinitionRepository, new MavenProperties()) ; + } + + + @Test + public void testGetSchedules() { + Mockito.when(this.cloudFoundryOperations.applications()).thenReturn(new TestApplications()); + Mockito.when(this.cloudFoundryOperations.spaces()).thenReturn(new TestSpaces()); + Mockito.when(this.schedulerClient.jobs()).thenReturn(new TestJobs()); + List convertScheduleInfos = this.cfConvertSchedulerService.getSchedules(1); + assertThat(convertScheduleInfos.size()).isEqualTo(2); + + ConvertScheduleInfo convertScheduleInfo = convertScheduleInfos.get(0); + baseTests(convertScheduleInfo,"JOB1", DEFAULT_APPLICATION_ID); + assertThat(convertScheduleInfo.getCommandLineArgs().size()).isEqualTo(1); + assertThat(convertScheduleInfo.getCommandLineArgs().get(0)).isEqualTo(DEFAULT_COMMAND_ARG_PROP); + + convertScheduleInfo = convertScheduleInfos.get(1); + assertThat(convertScheduleInfo.getCommandLineArgs().size()).isEqualTo(0); + baseTests(convertScheduleInfo,"JOB2", DEFAULT_APPLICATION_ID); + } + + private void baseTests(ConvertScheduleInfo convertScheduleInfo, String scheduleName, String taskDefinitionName) { + assertThat(convertScheduleInfo.getScheduleProperties().get("spring.cloud.scheduler.cron.expression")).isEqualTo("*/1 * ? * *"); + assertThat(convertScheduleInfo.getScheduleName()).isEqualTo(scheduleName); + assertThat(convertScheduleInfo.getTaskDefinitionName()).isEqualTo(taskDefinitionName); + } + + private static class TestApplications extends AbstractApplications { + @Override + public Mono get(GetApplicationRequest request) { + return null; + } + + @Override + public Mono getApplicationManifest(GetApplicationManifestRequest request) { + return null; + } + + @Override + public Mono getEnvironments(GetApplicationEnvironmentsRequest request) { + return null; + } + + @Override + public Mono getHealthCheck(GetApplicationHealthCheckRequest request) { + return null; + } + + @Override + public Flux list() { + ApplicationSummary applicationSummary = ApplicationSummary.builder(). + id(DEFAULT_APPLICATION_ID). + diskQuota(1024). + instances(1). + memoryLimit(1024). + name(DEFAULT_APPLICATION_ID). + requestedState("GOOD"). + runningInstances(1). + build(); + Flux applicationSummaries = Flux.just(applicationSummary); + return applicationSummaries; + } + } + + private static class TestSpaces implements Spaces { + + @Override + public Mono allowSsh(AllowSpaceSshRequest request) { + return null; + } + + @Override + public Mono create(CreateSpaceRequest request) { + return null; + } + + @Override + public Mono delete(DeleteSpaceRequest request) { + return null; + } + + @Override + public Mono disallowSsh(DisallowSpaceSshRequest request) { + return null; + } + + @Override + public Mono get(GetSpaceRequest request) { + return null; + } + + @Override + public Flux list() { + SpaceSummary spaceSummary = SpaceSummary.builder().id(DEFAULT_SPACE).name(DEFAULT_SPACE).build(); + Flux spaceSummaries = Flux.just(spaceSummary); + return spaceSummaries; + } + + @Override + public Mono rename(RenameSpaceRequest request) { + return null; + } + + @Override + public Mono sshAllowed(SpaceSshAllowedRequest request) { + return null; + } + } + + public static class TestJobs implements Jobs { + + @Override + public Mono create(CreateJobRequest createJobRequest) { + return null; + } + + @Override + public Mono delete(DeleteJobRequest deleteJobRequest) { + return null; + } + + @Override + public Mono deleteSchedule(DeleteJobScheduleRequest deleteJobScheduleRequest) { + return null; + } + + @Override + public Mono execute(ExecuteJobRequest executeJobRequest) { + return null; + } + + @Override + public Mono get(GetJobRequest getJobRequest) { + return null; + } + + @Override + public Mono list(ListJobsRequest listJobsRequest) { + Job job = Job.builder().id("JOB1").name("JOB1"). + command(DEFAULT_COMMAND_ARG). + applicationId(DEFAULT_APPLICATION_ID). + jobSchedule(JobSchedule.builder(). + expression("*/1 * ? * *"). + build()). + build(); + List jobList = new ArrayList<>(); + jobList.add(job); + + job = Job.builder().id("JOB2").name("JOB2"). + applicationId(DEFAULT_APPLICATION_ID). + jobSchedule(JobSchedule.builder(). + expression(DEFAULT_SCHEDULE_EXPRESSION). + build()). + build(); + jobList.add(job); + ListJobsResponse listJobsResponse = ListJobsResponse.builder().addAllResources(jobList).build(); + return Mono.just(listJobsResponse); + } + + @Override + public Mono listHistories(ListJobHistoriesRequest listJobHistoriesRequest) { + return null; + } + + @Override + public Mono listScheduleHistories(ListJobScheduleHistoriesRequest listJobScheduleHistoriesRequest) { + return null; + } + + @Override + public Mono listSchedules(ListJobSchedulesRequest listJobSchedulesRequest) { + return null; + } + + @Override + public Mono schedule(ScheduleJobRequest scheduleJobRequest) { + return null; + } + } + +} diff --git a/dataflow-migrate-schedules/src/test/java/io/spring/migrateschedule/CFMigrateScheduleConfigurationTests.java b/dataflow-migrate-schedules/src/test/java/io/spring/migrateschedule/CFMigrateScheduleConfigurationTests.java new file mode 100644 index 0000000..248cd49 --- /dev/null +++ b/dataflow-migrate-schedules/src/test/java/io/spring/migrateschedule/CFMigrateScheduleConfigurationTests.java @@ -0,0 +1,220 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.spring.migrateschedule; + + +import java.util.HashMap; + +import io.pivotal.scheduler.SchedulerClient; +import io.spring.migrateschedule.service.ConvertScheduleInfo; +import io.spring.migrateschedule.service.MigrateProperties; +import io.spring.migrateschedule.service.CFMigrateSchedulerService; +import io.spring.migrateschedule.service.TaskDefinitionRepository; +import org.cloudfoundry.operations.CloudFoundryOperations; +import org.cloudfoundry.operations.applications.ApplicationDetail; +import org.cloudfoundry.operations.applications.ApplicationEnvironments; +import org.cloudfoundry.operations.applications.ApplicationHealthCheck; +import org.cloudfoundry.operations.applications.ApplicationManifest; +import org.cloudfoundry.operations.applications.GetApplicationEnvironmentsRequest; +import org.cloudfoundry.operations.applications.GetApplicationHealthCheckRequest; +import org.cloudfoundry.operations.applications.GetApplicationManifestRequest; +import org.cloudfoundry.operations.applications.GetApplicationRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import reactor.core.publisher.Mono; + +import org.springframework.cloud.dataflow.core.TaskDefinition; +import org.springframework.cloud.deployer.resource.maven.MavenProperties; +import org.springframework.cloud.deployer.spi.cloudfoundry.CloudFoundryConnectionProperties; +import org.springframework.cloud.deployer.spi.scheduler.ScheduleRequest; +import org.springframework.cloud.deployer.spi.scheduler.Scheduler; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class CFMigrateScheduleConfigurationTests { + + public static final String DEFAULT_SCHEDULE_NAME = "defaultScheduleName"; + public static final String DEFAULT_TASK_DEFINITION_NAME = "defaultTaskDefinitionName"; + public static final String DEFAULT_APP_NAME = "defaultAppName"; + public static final String DEFAULT_CMD_ARG = "defaultCmd=WOW"; + public static final String DEFAULT_BUILD_PACK = "defaultBuildPack"; + + private CFMigrateSchedulerService cfConvertSchedulerService; + private CloudFoundryOperations cloudFoundryOperations; + private CloudFoundryConnectionProperties cloudFoundryConnectionProperties; + private MigrateProperties migrateProperties; + private TaskDefinitionRepository taskDefinitionRepository; + private SchedulerClient schedulerClient; + private Scheduler scheduler; + + @BeforeEach + public void setup() { + this.cloudFoundryOperations = Mockito.mock(CloudFoundryOperations.class); + this.schedulerClient = Mockito.mock(SchedulerClient.class); + this.cloudFoundryConnectionProperties = new CloudFoundryConnectionProperties(); + this.migrateProperties = new MigrateProperties(); + this.taskDefinitionRepository = Mockito.mock(TaskDefinitionRepository.class); + this.scheduler = Mockito.mock(Scheduler.class); + this.cfConvertSchedulerService = new CFMigrateSchedulerService(this.cloudFoundryOperations, + this.schedulerClient, + this.cloudFoundryConnectionProperties, this.migrateProperties, + this.taskDefinitionRepository, new MavenProperties()) ; + + } + + @Test + public void testEnrichment() { + ConvertScheduleInfo scheduleInfo = createFoundationConvertScheduleInfo(); + assertThat(scheduleInfo.getAppProperties().keySet().size()).isEqualTo(5); + assertThat(scheduleInfo.getAppProperties().get("tasklauncher.app.defaultAppName.foo")).isEqualTo("bar"); + assertThat(scheduleInfo.getAppProperties().get("spring.cloud.dataflow.client.serverUri")).isEqualTo("http://localhost:9393"); + assertThat(scheduleInfo.getAppProperties().get("tasklauncher.deployer.defaultAppName.cloudfoundry.memory")).isEqualTo("1024m"); + assertThat(scheduleInfo.getAppProperties().get("tasklauncher.deployer.defaultAppName.cloudfoundry.health-check")).isEqualTo("port"); + assertThat(scheduleInfo.getAppProperties().get("tasklauncher.deployer.defaultAppName.cloudfoundry.disk")).isEqualTo("1024m"); + + assertThat(scheduleInfo.getCommandLineArgs().size()).isEqualTo(2); + assertThat(scheduleInfo.getCommandLineArgs().get(0)).isEqualTo("cmdarg.tasklauncher.defaultCmd=WOW"); + assertThat(scheduleInfo.getCommandLineArgs().get(1)).isEqualTo("--spring.cloud.scheduler.task.launcher.taskName=defaultTaskDefinitionName"); + } + + @Test + public void testEnrichmentNoProps() { + ConvertScheduleInfo scheduleInfo = new ConvertScheduleInfo(); + scheduleInfo.setScheduleName(DEFAULT_SCHEDULE_NAME); + scheduleInfo.setTaskDefinitionName(DEFAULT_TASK_DEFINITION_NAME); + scheduleInfo.setScheduleProperties(new HashMap<>()); + scheduleInfo.setRegisteredAppName(DEFAULT_APP_NAME); + Mockito.when(cloudFoundryOperations.applications()).thenReturn(new NoPropertyApplication()); + TaskDefinition taskDefinition = TaskDefinition.TaskDefinitionBuilder + .from(new TaskDefinition("fooTask", "foo")) + .setTaskName(DEFAULT_TASK_DEFINITION_NAME) + .setRegisteredAppName(DEFAULT_APP_NAME) + .build(); + Mockito.when(this.taskDefinitionRepository.findByTaskName(Mockito.any())).thenReturn(taskDefinition); + scheduleInfo = this.cfConvertSchedulerService.enrichScheduleMetadata(scheduleInfo); + assertThat(scheduleInfo.getAppProperties().keySet().size()).isEqualTo(1); + assertThat(scheduleInfo.getAppProperties().get("spring.cloud.dataflow.client.serverUri")).isEqualTo("http://localhost:9393"); + assertThat(scheduleInfo.getCommandLineArgs().size()).isEqualTo(1); + assertThat(scheduleInfo.getCommandLineArgs().get(0)).isEqualTo("--spring.cloud.scheduler.task.launcher.taskName=defaultTaskDefinitionName"); + } + + @Test + public void testNoTaskDefinition() { + ConvertScheduleInfo scheduleInfo = new ConvertScheduleInfo(); + scheduleInfo.setScheduleName(DEFAULT_SCHEDULE_NAME); + scheduleInfo.setTaskDefinitionName(DEFAULT_TASK_DEFINITION_NAME); + scheduleInfo.setScheduleProperties(new HashMap<>()); + scheduleInfo.setRegisteredAppName(DEFAULT_APP_NAME); + scheduleInfo.getCommandLineArgs().add(DEFAULT_CMD_ARG); + Mockito.when(cloudFoundryOperations.applications()).thenReturn(new SinglePropertyApplication()); + assertThrows(IllegalStateException.class, () -> this.cfConvertSchedulerService.enrichScheduleMetadata(scheduleInfo)); + } + + + @Test + public void testMigrate() { + ConvertScheduleInfo scheduleInfo = createFoundationConvertScheduleInfo(); + this.cfConvertSchedulerService.migrateSchedule(this.scheduler, scheduleInfo); + final ArgumentCaptor scheduleRequestArgument = ArgumentCaptor.forClass(ScheduleRequest.class); + final ArgumentCaptor scheduleNameArg = ArgumentCaptor.forClass(String.class); + verify(this.scheduler, times(1)).schedule(scheduleRequestArgument.capture()); + verify(this.scheduler, times(1)).unschedule(scheduleNameArg.capture()); + assertThat(scheduleRequestArgument.getValue().getScheduleName()).isEqualTo("defaultScheduleName-scdf-defaultTaskDefinitionName"); + assertThat(scheduleNameArg.getValue()).isEqualTo(DEFAULT_SCHEDULE_NAME); + } + + private ConvertScheduleInfo createFoundationConvertScheduleInfo() { + ConvertScheduleInfo scheduleInfo = new ConvertScheduleInfo(); + scheduleInfo.setScheduleName(DEFAULT_SCHEDULE_NAME); + scheduleInfo.setTaskDefinitionName(DEFAULT_TASK_DEFINITION_NAME); + scheduleInfo.setScheduleProperties(new HashMap<>()); + scheduleInfo.setRegisteredAppName(DEFAULT_APP_NAME); + scheduleInfo.getCommandLineArgs().add(DEFAULT_CMD_ARG); + Mockito.when(cloudFoundryOperations.applications()).thenReturn(new SinglePropertyApplication()); + TaskDefinition taskDefinition = TaskDefinition.TaskDefinitionBuilder + .from(new TaskDefinition("fooTask", "foo")) + .setTaskName(DEFAULT_TASK_DEFINITION_NAME) + .setRegisteredAppName(DEFAULT_APP_NAME) + .build(); + Mockito.when(this.taskDefinitionRepository.findByTaskName(Mockito.any())).thenReturn(taskDefinition); + return this.cfConvertSchedulerService.enrichScheduleMetadata(scheduleInfo); + } + + public static class SinglePropertyApplication extends AbstractApplications { + @Override + public Mono get(GetApplicationRequest getApplicationRequest) { + ApplicationDetail applicationDetail = ApplicationDetail.builder() + .buildpack(DEFAULT_BUILD_PACK) + .stack("defaultstack") + .diskQuota(1024) + .id("defaultappidone") + .memoryLimit(1024) + .instances(1) + .name(DEFAULT_TASK_DEFINITION_NAME) + .requestedState("requestedState") + .runningInstances(1) + .build(); + return Mono.just(applicationDetail); + } + + @Override + public Mono getEnvironments(GetApplicationEnvironmentsRequest getApplicationEnvironmentsRequest) { + return Mono.just(ApplicationEnvironments.builder().userProvided("SPRING_APPLICATION_JSON", "{\"foo\":\"bar\"}").build()); + } + @Override + public Mono getHealthCheck(GetApplicationHealthCheckRequest getApplicationHealthCheckRequest) { + return Mono.just(ApplicationHealthCheck.PORT); + } + + @Override + public Mono getApplicationManifest(GetApplicationManifestRequest getApplicationManifestRequest) { + return Mono.just(ApplicationManifest.builder().name(DEFAULT_TASK_DEFINITION_NAME) + .disk(1024) + .memory(1024) + .healthCheckType(ApplicationHealthCheck.PORT) + .build()); + } + + } + public static class NoPropertyApplication extends AbstractApplications { + + @Override + public Mono get(GetApplicationRequest getApplicationRequest) { + return Mono.empty(); + } + + @Override + public Mono getEnvironments(GetApplicationEnvironmentsRequest getApplicationEnvironmentsRequest) { + return Mono.empty(); + } + + @Override + public Mono getHealthCheck(GetApplicationHealthCheckRequest getApplicationHealthCheckRequest) { + return Mono.empty(); + } + + @Override + public Mono getApplicationManifest(GetApplicationManifestRequest getApplicationManifestRequest) { + return Mono.empty(); + } + } +}