diff --git a/dataflow-template-load-generator/README.adoc b/dataflow-template-load-generator/README.adoc new file mode 100644 index 0000000..24eaaca --- /dev/null +++ b/dataflow-template-load-generator/README.adoc @@ -0,0 +1,22 @@ += Data Flow Template Load Generator + +This example uses the Spring Cloud Data Flow Template to generator a set of Task- and Batch Executions. + +This examples, will: + +- Import https://cloud.spring.io/spring-cloud-task-app-starters/[Task App Starter applications] +- Import https://cloud.spring.io/spring-cloud-stream-app-starters/[Stream App Starter applications] +- Create several Task Applications +- Execute each Task Application several times + +In order to get everything running we need to setup the following server instances: + +* Spring Cloud Skipper +* Spring Cloud Data Flow + +== Build + Run + +Configure the example application in `DataflowTemplateApplication.java`. Then build +using `$ mvn clean package` and run using +`$ java -jar target/dataflow-template-load-generator-1.0.0.BUILD-SNAPSHOT.jar` + diff --git a/dataflow-template-load-generator/mvnw.cmd b/dataflow-template-load-generator/mvnw.cmd new file mode 100644 index 0000000..86846ae --- /dev/null +++ b/dataflow-template-load-generator/mvnw.cmd @@ -0,0 +1,143 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" + +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/dataflow-template-load-generator/pom.xml b/dataflow-template-load-generator/pom.xml new file mode 100644 index 0000000..06d6cc4 --- /dev/null +++ b/dataflow-template-load-generator/pom.xml @@ -0,0 +1,90 @@ + + + 4.0.0 + + org.springframework.cloud.samples + dataflow-template-load-generator + 1.0.0.BUILD-SNAPSHOT + jar + + ldapserver + Utility app to generate Task and Batch Execution data for Spring Cloud Data Flow + + + org.springframework.boot + spring-boot-starter-parent + 2.1.0.RELEASE + + + + + UTF-8 + UTF-8 + 1.8 + + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud + spring-cloud-dataflow-rest-client + 2.1.0.RELEASE + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/snapshot + + true + + + + spring-milestones + Spring Milestones + https://repo.spring.io/milestone + + false + + + + + + + spring-snapshots + Spring Snapshots + https://repo.spring.io/snapshot + + true + + + + spring-milestones + Spring Milestones + https://repo.spring.io/milestone + + false + + + + diff --git a/dataflow-template-load-generator/src/main/java/org/springframework/cloud/samples/dataflowtemplate/LoadGeneratorApplication.java b/dataflow-template-load-generator/src/main/java/org/springframework/cloud/samples/dataflowtemplate/LoadGeneratorApplication.java new file mode 100644 index 0000000..b3cc386 --- /dev/null +++ b/dataflow-template-load-generator/src/main/java/org/springframework/cloud/samples/dataflowtemplate/LoadGeneratorApplication.java @@ -0,0 +1,152 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.samples.dataflowtemplate; + +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.dataflow.rest.client.DataFlowTemplate; +import org.springframework.cloud.dataflow.rest.client.TaskOperations; +import org.springframework.cloud.dataflow.rest.client.config.DataFlowClientAutoConfiguration; +import org.springframework.cloud.dataflow.rest.resource.AppRegistrationResource; +import org.springframework.cloud.dataflow.rest.resource.CurrentTaskExecutionsResource; +import org.springframework.cloud.dataflow.rest.resource.TaskDefinitionResource; +import org.springframework.cloud.dataflow.rest.util.HttpClientConfigurer; +import org.springframework.hateoas.PagedResources; +import org.springframework.web.client.RestTemplate; + +/** + * + * @author Gunnar Hillert + * + */ +@SpringBootApplication(exclude = { DataFlowClientAutoConfiguration.class }) +public class LoadGeneratorApplication implements ApplicationRunner { + + public static void main(String[] args) { + SpringApplication.run(LoadGeneratorApplication.class, args); + } + + private final Logger logger = LoggerFactory.getLogger(LoadGeneratorApplication.class); + + @Override + public void run(ApplicationArguments applicationArguments) throws Exception { + final DataFlowTemplate dataFlowTemplate = getDataFlowTemplate(null, null, "http://localhost:9393"); + + importApps(dataFlowTemplate); + + final Map templateTaskDefinitions = new HashMap<>(); + templateTaskDefinitions.put("timestamp", "timestamp"); + templateTaskDefinitions.put("timestamp-batch", "timestamp-batch"); + + final TaskOperations taskOperations = dataFlowTemplate.taskOperations(); + + final int numberOfDesiredTaskDefinitions = 100; + + final Set taskDefinitions = new HashSet<>(numberOfDesiredTaskDefinitions*templateTaskDefinitions.size()); + + for (java.util.Map.Entry entry : templateTaskDefinitions.entrySet()) { + for (int i = 1; i <= numberOfDesiredTaskDefinitions; i++) { + final TaskDefinitionResource createdTaskDefinition = taskOperations.create(entry.getKey() + "_" + i, entry.getValue()); + taskDefinitions.add(createdTaskDefinition); + } + } + + logger.info("\n\nCreated a total of {} Task Definitions.\n", taskDefinitions.size()); + + final Set executions = new HashSet<>(0); + int numberOfDesiredExecutions = 20; + + for (TaskDefinitionResource taskDefinition : taskDefinitions) { + final Set executionsPerTaskDefinition = new HashSet<>(0); + while (executionsPerTaskDefinition.size() < numberOfDesiredExecutions) { + CurrentTaskExecutionsResource c = taskOperations.currentTaskExecutions().iterator().next(); + long runningExecutions = c.getRunningExecutionCount(); + long maxExecutions = c.getMaximumTaskExecutions(); + long delta = maxExecutions - runningExecutions; + + if (delta > 0) { + long executionId = taskOperations.launch(taskDefinition.getName(), new HashMap<>(), new ArrayList<>()); + executionsPerTaskDefinition.add(executionId); + } + + Thread.sleep(200); + } + executions.addAll(executionsPerTaskDefinition); + } + + logger.info("\n\nCreated a total of {} task executions.", executions.size()); + } + + private void importApps(DataFlowTemplate dataFlowTemplate) { + final String taskAppRegistrationUrl = "https://dataflow.spring.io/Elston-GA-task-applications-maven"; + final String appRegistrationUrl = "https://dataflow.spring.io/Einstein-SR3-stream-applications-rabbit-maven"; + logger.info("\n\nImporting Stream Applications from: {}.\n", appRegistrationUrl); + logger.info("\n\nImporting Task Applications from: {}.\n", taskAppRegistrationUrl); + + final PagedResources appRegistrations = + dataFlowTemplate.appRegistryOperations().importFromResource(appRegistrationUrl, false); + + final PagedResources taskAppRegistrations = + dataFlowTemplate.appRegistryOperations().importFromResource(taskAppRegistrationUrl, false); + + logger.info("Imported {} Stream Applications. Showing the first {} from page {}", + appRegistrations.getMetadata().getTotalElements(), + appRegistrations.getMetadata().getSize(), + appRegistrations.getMetadata().getNumber()); + + for (AppRegistrationResource app : appRegistrations.getContent()) { + logger.info("{} (type: {})", app.getName(), app.getType()); + } + + logger.info("Imported {} Task Applications. Showing the first {} from page {}", + taskAppRegistrations.getMetadata().getTotalElements(), + taskAppRegistrations.getMetadata().getSize(), + taskAppRegistrations.getMetadata().getNumber()); + + for (AppRegistrationResource app : taskAppRegistrations.getContent()) { + logger.info("{} (type: {})", app.getName(), app.getType()); + } + + } + + private static DataFlowTemplate getDataFlowTemplate(String username, String password, String target) { + final URI targetUri = URI.create(target); + + final RestTemplate restTemplate = DataFlowTemplate.getDefaultDataflowRestTemplate(); + + if (username != null && password != null) { + final HttpClientConfigurer httpClientConfigurer = HttpClientConfigurer.create(targetUri); + httpClientConfigurer.basicAuthCredentials(username, password); + restTemplate.setRequestFactory(httpClientConfigurer.buildClientHttpRequestFactory()); + } + + final DataFlowTemplate dataFlowTemplate = new DataFlowTemplate(targetUri, restTemplate); + + return dataFlowTemplate; + } +} diff --git a/dataflow-template-load-generator/src/main/resources/application.yml b/dataflow-template-load-generator/src/main/resources/application.yml new file mode 100644 index 0000000..e69de29