Add Data Flow Load Generator Example
The `dataflow-template-load-generator` will be able to create a parameterized amount of task applications and generate a definable amount of task/batch executions.
This commit is contained in:
Gunnar Hillert
22
dataflow-template-load-generator/README.adoc
Normal file
22
dataflow-template-load-generator/README.adoc
Normal file
@@ -0,0 +1,22 @@
|
||||
= Data Flow Template Load Generator
|
||||
|
||||
This example uses the Spring Cloud Data Flow Template to generator a set of Task- and Batch Executions.
|
||||
|
||||
This examples, will:
|
||||
|
||||
- Import https://cloud.spring.io/spring-cloud-task-app-starters/[Task App Starter applications]
|
||||
- Import https://cloud.spring.io/spring-cloud-stream-app-starters/[Stream App Starter applications]
|
||||
- Create several Task Applications
|
||||
- Execute each Task Application several times
|
||||
|
||||
In order to get everything running we need to setup the following server instances:
|
||||
|
||||
* Spring Cloud Skipper
|
||||
* Spring Cloud Data Flow
|
||||
|
||||
== Build + Run
|
||||
|
||||
Configure the example application in `DataflowTemplateApplication.java`. Then build
|
||||
using `$ mvn clean package` and run using
|
||||
`$ java -jar target/dataflow-template-load-generator-1.0.0.BUILD-SNAPSHOT.jar`
|
||||
|
||||
143
dataflow-template-load-generator/mvnw.cmd
vendored
Normal file
143
dataflow-template-load-generator/mvnw.cmd
vendored
Normal file
@@ -0,0 +1,143 @@
|
||||
@REM ----------------------------------------------------------------------------
|
||||
@REM Licensed to the Apache Software Foundation (ASF) under one
|
||||
@REM or more contributor license agreements. See the NOTICE file
|
||||
@REM distributed with this work for additional information
|
||||
@REM regarding copyright ownership. The ASF licenses this file
|
||||
@REM to you under the Apache License, Version 2.0 (the
|
||||
@REM "License"); you may not use this file except in compliance
|
||||
@REM with the License. You may obtain a copy of the License at
|
||||
@REM
|
||||
@REM https://www.apache.org/licenses/LICENSE-2.0
|
||||
@REM
|
||||
@REM Unless required by applicable law or agreed to in writing,
|
||||
@REM software distributed under the License is distributed on an
|
||||
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
@REM KIND, either express or implied. See the License for the
|
||||
@REM specific language governing permissions and limitations
|
||||
@REM under the License.
|
||||
@REM ----------------------------------------------------------------------------
|
||||
|
||||
@REM ----------------------------------------------------------------------------
|
||||
@REM Maven2 Start Up Batch script
|
||||
@REM
|
||||
@REM Required ENV vars:
|
||||
@REM JAVA_HOME - location of a JDK home dir
|
||||
@REM
|
||||
@REM Optional ENV vars
|
||||
@REM M2_HOME - location of maven2's installed home dir
|
||||
@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
|
||||
@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending
|
||||
@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
|
||||
@REM e.g. to debug Maven itself, use
|
||||
@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
|
||||
@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
|
||||
@REM ----------------------------------------------------------------------------
|
||||
|
||||
@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
|
||||
@echo off
|
||||
@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on'
|
||||
@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
|
||||
|
||||
@REM set %HOME% to equivalent of $HOME
|
||||
if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
|
||||
|
||||
@REM Execute a user defined script before this one
|
||||
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
|
||||
@REM check for pre script, once with legacy .bat ending and once with .cmd ending
|
||||
if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
|
||||
if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
|
||||
:skipRcPre
|
||||
|
||||
@setlocal
|
||||
|
||||
set ERROR_CODE=0
|
||||
|
||||
@REM To isolate internal variables from possible post scripts, we use another setlocal
|
||||
@setlocal
|
||||
|
||||
@REM ==== START VALIDATION ====
|
||||
if not "%JAVA_HOME%" == "" goto OkJHome
|
||||
|
||||
echo.
|
||||
echo Error: JAVA_HOME not found in your environment. >&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the >&2
|
||||
echo location of your Java installation. >&2
|
||||
echo.
|
||||
goto error
|
||||
|
||||
:OkJHome
|
||||
if exist "%JAVA_HOME%\bin\java.exe" goto init
|
||||
|
||||
echo.
|
||||
echo Error: JAVA_HOME is set to an invalid directory. >&2
|
||||
echo JAVA_HOME = "%JAVA_HOME%" >&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the >&2
|
||||
echo location of your Java installation. >&2
|
||||
echo.
|
||||
goto error
|
||||
|
||||
@REM ==== END VALIDATION ====
|
||||
|
||||
:init
|
||||
|
||||
@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
|
||||
@REM Fallback to current working directory if not found.
|
||||
|
||||
set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
|
||||
IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
|
||||
|
||||
set EXEC_DIR=%CD%
|
||||
set WDIR=%EXEC_DIR%
|
||||
:findBaseDir
|
||||
IF EXIST "%WDIR%"\.mvn goto baseDirFound
|
||||
cd ..
|
||||
IF "%WDIR%"=="%CD%" goto baseDirNotFound
|
||||
set WDIR=%CD%
|
||||
goto findBaseDir
|
||||
|
||||
:baseDirFound
|
||||
set MAVEN_PROJECTBASEDIR=%WDIR%
|
||||
cd "%EXEC_DIR%"
|
||||
goto endDetectBaseDir
|
||||
|
||||
:baseDirNotFound
|
||||
set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
|
||||
cd "%EXEC_DIR%"
|
||||
|
||||
:endDetectBaseDir
|
||||
|
||||
IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
|
||||
|
||||
@setlocal EnableExtensions EnableDelayedExpansion
|
||||
for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
|
||||
@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
|
||||
|
||||
:endReadAdditionalConfig
|
||||
|
||||
SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
|
||||
|
||||
set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
|
||||
set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
|
||||
|
||||
%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
|
||||
if ERRORLEVEL 1 goto error
|
||||
goto end
|
||||
|
||||
:error
|
||||
set ERROR_CODE=1
|
||||
|
||||
:end
|
||||
@endlocal & set ERROR_CODE=%ERROR_CODE%
|
||||
|
||||
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
|
||||
@REM check for post script, once with legacy .bat ending and once with .cmd ending
|
||||
if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
|
||||
if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
|
||||
:skipRcPost
|
||||
|
||||
@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
|
||||
if "%MAVEN_BATCH_PAUSE%" == "on" pause
|
||||
|
||||
if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
|
||||
|
||||
exit /B %ERROR_CODE%
|
||||
90
dataflow-template-load-generator/pom.xml
Normal file
90
dataflow-template-load-generator/pom.xml
Normal file
@@ -0,0 +1,90 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.springframework.cloud.samples</groupId>
|
||||
<artifactId>dataflow-template-load-generator</artifactId>
|
||||
<version>1.0.0.BUILD-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>ldapserver</name>
|
||||
<description>Utility app to generate Task and Batch Execution data for Spring Cloud Data Flow</description>
|
||||
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-parent</artifactId>
|
||||
<version>2.1.0.RELEASE</version>
|
||||
<relativePath/> <!-- lookup parent from repository -->
|
||||
</parent>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
<java.version>1.8</java.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-dataflow-rest-client</artifactId>
|
||||
<version>2.1.0.RELEASE</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>spring-snapshots</id>
|
||||
<name>Spring Snapshots</name>
|
||||
<url>https://repo.spring.io/snapshot</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
<repository>
|
||||
<id>spring-milestones</id>
|
||||
<name>Spring Milestones</name>
|
||||
<url>https://repo.spring.io/milestone</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
<pluginRepositories>
|
||||
<pluginRepository>
|
||||
<id>spring-snapshots</id>
|
||||
<name>Spring Snapshots</name>
|
||||
<url>https://repo.spring.io/snapshot</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
</pluginRepository>
|
||||
<pluginRepository>
|
||||
<id>spring-milestones</id>
|
||||
<name>Spring Milestones</name>
|
||||
<url>https://repo.spring.io/milestone</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
</project>
|
||||
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* Copyright 2019 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.samples.dataflowtemplate;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.cloud.dataflow.rest.client.DataFlowTemplate;
|
||||
import org.springframework.cloud.dataflow.rest.client.TaskOperations;
|
||||
import org.springframework.cloud.dataflow.rest.client.config.DataFlowClientAutoConfiguration;
|
||||
import org.springframework.cloud.dataflow.rest.resource.AppRegistrationResource;
|
||||
import org.springframework.cloud.dataflow.rest.resource.CurrentTaskExecutionsResource;
|
||||
import org.springframework.cloud.dataflow.rest.resource.TaskDefinitionResource;
|
||||
import org.springframework.cloud.dataflow.rest.util.HttpClientConfigurer;
|
||||
import org.springframework.hateoas.PagedResources;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Gunnar Hillert
|
||||
*
|
||||
*/
|
||||
@SpringBootApplication(exclude = { DataFlowClientAutoConfiguration.class })
|
||||
public class LoadGeneratorApplication implements ApplicationRunner {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(LoadGeneratorApplication.class, args);
|
||||
}
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(LoadGeneratorApplication.class);
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments applicationArguments) throws Exception {
|
||||
final DataFlowTemplate dataFlowTemplate = getDataFlowTemplate(null, null, "http://localhost:9393");
|
||||
|
||||
importApps(dataFlowTemplate);
|
||||
|
||||
final Map<String, String> templateTaskDefinitions = new HashMap<>();
|
||||
templateTaskDefinitions.put("timestamp", "timestamp");
|
||||
templateTaskDefinitions.put("timestamp-batch", "timestamp-batch");
|
||||
|
||||
final TaskOperations taskOperations = dataFlowTemplate.taskOperations();
|
||||
|
||||
final int numberOfDesiredTaskDefinitions = 100;
|
||||
|
||||
final Set<TaskDefinitionResource> taskDefinitions = new HashSet<>(numberOfDesiredTaskDefinitions*templateTaskDefinitions.size());
|
||||
|
||||
for (java.util.Map.Entry<String, String> entry : templateTaskDefinitions.entrySet()) {
|
||||
for (int i = 1; i <= numberOfDesiredTaskDefinitions; i++) {
|
||||
final TaskDefinitionResource createdTaskDefinition = taskOperations.create(entry.getKey() + "_" + i, entry.getValue());
|
||||
taskDefinitions.add(createdTaskDefinition);
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("\n\nCreated a total of {} Task Definitions.\n", taskDefinitions.size());
|
||||
|
||||
final Set<Long> executions = new HashSet<>(0);
|
||||
int numberOfDesiredExecutions = 20;
|
||||
|
||||
for (TaskDefinitionResource taskDefinition : taskDefinitions) {
|
||||
final Set<Long> executionsPerTaskDefinition = new HashSet<>(0);
|
||||
while (executionsPerTaskDefinition.size() < numberOfDesiredExecutions) {
|
||||
CurrentTaskExecutionsResource c = taskOperations.currentTaskExecutions().iterator().next();
|
||||
long runningExecutions = c.getRunningExecutionCount();
|
||||
long maxExecutions = c.getMaximumTaskExecutions();
|
||||
long delta = maxExecutions - runningExecutions;
|
||||
|
||||
if (delta > 0) {
|
||||
long executionId = taskOperations.launch(taskDefinition.getName(), new HashMap<>(), new ArrayList<>());
|
||||
executionsPerTaskDefinition.add(executionId);
|
||||
}
|
||||
|
||||
Thread.sleep(200);
|
||||
}
|
||||
executions.addAll(executionsPerTaskDefinition);
|
||||
}
|
||||
|
||||
logger.info("\n\nCreated a total of {} task executions.", executions.size());
|
||||
}
|
||||
|
||||
private void importApps(DataFlowTemplate dataFlowTemplate) {
|
||||
final String taskAppRegistrationUrl = "https://dataflow.spring.io/Elston-GA-task-applications-maven";
|
||||
final String appRegistrationUrl = "https://dataflow.spring.io/Einstein-SR3-stream-applications-rabbit-maven";
|
||||
logger.info("\n\nImporting Stream Applications from: {}.\n", appRegistrationUrl);
|
||||
logger.info("\n\nImporting Task Applications from: {}.\n", taskAppRegistrationUrl);
|
||||
|
||||
final PagedResources<AppRegistrationResource> appRegistrations =
|
||||
dataFlowTemplate.appRegistryOperations().importFromResource(appRegistrationUrl, false);
|
||||
|
||||
final PagedResources<AppRegistrationResource> taskAppRegistrations =
|
||||
dataFlowTemplate.appRegistryOperations().importFromResource(taskAppRegistrationUrl, false);
|
||||
|
||||
logger.info("Imported {} Stream Applications. Showing the first {} from page {}",
|
||||
appRegistrations.getMetadata().getTotalElements(),
|
||||
appRegistrations.getMetadata().getSize(),
|
||||
appRegistrations.getMetadata().getNumber());
|
||||
|
||||
for (AppRegistrationResource app : appRegistrations.getContent()) {
|
||||
logger.info("{} (type: {})", app.getName(), app.getType());
|
||||
}
|
||||
|
||||
logger.info("Imported {} Task Applications. Showing the first {} from page {}",
|
||||
taskAppRegistrations.getMetadata().getTotalElements(),
|
||||
taskAppRegistrations.getMetadata().getSize(),
|
||||
taskAppRegistrations.getMetadata().getNumber());
|
||||
|
||||
for (AppRegistrationResource app : taskAppRegistrations.getContent()) {
|
||||
logger.info("{} (type: {})", app.getName(), app.getType());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static DataFlowTemplate getDataFlowTemplate(String username, String password, String target) {
|
||||
final URI targetUri = URI.create(target);
|
||||
|
||||
final RestTemplate restTemplate = DataFlowTemplate.getDefaultDataflowRestTemplate();
|
||||
|
||||
if (username != null && password != null) {
|
||||
final HttpClientConfigurer httpClientConfigurer = HttpClientConfigurer.create(targetUri);
|
||||
httpClientConfigurer.basicAuthCredentials(username, password);
|
||||
restTemplate.setRequestFactory(httpClientConfigurer.buildClientHttpRequestFactory());
|
||||
}
|
||||
|
||||
final DataFlowTemplate dataFlowTemplate = new DataFlowTemplate(targetUri, restTemplate);
|
||||
|
||||
return dataFlowTemplate;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user