#78 - Allow selective benchmark execution.
This commit is contained in:
Mark Paluch
@@ -15,314 +15,25 @@
|
||||
*/
|
||||
package org.springframework.data.microbenchmark.common;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.openjdk.jmh.annotations.Fork;
|
||||
import org.openjdk.jmh.annotations.Measurement;
|
||||
import org.openjdk.jmh.annotations.Scope;
|
||||
import org.openjdk.jmh.annotations.State;
|
||||
import org.openjdk.jmh.annotations.Warmup;
|
||||
import org.openjdk.jmh.results.RunResult;
|
||||
import org.openjdk.jmh.results.format.ResultFormatType;
|
||||
import org.openjdk.jmh.runner.Runner;
|
||||
import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
|
||||
import org.openjdk.jmh.runner.options.OptionsBuilder;
|
||||
import org.openjdk.jmh.runner.options.TimeValue;
|
||||
import org.springframework.core.env.StandardEnvironment;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.ResourceUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* Base class for microbenchmarks providing default JMH settings and allowing execution through JUnit.
|
||||
*
|
||||
* @author Christoph Strobl
|
||||
* @author Mark Paluch
|
||||
* @see Microbenchmark
|
||||
*/
|
||||
@Warmup(iterations = AbstractMicrobenchmark.WARMUP_ITERATIONS)
|
||||
@Measurement(iterations = AbstractMicrobenchmark.MEASUREMENT_ITERATIONS)
|
||||
@Fork(AbstractMicrobenchmark.FORKS)
|
||||
@Warmup(iterations = JmhSupport.WARMUP_ITERATIONS)
|
||||
@Measurement(iterations = JmhSupport.MEASUREMENT_ITERATIONS)
|
||||
@Fork(JmhSupport.FORKS)
|
||||
@State(Scope.Thread)
|
||||
public class AbstractMicrobenchmark {
|
||||
@RunWith(Microbenchmark.class)
|
||||
public abstract class AbstractMicrobenchmark {
|
||||
|
||||
static final int WARMUP_ITERATIONS = 5;
|
||||
static final int MEASUREMENT_ITERATIONS = 10;
|
||||
static final int FORKS = 1;
|
||||
static final String[] JVM_ARGS = { "-server", "-XX:+HeapDumpOnOutOfMemoryError", "-Xms1024m", "-Xmx1024m",
|
||||
"-XX:MaxDirectMemorySize=1024m" };
|
||||
|
||||
private final StandardEnvironment environment = new StandardEnvironment();
|
||||
|
||||
/**
|
||||
* Run matching {@link org.openjdk.jmh.annotations.Benchmark} methods with options collected from
|
||||
* {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @throws Exception
|
||||
* @see #options(String)
|
||||
*/
|
||||
@Test
|
||||
public void run() throws Exception {
|
||||
|
||||
String includes = includes();
|
||||
|
||||
if (!includes.contains(org.springframework.util.ClassUtils.getShortName(getClass()))) {
|
||||
return;
|
||||
}
|
||||
|
||||
publishResults(new Runner(options(includes).build()).run());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the regex for all benchmarks to be included in the run. By default every benchmark within classes matching the
|
||||
* current ones short name. <br />
|
||||
* The {@literal benchmark} command line argument allows overriding the defaults using {@code #} as class / method
|
||||
* name separator.
|
||||
*
|
||||
* @return never {@literal null}.
|
||||
* @see org.springframework.util.ClassUtils#getShortName(Class)
|
||||
*/
|
||||
protected String includes() {
|
||||
|
||||
String tests = environment.getProperty("benchmark", String.class);
|
||||
|
||||
if (!StringUtils.hasText(tests)) {
|
||||
return ".*" + org.springframework.util.ClassUtils.getShortName(getClass()) + ".*";
|
||||
}
|
||||
|
||||
if (!tests.contains("#")) {
|
||||
return ".*" + tests + ".*";
|
||||
}
|
||||
|
||||
String[] args = tests.split("#");
|
||||
return ".*" + args[0] + "." + args[1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect all options for the {@link Runner}.
|
||||
*
|
||||
* @param includes regex for matching benchmarks to be included in the run.
|
||||
* @return never {@literal null}.
|
||||
* @throws Exception
|
||||
*/
|
||||
protected ChainedOptionsBuilder options(String includes) throws Exception {
|
||||
|
||||
ChainedOptionsBuilder optionsBuilder = new OptionsBuilder().include(includes).jvmArgs(jvmArgs());
|
||||
|
||||
optionsBuilder = warmup(optionsBuilder);
|
||||
optionsBuilder = measure(optionsBuilder);
|
||||
optionsBuilder = forks(optionsBuilder);
|
||||
optionsBuilder = report(optionsBuilder);
|
||||
|
||||
return optionsBuilder;
|
||||
}
|
||||
|
||||
/**
|
||||
* JVM args to apply to {@link Runner} via its {@link org.openjdk.jmh.runner.options.Options}.
|
||||
*
|
||||
* @return {@link #JVM_ARGS} by default.
|
||||
*/
|
||||
protected String[] jvmArgs() {
|
||||
|
||||
String[] args = new String[JVM_ARGS.length];
|
||||
System.arraycopy(JVM_ARGS, 0, args, 0, JVM_ARGS.length);
|
||||
return args;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read {@code warmupIterations} property from {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @return -1 if not set.
|
||||
*/
|
||||
protected int getWarmupIterations() {
|
||||
return environment.getProperty("warmupIterations", Integer.class, -1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read {@code measurementIterations} property from {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @return -1 if not set.
|
||||
*/
|
||||
protected int getMeasurementIterations() {
|
||||
return environment.getProperty("measurementIterations", Integer.class, -1);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Read {@code forks} property from {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @return -1 if not set.
|
||||
*/
|
||||
protected int getForksCount() {
|
||||
return environment.getProperty("forks", Integer.class, -1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read {@code benchmarkReportDir} property from {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @return {@literal null} if not set.
|
||||
*/
|
||||
protected String getReportDirectory() {
|
||||
return environment.getProperty("benchmarkReportDir");
|
||||
}
|
||||
|
||||
/**
|
||||
* Read {@code measurementTime} property from {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @return -1 if not set.
|
||||
*/
|
||||
protected long getMeasurementTime() {
|
||||
return environment.getProperty("measurementTime", Long.class, -1L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read {@code warmupTime} property from {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @return -1 if not set.
|
||||
*/
|
||||
protected long getWarmupTime() {
|
||||
return environment.getProperty("warmupTime", Long.class, -1L);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@code project.version_yyyy-MM-dd_ClassName.json} eg.
|
||||
* {@literal 1.11.0.BUILD-SNAPSHOT_2017-03-07_MappingMongoConverterBenchmark.json}
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
protected String reportFilename() {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
if (environment.containsProperty("project.version")) {
|
||||
|
||||
sb.append(environment.getProperty("project.version"));
|
||||
sb.append("_");
|
||||
}
|
||||
|
||||
sb.append(new SimpleDateFormat("yyyy-MM-dd").format(new Date()));
|
||||
sb.append("_");
|
||||
sb.append(org.springframework.util.ClassUtils.getShortName(getClass()));
|
||||
sb.append(".json");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply measurement options to {@link ChainedOptionsBuilder}.
|
||||
*
|
||||
* @param optionsBuilder must not be {@literal null}.
|
||||
* @return {@link ChainedOptionsBuilder} with options applied.
|
||||
* @see #getMeasurementIterations()
|
||||
* @see #getMeasurementTime()
|
||||
*/
|
||||
private ChainedOptionsBuilder measure(ChainedOptionsBuilder optionsBuilder) {
|
||||
|
||||
int measurementIterations = getMeasurementIterations();
|
||||
long measurementTime = getMeasurementTime();
|
||||
|
||||
if (measurementIterations > 0) {
|
||||
optionsBuilder = optionsBuilder.measurementIterations(measurementIterations);
|
||||
}
|
||||
|
||||
if (measurementTime > 0) {
|
||||
optionsBuilder = optionsBuilder.measurementTime(TimeValue.seconds(measurementTime));
|
||||
}
|
||||
|
||||
return optionsBuilder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply warmup options to {@link ChainedOptionsBuilder}.
|
||||
*
|
||||
* @param optionsBuilder must not be {@literal null}.
|
||||
* @return {@link ChainedOptionsBuilder} with options applied.
|
||||
* @see #getWarmupIterations()
|
||||
* @see #getWarmupTime()
|
||||
*/
|
||||
private ChainedOptionsBuilder warmup(ChainedOptionsBuilder optionsBuilder) {
|
||||
|
||||
int warmupIterations = getWarmupIterations();
|
||||
long warmupTime = getWarmupTime();
|
||||
|
||||
if (warmupIterations > 0) {
|
||||
optionsBuilder = optionsBuilder.warmupIterations(warmupIterations);
|
||||
}
|
||||
|
||||
if (warmupTime > 0) {
|
||||
optionsBuilder = optionsBuilder.warmupTime(TimeValue.seconds(warmupTime));
|
||||
}
|
||||
|
||||
return optionsBuilder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply forks option to {@link ChainedOptionsBuilder}.
|
||||
*
|
||||
* @param optionsBuilder must not be {@literal null}.
|
||||
* @return {@link ChainedOptionsBuilder} with options applied.
|
||||
* @see #getForksCount()
|
||||
*/
|
||||
private ChainedOptionsBuilder forks(ChainedOptionsBuilder optionsBuilder) {
|
||||
|
||||
int forks = getForksCount();
|
||||
|
||||
if (forks <= 0) {
|
||||
return optionsBuilder;
|
||||
}
|
||||
|
||||
return optionsBuilder.forks(forks);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply report option to {@link ChainedOptionsBuilder}.
|
||||
*
|
||||
* @param optionsBuilder must not be {@literal null}.
|
||||
* @return {@link ChainedOptionsBuilder} with options applied.
|
||||
* @throws IOException if report file cannot be created.
|
||||
* @see #getReportDirectory()
|
||||
*/
|
||||
private ChainedOptionsBuilder report(ChainedOptionsBuilder optionsBuilder) throws IOException {
|
||||
|
||||
String reportDir = getReportDirectory();
|
||||
|
||||
if (!StringUtils.hasText(reportDir)) {
|
||||
return optionsBuilder;
|
||||
}
|
||||
|
||||
String reportFilePath = reportDir + (reportDir.endsWith(File.separator) ? "" : File.separator) + reportFilename();
|
||||
File file = ResourceUtils.getFile(reportFilePath);
|
||||
|
||||
if (file.exists()) {
|
||||
file.delete();
|
||||
} else {
|
||||
|
||||
file.getParentFile().mkdirs();
|
||||
file.createNewFile();
|
||||
}
|
||||
|
||||
optionsBuilder.resultFormat(ResultFormatType.JSON);
|
||||
optionsBuilder.result(reportFilePath);
|
||||
|
||||
return optionsBuilder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish results to an external system.
|
||||
*
|
||||
* @param results must not be {@literal null}.
|
||||
*/
|
||||
private void publishResults(Collection<RunResult> results) {
|
||||
|
||||
if (CollectionUtils.isEmpty(results) || !environment.containsProperty("publishTo")) {
|
||||
return;
|
||||
}
|
||||
|
||||
String uri = environment.getProperty("publishTo");
|
||||
try {
|
||||
ResultsWriter.forUri(uri).write(results);
|
||||
} catch (Exception e) {
|
||||
System.err.println(String.format("Cannot save benchmark results to '%s'. Error was %s.", uri, e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
/*
|
||||
* Copyright 2018 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.data.microbenchmark.common;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.junit.runners.model.FrameworkMethod;
|
||||
import org.openjdk.jmh.results.RunResult;
|
||||
import org.openjdk.jmh.results.format.ResultFormatType;
|
||||
import org.openjdk.jmh.runner.Runner;
|
||||
import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
|
||||
import org.openjdk.jmh.runner.options.OptionsBuilder;
|
||||
import org.openjdk.jmh.runner.options.TimeValue;
|
||||
import org.springframework.core.env.StandardEnvironment;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.ResourceUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* @author Christoph Strobl
|
||||
* @author Mark Paluch
|
||||
*/
|
||||
class JmhSupport {
|
||||
|
||||
static final int WARMUP_ITERATIONS = 5;
|
||||
static final int MEASUREMENT_ITERATIONS = 10;
|
||||
static final int FORKS = 1;
|
||||
static final String[] JVM_ARGS = { "-server", "-XX:+HeapDumpOnOutOfMemoryError", "-Xms1024m", "-Xmx1024m",
|
||||
"-XX:MaxDirectMemorySize=1024m" };
|
||||
|
||||
private final StandardEnvironment environment = new StandardEnvironment();
|
||||
|
||||
/**
|
||||
* Get the regex for all benchmarks to be included in the run. By default every benchmark within classes matching the
|
||||
* fqcn. <br />
|
||||
* The {@literal benchmark} command line argument allows overriding the defaults using {@code #} as class / method
|
||||
* name separator.
|
||||
*
|
||||
* @return never {@literal null}.
|
||||
* @param methods
|
||||
*/
|
||||
protected List<String> includes(Collection<FrameworkMethod> methods) {
|
||||
|
||||
String tests = environment.getProperty("benchmark", String.class);
|
||||
|
||||
if (!StringUtils.hasText(tests)) {
|
||||
|
||||
return methods.stream().map(it -> Pattern.quote(it.getDeclaringClass().getName()) + "\\." + it.getName())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
if (!tests.contains("#")) {
|
||||
return Collections.singletonList(".*" + tests + ".*");
|
||||
}
|
||||
|
||||
String[] args = tests.split("#");
|
||||
return Collections.singletonList(".*" + args[0] + "." + args[1]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect all options for the {@link Runner}.
|
||||
*
|
||||
* @return never {@literal null}.
|
||||
* @throws Exception
|
||||
*/
|
||||
protected ChainedOptionsBuilder options() throws Exception {
|
||||
|
||||
ChainedOptionsBuilder optionsBuilder = new OptionsBuilder().jvmArgs(jvmArgs());
|
||||
|
||||
optionsBuilder = warmup(optionsBuilder);
|
||||
optionsBuilder = measure(optionsBuilder);
|
||||
optionsBuilder = forks(optionsBuilder);
|
||||
optionsBuilder = report(optionsBuilder);
|
||||
|
||||
return optionsBuilder;
|
||||
}
|
||||
|
||||
/**
|
||||
* JVM args to apply to {@link Runner} via its {@link org.openjdk.jmh.runner.options.Options}.
|
||||
*
|
||||
* @return {@link #JVM_ARGS} by default.
|
||||
*/
|
||||
protected String[] jvmArgs() {
|
||||
|
||||
String[] args = new String[JVM_ARGS.length];
|
||||
System.arraycopy(JVM_ARGS, 0, args, 0, JVM_ARGS.length);
|
||||
return args;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read {@code warmupIterations} property from {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @return -1 if not set.
|
||||
*/
|
||||
protected int getWarmupIterations() {
|
||||
return environment.getProperty("warmupIterations", Integer.class, -1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read {@code measurementIterations} property from {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @return -1 if not set.
|
||||
*/
|
||||
protected int getMeasurementIterations() {
|
||||
return environment.getProperty("measurementIterations", Integer.class, -1);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Read {@code forks} property from {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @return -1 if not set.
|
||||
*/
|
||||
protected int getForksCount() {
|
||||
return environment.getProperty("forks", Integer.class, -1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read {@code benchmarkReportDir} property from {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @return {@literal null} if not set.
|
||||
*/
|
||||
protected String getReportDirectory() {
|
||||
return environment.getProperty("benchmarkReportDir");
|
||||
}
|
||||
|
||||
/**
|
||||
* Read {@code measurementTime} property from {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @return -1 if not set.
|
||||
*/
|
||||
protected long getMeasurementTime() {
|
||||
return environment.getProperty("measurementTime", Long.class, -1L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read {@code warmupTime} property from {@link org.springframework.core.env.Environment}.
|
||||
*
|
||||
* @return -1 if not set.
|
||||
*/
|
||||
protected long getWarmupTime() {
|
||||
return environment.getProperty("warmupTime", Long.class, -1L);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@code project.version_yyyy-MM-dd_ClassName.json} eg.
|
||||
* {@literal 1.11.0.BUILD-SNAPSHOT_2017-03-07_MappingMongoConverterBenchmark.json}
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
protected String reportFilename() {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
if (environment.containsProperty("project.version")) {
|
||||
|
||||
sb.append(environment.getProperty("project.version"));
|
||||
sb.append("_");
|
||||
}
|
||||
|
||||
sb.append(new SimpleDateFormat("yyyy-MM-dd").format(new Date()));
|
||||
sb.append("_");
|
||||
sb.append(org.springframework.util.ClassUtils.getShortName(getClass()));
|
||||
sb.append(".json");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply measurement options to {@link ChainedOptionsBuilder}.
|
||||
*
|
||||
* @param optionsBuilder must not be {@literal null}.
|
||||
* @return {@link ChainedOptionsBuilder} with options applied.
|
||||
* @see #getMeasurementIterations()
|
||||
* @see #getMeasurementTime()
|
||||
*/
|
||||
private ChainedOptionsBuilder measure(ChainedOptionsBuilder optionsBuilder) {
|
||||
|
||||
int measurementIterations = getMeasurementIterations();
|
||||
long measurementTime = getMeasurementTime();
|
||||
|
||||
if (measurementIterations > 0) {
|
||||
optionsBuilder = optionsBuilder.measurementIterations(measurementIterations);
|
||||
}
|
||||
|
||||
if (measurementTime > 0) {
|
||||
optionsBuilder = optionsBuilder.measurementTime(TimeValue.seconds(measurementTime));
|
||||
}
|
||||
|
||||
return optionsBuilder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply warmup options to {@link ChainedOptionsBuilder}.
|
||||
*
|
||||
* @param optionsBuilder must not be {@literal null}.
|
||||
* @return {@link ChainedOptionsBuilder} with options applied.
|
||||
* @see #getWarmupIterations()
|
||||
* @see #getWarmupTime()
|
||||
*/
|
||||
private ChainedOptionsBuilder warmup(ChainedOptionsBuilder optionsBuilder) {
|
||||
|
||||
int warmupIterations = getWarmupIterations();
|
||||
long warmupTime = getWarmupTime();
|
||||
|
||||
if (warmupIterations > 0) {
|
||||
optionsBuilder = optionsBuilder.warmupIterations(warmupIterations);
|
||||
}
|
||||
|
||||
if (warmupTime > 0) {
|
||||
optionsBuilder = optionsBuilder.warmupTime(TimeValue.seconds(warmupTime));
|
||||
}
|
||||
|
||||
return optionsBuilder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply forks option to {@link ChainedOptionsBuilder}.
|
||||
*
|
||||
* @param optionsBuilder must not be {@literal null}.
|
||||
* @return {@link ChainedOptionsBuilder} with options applied.
|
||||
* @see #getForksCount()
|
||||
*/
|
||||
private ChainedOptionsBuilder forks(ChainedOptionsBuilder optionsBuilder) {
|
||||
|
||||
int forks = getForksCount();
|
||||
|
||||
if (forks <= 0) {
|
||||
return optionsBuilder;
|
||||
}
|
||||
|
||||
return optionsBuilder.forks(forks);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply report option to {@link ChainedOptionsBuilder}.
|
||||
*
|
||||
* @param optionsBuilder must not be {@literal null}.
|
||||
* @return {@link ChainedOptionsBuilder} with options applied.
|
||||
* @throws IOException if report file cannot be created.
|
||||
* @see #getReportDirectory()
|
||||
*/
|
||||
private ChainedOptionsBuilder report(ChainedOptionsBuilder optionsBuilder) throws IOException {
|
||||
|
||||
String reportDir = getReportDirectory();
|
||||
|
||||
if (!StringUtils.hasText(reportDir)) {
|
||||
return optionsBuilder;
|
||||
}
|
||||
|
||||
String reportFilePath = reportDir + (reportDir.endsWith(File.separator) ? "" : File.separator) + reportFilename();
|
||||
File file = ResourceUtils.getFile(reportFilePath);
|
||||
|
||||
if (file.exists()) {
|
||||
file.delete();
|
||||
} else {
|
||||
|
||||
file.getParentFile().mkdirs();
|
||||
file.createNewFile();
|
||||
}
|
||||
|
||||
optionsBuilder.resultFormat(ResultFormatType.JSON);
|
||||
optionsBuilder.result(reportFilePath);
|
||||
|
||||
return optionsBuilder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish results to an external system.
|
||||
*
|
||||
* @param results must not be {@literal null}.
|
||||
*/
|
||||
void publishResults(Collection<RunResult> results) {
|
||||
|
||||
if (CollectionUtils.isEmpty(results) || !environment.containsProperty("publishTo")) {
|
||||
return;
|
||||
}
|
||||
|
||||
String uri = environment.getProperty("publishTo");
|
||||
try {
|
||||
ResultsWriter.forUri(uri).write(results);
|
||||
} catch (Exception e) {
|
||||
System.err.println(String.format("Cannot save benchmark results to '%s'. Error was %s.", uri, e));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,475 @@
|
||||
/*
|
||||
* Copyright 2018 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.data.microbenchmark.common;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.junit.runner.Description;
|
||||
import org.junit.runner.manipulation.Filter;
|
||||
import org.junit.runner.manipulation.NoTestsRemainException;
|
||||
import org.junit.runner.manipulation.Sorter;
|
||||
import org.junit.runner.notification.Failure;
|
||||
import org.junit.runner.notification.RunNotifier;
|
||||
import org.junit.runners.BlockJUnit4ClassRunner;
|
||||
import org.junit.runners.model.FrameworkMethod;
|
||||
import org.junit.runners.model.InitializationError;
|
||||
import org.junit.runners.model.Statement;
|
||||
import org.openjdk.jmh.annotations.Benchmark;
|
||||
import org.openjdk.jmh.infra.BenchmarkParams;
|
||||
import org.openjdk.jmh.infra.IterationParams;
|
||||
import org.openjdk.jmh.results.BenchmarkResult;
|
||||
import org.openjdk.jmh.results.IterationResult;
|
||||
import org.openjdk.jmh.results.RunResult;
|
||||
import org.openjdk.jmh.runner.Defaults;
|
||||
import org.openjdk.jmh.runner.Runner;
|
||||
import org.openjdk.jmh.runner.format.OutputFormat;
|
||||
import org.openjdk.jmh.runner.format.OutputFormatFactory;
|
||||
import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
|
||||
import org.openjdk.jmh.runner.options.Options;
|
||||
import org.openjdk.jmh.util.UnCloseablePrintStream;
|
||||
import org.openjdk.jmh.util.Utils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* JMH Microbenchmark runner that turns methods annotated with {@link Benchmark} into runnable methods allowing
|
||||
* execution through JUnit.
|
||||
*
|
||||
* @author Mark Paluch
|
||||
*/
|
||||
public class Microbenchmark extends BlockJUnit4ClassRunner {
|
||||
|
||||
private final Object childrenLock = new Object();
|
||||
private final JmhSupport jmhRunner = new JmhSupport();
|
||||
|
||||
private Collection<FrameworkMethod> filteredChildren;
|
||||
|
||||
/**
|
||||
* Creates a {@link Microbenchmark} to run {@link Class test class}.
|
||||
*
|
||||
* @param testClass
|
||||
* @throws InitializationError if the test class is malformed.
|
||||
*/
|
||||
public Microbenchmark(Class<?> testClass) throws InitializationError {
|
||||
super(testClass);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ignore JUnit validation as we're using JMH here.
|
||||
*
|
||||
* @param errors
|
||||
*/
|
||||
@Override
|
||||
protected void collectInitializationErrors(List<Throwable> errors) {}
|
||||
|
||||
/**
|
||||
* JMH has no means to exclude benchmark methods.
|
||||
*/
|
||||
@Override
|
||||
protected boolean isIgnored(FrameworkMethod child) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the methods that run tests. Default implementation returns all methods annotated with {@code @Test} on this
|
||||
* class and superclasses that are not overridden.
|
||||
*/
|
||||
protected List<FrameworkMethod> computeTestMethods() {
|
||||
|
||||
List<FrameworkMethod> annotatedMethods = new ArrayList<>(getTestClass().getAnnotatedMethods(Benchmark.class));
|
||||
|
||||
annotatedMethods.sort(Comparator.comparing(Microbenchmark::getBenchmarkName));
|
||||
|
||||
return Collections.unmodifiableList(annotatedMethods);
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.junit.runners.ParentRunner#classBlock(org.junit.runner.notification.RunNotifier)
|
||||
*/
|
||||
@Override
|
||||
protected Statement classBlock(RunNotifier notifier) {
|
||||
return childrenInvoker(notifier);
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.junit.runners.ParentRunner#getDescription()
|
||||
*/
|
||||
@Override
|
||||
public Description getDescription() {
|
||||
|
||||
Description description = Description.createSuiteDescription(getName(), getRunnerAnnotations());
|
||||
|
||||
getFilteredChildren().stream().map(this::describeChild).forEach(description::addChild);
|
||||
|
||||
return description;
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.junit.runners.ParentRunner#filter(org.junit.runner.manipulation.Filter)
|
||||
*/
|
||||
public void filter(Filter filter) throws NoTestsRemainException {
|
||||
|
||||
synchronized (childrenLock) {
|
||||
|
||||
List<FrameworkMethod> children = new ArrayList<>(getFilteredChildren());
|
||||
List<FrameworkMethod> filtered = children.stream().filter(it -> {
|
||||
|
||||
if (filter.shouldRun(describeChild(it))) {
|
||||
try {
|
||||
filter.apply(it);
|
||||
} catch (NoTestsRemainException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}).collect(Collectors.toList());
|
||||
|
||||
if (filtered.isEmpty()) {
|
||||
throw new NoTestsRemainException();
|
||||
}
|
||||
|
||||
filteredChildren = Collections.unmodifiableCollection(filtered);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.junit.runners.ParentRunner#sort(org.junit.runner.manipulation.Sorter)
|
||||
*/
|
||||
public void sort(Sorter sorter) {
|
||||
|
||||
synchronized (childrenLock) {
|
||||
|
||||
getFilteredChildren().forEach(sorter::apply);
|
||||
|
||||
List<FrameworkMethod> sortedChildren = new ArrayList<>(getFilteredChildren());
|
||||
|
||||
sortedChildren.sort((o1, o2) -> sorter.compare(describeChild(o1), describeChild(o2)));
|
||||
|
||||
filteredChildren = Collections.unmodifiableCollection(sortedChildren);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run matching {@link org.openjdk.jmh.annotations.Benchmark} methods with options collected from
|
||||
* {@link org.springframework.core.env.Environment}.
|
||||
*/
|
||||
@Override
|
||||
protected Statement childrenInvoker(RunNotifier notifier) {
|
||||
|
||||
Collection<FrameworkMethod> methods = getFilteredChildren();
|
||||
CacheFunction cache = new CacheFunction(methods, this::describeChild);
|
||||
|
||||
return new Statement() {
|
||||
|
||||
@Override
|
||||
public void evaluate() throws Throwable {
|
||||
doRun(notifier, methods, cache);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private void doRun(RunNotifier notifier, Collection<FrameworkMethod> methods, CacheFunction cache) throws Exception {
|
||||
|
||||
List<String> includes = jmhRunner.includes(methods);
|
||||
|
||||
ChainedOptionsBuilder optionsBuilder = jmhRunner.options();
|
||||
|
||||
includes.forEach(optionsBuilder::include);
|
||||
|
||||
Options options = optionsBuilder.build();
|
||||
NotifyingOutputFormat notifyingOutputFormat = new NotifyingOutputFormat(notifier, cache,
|
||||
createOutputFormat(options));
|
||||
|
||||
jmhRunner.publishResults(new Runner(options, notifyingOutputFormat).run());
|
||||
}
|
||||
|
||||
private Collection<FrameworkMethod> getFilteredChildren() {
|
||||
|
||||
if (filteredChildren == null) {
|
||||
synchronized (childrenLock) {
|
||||
if (filteredChildren == null) {
|
||||
filteredChildren = Collections.unmodifiableCollection(getChildren());
|
||||
}
|
||||
}
|
||||
}
|
||||
return filteredChildren;
|
||||
}
|
||||
|
||||
private static OutputFormat createOutputFormat(Options options) {
|
||||
|
||||
// sadly required here as the check cannot be made before calling this method in constructor
|
||||
if (options == null) {
|
||||
throw new IllegalArgumentException("Options not allowed to be null.");
|
||||
}
|
||||
|
||||
PrintStream out;
|
||||
if (options.getOutput().hasValue()) {
|
||||
try {
|
||||
out = new PrintStream(options.getOutput().get());
|
||||
} catch (FileNotFoundException ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
} else {
|
||||
// Protect the System.out from accidental closing
|
||||
try {
|
||||
out = new UnCloseablePrintStream(System.out, Utils.guessConsoleEncoding());
|
||||
} catch (UnsupportedEncodingException ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
return OutputFormatFactory.createFormatInstance(out, options.verbosity().orElse(Defaults.VERBOSITY));
|
||||
}
|
||||
|
||||
private static String getBenchmarkName(FrameworkMethod it) {
|
||||
return it.getDeclaringClass().getName() + "." + it.getName();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@link OutputFormat} that delegates to another {@link OutputFormat} and notifies {@link RunNotifier} about the
|
||||
* progress.
|
||||
*/
|
||||
static class NotifyingOutputFormat implements OutputFormat {
|
||||
|
||||
private final RunNotifier notifier;
|
||||
private final Function<String, Description> descriptionResolver;
|
||||
private final OutputFormat delegate;
|
||||
private final List<String> log = new CopyOnWriteArrayList<>();
|
||||
|
||||
private volatile String lastKnownBenchmark;
|
||||
private volatile boolean recordOutput;
|
||||
|
||||
NotifyingOutputFormat(RunNotifier notifier, Function<String, Description> methods, OutputFormat delegate) {
|
||||
this.notifier = notifier;
|
||||
this.descriptionResolver = methods;
|
||||
this.delegate = delegate;
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#iteration(org.openjdk.jmh.infra.BenchmarkParams, org.openjdk.jmh.infra.IterationParams, int)
|
||||
*/
|
||||
@Override
|
||||
public void iteration(BenchmarkParams benchParams, IterationParams params, int iteration) {
|
||||
delegate.iteration(benchParams, params, iteration);
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#iterationResult(org.openjdk.jmh.infra.BenchmarkParams, org.openjdk.jmh.infra.IterationParams, int, org.openjdk.jmh.results.IterationResult)
|
||||
*/
|
||||
@Override
|
||||
public void iterationResult(BenchmarkParams benchParams, IterationParams params, int iteration,
|
||||
IterationResult data) {
|
||||
delegate.iterationResult(benchParams, params, iteration, data);
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#startBenchmark(org.openjdk.jmh.infra.BenchmarkParams)
|
||||
*/
|
||||
@Override
|
||||
public void startBenchmark(BenchmarkParams benchParams) {
|
||||
|
||||
log.clear();
|
||||
|
||||
lastKnownBenchmark = benchParams.getBenchmark();
|
||||
notifier.fireTestStarted(descriptionResolver.apply(benchParams.getBenchmark()));
|
||||
|
||||
delegate.startBenchmark(benchParams);
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#endBenchmark(org.openjdk.jmh.results.BenchmarkResult)
|
||||
*/
|
||||
@Override
|
||||
public void endBenchmark(BenchmarkResult result) {
|
||||
|
||||
recordOutput = false;
|
||||
String lastKnownBenchmark = this.lastKnownBenchmark;
|
||||
if (result != null) {
|
||||
notifier.fireTestFinished(descriptionResolver.apply(result.getParams().getBenchmark()));
|
||||
} else if (lastKnownBenchmark != null) {
|
||||
|
||||
String output = StringUtils.collectionToDelimitedString(log, System.getProperty("line.separator"));
|
||||
notifier.fireTestFailure(
|
||||
new Failure(descriptionResolver.apply(lastKnownBenchmark), new JmhRunnerException(output)));
|
||||
}
|
||||
|
||||
log.clear();
|
||||
delegate.endBenchmark(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#startRun()
|
||||
*/
|
||||
@Override
|
||||
public void startRun() {
|
||||
delegate.startRun();
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#endRun(java.util.Collection)
|
||||
*/
|
||||
@Override
|
||||
public void endRun(Collection<RunResult> result) {
|
||||
delegate.endRun(result);
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#print(java.lang.String)
|
||||
*/
|
||||
@Override
|
||||
public void print(String s) {
|
||||
delegate.print(s);
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#println(java.lang.String)
|
||||
*/
|
||||
@Override
|
||||
public void println(String s) {
|
||||
|
||||
if (recordOutput && StringUtils.hasText(s)) {
|
||||
log.add(s);
|
||||
}
|
||||
|
||||
if (s.equals("<failure>")) {
|
||||
recordOutput = true;
|
||||
}
|
||||
|
||||
delegate.println(s);
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#flush()
|
||||
*/
|
||||
@Override
|
||||
public void flush() {
|
||||
delegate.flush();
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#close()
|
||||
*/
|
||||
@Override
|
||||
public void close() {
|
||||
delegate.close();
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#verbosePrintln(java.lang.String)
|
||||
*/
|
||||
@Override
|
||||
public void verbosePrintln(String s) {
|
||||
delegate.verbosePrintln(s);
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#write(int)
|
||||
*/
|
||||
@Override
|
||||
public void write(int b) {
|
||||
delegate.write(b);
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see org.openjdk.jmh.runner.format.OutputFormat#write(byte[])
|
||||
*/
|
||||
@Override
|
||||
public void write(byte[] b) throws IOException {
|
||||
delegate.write(b);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Exception proxy without stack trace.
|
||||
*/
|
||||
static class JmhRunnerException extends RuntimeException {
|
||||
|
||||
private static final long serialVersionUID = -1385006784559013618L;
|
||||
|
||||
JmhRunnerException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see java.lang.Throwable#fillInStackTrace()
|
||||
*/
|
||||
@Override
|
||||
public synchronized Throwable fillInStackTrace() {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache {@link Function} for benchmark names to {@link Description}.
|
||||
*/
|
||||
static class CacheFunction implements Function<String, Description> {
|
||||
|
||||
private final Map<String, FrameworkMethod> methodMap = new ConcurrentHashMap<>();
|
||||
private final Collection<FrameworkMethod> methods;
|
||||
private final Function<FrameworkMethod, Description> describeFunction;
|
||||
|
||||
CacheFunction(Collection<FrameworkMethod> methods, Function<FrameworkMethod, Description> describeFunction) {
|
||||
this.methods = methods;
|
||||
this.describeFunction = describeFunction;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a benchmark name (fqcn + "." + method name) to a {@link Description}.
|
||||
*
|
||||
* @param benchmarkName
|
||||
* @return
|
||||
*/
|
||||
public Description apply(String benchmarkName) {
|
||||
|
||||
FrameworkMethod frameworkMethod = methodMap.computeIfAbsent(benchmarkName, key -> {
|
||||
return methods.stream().filter(method -> getBenchmarkName(method).equals(key)).findFirst().get();
|
||||
});
|
||||
|
||||
return describeFunction.apply(frameworkMethod);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user