semantic segmentation processor

applications/processor/semantic-segmentation-processor/src/main/java/org/springframework/cloud/stream/app/processor/semantic/segmentation/SemanticSegmentationProcessorConfiguration.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2020-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.stream.app.processor.semantic.segmentation;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.function.Function;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.cloud.fn.common.tensorflow.deprecated.JsonMapperFunction;
+import org.springframework.cloud.fn.semantic.segmentation.SegmentationColorMap;
+import org.springframework.cloud.fn.semantic.segmentation.SemanticSegmentation;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.integration.support.MessageBuilder;
+import org.springframework.messaging.Message;
+
+/**
+ * @author Christian Tzolov
+ */
+@Configuration
+@EnableConfigurationProperties(SemanticSegmentationProcessorProperties.class)
+public class SemanticSegmentationProcessorConfiguration {
+
+	private static final Log logger = LogFactory.getLog(SemanticSegmentationProcessorConfiguration.class);
+
+	/**
+	 * Output header name.
+	 */
+	public static final String SEMANTIC_SEGMENTATION_HEADER = "semantic_segmentation";
+
+	@Bean
+	public SemanticSegmentation semanticSegmentation(SemanticSegmentationProcessorProperties properties) {
+		return new SemanticSegmentation(properties.getModel(),
+				SegmentationColorMap.loadColorMap(properties.getColorMapUri()), null,
+				properties.getMaskTransparency());
+	}
+
+	@Bean
+	public Function<Message<byte[]>, Message<byte[]>> semanticSegmentationFunction(
+			SemanticSegmentation semanticSegmentation,
+			SemanticSegmentationProcessorProperties properties) {
+
+		return input -> {
+			// You can use file:, http: or classpath: to provide the path to the input image.
+			byte[] inputImage = input.getPayload();
+
+			byte[] outputImage = (properties.getOutputType() == SemanticSegmentationProcessorProperties.OutputType.blended) ?
+					semanticSegmentation.blendMask(inputImage) : semanticSegmentation.maskImage(inputImage);
+
+			long[][] maskPixels = semanticSegmentation.maskPixels(inputImage);
+			String jsonMaskPixels = new JsonMapperFunction().apply(maskPixels);
+
+			Message<byte[]> outMessage = MessageBuilder
+					.withPayload(outputImage)
+					.setHeader(SEMANTIC_SEGMENTATION_HEADER, jsonMaskPixels)
+					.build();
+
+			if (properties.isDebugOutput()) {
+				try {
+					IOUtils.write(outputImage, new FileOutputStream(properties.getDebugOutputPath()));
+				}
+				catch (IOException e) {
+					logger.warn("Cloud not produce debug output", e);
+				}
+			}
+
+			return outMessage;
+		};
+	}
+}

applications/processor/semantic-segmentation-processor/src/main/java/org/springframework/cloud/stream/app/processor/semantic/segmentation/SemanticSegmentationProcessorProperties.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2020-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.stream.app.processor.semantic.segmentation;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.validation.annotation.Validated;
+
+/**
+ * @author Christian Tzolov
+ */
+@ConfigurationProperties("semantic.segmentation")
+@Validated
+public class SemanticSegmentationProcessorProperties {
+
+	enum OutputType {
+		/** Input image augmented with the segmentation mask on top. */
+		blended,
+		/** Image of the segmentation mask. */
+		mask
+	}
+
+	/**
+	 * pre-trained tensorflow semantic segmentation model.
+	 */
+	private String model = "http://download.tensorflow.org/models/deeplabv3_mnv2_cityscapes_train_2018_02_05.tar.gz#frozen_inference_graph.pb";
+
+	/**
+	 * Specifies the output image type. You can return either the input image with the computed mask overlay, or
+	 * the mask alone.
+	 */
+	private OutputType outputType = OutputType.blended;
+
+	/**
+	 * Every pre-trained model is based on certain object color maps.
+	 * The pre-defined options are:
+	 *  - classpath:/colormap/citymap_colormap.json
+	 *  - classpath:/colormap/ade20k_colormap.json
+	 *  - classpath:/colormap/black_white_colormap.json
+	 *  - classpath:/colormap/mapillary_colormap.json
+	 */
+	private String colorMapUri = "classpath:/colormap/citymap_colormap.json";
+
+	/**
+	 * The alpha color of the computed segmentation mask image.
+	 */
+	private float maskTransparency = 0.45f;
+
+	/**
+	 * save output image inn the local debugOutputPath path.
+	 */
+	private boolean debugOutput = false;
+
+	private String debugOutputPath = "semantic-segmentation-result.png";
+
+	public OutputType getOutputType() {
+		return outputType;
+	}
+
+	public void setOutputType(OutputType outputType) {
+		this.outputType = outputType;
+	}
+
+	public boolean isDebugOutput() {
+		return debugOutput;
+	}
+
+	public void setDebugOutput(boolean debugOutput) {
+		this.debugOutput = debugOutput;
+	}
+
+	public String getDebugOutputPath() {
+		return debugOutputPath;
+	}
+
+	public void setDebugOutputPath(String debugOutputPath) {
+		this.debugOutputPath = debugOutputPath;
+	}
+
+	public String getModel() {
+		return model;
+	}
+
+	public void setModel(String model) {
+		this.model = model;
+	}
+
+	public String getColorMapUri() {
+		return colorMapUri;
+	}
+
+	public void setColorMapUri(String colorMapUri) {
+		this.colorMapUri = colorMapUri;
+	}
+
+	public float getMaskTransparency() {
+		return maskTransparency;
+	}
+
+	public void setMaskTransparency(float maskTransparency) {
+		this.maskTransparency = maskTransparency;
+	}
+
+}

applications/processor/semantic-segmentation-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties

@@ -0,0 +1,2 @@
+configuration-properties.classes=\
+  org.springframework.cloud.stream.app.processor.semantic.segmentation.SemanticSegmentationProcessorProperties

applications/processor/semantic-segmentation-processor/src/test/java/org/springframework/cloud/stream/app/processor/semantic/segmentation/SemanticSegmentationProcessorTests.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2020-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.stream.app.processor.semantic.segmentation;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.boot.WebApplicationType;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils;
+import org.springframework.cloud.stream.binder.test.InputDestination;
+import org.springframework.cloud.stream.binder.test.OutputDestination;
+import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.Import;
+import org.springframework.messaging.Message;
+import org.springframework.messaging.support.GenericMessage;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author Christian Tzolov
+ */
+public class SemanticSegmentationProcessorTests {
+
+	@Test
+	public void testSemanticSegmentationProcessor() throws IOException {
+		try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
+				TestChannelBinderConfiguration.getCompleteConfiguration(SemanticSegmentationTestApplication.class))
+				.web(WebApplicationType.NONE)
+				.run("--spring.cloud.function.definition=semanticSegmentationFunction",
+						"--semantic.segmentation.model=http://download.tensorflow.org/models/deeplabv3_mnv2_cityscapes_train_2018_02_05.tar.gz#frozen_inference_graph.pb",
+						"--semantic.segmentation.colorMapUri=classpath:/colormap/citymap_colormap.json",
+						"--semantic.segmentation.outputType=blended",
+						"--semantic.segmentation.debugOutput=true",
+						"--semantic.segmentation.debugOutputPath=./target/semantic-segmentation-1.png")) {
+
+			InputDestination processorInput = context.getBean(InputDestination.class);
+			OutputDestination processorOutput = context.getBean(OutputDestination.class);
+
+			byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/amsterdam-cityscape1.jpg");
+			processorInput.send(new GenericMessage<>(inputImage));
+			Message<byte[]> sourceMessage = processorOutput.receive(10000);
+			String jsonRecognizedObjects = (String) sourceMessage.getHeaders().get(
+					SemanticSegmentationProcessorConfiguration.SEMANTIC_SEGMENTATION_HEADER);
+			assertThat(jsonRecognizedObjects).isNotEmpty();
+			//assertThat(jsonRecognizedObjects)
+			//		.isEqualTo("[{\"label\":\"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\",\"probability\":0.962329626083374}," +
+			//				"{\"label\":\"badger\",\"probability\":0.006058811210095882}," +
+			//				"{\"label\":\"ram, tup\",\"probability\":0.0010668420000001788}]");
+		}
+	}
+
+
+	@SpringBootApplication
+	@Import(SemanticSegmentationProcessorConfiguration.class)
+	public static class SemanticSegmentationTestApplication {
+	}
+
+}