diff --git a/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/ProtoBufGraphDefinition.java b/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/ProtoBufGraphDefinition.java index a1cb7d49..e79be4b1 100644 --- a/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/ProtoBufGraphDefinition.java +++ b/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/ProtoBufGraphDefinition.java @@ -70,6 +70,5 @@ public class ProtoBufGraphDefinition implements GraphDefinition { while (ops.hasNext()) { System.out.println(ops.next().name()); } - System.out.println("Boza"); } } diff --git a/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SegmentationColorMap.java b/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SegmentationColorMap.java index 3a958cf1..7c42e179 100644 --- a/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SegmentationColorMap.java +++ b/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SegmentationColorMap.java @@ -16,6 +16,13 @@ package org.springframework.cloud.fn.semantic.segmentation; +import java.io.InputStream; +import java.util.Arrays; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.springframework.core.io.DefaultResourceLoader; + /** * * Visualizes the segmentation results via specified color map. @@ -303,4 +310,54 @@ public final class SegmentationColorMap { System.arraycopy(_CITYMAP_COLORMAP[i], 0, CITYMAP_COLORMAP[i], 0, _CITYMAP_COLORMAP[i].length); } } + + public static int[][] loadColorMap(String resourceUri) { + try { + InputStream colorMapIs = new DefaultResourceLoader().getResource(resourceUri).getInputStream(); + ColorMap colorMap = new ObjectMapper().readValue(colorMapIs, ColorMap.class); + return colorMap.getColormap(); + } + catch (Exception exception) { + throw new RuntimeException(exception); + } + } + + public static class ColorMap { + private String name; + private String info; + private int[][] colormap; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getInfo() { + return info; + } + + public void setInfo(String info) { + this.info = info; + } + + public int[][] getColormap() { + return colormap; + } + + public void setColormap(int[][] colormap) { + this.colormap = colormap; + } + + @Override + public String toString() { + return "ColorMap{" + + "name='" + name + '\'' + + "info='" + info + '\'' + + ", colormap=" + Arrays.deepToString(colormap) + + '}'; + } + } } diff --git a/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SemanticSegmentation.java b/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SemanticSegmentation.java index bfe5e317..fb872ae4 100644 --- a/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SemanticSegmentation.java +++ b/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SemanticSegmentation.java @@ -25,6 +25,7 @@ import java.util.Map; import javax.imageio.ImageIO; +import com.fasterxml.jackson.databind.ObjectMapper; import org.tensorflow.Operand; import org.tensorflow.Tensor; import org.tensorflow.op.Ops; @@ -219,33 +220,31 @@ public class SemanticSegmentation implements AutoCloseable { public static void main(String[] args) throws IOException { - //String inputImageUri = "file:/Users/ctzolov/Dev/projects/mindmodel/mind-model-services/semantic-segmentation/src/test/resources/images/VikiMaxiAdi.jpg"; - String outputBlendedImagePath = "./semantic-segmentation/target/blendedImage.png"; - String outputMaskImagePath = "./semantic-segmentation/target/maskImage.png"; - - try (SemanticSegmentation segmentationService = new SemanticSegmentation( "http://download.tensorflow.org/models/deeplabv3_mnv2_cityscapes_train_2018_02_05.tar.gz#frozen_inference_graph.pb", - SegmentationColorMap.CITYMAP_COLORMAP, null, 0.45f) + SegmentationColorMap.loadColorMap("classpath:/colormap/citymap_colormap.json"), null, 0.45f) ) { byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/amsterdam-cityscape1.jpg"); // 1. Mask pixels long[][] maskPixels = segmentationService.maskPixels(inputImage); + String json = new ObjectMapper().writeValueAsString(maskPixels); + // 2. Alpha Blending byte[] blended = segmentationService.blendMask(inputImage); - ImageIO.write(ImageIO.read(new ByteArrayInputStream(blended)), "png", new File(outputBlendedImagePath)); + ImageIO.write(ImageIO.read(new ByteArrayInputStream(blended)), "png", + new File("./functions/function/semantic-segmentation-function/target/blendedImage.png")); // 3. Mask Image byte[] maskImage = segmentationService.maskImage(inputImage); - ImageIO.write(ImageIO.read(new ByteArrayInputStream(maskImage)), "png", new File(outputMaskImagePath)); + ImageIO.write(ImageIO.read(new ByteArrayInputStream(maskImage)), "png", + new File("./functions/function/semantic-segmentation-function/target/maskImage.png")); } - try (SemanticSegmentation segmentationService = new SemanticSegmentation( "http://download.tensorflow.org/models/deeplabv3_xception_ade20k_train_2018_05_29.tar.gz#frozen_inference_graph.pb", - SegmentationColorMap.ADE20K_COLORMAP, null, 0.45f) + SegmentationColorMap.loadColorMap("classpath:/colormap/ade20k_colormap.json"), null, 0.45f) ) { byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/interior.jpg"); @@ -255,17 +254,17 @@ public class SemanticSegmentation implements AutoCloseable { // 2. Alpha Blending byte[] blended = segmentationService.blendMask(inputImage); ImageIO.write(ImageIO.read(new ByteArrayInputStream(blended)), "png", - new File("./semantic-segmentation/target/inventory-blendedImage.png")); + new File("./functions/function/semantic-segmentation-function/target/inventory-blendedImage.png")); // 3. Mask Image byte[] maskImage = segmentationService.maskImage(inputImage); ImageIO.write(ImageIO.read(new ByteArrayInputStream(maskImage)), "png", - new File("./semantic-segmentation/target/inventory-MaskImage.png")); + new File("./functions/function/semantic-segmentation-function/target/inventory-MaskImage.png")); } try (SemanticSegmentation segmentationService = new SemanticSegmentation( "http://download.tensorflow.org/models/deeplabv3_mnv2_pascal_trainval_2018_01_29.tar.gz#frozen_inference_graph.pb", - SegmentationColorMap.BLACK_WHITE_COLORMAP, null, 0.45f) + SegmentationColorMap.loadColorMap("classpath:/colormap/black_white_colormap.json"), null, 0.45f) ) { byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/VikiMaxiAdi.jpg"); @@ -275,12 +274,12 @@ public class SemanticSegmentation implements AutoCloseable { // 2. Alpha Blending byte[] blended = segmentationService.blendMask(inputImage); ImageIO.write(ImageIO.read(new ByteArrayInputStream(blended)), "png", - new File("./semantic-segmentation/target/pascal-blendedImage.png")); + new File("./functions/function/semantic-segmentation-function/target/pascal-blendedImage.png")); // 3. Mask Image byte[] maskImage = segmentationService.maskImage(inputImage); ImageIO.write(ImageIO.read(new ByteArrayInputStream(maskImage)), "png", - new File("./semantic-segmentation/target/pascal-MaskImage.png")); + new File("./functions/function/semantic-segmentation-function/target/pascal-MaskImage.png")); } } diff --git a/function/semantic-segmentation-function/src/main/resources/colormap/ade20k_colormap.json b/function/semantic-segmentation-function/src/main/resources/colormap/ade20k_colormap.json new file mode 100644 index 00000000..51648f4e --- /dev/null +++ b/function/semantic-segmentation-function/src/main/resources/colormap/ade20k_colormap.json @@ -0,0 +1,156 @@ +{ + "name" : "ade20k", + "info" : "ADE20K (http://groups.csail.mit.edu/vision/datasets/ADE20K/)", + "colormap" :[ + [ 0, 0, 0 ], + [ 120, 120, 120 ], + [ 180, 120, 120 ], + [ 6, 230, 230 ], + [ 80, 50, 50 ], + [ 4, 200, 3 ], + [ 120, 120, 80 ], + [ 140, 140, 140 ], + [ 204, 5, 255 ], + [ 230, 230, 230 ], + [ 4, 250, 7 ], + [ 224, 5, 255 ], + [ 235, 255, 7 ], + [ 150, 5, 61 ], + [ 120, 120, 70 ], + [ 8, 255, 51 ], + [ 255, 6, 82 ], + [ 143, 255, 140 ], + [ 204, 255, 4 ], + [ 255, 51, 7 ], + [ 204, 70, 3 ], + [ 0, 102, 200 ], + [ 61, 230, 250 ], + [ 255, 6, 51 ], + [ 11, 102, 255 ], + [ 255, 7, 71 ], + [ 255, 9, 224 ], + [ 9, 7, 230 ], + [ 220, 220, 220 ], + [ 255, 9, 92 ], + [ 112, 9, 255 ], + [ 8, 255, 214 ], + [ 7, 255, 224 ], + [ 255, 184, 6 ], + [ 10, 255, 71 ], + [ 255, 41, 10 ], + [ 7, 255, 255 ], + [ 224, 255, 8 ], + [ 102, 8, 255 ], + [ 255, 61, 6 ], + [ 255, 194, 7 ], + [ 255, 122, 8 ], + [ 0, 255, 20 ], + [ 255, 8, 41 ], + [ 255, 5, 153 ], + [ 6, 51, 255 ], + [ 235, 12, 255 ], + [ 160, 150, 20 ], + [ 0, 163, 255 ], + [ 140, 140, 140 ], + [ 250, 10, 15 ], + [ 20, 255, 0 ], + [ 31, 255, 0 ], + [ 255, 31, 0 ], + [ 255, 224, 0 ], + [ 153, 255, 0 ], + [ 0, 0, 255 ], + [ 255, 71, 0 ], + [ 0, 235, 255 ], + [ 0, 173, 255 ], + [ 31, 0, 255 ], + [ 11, 200, 200 ], + [ 255, 82, 0 ], + [ 0, 255, 245 ], + [ 0, 61, 255 ], + [ 0, 255, 112 ], + [ 0, 255, 133 ], + [ 255, 0, 0 ], + [ 255, 163, 0 ], + [ 255, 102, 0 ], + [ 194, 255, 0 ], + [ 0, 143, 255 ], + [ 51, 255, 0 ], + [ 0, 82, 255 ], + [ 0, 255, 41 ], + [ 0, 255, 173 ], + [ 10, 0, 255 ], + [ 173, 255, 0 ], + [ 0, 255, 153 ], + [ 255, 92, 0 ], + [ 255, 0, 255 ], + [ 255, 0, 245 ], + [ 255, 0, 102 ], + [ 255, 173, 0 ], + [ 255, 0, 20 ], + [ 255, 184, 184 ], + [ 0, 31, 255 ], + [ 0, 255, 61 ], + [ 0, 71, 255 ], + [ 255, 0, 204 ], + [ 0, 255, 194 ], + [ 0, 255, 82 ], + [ 0, 10, 255 ], + [ 0, 112, 255 ], + [ 51, 0, 255 ], + [ 0, 194, 255 ], + [ 0, 122, 255 ], + [ 0, 255, 163 ], + [ 255, 153, 0 ], + [ 0, 255, 10 ], + [ 255, 112, 0 ], + [ 143, 255, 0 ], + [ 82, 0, 255 ], + [ 163, 255, 0 ], + [ 255, 235, 0 ], + [ 8, 184, 170 ], + [ 133, 0, 255 ], + [ 0, 255, 92 ], + [ 184, 0, 255 ], + [ 255, 0, 31 ], + [ 0, 184, 255 ], + [ 0, 214, 255 ], + [ 255, 0, 112 ], + [ 92, 255, 0 ], + [ 0, 224, 255 ], + [ 112, 224, 255 ], + [ 70, 184, 160 ], + [ 163, 0, 255 ], + [ 153, 0, 255 ], + [ 71, 255, 0 ], + [ 255, 0, 163 ], + [ 255, 204, 0 ], + [ 255, 0, 143 ], + [ 0, 255, 235 ], + [ 133, 255, 0 ], + [ 255, 0, 235 ], + [ 245, 0, 255 ], + [ 255, 0, 122 ], + [ 255, 245, 0 ], + [ 10, 190, 212 ], + [ 214, 255, 0 ], + [ 0, 204, 255 ], + [ 20, 0, 255 ], + [ 255, 255, 0 ], + [ 0, 153, 255 ], + [ 0, 41, 255 ], + [ 0, 255, 204 ], + [ 41, 0, 255 ], + [ 41, 255, 0 ], + [ 173, 0, 255 ], + [ 0, 245, 255 ], + [ 71, 0, 255 ], + [ 122, 0, 255 ], + [ 0, 255, 184 ], + [ 0, 92, 255 ], + [ 184, 255, 0 ], + [ 0, 133, 255 ], + [ 255, 214, 0 ], + [ 25, 194, 194 ], + [ 102, 255, 0 ], + [ 92, 0, 255 ]] +} diff --git a/function/semantic-segmentation-function/src/main/resources/colormap/black_white_colormap.json b/function/semantic-segmentation-function/src/main/resources/colormap/black_white_colormap.json new file mode 100644 index 00000000..ca1c33ea --- /dev/null +++ b/function/semantic-segmentation-function/src/main/resources/colormap/black_white_colormap.json @@ -0,0 +1,8 @@ +{ + "name" : "black_white", + "info" : "Black and white color map", + "colormap" :[ + [ 0, 0, 0 ], + [ 127, 127, 127 ], + [ 255, 255, 255 ]] +} diff --git a/function/semantic-segmentation-function/src/main/resources/colormap/citymap_colormap.json b/function/semantic-segmentation-function/src/main/resources/colormap/citymap_colormap.json new file mode 100644 index 00000000..dbd1ef2c --- /dev/null +++ b/function/semantic-segmentation-function/src/main/resources/colormap/citymap_colormap.json @@ -0,0 +1,24 @@ +{ + "name" : "citymap", + "info" : "Cityscapes dataset (https://www.cityscapes-dataset.com).", + "colormap" :[ + [ 128, 64, 128 ], + [ 244, 35, 232 ], + [ 70, 70, 70 ], + [ 102, 102, 156 ], + [ 190, 153, 153 ], + [ 153, 153, 153 ], + [ 250, 170, 30 ], + [ 220, 220, 0 ], + [ 107, 142, 35 ], + [ 152, 251, 152 ], + [ 70, 130, 180 ], + [ 220, 20, 60 ], + [ 255, 0, 0 ], + [ 0, 0, 142 ], + [ 0, 0, 70 ], + [ 0, 60, 100 ], + [ 0, 80, 100 ], + [ 0, 0, 230 ], + [ 119, 11, 32 ]] +} diff --git a/function/semantic-segmentation-function/src/main/resources/colormap/mapillary_colormap.json b/function/semantic-segmentation-function/src/main/resources/colormap/mapillary_colormap.json new file mode 100644 index 00000000..f089268b --- /dev/null +++ b/function/semantic-segmentation-function/src/main/resources/colormap/mapillary_colormap.json @@ -0,0 +1,71 @@ +{ + "name" : "mapillary", + "info" : "Mapillary Vistas (https://research.mapillary.com).", + "colormap" :[ + [ 165, 42, 42 ], + [ 0, 192, 0 ], + [ 196, 196, 196 ], + [ 190, 153, 153 ], + [ 180, 165, 180 ], + [ 102, 102, 156 ], + [ 102, 102, 156 ], + [ 128, 64, 255 ], + [ 140, 140, 200 ], + [ 170, 170, 170 ], + [ 250, 170, 160 ], + [ 96, 96, 96 ], + [ 230, 150, 140 ], + [ 128, 64, 128 ], + [ 110, 110, 110 ], + [ 244, 35, 232 ], + [ 150, 100, 100 ], + [ 70, 70, 70 ], + [ 150, 120, 90 ], + [ 220, 20, 60 ], + [ 255, 0, 0 ], + [ 255, 0, 0 ], + [ 255, 0, 0 ], + [ 200, 128, 128 ], + [ 255, 255, 255 ], + [ 64, 170, 64 ], + [ 128, 64, 64 ], + [ 70, 130, 180 ], + [ 255, 255, 255 ], + [ 152, 251, 152 ], + [ 107, 142, 35 ], + [ 0, 170, 30 ], + [ 255, 255, 128 ], + [ 250, 0, 30 ], + [ 0, 0, 0 ], + [ 220, 220, 220 ], + [ 170, 170, 170 ], + [ 222, 40, 40 ], + [ 100, 170, 30 ], + [ 40, 40, 40 ], + [ 33, 33, 33 ], + [ 170, 170, 170 ], + [ 0, 0, 142 ], + [ 170, 170, 170 ], + [ 210, 170, 100 ], + [ 153, 153, 153 ], + [ 128, 128, 128 ], + [ 0, 0, 142 ], + [ 250, 170, 30 ], + [ 192, 192, 192 ], + [ 220, 220, 0 ], + [ 180, 165, 180 ], + [ 119, 11, 32 ], + [ 0, 0, 142 ], + [ 0, 60, 100 ], + [ 0, 0, 142 ], + [ 0, 0, 90 ], + [ 0, 0, 230 ], + [ 0, 80, 100 ], + [ 128, 64, 64 ], + [ 0, 0, 110 ], + [ 0, 0, 70 ], + [ 0, 0, 192 ], + [ 32, 32, 32 ], + [ 0, 0, 0 ], + [ 0, 0, 0 ]] +} diff --git a/function/semantic-segmentation-function/src/main/resources/images/VikiMaxiAdi.jpg b/function/semantic-segmentation-function/src/main/resources/images/VikiMaxiAdi.jpg new file mode 100644 index 00000000..5d901702 Binary files /dev/null and b/function/semantic-segmentation-function/src/main/resources/images/VikiMaxiAdi.jpg differ diff --git a/function/semantic-segmentation-function/src/main/resources/images/amsterdam-cityscape1.jpg b/function/semantic-segmentation-function/src/main/resources/images/amsterdam-cityscape1.jpg new file mode 100644 index 00000000..d77cae40 Binary files /dev/null and b/function/semantic-segmentation-function/src/main/resources/images/amsterdam-cityscape1.jpg differ diff --git a/function/semantic-segmentation-function/src/main/resources/images/giant_panda_in_beijing_zoo_1.jpg b/function/semantic-segmentation-function/src/main/resources/images/giant_panda_in_beijing_zoo_1.jpg new file mode 100644 index 00000000..e7a44cae Binary files /dev/null and b/function/semantic-segmentation-function/src/main/resources/images/giant_panda_in_beijing_zoo_1.jpg differ diff --git a/function/semantic-segmentation-function/src/main/resources/images/interior.jpg b/function/semantic-segmentation-function/src/main/resources/images/interior.jpg new file mode 100644 index 00000000..f5516015 Binary files /dev/null and b/function/semantic-segmentation-function/src/main/resources/images/interior.jpg differ