SpringCloud/spring-functions-catalog
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-7: Add checkstyle and javaformat plugins

Browse Source 
Fixes: #7

* Run `./gradlew format`
* Updates from PR review suggestions
This commit is contained in:
Chris Bono
2023-12-29 21:08:58 -06:00
committed by Artem Bilan
parent 84e732da08
commit 836708f0f2
357 changed files with 4344 additions and 4519 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
function/spring-semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/NativeImageUtils.java
View File

@@ -43,11 +43,10 @@ public final class NativeImageUtils {
* https://github.com/tensorflow/tensorflow/blob/r1.13/tensorflow/python/ops/image_ops_impl.py#L1536
*/
public static <T> Operand<T> grayscaleToRgb(Ops tf, Operand<T> images) {
ExpandDims<Integer> rank_1 = tf.expandDims(
tf.math.sub(tf.rank(images), tf.constant(1)),
tf.constant(0));
ExpandDims<Integer> rank_1 = tf.expandDims(tf.math.sub(tf.rank(images), tf.constant(1)), tf.constant(0));
// Create once 1D vector of the shape defined by the rank_1.
// E.g. for rank [2] will produce matrix [1, 1]. For [3] rank will produce a cube [1, 1, 1]
// E.g. for rank [2] will produce matrix [1, 1]. For [3] rank will produce a cube
// [1, 1, 1]
Add<Integer> ones = tf.math.add(tf.zeros(rank_1, Integer.class), tf.constant(1));
// Convert scalar 3 into 1D array [3]
ExpandDims<Integer> channelsAs1D = tf.expandDims(tf.constant(3), tf.constant(0));
@@ -59,49 +58,57 @@ public final class NativeImageUtils {
public static Operand<Float> normalizeMask(Ops tf, Operand<Float> mask, float newValue) {
// generate array representing the axis indexes.
// For example of tensor of rank K the axisRange is {0, 1, 2 ...K}
Range<Integer> axisRange = tf.range(tf.constant(0), // from
Range<Integer> axisRange = tf.range(tf.constant(0), // from
tf.dtypes.cast(tf.rank(mask), Integer.class), // to
tf.constant(1)); // step
ReduceMax<Float> max = tf.reduceMax(mask, axisRange);
//Mul<Float> input2Float1 = tf.math.mul(tf.math.div(input2Float, max), tf.constant(1f));
// Mul<Float> input2Float1 = tf.math.mul(tf.math.div(input2Float, max),
// tf.constant(1f));
Mul<Float> normalizedMask = tf.math.mul(tf.math.div(mask, max), tf.constant(newValue));
return normalizedMask;
}
/**
* Alpha Blending .
* https://en.wikipedia.org/wiki/Alpha_compositing#Alpha_blending
* Alpha Blending . https://en.wikipedia.org/wiki/Alpha_compositing#Alpha_blending
*/
public static Operand<Float> alphaBlending(Ops tf, Operand<Float> srcRgb, Operand<Float> dstRgb, Operand<Float> srcAlpha) {
public static Operand<Float> alphaBlending(Ops tf, Operand<Float> srcRgb, Operand<Float> dstRgb,
Operand<Float> srcAlpha) {
Sub<Float> alpha = tf.math.sub(tf.onesLike(srcRgb), srcAlpha);
Mul<Float> src = tf.math.mul(srcRgb, alpha);
Mul<Float> dst = tf.math.mul(dstRgb, tf.math.sub(tf.constant(1.0f), alpha));
Add<Float> out = tf.math.add(dst, src);
//Mul<Float> out = tf.math.mul(srcRgbNormalized, dstRgb);
//Squeeze<Float> squeeze = tf.withName("squeeze").squeeze(out, Squeeze.axis(Arrays.asList(0L)));
// Mul<Float> out = tf.math.mul(srcRgbNormalized, dstRgb);
// Squeeze<Float> squeeze = tf.withName("squeeze").squeeze(out,
// Squeeze.axis(Arrays.asList(0L)));
return out;
}
/**
* The mask can contain label values larger than the list of colors provided in the color map.
* To avoid out-of-index errors we will "normalize" the label values in the mask to MOD max-color-table-value.
* The mask can contain label values larger than the list of colors provided in the
* color map. To avoid out-of-index errors we will "normalize" the label values in the
* mask to MOD max-color-table-value.
* @param tf - tensorflow
* @param colorTable Color map of shape [n, 3]. n is the count of label entries and 3 is the RGB color assigned
* to that label.
* @param colorTable Color map of shape [n, 3]. n is the count of label entries and 3
* is the RGB color assigned to that label.
* @param mask Mask of shape [h, w] containing label vales.
* @return Mask of shape [h, w] fromMemory values normalized between [0, n]
*/
public static Operand<Long> normalizeMaskLabels(Ops tf, Operand<Integer> colorTable, Operand<Long> mask) {
// The mask can contain label values larger than the list of colors provided in the color map.
// To avoid out-of-index errors we will "normalize" the label values in the mask to MOD max-color-table-value.
// The mask can contain label values larger than the list of colors provided in
// the color map.
// To avoid out-of-index errors we will "normalize" the label values in the mask
// to MOD max-color-table-value.
Sub<Long> colorTableShape = tf.math.sub(tf.shape(colorTable, Long.class), tf.constant(1L));
// Color tables have shape [N, 3], where N is the count of label entries. Therefore the max label id is (N - 1).
// Color tables have shape [N, 3], where N is the count of label entries.
// Therefore the max label id is (N - 1).
Gather<Long> colorTableSize = tf.gather(colorTableShape, tf.constant(new int[] { 0 }), tf.constant(0));
// Normalize the label values in the mask so they don't exceed the max value in the color map.
// Normalize the label values in the mask so they don't exceed the max value in
// the color map.
return tf.math.mod(mask, colorTableSize);
}
}

319
function/spring-semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SegmentationColorMap.java
View File

@@ -25,16 +25,17 @@ import org.springframework.core.io.DefaultResourceLoader;
/**
*
* Visualizes the segmentation results via specified color map.
* Color maps helping to visualize the semantic segmentation results for the different datasets.
* Visualizes the segmentation results via specified color map. Color maps helping to
* visualize the semantic segmentation results for the different datasets.
*
* Supported colormaps are:
* - ADE20K (http://groups.csail.mit.edu/vision/datasets/ADE20K/).
* - Cityscapes dataset (https://www.cityscapes-dataset.com).
* - Mapillary Vistas (https://research.mapillary.com).
* - PASCAL VOC 2012 (http://host.robots.ox.ac.uk/pascal/VOC/).
* Supported colormaps are: - ADE20K
* (http://groups.csail.mit.edu/vision/datasets/ADE20K/). - Cityscapes dataset
* (https://www.cityscapes-dataset.com). - Mapillary Vistas
* (https://research.mapillary.com). - PASCAL VOC 2012
* (http://host.robots.ox.ac.uk/pascal/VOC/).
*
* Based on: https://github.com/tensorflow/models/blob/master/research/deeplab/utils/get_dataset_colormap.py
* Based on:
* https://github.com/tensorflow/models/blob/master/research/deeplab/utils/get_dataset_colormap.py
*
* @author Christian Tzolov
*/
@@ -45,238 +46,51 @@ public final class SegmentationColorMap {
}
/** MAPILLARY_COLORMAP . */
public static final int[][] MAPILLARY_COLORMAP = new int[][] {
{ 165, 42, 42 },
{ 0, 192, 0 },
{ 196, 196, 196 },
{ 190, 153, 153 },
{ 180, 165, 180 },
{ 102, 102, 156 },
{ 102, 102, 156 },
{ 128, 64, 255 },
{ 140, 140, 200 },
{ 170, 170, 170 },
{ 250, 170, 160 },
{ 96, 96, 96 },
{ 230, 150, 140 },
{ 128, 64, 128 },
{ 110, 110, 110 },
{ 244, 35, 232 },
{ 150, 100, 100 },
{ 70, 70, 70 },
{ 150, 120, 90 },
{ 220, 20, 60 },
{ 255, 0, 0 },
{ 255, 0, 0 },
{ 255, 0, 0 },
{ 200, 128, 128 },
{ 255, 255, 255 },
{ 64, 170, 64 },
{ 128, 64, 64 },
{ 70, 130, 180 },
{ 255, 255, 255 },
{ 152, 251, 152 },
{ 107, 142, 35 },
{ 0, 170, 30 },
{ 255, 255, 128 },
{ 250, 0, 30 },
{ 0, 0, 0 },
{ 220, 220, 220 },
{ 170, 170, 170 },
{ 222, 40, 40 },
{ 100, 170, 30 },
{ 40, 40, 40 },
{ 33, 33, 33 },
{ 170, 170, 170 },
{ 0, 0, 142 },
{ 170, 170, 170 },
{ 210, 170, 100 },
{ 153, 153, 153 },
{ 128, 128, 128 },
{ 0, 0, 142 },
{ 250, 170, 30 },
{ 192, 192, 192 },
{ 220, 220, 0 },
{ 180, 165, 180 },
{ 119, 11, 32 },
{ 0, 0, 142 },
{ 0, 60, 100 },
{ 0, 0, 142 },
{ 0, 0, 90 },
{ 0, 0, 230 },
{ 0, 80, 100 },
{ 128, 64, 64 },
{ 0, 0, 110 },
{ 0, 0, 70 },
{ 0, 0, 192 },
{ 32, 32, 32 },
{ 0, 0, 0 },
{ 0, 0, 0 },
};
public static final int[][] MAPILLARY_COLORMAP = new int[][] { { 165, 42, 42 }, { 0, 192, 0 }, { 196, 196, 196 },
{ 190, 153, 153 }, { 180, 165, 180 }, { 102, 102, 156 }, { 102, 102, 156 }, { 128, 64, 255 },
{ 140, 140, 200 }, { 170, 170, 170 }, { 250, 170, 160 }, { 96, 96, 96 }, { 230, 150, 140 },
{ 128, 64, 128 }, { 110, 110, 110 }, { 244, 35, 232 }, { 150, 100, 100 }, { 70, 70, 70 }, { 150, 120, 90 },
{ 220, 20, 60 }, { 255, 0, 0 }, { 255, 0, 0 }, { 255, 0, 0 }, { 200, 128, 128 }, { 255, 255, 255 },
{ 64, 170, 64 }, { 128, 64, 64 }, { 70, 130, 180 }, { 255, 255, 255 }, { 152, 251, 152 }, { 107, 142, 35 },
{ 0, 170, 30 }, { 255, 255, 128 }, { 250, 0, 30 }, { 0, 0, 0 }, { 220, 220, 220 }, { 170, 170, 170 },
{ 222, 40, 40 }, { 100, 170, 30 }, { 40, 40, 40 }, { 33, 33, 33 }, { 170, 170, 170 }, { 0, 0, 142 },
{ 170, 170, 170 }, { 210, 170, 100 }, { 153, 153, 153 }, { 128, 128, 128 }, { 0, 0, 142 }, { 250, 170, 30 },
{ 192, 192, 192 }, { 220, 220, 0 }, { 180, 165, 180 }, { 119, 11, 32 }, { 0, 0, 142 }, { 0, 60, 100 },
{ 0, 0, 142 }, { 0, 0, 90 }, { 0, 0, 230 }, { 0, 80, 100 }, { 128, 64, 64 }, { 0, 0, 110 }, { 0, 0, 70 },
{ 0, 0, 192 }, { 32, 32, 32 }, { 0, 0, 0 }, { 0, 0, 0 }, };
/**
* Label colormap used in ADE20K segmentation benchmark.
*/
public static final int[][] ADE20K_COLORMAP = new int[][] {
{ 0, 0, 0 },
{ 120, 120, 120 },
{ 180, 120, 120 },
{ 6, 230, 230 },
{ 80, 50, 50 },
{ 4, 200, 3 },
{ 120, 120, 80 },
{ 140, 140, 140 },
{ 204, 5, 255 },
{ 230, 230, 230 },
{ 4, 250, 7 },
{ 224, 5, 255 },
{ 235, 255, 7 },
{ 150, 5, 61 },
{ 120, 120, 70 },
{ 8, 255, 51 },
{ 255, 6, 82 },
{ 143, 255, 140 },
{ 204, 255, 4 },
{ 255, 51, 7 },
{ 204, 70, 3 },
{ 0, 102, 200 },
{ 61, 230, 250 },
{ 255, 6, 51 },
{ 11, 102, 255 },
{ 255, 7, 71 },
{ 255, 9, 224 },
{ 9, 7, 230 },
{ 220, 220, 220 },
{ 255, 9, 92 },
{ 112, 9, 255 },
{ 8, 255, 214 },
{ 7, 255, 224 },
{ 255, 184, 6 },
{ 10, 255, 71 },
{ 255, 41, 10 },
{ 7, 255, 255 },
{ 224, 255, 8 },
{ 102, 8, 255 },
{ 255, 61, 6 },
{ 255, 194, 7 },
{ 255, 122, 8 },
{ 0, 255, 20 },
{ 255, 8, 41 },
{ 255, 5, 153 },
{ 6, 51, 255 },
{ 235, 12, 255 },
{ 160, 150, 20 },
{ 0, 163, 255 },
{ 140, 140, 140 },
{ 250, 10, 15 },
{ 20, 255, 0 },
{ 31, 255, 0 },
{ 255, 31, 0 },
{ 255, 224, 0 },
{ 153, 255, 0 },
{ 0, 0, 255 },
{ 255, 71, 0 },
{ 0, 235, 255 },
{ 0, 173, 255 },
{ 31, 0, 255 },
{ 11, 200, 200 },
{ 255, 82, 0 },
{ 0, 255, 245 },
{ 0, 61, 255 },
{ 0, 255, 112 },
{ 0, 255, 133 },
{ 255, 0, 0 },
{ 255, 163, 0 },
{ 255, 102, 0 },
{ 194, 255, 0 },
{ 0, 143, 255 },
{ 51, 255, 0 },
{ 0, 82, 255 },
{ 0, 255, 41 },
{ 0, 255, 173 },
{ 10, 0, 255 },
{ 173, 255, 0 },
{ 0, 255, 153 },
{ 255, 92, 0 },
{ 255, 0, 255 },
{ 255, 0, 245 },
{ 255, 0, 102 },
{ 255, 173, 0 },
{ 255, 0, 20 },
{ 255, 184, 184 },
{ 0, 31, 255 },
{ 0, 255, 61 },
{ 0, 71, 255 },
{ 255, 0, 204 },
{ 0, 255, 194 },
{ 0, 255, 82 },
{ 0, 10, 255 },
{ 0, 112, 255 },
{ 51, 0, 255 },
{ 0, 194, 255 },
{ 0, 122, 255 },
{ 0, 255, 163 },
{ 255, 153, 0 },
{ 0, 255, 10 },
{ 255, 112, 0 },
{ 143, 255, 0 },
{ 82, 0, 255 },
{ 163, 255, 0 },
{ 255, 235, 0 },
{ 8, 184, 170 },
{ 133, 0, 255 },
{ 0, 255, 92 },
{ 184, 0, 255 },
{ 255, 0, 31 },
{ 0, 184, 255 },
{ 0, 214, 255 },
{ 255, 0, 112 },
{ 92, 255, 0 },
{ 0, 224, 255 },
{ 112, 224, 255 },
{ 70, 184, 160 },
{ 163, 0, 255 },
{ 153, 0, 255 },
{ 71, 255, 0 },
{ 255, 0, 163 },
{ 255, 204, 0 },
{ 255, 0, 143 },
{ 0, 255, 235 },
{ 133, 255, 0 },
{ 255, 0, 235 },
{ 245, 0, 255 },
{ 255, 0, 122 },
{ 255, 245, 0 },
{ 10, 190, 212 },
{ 214, 255, 0 },
{ 0, 204, 255 },
{ 20, 0, 255 },
{ 255, 255, 0 },
{ 0, 153, 255 },
{ 0, 41, 255 },
{ 0, 255, 204 },
{ 41, 0, 255 },
{ 41, 255, 0 },
{ 173, 0, 255 },
{ 0, 245, 255 },
{ 71, 0, 255 },
{ 122, 0, 255 },
{ 0, 255, 184 },
{ 0, 92, 255 },
{ 184, 255, 0 },
{ 0, 133, 255 },
{ 255, 214, 0 },
{ 25, 194, 194 },
{ 102, 255, 0 },
{ 92, 0, 255 },
};
public static final int[][] ADE20K_COLORMAP = new int[][] { { 0, 0, 0 }, { 120, 120, 120 }, { 180, 120, 120 },
{ 6, 230, 230 }, { 80, 50, 50 }, { 4, 200, 3 }, { 120, 120, 80 }, { 140, 140, 140 }, { 204, 5, 255 },
{ 230, 230, 230 }, { 4, 250, 7 }, { 224, 5, 255 }, { 235, 255, 7 }, { 150, 5, 61 }, { 120, 120, 70 },
{ 8, 255, 51 }, { 255, 6, 82 }, { 143, 255, 140 }, { 204, 255, 4 }, { 255, 51, 7 }, { 204, 70, 3 },
{ 0, 102, 200 }, { 61, 230, 250 }, { 255, 6, 51 }, { 11, 102, 255 }, { 255, 7, 71 }, { 255, 9, 224 },
{ 9, 7, 230 }, { 220, 220, 220 }, { 255, 9, 92 }, { 112, 9, 255 }, { 8, 255, 214 }, { 7, 255, 224 },
{ 255, 184, 6 }, { 10, 255, 71 }, { 255, 41, 10 }, { 7, 255, 255 }, { 224, 255, 8 }, { 102, 8, 255 },
{ 255, 61, 6 }, { 255, 194, 7 }, { 255, 122, 8 }, { 0, 255, 20 }, { 255, 8, 41 }, { 255, 5, 153 },
{ 6, 51, 255 }, { 235, 12, 255 }, { 160, 150, 20 }, { 0, 163, 255 }, { 140, 140, 140 }, { 250, 10, 15 },
{ 20, 255, 0 }, { 31, 255, 0 }, { 255, 31, 0 }, { 255, 224, 0 }, { 153, 255, 0 }, { 0, 0, 255 },
{ 255, 71, 0 }, { 0, 235, 255 }, { 0, 173, 255 }, { 31, 0, 255 }, { 11, 200, 200 }, { 255, 82, 0 },
{ 0, 255, 245 }, { 0, 61, 255 }, { 0, 255, 112 }, { 0, 255, 133 }, { 255, 0, 0 }, { 255, 163, 0 },
{ 255, 102, 0 }, { 194, 255, 0 }, { 0, 143, 255 }, { 51, 255, 0 }, { 0, 82, 255 }, { 0, 255, 41 },
{ 0, 255, 173 }, { 10, 0, 255 }, { 173, 255, 0 }, { 0, 255, 153 }, { 255, 92, 0 }, { 255, 0, 255 },
{ 255, 0, 245 }, { 255, 0, 102 }, { 255, 173, 0 }, { 255, 0, 20 }, { 255, 184, 184 }, { 0, 31, 255 },
{ 0, 255, 61 }, { 0, 71, 255 }, { 255, 0, 204 }, { 0, 255, 194 }, { 0, 255, 82 }, { 0, 10, 255 },
{ 0, 112, 255 }, { 51, 0, 255 }, { 0, 194, 255 }, { 0, 122, 255 }, { 0, 255, 163 }, { 255, 153, 0 },
{ 0, 255, 10 }, { 255, 112, 0 }, { 143, 255, 0 }, { 82, 0, 255 }, { 163, 255, 0 }, { 255, 235, 0 },
{ 8, 184, 170 }, { 133, 0, 255 }, { 0, 255, 92 }, { 184, 0, 255 }, { 255, 0, 31 }, { 0, 184, 255 },
{ 0, 214, 255 }, { 255, 0, 112 }, { 92, 255, 0 }, { 0, 224, 255 }, { 112, 224, 255 }, { 70, 184, 160 },
{ 163, 0, 255 }, { 153, 0, 255 }, { 71, 255, 0 }, { 255, 0, 163 }, { 255, 204, 0 }, { 255, 0, 143 },
{ 0, 255, 235 }, { 133, 255, 0 }, { 255, 0, 235 }, { 245, 0, 255 }, { 255, 0, 122 }, { 255, 245, 0 },
{ 10, 190, 212 }, { 214, 255, 0 }, { 0, 204, 255 }, { 20, 0, 255 }, { 255, 255, 0 }, { 0, 153, 255 },
{ 0, 41, 255 }, { 0, 255, 204 }, { 41, 0, 255 }, { 41, 255, 0 }, { 173, 0, 255 }, { 0, 245, 255 },
{ 71, 0, 255 }, { 122, 0, 255 }, { 0, 255, 184 }, { 0, 92, 255 }, { 184, 255, 0 }, { 0, 133, 255 },
{ 255, 214, 0 }, { 25, 194, 194 }, { 102, 255, 0 }, { 92, 0, 255 }, };
/** BLACK_WHITE_COLORMAP . */
public static int[][] BLACK_WHITE_COLORMAP = new int[][] {
{ 0, 0, 0 },
{ 127, 127, 127 },
{ 255, 255, 255 },
};
public static int[][] BLACK_WHITE_COLORMAP = new int[][] { { 0, 0, 0 }, { 127, 127, 127 }, { 255, 255, 255 }, };
/** CITYMAP_COLORMAP . */
public static final int[][] CITYMAP_COLORMAP = new int[255][3];
@@ -284,27 +98,10 @@ public final class SegmentationColorMap {
static {
// Initialize citymap
int[][] _CITYMAP_COLORMAP = new int[][] {
{ 128, 64, 128 },
{ 244, 35, 232 },
{ 70, 70, 70 },
{ 102, 102, 156 },
{ 190, 153, 153 },
{ 153, 153, 153 },
{ 250, 170, 30 },
{ 220, 220, 0 },
{ 107, 142, 35 },
{ 152, 251, 152 },
{ 70, 130, 180 },
{ 220, 20, 60 },
{ 255, 0, 0 },
{ 0, 0, 142 },
{ 0, 0, 70 },
{ 0, 60, 100 },
{ 0, 80, 100 },
{ 0, 0, 230 },
{ 119, 11, 32 }
};
int[][] _CITYMAP_COLORMAP = new int[][] { { 128, 64, 128 }, { 244, 35, 232 }, { 70, 70, 70 }, { 102, 102, 156 },
{ 190, 153, 153 }, { 153, 153, 153 }, { 250, 170, 30 }, { 220, 220, 0 }, { 107, 142, 35 },
{ 152, 251, 152 }, { 70, 130, 180 }, { 220, 20, 60 }, { 255, 0, 0 }, { 0, 0, 142 }, { 0, 0, 70 },
{ 0, 60, 100 }, { 0, 80, 100 }, { 0, 0, 230 }, { 119, 11, 32 } };
for (int i = 0; i < _CITYMAP_COLORMAP.length; i++) {
System.arraycopy(_CITYMAP_COLORMAP[i], 0, CITYMAP_COLORMAP[i], 0, _CITYMAP_COLORMAP[i].length);
@@ -323,8 +120,11 @@ public final class SegmentationColorMap {
}
public static class ColorMap {
private String name;
private String info;
private int[][] colormap;
public String getName() {
@@ -353,11 +153,10 @@ public final class SegmentationColorMap {
@Override
public String toString() {
return "ColorMap{" +
"name='" + name + '\'' +
"info='" + info + '\'' +
", colormap=" + Arrays.deepToString(colormap) +
'}';
return "ColorMap{" + "name='" + name + '\'' + "info='" + info + '\'' + ", colormap="
+ Arrays.deepToString(colormap) + '}';
}
}
}

224
function/spring-semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SemanticSegmentation.java
View File

@@ -54,12 +54,19 @@ import org.springframework.core.io.DefaultResourceLoader;
public class SemanticSegmentation implements AutoCloseable {
private static final long CHANNELS = 3;
private static final float REQUIRED_INPUT_IMAGE_SIZE = 513f;
private final GraphRunner imageNormalization;
private final GraphRunner semanticSegmentation;
private final GraphRunner maskImageEncoding;
private final GraphRunner alphaBlending;
private final Tensor<Integer> colorMapTensor;
private final Tensor<Float> maskTransparencyTensor;
@Override
@@ -75,95 +82,134 @@ public class SemanticSegmentation implements AutoCloseable {
public SemanticSegmentation(String modelUrl, int[][] colorMap, long[] labelFilter, float maskTransparency) {
this.imageNormalization = new GraphRunner("input_image", "resized_image")
.withGraphDefinition(tf -> {
Placeholder<String> input = tf.withName("input_image").placeholder(String.class);
ExtractJpegShape<Integer> imageShapeAndChannel = tf.image.extractJpegShape(input);
Gather<Integer> imageShape = tf.gather(imageShapeAndChannel, tf.constant(new int[] { 0, 1 }), tf.constant(0));
this.imageNormalization = new GraphRunner("input_image", "resized_image").withGraphDefinition(tf -> {
Placeholder<String> input = tf.withName("input_image").placeholder(String.class);
ExtractJpegShape<Integer> imageShapeAndChannel = tf.image.extractJpegShape(input);
Gather<Integer> imageShape = tf.gather(imageShapeAndChannel, tf.constant(new int[] { 0, 1 }),
tf.constant(0));
Cast<Float> maxSize = tf.dtypes.cast(tf.max(imageShape, tf.constant(0)), Float.class);
Div<Float> scale = tf.math.div(tf.constant(REQUIRED_INPUT_IMAGE_SIZE), maxSize);
Cast<Integer> newSize = tf.dtypes.cast(tf.math.mul(scale, tf.dtypes.cast(imageShape, Float.class)), Integer.class);
Cast<Float> maxSize = tf.dtypes.cast(tf.max(imageShape, tf.constant(0)), Float.class);
Div<Float> scale = tf.math.div(tf.constant(REQUIRED_INPUT_IMAGE_SIZE), maxSize);
Cast<Integer> newSize = tf.dtypes.cast(tf.math.mul(scale, tf.dtypes.cast(imageShape, Float.class)),
Integer.class);
final Operand<Float> decodedImage =
tf.dtypes.cast(tf.image.decodeJpeg(input, DecodeJpeg.channels(CHANNELS)), Float.class);
final Operand<Float> decodedImage = tf.dtypes
.cast(tf.image.decodeJpeg(input, DecodeJpeg.channels(CHANNELS)), Float.class);
final Operand<Float> resizedImageFloat =
tf.image.resizeBilinear(tf.expandDims(decodedImage, tf.constant(0)), newSize);
final Operand<Float> resizedImageFloat = tf.image
.resizeBilinear(tf.expandDims(decodedImage, tf.constant(0)), newSize);
tf.withName("resized_image").dtypes.cast(resizedImageFloat, UInt8.class);
});
tf.withName("resized_image").dtypes.cast(resizedImageFloat, UInt8.class);
});
this.semanticSegmentation = new GraphRunner("ImageTensor:0", "SemanticPredictions:0")
.withGraphDefinition(new ProtoBufGraphDefinition(new DefaultResourceLoader().getResource(modelUrl), true));
.withGraphDefinition(new ProtoBufGraphDefinition(new DefaultResourceLoader().getResource(modelUrl), true));
this.colorMapTensor = Tensor.create(colorMap).expect(Integer.class);
this.maskImageEncoding = new GraphRunner(Arrays.asList("color_map", "mask_pixels"), Arrays.asList("mask_png", "mask_rgb"))
.withGraphDefinition(tf -> {
Placeholder<Integer> colorTable = tf.withName("color_map").placeholder(Integer.class);
this.maskImageEncoding = new GraphRunner(Arrays.asList("color_map", "mask_pixels"),
Arrays.asList("mask_png", "mask_rgb"))
.withGraphDefinition(tf -> {
Placeholder<Integer> colorTable = tf.withName("color_map").placeholder(Integer.class);
Placeholder<Long> batchedMask = tf.withName("mask_pixels").placeholder(Long.class);
// Remove batch dimension
Squeeze<Long> mask = tf.squeeze(batchedMask, Squeeze.axis(Arrays.asList(0L)));
Placeholder<Long> batchedMask = tf.withName("mask_pixels").placeholder(Long.class);
// Remove batch dimension
Squeeze<Long> mask = tf.squeeze(batchedMask, Squeeze.axis(Arrays.asList(0L)));
Operand<Long> filteredMask = labelFilter(tf, mask, labelFilter);
Operand<Long> filteredMask = labelFilter(tf, mask, labelFilter);
// The mask can contain label values larger than the list of colors provided in the color map.
// To avoid out-of-index errors we will "normalize" the label values in the mask to MOD max-color-table-value.
Operand<Long> mask3 = NativeImageUtils.normalizeMaskLabels(tf, colorTable, filteredMask);
// The mask can contain label values larger than the list of colors
// provided in the color map.
// To avoid out-of-index errors we will "normalize" the label values in
// the mask to MOD max-color-table-value.
Operand<Long> mask3 = NativeImageUtils.normalizeMaskLabels(tf, colorTable, filteredMask);
Gather<Integer> maskRgb = tf.withName("mask_rgb").gather(colorTable, mask3, tf.constant(0));
Gather<Integer> maskRgb = tf.withName("mask_rgb").gather(colorTable, mask3, tf.constant(0));
Operand<String> png = tf.withName("mask_png").image.encodePng(tf.dtypes.cast(maskRgb, UInt8.class));
Operand<String> png = tf.withName("mask_png").image.encodePng(tf.dtypes.cast(maskRgb, UInt8.class));
});
});
this.maskTransparencyTensor = Tensor.create(maskTransparency).expect(Float.class);
this.alphaBlending = new GraphRunner(
Arrays.asList("input_image", "mask_image", "mask_transparency"), Arrays.asList("blended_png"))
.withGraphDefinition(tf -> {
// Input image [B, H, W, 3]
Cast<Float> inputImageRgb = tf.dtypes.cast(tf.withName("input_image").placeholder(UInt8.class), Float.class);
this.alphaBlending = new GraphRunner(Arrays.asList("input_image", "mask_image", "mask_transparency"),
Arrays.asList("blended_png"))
.withGraphDefinition(tf -> {
// Input image [B, H, W, 3]
Cast<Float> inputImageRgb = tf.dtypes.cast(tf.withName("input_image").placeholder(UInt8.class),
Float.class);
Placeholder<Integer> a = tf.withName("mask_image").placeholder(Integer.class);
Cast<Float> maskRgb = tf.dtypes.cast(a, Float.class);
Placeholder<Integer> a = tf.withName("mask_image").placeholder(Integer.class);
Cast<Float> maskRgb = tf.dtypes.cast(a, Float.class);
Squeeze<Float> inputImageRgb2 = tf.squeeze(inputImageRgb, Squeeze.axis(Arrays.asList(0L)));
Squeeze<Float> inputImageRgb2 = tf.squeeze(inputImageRgb, Squeeze.axis(Arrays.asList(0L)));
Placeholder<Float> maskTransparencyHolder = tf.withName("mask_transparency").placeholder(Float.class);
Placeholder<Float> maskTransparencyHolder = tf.withName("mask_transparency").placeholder(Float.class);
// Blend the transparent maskImage on top of the input image.
Operand<Float> blended = NativeImageUtils.alphaBlending(tf, maskRgb, inputImageRgb2, maskTransparencyHolder);
// Blend the transparent maskImage on top of the input image.
Operand<Float> blended = NativeImageUtils.alphaBlending(tf, maskRgb, inputImageRgb2,
maskTransparencyHolder);
// Cut
//Operand<Boolean> condition = tf.math.equal(a, tf.zerosLike(a));
//Operand<Float> blended = tf.where3(condition, tf.zerosLike(maskRgb), inputImageRgb2);
// Cut
// Operand<Boolean> condition = tf.math.equal(a, tf.zerosLike(a));
// Operand<Float> blended = tf.where3(condition, tf.zerosLike(maskRgb),
// inputImageRgb2);
// Encode PNG
tf.withName("blended_png").image.encodePng(tf.dtypes.cast(blended, UInt8.class));
// Encode PNG
tf.withName("blended_png").image.encodePng(tf.dtypes.cast(blended, UInt8.class));
});
});
}
public byte[] blendMask(byte[] image) {
try (Tensor inputTensor = Tensor.create(image); GraphRunnerMemory memory = new GraphRunnerMemory()) {
Map<String, Tensor<?>> blendedTensors =
this.imageNormalization.andThen(memory) // (input_image) -> (resized_image) and memorize (resized_image)
.andThen(this.semanticSegmentation).andThen(memory) // (ImageTensor:0) -> (SemanticPredictions:0) and memorize (SemanticPredictions:0)
.andThen(Functions.rename("SemanticPredictions:0", "mask_pixels")) // (SemanticPredictions:0) -> (mask_pixels)
.andThen(Functions.enrichWith("color_map", this.colorMapTensor)) // (mask_pixels) -> (mask_pixels, color_map)
.andThen(this.maskImageEncoding).andThen(memory) // (color_map, mask_pixels) -> (mask_png, mask_rgb) and memorize (mask_png, mask_rgb)
.andThen(Functions.enrichFromMemory(
memory, "resized_image")) // (mask_png, mask_rgb) -> (mask_png, mask_rgb, resized_image), e.g. join the normalizedImageTensor
.andThen(Functions.rename(
"resized_image", "input_image",
"mask_rgb", "mask_image")) // (mask_png, mask_rgb, resized_image) -> (mask_image, input_image)
.andThen(Functions.enrichWith("mask_transparency", this.maskTransparencyTensor)) // (mask_image, input_image) -> (mask_image, input_image, mask_transparency)
.andThen(this.alphaBlending).andThen(memory) // (mask_image, input_image, mask_transparency) -> (blended_png)
.apply(Collections.singletonMap("input_image", inputTensor)); // () -> (input_image)
Map<String, Tensor<?>> blendedTensors = this.imageNormalization.andThen(memory) // (input_image)
// ->
// (resized_image)
// and
// memorize
// (resized_image)
.andThen(this.semanticSegmentation)
.andThen(memory) // (ImageTensor:0) -> (SemanticPredictions:0) and
// memorize (SemanticPredictions:0)
.andThen(Functions.rename("SemanticPredictions:0", "mask_pixels")) // (SemanticPredictions:0)
// ->
// (mask_pixels)
.andThen(Functions.enrichWith("color_map", this.colorMapTensor)) // (mask_pixels)
// ->
// (mask_pixels,
// color_map)
.andThen(this.maskImageEncoding)
.andThen(memory) // (color_map, mask_pixels) -> (mask_png, mask_rgb) and
// memorize (mask_png, mask_rgb)
.andThen(Functions.enrichFromMemory(memory, "resized_image")) // (mask_png,
// mask_rgb)
// ->
// (mask_png,
// mask_rgb,
// resized_image),
// e.g.
// join
// the
// normalizedImageTensor
.andThen(Functions.rename("resized_image", "input_image", "mask_rgb", "mask_image")) // (mask_png,
// mask_rgb,
// resized_image)
// ->
// (mask_image,
// input_image)
.andThen(Functions.enrichWith("mask_transparency", this.maskTransparencyTensor)) // (mask_image,
// input_image)
// ->
// (mask_image,
// input_image,
// mask_transparency)
.andThen(this.alphaBlending)
.andThen(memory) // (mask_image, input_image, mask_transparency) ->
// (blended_png)
.apply(Collections.singletonMap("input_image", inputTensor)); // () ->
// (input_image)
byte[] blendedImage = blendedTensors.get("blended_png").bytesValue();
@@ -176,15 +222,21 @@ public class SemanticSegmentation implements AutoCloseable {
public long[][] maskPixels(byte[] image) {
try (Tensor inputTensor = Tensor.create(image); GraphRunnerMemory memory = new GraphRunnerMemory()) {
return this.imageNormalization.andThen(memory) // (input_image) -> (resized_image) and memorize (resized_image)
.andThen(this.semanticSegmentation).andThen(memory) // (ImageTensor:0) -> (SemanticPredictions:0) and memorize (SemanticPredictions:0)
.andThen(tensorMap -> {
Tensor<?> maskTensor = tensorMap.get("SemanticPredictions:0");
int width = (int) maskTensor.shape()[1];
int height = (int) maskTensor.shape()[2];
return maskTensor.copyTo(new long[1][width][height])[0]; // 1 == batch size
})
.apply(Collections.singletonMap("input_image", inputTensor)); // () -> (input_image)
return this.imageNormalization.andThen(memory) // (input_image) ->
// (resized_image) and
// memorize (resized_image)
.andThen(this.semanticSegmentation)
.andThen(memory) // (ImageTensor:0) -> (SemanticPredictions:0) and
// memorize (SemanticPredictions:0)
.andThen(tensorMap -> {
Tensor<?> maskTensor = tensorMap.get("SemanticPredictions:0");
int width = (int) maskTensor.shape()[1];
int height = (int) maskTensor.shape()[2];
return maskTensor.copyTo(new long[1][width][height])[0]; // 1 == batch
// size
})
.apply(Collections.singletonMap("input_image", inputTensor)); // () ->
// (input_image)
}
}
@@ -192,13 +244,25 @@ public class SemanticSegmentation implements AutoCloseable {
try (Tensor inputTensor = Tensor.create(image); GraphRunnerMemory memory = new GraphRunnerMemory()) {
return this.imageNormalization.andThen(memory) // (input_image) -> (resized_image) and memorize (resized_image)
.andThen(this.semanticSegmentation).andThen(memory) // (ImageTensor:0) -> (SemanticPredictions:0) and memorize (SemanticPredictions:0)
.andThen(Functions.rename("SemanticPredictions:0", "mask_pixels")) // (SemanticPredictions:0) -> (mask_pixels)
.andThen(Functions.enrichWith("color_map", this.colorMapTensor)) // (mask_pixels) -> (mask_pixels, color_map)
.andThen(this.maskImageEncoding).andThen(memory) // (color_map, mask_pixels) -> (mask_png, mask_rgb) and memorize (mask_png, mask_rgb)
.andThen(tensorMap -> tensorMap.get("mask_png").bytesValue())
.apply(Collections.singletonMap("input_image", inputTensor)); // () -> (input_image)
return this.imageNormalization.andThen(memory) // (input_image) ->
// (resized_image) and
// memorize (resized_image)
.andThen(this.semanticSegmentation)
.andThen(memory) // (ImageTensor:0) -> (SemanticPredictions:0) and
// memorize (SemanticPredictions:0)
.andThen(Functions.rename("SemanticPredictions:0", "mask_pixels")) // (SemanticPredictions:0)
// ->
// (mask_pixels)
.andThen(Functions.enrichWith("color_map", this.colorMapTensor)) // (mask_pixels)
// ->
// (mask_pixels,
// color_map)
.andThen(this.maskImageEncoding)
.andThen(memory) // (color_map, mask_pixels) -> (mask_png, mask_rgb) and
// memorize (mask_png, mask_rgb)
.andThen(tensorMap -> tensorMap.get("mask_png").bytesValue())
.apply(Collections.singletonMap("input_image", inputTensor)); // () ->
// (input_image)
}
}
@@ -222,8 +286,7 @@ public class SemanticSegmentation implements AutoCloseable {
try (SemanticSegmentation segmentationService = new SemanticSegmentation(
"https://download.tensorflow.org/models/deeplabv3_mnv2_cityscapes_train_2018_02_05.tar.gz#frozen_inference_graph.pb",
SegmentationColorMap.loadColorMap("classpath:/colormap/citymap_colormap.json"), null, 0.45f)
) {
SegmentationColorMap.loadColorMap("classpath:/colormap/citymap_colormap.json"), null, 0.45f)) {
byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/amsterdam-cityscape1.jpg");
// 1. Mask pixels
@@ -244,8 +307,7 @@ public class SemanticSegmentation implements AutoCloseable {
try (SemanticSegmentation segmentationService = new SemanticSegmentation(
"https://download.tensorflow.org/models/deeplabv3_xception_ade20k_train_2018_05_29.tar.gz#frozen_inference_graph.pb",
SegmentationColorMap.loadColorMap("classpath:/colormap/ade20k_colormap.json"), null, 0.45f)
) {
SegmentationColorMap.loadColorMap("classpath:/colormap/ade20k_colormap.json"), null, 0.45f)) {
byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/interior.jpg");
// 1. Mask pixels
@@ -264,8 +326,7 @@ public class SemanticSegmentation implements AutoCloseable {
try (SemanticSegmentation segmentationService = new SemanticSegmentation(
"https://download.tensorflow.org/models/deeplabv3_mnv2_pascal_trainval_2018_01_29.tar.gz#frozen_inference_graph.pb",
SegmentationColorMap.loadColorMap("classpath:/colormap/black_white_colormap.json"), null, 0.45f)
) {
SegmentationColorMap.loadColorMap("classpath:/colormap/black_white_colormap.json"), null, 0.45f)) {
byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/VikiMaxiAdi.jpg");
// 1. Mask pixels
@@ -283,4 +344,5 @@ public class SemanticSegmentation implements AutoCloseable {
}
}
}

56
function/spring-semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/attic/SemanticSegmentationUtils.java
View File

@@ -44,8 +44,8 @@ import static java.awt.image.BufferedImage.TYPE_3BYTE_BGR;
/**
*
* Semantic image segmentation - the task of assigning a semantic label, such as "road", "sky", "person", "dog", to
* every pixel in an image.
* Semantic image segmentation - the task of assigning a semantic label, such as "road",
* "sky", "person", "dog", to every pixel in an image.
*
* https://ai.googleblog.com/2018/03/semantic-image-segmentation-with.html
* https://github.com/tensorflow/models/blob/master/research/deeplab/g3doc/model_zoo.md
@@ -65,11 +65,14 @@ public class SemanticSegmentationUtils {
/** INPUT_TENSOR_NAME . */
public static final String INPUT_TENSOR_NAME = "ImageTensor:0";
/** OUTPUT_TENSOR_NAME . */
public static final String OUTPUT_TENSOR_NAME = "SemanticPredictions:0";
private static final int BATCH_SIZE = 1;
private static final long CHANNELS = 3;
private static final int REQUIRED_INPUT_IMAGE_SIZE = 513;
public static BufferedImage scaledImage(String imagePath) {
@@ -100,9 +103,11 @@ public class SemanticSegmentationUtils {
int newHeight = (int) (originalImage.getHeight() * scale);
Image tmpImage = originalImage.getScaledInstance(newWidth, newHeight, Image.SCALE_DEFAULT);
//BufferedImage resizedImage = new BufferedImage(newWidth, newHeight, TYPE_INT_BGR);
// BufferedImage resizedImage = new BufferedImage(newWidth, newHeight,
// TYPE_INT_BGR);
BufferedImage resizedImage = new BufferedImage(newWidth, newHeight, TYPE_3BYTE_BGR);
//BufferedImage resizedImage = new BufferedImage(newWidth, newHeight, originalImage.getType());
// BufferedImage resizedImage = new BufferedImage(newWidth, newHeight,
// originalImage.getType());
Graphics2D g2d = resizedImage.createGraphics();
g2d.drawImage(tmpImage, 0, 0, null);
@@ -125,7 +130,8 @@ public class SemanticSegmentationUtils {
// ImageIO.read produces BGR-encoded images, while the model expects RGB.
byte[] data = bgrToRgb(toBytes(scaledImage));
// Expand dimensions since the model expects images to have shape: [1, None, None, 3]
// Expand dimensions since the model expects images to have shape: [1, None, None,
// 3]
long[] shape = new long[] { BATCH_SIZE, scaledImage.getHeight(), scaledImage.getWidth(), CHANNELS };
return Tensor.create(UInt8.class, shape, ByteBuffer.wrap(data));
@@ -201,7 +207,8 @@ public class SemanticSegmentationUtils {
public String serializeToJson(int[][] pixels) {
String masksBase64 = Base64.getEncoder().encodeToString(toBytes(pixels));
return String.format("{ \"columns\":%d, \"rows\":%d, \"masks\":\"%s\"}", pixels.length, pixels[0].length, masksBase64);
return String.format("{ \"columns\":%d, \"rows\":%d, \"masks\":\"%s\"}", pixels.length, pixels[0].length,
masksBase64);
}
public int[][] deserializeToMasks(String json) throws IOException {
@@ -221,7 +228,7 @@ public class SemanticSegmentationUtils {
b[bi + 0] = (byte) (i >> 24);
b[bi + 1] = (byte) (i >> 16);
b[bi + 2] = (byte) (i >> 8);
b[bi + 3] = (byte) (i /*>> 0*/);
b[bi + 3] = (byte) (i /* >> 0 */);
bi = bi + 4;
}
}
@@ -233,10 +240,7 @@ public class SemanticSegmentationUtils {
int bi = 0;
for (int i = 0; i < ic; i++) {
for (int j = 0; j < jc; j++) {
intResult[i][j] = (b[bi] << 24) +
(b[bi + 1] << 16) +
(b[bi + 2] << 8) +
b[bi + 3];
intResult[i][j] = (b[bi] << 24) + (b[bi + 1] << 16) + (b[bi + 2] << 8) + b[bi + 3];
bi = bi + 4;
}
}
@@ -246,14 +250,16 @@ public class SemanticSegmentationUtils {
public static void main(String[] args) throws IOException {
// PASCAL VOC 2012
//String tensorflowModelLocation = "file:/Users/ctzolov/Downloads/deeplabv3_mnv2_pascal_train_aug/frozen_inference_graph.pb";
//String imagePath = "classpath:/images/VikiMaxiAdi.jpg";
// String tensorflowModelLocation =
// "file:/Users/ctzolov/Downloads/deeplabv3_mnv2_pascal_train_aug/frozen_inference_graph.pb";
// String imagePath = "classpath:/images/VikiMaxiAdi.jpg";
// CITYSCAPE
//String tensorflowModelLocation = "file:/Users/ctzolov/Downloads/deeplabv3_mnv2_cityscapes_train/frozen_inference_graph.pb";
//String imagePath = "classpath:/images/amsterdam-cityscape1.jpg";
//String imagePath = "classpath:/images/amsterdam-channel.jpg";
//String imagePath = "classpath:/images/landsmeer.png";
// String tensorflowModelLocation =
// "file:/Users/ctzolov/Downloads/deeplabv3_mnv2_cityscapes_train/frozen_inference_graph.pb";
// String imagePath = "classpath:/images/amsterdam-cityscape1.jpg";
// String imagePath = "classpath:/images/amsterdam-channel.jpg";
// String imagePath = "classpath:/images/landsmeer.png";
// ADE20K
String tensorflowModelLocation = "file:/Users/ctzolov/Downloads/deeplabv3_xception_ade20k_train/frozen_inference_graph.pb";
@@ -261,7 +267,8 @@ public class SemanticSegmentationUtils {
BufferedImage inputImage = ImageIO.read(new DefaultResourceLoader().getResource(imagePath).getInputStream());
TensorFlowService tf = new TensorFlowService(new DefaultResourceLoader().getResource(tensorflowModelLocation), Arrays.asList(OUTPUT_TENSOR_NAME));
TensorFlowService tf = new TensorFlowService(new DefaultResourceLoader().getResource(tensorflowModelLocation),
Arrays.asList(OUTPUT_TENSOR_NAME));
SemanticSegmentationUtils segmentationService = new SemanticSegmentationUtils();
@@ -275,15 +282,24 @@ public class SemanticSegmentationUtils {
int height = (int) maskPixelsTensor.shape()[1];
int width = (int) maskPixelsTensor.shape()[2];
long[][] maskPixels = maskPixelsTensor.copyTo(new long[BATCH_SIZE][height][width])[0]; // take 0 because the batch size is 1.
long[][] maskPixels = maskPixelsTensor.copyTo(new long[BATCH_SIZE][height][width])[0]; // take
// 0
// because
// the
// batch
// size
// is
// 1.
int[][] maskPixelsInt = segmentationService.toIntArray(maskPixels);
BufferedImage maskImage = segmentationService.createMaskImage(maskPixelsInt, scaledImage.getWidth(), scaledImage.getHeight(), 0.35);
BufferedImage maskImage = segmentationService.createMaskImage(maskPixelsInt, scaledImage.getWidth(),
scaledImage.getHeight(), 0.35);
BufferedImage blended = segmentationService.blendMask(maskImage, scaledImage);
ImageIO.write(maskImage, "png", new File("./semantic-segmentation/target/java2Dmask.jpg"));
ImageIO.write(blended, "png", new File("./semantic-segmentation/target/java2Dblended.jpg"));
}
}