SpringCloud/spring-functions-catalog
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleann the tensorflow-common dependencies

Browse Source
This commit is contained in:
Christian Tzolov
2020-07-09 14:09:51 +02:00
parent 0567e39a91
commit 94f360e06f
6 changed files with 52 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
common/tensorflow-common/pom.xml
View File

@@ -19,15 +19,23 @@
<properties>
<checkstyle.skip>true</checkstyle.skip>
<spring-framework.version>5.1.6.RELEASE</spring-framework.version>
<tensorflow.version>1.15.0</tensorflow.version>
<spring-core.version>5.2.7.RELEASE</spring-core.version>
<jackson.version>2.11.0</jackson.version>
<apache.commons.compress>1.20</apache.commons.compress>
<commons-io.version>2.7</commons-io.version>
<tensorflow.version>1.15.0</tensorflow.version>
<commons-lang3.version>3.10</commons-lang3.version>
<pcollections.version>3.0.3</pcollections.version>
<slf4j-api.version>1.7.26</slf4j-api.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
<version>${spring-core.version}</version>
</dependency>
<dependency>
<groupId>org.tensorflow</groupId>
<artifactId>tensorflow</artifactId>
@@ -50,6 +58,11 @@
<artifactId>commons-io</artifactId>
<version>${commons-io.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -69,45 +82,12 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.26</version>
<version>${slf4j-api.version}</version>
</dependency>
<dependency>
<groupId>org.pcollections</groupId>
<artifactId>pcollections</artifactId>
<version>3.0.3</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-validation</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-messaging</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-webflux</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<scope>provided</scope>
<version>${pcollections.version}</version>
</dependency>
</dependencies>
@@ -120,6 +100,4 @@
</snapshots>
</repository>
</repositories>
</project>

9
common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AbstractGraphRunner.java
View File

@@ -22,11 +22,10 @@ import java.util.List;
import java.util.Map;
import java.util.function.Function;
import org.apache.commons.lang3.Validate;
import org.tensorflow.Session;
import org.tensorflow.Tensor;
import org.springframework.util.Assert;
/**
* @author Christian Tzolov
*/
@@ -103,8 +102,8 @@ public abstract class AbstractGraphRunner implements Function<Map<String, Tensor
return this.feedNames;
}
public String getSingleøøƶFeedName() {
Assert.isTrue(feedNames.size() == 1, "Assumes a single feed input");
public String getSingleFeedName() {
Validate.isTrue(feedNames.size() == 1, "Assumes a single feed input");
return this.feedNames.get(0);
}
@@ -113,7 +112,7 @@ public abstract class AbstractGraphRunner implements Function<Map<String, Tensor
}
public String getSingleFetchName() {
Assert.isTrue(this.fetchNames.size() == 1, "Assumes a single fetch output");
Validate.isTrue(this.fetchNames.size() == 1, "Assumes a single fetch output");
return this.fetchNames.get(0);
}

7
common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunner.java
View File

@@ -19,11 +19,11 @@ package org.springframework.cloud.fn.common.tensorflow;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.Validate;
import org.tensorflow.SavedModelBundle;
import org.tensorflow.Session;
import org.tensorflow.op.Ops;
import org.springframework.util.Assert;
/**
* @author Christian Tzolov
@@ -36,6 +36,7 @@ public class GraphRunner extends AbstractGraphRunner implements AutoCloseable {
public GraphRunner(List<String> feedNames, String fetchedName) {
super(feedNames, Arrays.asList(fetchedName));
}
public GraphRunner(String feedName, List<String> fetchedNames) {
super(Arrays.asList(feedName), fetchedNames);
}
@@ -68,7 +69,7 @@ public class GraphRunner extends AbstractGraphRunner implements AutoCloseable {
}
public GraphRunner withGraphDefinition(GraphDefinition graphDefinition) {
Assert.isNull(this.savedModelBundle, "Either SavedModel or GraphDefinition can be set! " +
Validate.isTrue(this.savedModelBundle == null, "Either SavedModel or GraphDefinition can be set! " +
"SavedModelBundle is found: " + this.savedModelBundle);
this.autoCloseableSession = new AutoCloseableSession() {
@@ -82,7 +83,7 @@ public class GraphRunner extends AbstractGraphRunner implements AutoCloseable {
}
public GraphRunner withSavedModel(String savedModelDir, String... tags) {
Assert.isNull(this.autoCloseableSession, "Either SavedModel or GraphDefinition can be set! " +
Validate.isTrue(this.autoCloseableSession == null, "Either SavedModel or GraphDefinition can be set! " +
"AutoCloseableSession is found: " + this.autoCloseableSession);
this.savedModelBundle = SavedModelBundle.load(savedModelDir, tags);
return this;

8
common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/GraphicsUtils.java
View File

@@ -33,9 +33,11 @@ import java.io.InputStream;
import javax.imageio.ImageIO;
import org.apache.commons.io.IOUtils;
import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.core.io.Resource;
import org.springframework.util.StreamUtils;
/**
* Utility class used to provide some handy image manipulation functions. Among others it can provide contrast colors
@@ -617,7 +619,7 @@ public final class GraphicsUtils {
public static byte[] toImageToBytes(String imageUri) throws IOException {
try (InputStream is = new DefaultResourceLoader().getResource(imageUri).getInputStream()) {
return StreamUtils.copyToByteArray(is);
return IOUtils.toByteArray(is);
}
}
@@ -630,7 +632,7 @@ public final class GraphicsUtils {
public static byte[] loadAsByteArray(String resourceUri) throws IOException {
Resource expectedPoseResponse = new DefaultResourceLoader().getResource(resourceUri);
try (InputStream is = expectedPoseResponse.getInputStream()) {
return StreamUtils.copyToByteArray(is);
return IOUtils.toByteArray(is);
}
}

14
common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/CachedModelExtractor.java
View File

@@ -19,15 +19,15 @@ package org.springframework.cloud.fn.common.tensorflow.util;
import java.io.File;
import java.io.FileInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.core.io.Resource;
import org.springframework.util.Assert;
import org.springframework.util.FileCopyUtils;
import org.springframework.util.StreamUtils;
import org.springframework.util.StringUtils;
/**
* Extends the {@link ModelExtractor} to allow keeping a local copy (cache) of the loaded model (protobuf) files.
@@ -72,7 +72,7 @@ public class CachedModelExtractor extends ModelExtractor {
rootFolder.mkdirs();
}
Assert.isTrue(rootFolder.isDirectory(), "The cache root folder must be a Directory");
Validate.isTrue(rootFolder.isDirectory(), "The cache root folder must be a Directory");
String fileName = modelResource.getFilename();
String fragment = modelResource.getURI().getFragment();
@@ -80,13 +80,13 @@ public class CachedModelExtractor extends ModelExtractor {
new File(rootFolder, fileName + "_" + fragment);
if (cachedFile.exists()) {
logger.info("Load model " + modelResource.toString() + " from cache: " + cacheRootDirectory);
return StreamUtils.copyToByteArray(new FileInputStream(cachedFile));
return IOUtils.toByteArray(new FileInputStream(cachedFile));
}
byte[] model = super.getModel(modelResource);
// cache the file
FileCopyUtils.copy(model, cachedFile);
FileUtils.writeByteArrayToFile(cachedFile, model);
logger.info("Caching the " + modelResource.toString() + " model at: " + cachedFile);
return model;

26
common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/ModelExtractor.java
View File

@@ -36,12 +36,13 @@ import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.archivers.ArchiveStreamFactory;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.core.io.Resource;
import org.springframework.util.Assert;
import org.springframework.util.StreamUtils;
import org.springframework.util.StringUtils;
/**
* Extracts a pre-trained (frozen) Tensorflow model URI into byte array. The 'http://', 'file://' and 'classpath://'
@@ -79,7 +80,7 @@ public class ModelExtractor {
public byte[] getModel(Resource modelResource) {
Assert.notNull(modelResource, "Not null model resource is required!");
Validate.notNull(modelResource, "Not null model resource is required!");
try (InputStream is = modelResource.getInputStream(); InputStream bi = new BufferedInputStream(is)) {
@@ -88,27 +89,28 @@ public class ModelExtractor {
String compressor = archiveCompressor[1];
String fragment = modelResource.getURI().getFragment();
if (StringUtils.hasText(compressor)) {
if (StringUtils.isNotBlank(compressor)) {
try (CompressorInputStream cis = new CompressorStreamFactory().createCompressorInputStream(compressor, bi)) {
if (StringUtils.hasText(archive)) {
if (StringUtils.isNotBlank(archive)) {
try (ArchiveInputStream ais = new ArchiveStreamFactory().createArchiveInputStream(archive, cis)) {
// Compressor fromMemory Archive
return findInArchiveStream(fragment, ais);
}
}
else { // Compressor only
return StreamUtils.copyToByteArray(cis);
return IOUtils.toByteArray(cis);
}
}
}
else if (StringUtils.hasText(archive)) { // Archive only
else if (StringUtils.isNotBlank(archive)) { // Archive only
try (ArchiveInputStream ais = new ArchiveStreamFactory().createArchiveInputStream(archive, bi)) {
return findInArchiveStream(fragment, ais);
}
}
else {
// No compressor nor Archive
return StreamUtils.copyToByteArray(bi);
return IOUtils.toByteArray(bi);
}
}
catch (Exception e) {
@@ -131,9 +133,9 @@ public class ModelExtractor {
//System.out.println(entry.getName() + " : " + entry.isDirectory());
if (archive.canReadEntryData(entry) && !entry.isDirectory()) {
if ((StringUtils.hasText(modelFileNameInArchive) && entry.getName().endsWith(modelFileNameInArchive)) ||
(!StringUtils.hasText(modelFileNameInArchive) && entry.getName().endsWith(this.frozenGraphFileExtension))) {
return StreamUtils.copyToByteArray(archive);
if ((StringUtils.isNotBlank(modelFileNameInArchive) && entry.getName().endsWith(modelFileNameInArchive)) ||
(!StringUtils.isNotBlank(modelFileNameInArchive) && entry.getName().endsWith(this.frozenGraphFileExtension))) {
return IOUtils.toByteArray(archive);
}
}
}