Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Introduce checkstyle plugin

Browse Source 
- Based on https://github.com/spring-io/spring-javaformat
- In this iteration, checkstyles are only enabled for spring-ai-core
This commit is contained in:
Soby Chacko
2024-10-24 10:39:48 -04:00
committed by Mark Pollack
parent 33a72417e1
commit 8e758dbd00
1412 changed files with 26997 additions and 21963 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
models/spring-ai-transformers/src/main/java/org/springframework/ai/transformers/ResourceCacheService.java
View File

@@ -1,11 +1,11 @@
/*
* Copyright 2023 - 2024 the original author or authors.
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.transformers;
import java.io.File;

83
models/spring-ai-transformers/src/main/java/org/springframework/ai/transformers/TransformersEmbeddingModel.java
View File

@@ -1,11 +1,11 @@
/*
* Copyright 2023 - 2024 the original author or authors.
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.transformers;
import java.nio.FloatBuffer;
@@ -23,8 +24,22 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import ai.djl.huggingface.tokenizers.Encoding;
import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
import ai.djl.modality.nlp.preprocess.Tokenizer;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import ai.onnxruntime.OnnxTensor;
import ai.onnxruntime.OnnxValue;
import ai.onnxruntime.OrtEnvironment;
import ai.onnxruntime.OrtException;
import ai.onnxruntime.OrtSession;
import io.micrometer.observation.ObservationRegistry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.ai.document.Document;
import org.springframework.ai.document.MetadataMode;
import org.springframework.ai.embedding.AbstractEmbeddingModel;
@@ -43,20 +58,6 @@ import org.springframework.core.io.Resource;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
import ai.djl.huggingface.tokenizers.Encoding;
import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
import ai.djl.modality.nlp.preprocess.Tokenizer;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import ai.onnxruntime.OnnxTensor;
import ai.onnxruntime.OnnxValue;
import ai.onnxruntime.OrtEnvironment;
import ai.onnxruntime.OrtException;
import ai.onnxruntime.OrtSession;
import io.micrometer.observation.ObservationRegistry;
/**
* An implementation of the AbstractEmbeddingModel that uses ONNX-based Transformer models
* for text embeddings.
@@ -79,10 +80,6 @@ import io.micrometer.observation.ObservationRegistry;
*/
public class TransformersEmbeddingModel extends AbstractEmbeddingModel implements InitializingBean {
private static final Log logger = LogFactory.getLog(TransformersEmbeddingModel.class);
private static final EmbeddingModelObservationConvention DEFAULT_OBSERVATION_CONVENTION = new DefaultEmbeddingModelObservationConvention();
// ONNX tokenizer for the all-MiniLM-L6-v2 generative
public final static String DEFAULT_ONNX_TOKENIZER_URI = "https://raw.githubusercontent.com/spring-projects/spring-ai/main/models/spring-ai-transformers/src/main/resources/onnx/all-MiniLM-L6-v2/tokenizer.json";
@@ -92,8 +89,27 @@ public class TransformersEmbeddingModel extends AbstractEmbeddingModel implement
public final static String DEFAULT_MODEL_OUTPUT_NAME = "last_hidden_state";
private static final Log logger = LogFactory.getLog(TransformersEmbeddingModel.class);
private static final EmbeddingModelObservationConvention DEFAULT_OBSERVATION_CONVENTION = new DefaultEmbeddingModelObservationConvention();
private final static int EMBEDDING_AXIS = 1;
/**
* Specifies what parts of the {@link Document}'s content and metadata will be used
* for computing the embeddings. Applicable for the {@link #embed(Document)} method
* only. Has no effect on the {@link #embed(String)} or {@link #embed(List)}. Defaults
* to {@link MetadataMode#NONE}.
*/
private final MetadataMode metadataMode;
/**
* Observation registry used for instrumentation.
*/
private final ObservationRegistry observationRegistry;
public Map<String, String> tokenizerOptions = Map.of();
private Resource tokenizerResource = toResource(DEFAULT_ONNX_TOKENIZER_URI);
private Resource modelResource = toResource(DEFAULT_ONNX_MODEL_URI);
@@ -116,14 +132,6 @@ public class TransformersEmbeddingModel extends AbstractEmbeddingModel implement
*/
private OrtSession session;
/**
* Specifies what parts of the {@link Document}'s content and metadata will be used
* for computing the embeddings. Applicable for the {@link #embed(Document)} method
* only. Has no effect on the {@link #embed(String)} or {@link #embed(List)}. Defaults
* to {@link MetadataMode#NONE}.
*/
private final MetadataMode metadataMode;
/**
* Resource cache directory. Used to cache remote resources, such as the ONNX models,
* to the local file system.
@@ -143,17 +151,10 @@ public class TransformersEmbeddingModel extends AbstractEmbeddingModel implement
*/
private ResourceCacheService cacheService;
public Map<String, String> tokenizerOptions = Map.of();
private String modelOutputName = DEFAULT_MODEL_OUTPUT_NAME;
private Set<String> onnxModelInputs;
/**
* Observation registry used for instrumentation.
*/
private final ObservationRegistry observationRegistry;
/**
* Conventions to use for generating observations.
*/
@@ -174,6 +175,10 @@ public class TransformersEmbeddingModel extends AbstractEmbeddingModel implement
this.observationRegistry = observationRegistry;
}
private static Resource toResource(String uri) {
return new DefaultResourceLoader().getResource(uri);
}
public void setTokenizerOptions(Map<String, String> tokenizerOptions) {
this.tokenizerOptions = tokenizerOptions;
}
@@ -360,7 +365,7 @@ public class TransformersEmbeddingModel extends AbstractEmbeddingModel implement
return modelInputs.entrySet()
.stream()
.filter(a -> onnxModelInputs.contains(a.getKey()))
.filter(a -> this.onnxModelInputs.contains(a.getKey()))
.collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue()));
}
@@ -399,10 +404,6 @@ public class TransformersEmbeddingModel extends AbstractEmbeddingModel implement
return sumEmbeddings.div(sumMask);
}
private static Resource toResource(String uri) {
return new DefaultResourceLoader().getResource(uri);
}
/**
* Use the provided convention for reporting observation data
* @param observationConvention The provided convention
@@ -412,4 +413,4 @@ public class TransformersEmbeddingModel extends AbstractEmbeddingModel implement
this.observationConvention = observationConvention;
}
}
}

35
models/spring-ai-transformers/src/test/java/org/springframework/ai/transformers/ResourceCacheServiceTests.java
View File

@@ -1,11 +1,11 @@
/*
* Copyright 2023 - 2024 the original author or authors.
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.transformers;
import java.io.File;
@@ -37,27 +38,27 @@ public class ResourceCacheServiceTests {
@Test
public void fileResourcesAreExcludedByDefault() throws IOException {
var cache = new ResourceCacheService(tempDir);
var cache = new ResourceCacheService(this.tempDir);
var originalResourceUri = "file:src/main/resources/onnx/all-MiniLM-L6-v2/tokenizer.json";
var cachedResource = cache.getCachedResource(originalResourceUri);
assertThat(cachedResource).isEqualTo(new DefaultResourceLoader().getResource(originalResourceUri));
assertThat(Files.list(tempDir.toPath()).count()).isEqualTo(0);
assertThat(Files.list(this.tempDir.toPath()).count()).isEqualTo(0);
}
@Test
public void cacheFileResources() throws IOException {
var cache = new ResourceCacheService(tempDir);
var cache = new ResourceCacheService(this.tempDir);
cache.setExcludedUriSchemas(List.of()); // erase the excluded schema names,
// including 'file'.
// including 'file'.
var originalResourceUri = "file:src/main/resources/onnx/all-MiniLM-L6-v2/tokenizer.json";
var cachedResource1 = cache.getCachedResource(originalResourceUri);
assertThat(cachedResource1).isNotEqualTo(new DefaultResourceLoader().getResource(originalResourceUri));
assertThat(Files.list(tempDir.toPath()).count()).isEqualTo(1);
assertThat(Files.list(Files.list(tempDir.toPath()).iterator().next()).count()).isEqualTo(1);
assertThat(Files.list(this.tempDir.toPath()).count()).isEqualTo(1);
assertThat(Files.list(Files.list(this.tempDir.toPath()).iterator().next()).count()).isEqualTo(1);
// Attempt to cache the same resource again should return the already cached
// resource.
@@ -66,17 +67,17 @@ public class ResourceCacheServiceTests {
assertThat(cachedResource2).isNotEqualTo(new DefaultResourceLoader().getResource(originalResourceUri));
assertThat(cachedResource2).isEqualTo(cachedResource1);
assertThat(Files.list(tempDir.toPath()).count()).isEqualTo(1);
assertThat(Files.list(Files.list(tempDir.toPath()).iterator().next()).count()).isEqualTo(1);
assertThat(Files.list(this.tempDir.toPath()).count()).isEqualTo(1);
assertThat(Files.list(Files.list(this.tempDir.toPath()).iterator().next()).count()).isEqualTo(1);
}
@Test
public void cacheFileResourcesFromSameParentFolder() throws IOException {
var cache = new ResourceCacheService(tempDir);
var cache = new ResourceCacheService(this.tempDir);
cache.setExcludedUriSchemas(List.of()); // erase the excluded schema names,
// including 'file'.
// including 'file'.
var originalResourceUri1 = "file:src/main/resources/onnx/all-MiniLM-L6-v2/tokenizer.json";
var cachedResource1 = cache.getCachedResource(originalResourceUri1);
@@ -89,23 +90,23 @@ public class ResourceCacheServiceTests {
assertThat(cachedResource2).isNotEqualTo(new DefaultResourceLoader().getResource(originalResourceUri1));
assertThat(cachedResource2).isNotEqualTo(cachedResource1);
assertThat(Files.list(tempDir.toPath()).count()).isEqualTo(1)
assertThat(Files.list(this.tempDir.toPath()).count()).isEqualTo(1)
.describedAs(
"As both resources come from the same parent segments they should be cached in a single common parent.");
assertThat(Files.list(Files.list(tempDir.toPath()).iterator().next()).count()).isEqualTo(2);
assertThat(Files.list(Files.list(this.tempDir.toPath()).iterator().next()).count()).isEqualTo(2);
}
@Test
public void cacheHttpResources() throws IOException {
var cache = new ResourceCacheService(tempDir);
var cache = new ResourceCacheService(this.tempDir);
var originalResourceUri1 = "https://raw.githubusercontent.com/spring-projects/spring-ai/main/spring-ai-core/src/main/resources/embedding/embedding-model-dimensions.properties";
var cachedResource1 = cache.getCachedResource(originalResourceUri1);
assertThat(cachedResource1).isNotEqualTo(new DefaultResourceLoader().getResource(originalResourceUri1));
assertThat(Files.list(tempDir.toPath()).count()).isEqualTo(1);
assertThat(Files.list(Files.list(tempDir.toPath()).iterator().next()).count()).isEqualTo(1);
assertThat(Files.list(this.tempDir.toPath()).count()).isEqualTo(1);
assertThat(Files.list(Files.list(this.tempDir.toPath()).iterator().next()).count()).isEqualTo(1);
}
}

17
models/spring-ai-transformers/src/test/java/org/springframework/ai/transformers/TransformersEmbeddingModelObservationTests.java
View File

@@ -1,11 +1,11 @@
/*
* Copyright 2024 the original author or authors.
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,13 +13,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.transformers;
import static org.assertj.core.api.Assertions.assertThat;
package org.springframework.ai.transformers;
import java.util.List;
import io.micrometer.observation.tck.TestObservationRegistry;
import io.micrometer.observation.tck.TestObservationRegistryAssert;
import org.junit.jupiter.api.Test;
import org.springframework.ai.document.MetadataMode;
import org.springframework.ai.embedding.EmbeddingOptionsBuilder;
import org.springframework.ai.embedding.EmbeddingRequest;
@@ -35,8 +37,7 @@ import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import io.micrometer.observation.tck.TestObservationRegistry;
import io.micrometer.observation.tck.TestObservationRegistryAssert;
import static org.assertj.core.api.Assertions.assertThat;
/**
* Integration tests for observation instrumentation in {@link OpenAiEmbeddingModel}.
@@ -59,13 +60,13 @@ public class TransformersEmbeddingModelObservationTests {
EmbeddingRequest embeddingRequest = new EmbeddingRequest(List.of("Here comes the sun"), options);
EmbeddingResponse embeddingResponse = embeddingModel.call(embeddingRequest);
EmbeddingResponse embeddingResponse = this.embeddingModel.call(embeddingRequest);
assertThat(embeddingResponse.getResults()).isNotEmpty();
EmbeddingResponseMetadata responseMetadata = embeddingResponse.getMetadata();
assertThat(responseMetadata).isNotNull();
TestObservationRegistryAssert.assertThat(observationRegistry)
TestObservationRegistryAssert.assertThat(this.observationRegistry)
.doesNotHaveAnyRemainingCurrentObservation()
.hasObservationWithNameEqualTo(DefaultEmbeddingModelObservationConvention.DEFAULT_NAME)
.that()

5
models/spring-ai-transformers/src/test/java/org/springframework/ai/transformers/TransformersEmbeddingModelTests.java
View File

@@ -1,11 +1,11 @@
/*
* Copyright 2023 - 2024 the original author or authors.
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.transformers;
import java.text.DecimalFormat;

7
models/spring-ai-transformers/src/test/java/org/springframework/ai/transformers/samples/ONNXSample.java
View File

@@ -1,11 +1,11 @@
/*
* Copyright 2023 - 2024 the original author or authors.
* Copyright 2023-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.transformers.samples;
import java.nio.FloatBuffer;
@@ -125,4 +126,4 @@ public class ONNXSample {
return manager.create(buffer, new Shape(data.length, data[0].length, data[0][0].length));
}
}
}