Add Loader with JsonLoader impl. Add TextSplitter with TokenTextSplitter impl.

pom.xml

@@ -66,6 +66,7 @@
 		<stringtemplate.version>4.0.2</stringtemplate.version>
 		<open-ai-client.version>0.12.0</open-ai-client.version>
 		<azure-open-ai-client.version>1.0.0-beta.3</azure-open-ai-client.version>
+		<jtokkit.version>0.6.1</jtokkit.version>
 
         <!-- documentation dependencies -->
 		<asciidoctorj-pdf.version>1.6.2</asciidoctorj-pdf.version> <!-- FIXME build failure with version 2.3.9 -->

spring-ai-core/pom.xml

@@ -27,6 +27,22 @@
 			<version>${stringtemplate.version}</version>
 		</dependency>
 
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-core</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-json</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>com.knuddels</groupId>
+			<artifactId>jtokkit</artifactId>
+			<version>${jtokkit.version}</version>
+		</dependency>
+
 		<!-- test dependencies -->
 		<dependency>
 			<groupId>org.springframework.boot</groupId>

spring-ai-core/src/main/java/org/springframework/ai/core/loader/Document.java

@@ -0,0 +1,119 @@
+package org.springframework.ai.core.loader;
+
+import org.springframework.util.StringUtils;
+
+import java.util.*;
+
+public class Document {
+
+	private static String DEFAULT_TEXT_TEMPLATE = "{metadata_string}\n\n{text}";
+
+	private static String DEFAULT_METADATA_TEMPLATE = "{key}: {value}";
+
+	/**
+	 * Unique ID, creates UUID by default
+	 */
+	private String id;
+
+	// Embedding List<Float>
+
+	/**
+	 * Metadata for the document. It should not be nested and values should be restricted
+	 * to string, int, float, boolean for simple use with Vector Dbs.
+	 */
+	private Map<String, Object> metadata = new HashMap<>();
+
+	// Type; introduce when support images, now only text.
+
+	private String text;
+
+	private MetadataMode metadataMode = MetadataMode.NONE;
+
+	private List<String> excludedMetadataKeysForEmbedding;
+
+	private List<String> excludedMetadataKeysForLlm;
+
+	private List<String> relatedIds;
+
+	private String textTemplate = DEFAULT_TEXT_TEMPLATE;
+
+	private String metadataTemplate = DEFAULT_METADATA_TEMPLATE;
+
+	private String metadataSeperator;
+
+	public Document(String text) {
+		this.text = text;
+		this.metadata = metadata;
+	}
+
+	public Document(String text, Map<String, Object> metadata) {
+		this.text = text;
+		this.metadata = metadata;
+	}
+
+	public String getText() {
+		return this.text;
+	}
+
+	public String getContent(MetadataMode metadataMode) {
+		if (metadataMode == MetadataMode.NONE) {
+			return this.text;
+		}
+		String metadataString = getMetadataString(metadataMode);
+		if (!StringUtils.hasText(metadataString)) {
+			return this.text;
+		}
+		return getTextTemplate().replace("{metadata_string}", metadataString).replace("{text}", text);
+	}
+
+	public String getMetadataString() {
+		return getMetadataString(metadataMode);
+	}
+
+	public String getMetadataString(MetadataMode metadataMode) {
+		if (metadataMode == MetadataMode.NONE) {
+			return "";
+		}
+		Set<String> usableMetadataKeys = new HashSet<>(metadata.keySet());
+		if (metadataMode == MetadataMode.LLM) {
+			usableMetadataKeys.removeAll(this.excludedMetadataKeysForLlm);
+		}
+		else if (metadataMode == MetadataMode.EMBED) {
+			usableMetadataKeys.removeAll(this.excludedMetadataKeysForLlm);
+		}
+
+		List<String> metadataStringList = new ArrayList<>();
+
+		for (Map.Entry<String, Object> entry : metadata.entrySet()) {
+			String key = entry.getKey();
+			Object value = entry.getValue();
+			if (usableMetadataKeys.contains(key)) {
+				metadataStringList
+					.add(getMetadataTemplate().replace("{key}", key).replace("{value}", value.toString()));
+			}
+		}
+		return String.join(getMetadataSeperator(), metadataStringList);
+	}
+
+	public String getTextTemplate() {
+		return textTemplate;
+	}
+
+	public String getMetadataTemplate() {
+		return metadataTemplate;
+	}
+
+	public String getMetadataSeperator() {
+		return metadataSeperator;
+	}
+
+	public Map<String, Object> getMetadata() {
+		return metadata;
+	}
+
+	@Override
+	public String toString() {
+		return "Document{" + "id='" + id + '\'' + ", metadata=" + metadata + ", text='" + text + '\'' + '}';
+	}
+
+}

spring-ai-core/src/main/java/org/springframework/ai/core/loader/DocumentTransformer.java

@@ -0,0 +1,8 @@
+package org.springframework.ai.core.loader;
+
+import java.util.List;
+import java.util.function.Function;
+
+public interface DocumentTransformer extends Function<List<Document>, List<Document>> {
+
+}

spring-ai-core/src/main/java/org/springframework/ai/core/loader/Loader.java

@@ -0,0 +1,13 @@
+package org.springframework.ai.core.loader;
+
+import org.springframework.ai.core.loader.splitter.TextSplitter;
+
+import java.util.List;
+
+public interface Loader {
+
+	List<Document> load();
+
+	List<Document> load(TextSplitter textSplitter);
+
+}

spring-ai-core/src/main/java/org/springframework/ai/core/loader/MetadataMode.java

@@ -0,0 +1,7 @@
+package org.springframework.ai.core.loader;
+
+public enum MetadataMode {
+
+	ALL, EMBED, LLM, NONE;
+
+}

spring-ai-core/src/main/java/org/springframework/ai/core/loader/impl/JsonLoader.java

@@ -0,0 +1,67 @@
+package org.springframework.ai.core.loader.impl;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.ai.core.loader.Document;
+import org.springframework.ai.core.loader.Loader;
+import org.springframework.ai.core.loader.splitter.TextSplitter;
+import org.springframework.ai.core.loader.splitter.TokenTextSplitter;
+import org.springframework.core.io.Resource;
+
+import java.io.IOException;
+import java.util.*;
+
+public class JsonLoader implements Loader {
+
+	/**
+	 * The key from the JSON that we will use as the text to parse into the Document text
+	 */
+	private String textKey = "text";
+
+	private Resource resource;
+
+	public JsonLoader(Resource resource) {
+		Objects.requireNonNull(this.resource, "The Spring Resource must not be null");
+		this.resource = resource;
+	}
+
+	public JsonLoader(String textKey, Resource resource) {
+		Objects.requireNonNull(textKey, "textKey must not be null");
+		Objects.requireNonNull(resource, "The Spring Resource must not be null");
+		this.textKey = textKey;
+		this.resource = resource;
+	}
+
+	@Override
+	public List<Document> load() {
+		return load(new TokenTextSplitter());
+	}
+
+	@Override
+	public List<Document> load(TextSplitter textSplitter) {
+
+		ObjectMapper objectMapper = new ObjectMapper();
+		TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() {
+		};
+
+		List<Document> documents = new ArrayList<>();
+		try {
+			// TODO, not all json will be an array
+			List<Map<String, Object>> jsonData = objectMapper.readValue(this.resource.getInputStream(),
+					new TypeReference<List<Map<String, Object>>>() {
+					});
+			for (Map<String, Object> item : jsonData) {
+				if (item.containsKey(this.textKey)) {
+					Document document = new Document(item.get(this.textKey).toString());
+					documents.add(document);
+				}
+			}
+		}
+		catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+		List<Document> splitDocuments = textSplitter.apply(documents);
+		return splitDocuments;
+	}
+
+}

spring-ai-core/src/main/java/org/springframework/ai/core/loader/splitter/TextSplitter.java

@@ -0,0 +1,61 @@
+package org.springframework.ai.core.loader.splitter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.ai.core.loader.Document;
+import org.springframework.ai.core.loader.DocumentTransformer;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public abstract class TextSplitter implements DocumentTransformer {
+
+	private static final Logger logger = LoggerFactory.getLogger(TextSplitter.class);
+
+	@Override
+	public List<Document> apply(List<Document> documents) {
+		return doSplitDocuments(documents);
+	}
+
+	private List<Document> doSplitDocuments(List<Document> documents) {
+		List<String> texts = new ArrayList<>();
+		Map<String, Object> metadata = new HashMap<>();
+
+		for (Document doc : documents) {
+			texts.add(doc.getText());
+			metadata.putAll(doc.getMetadata());
+		}
+
+		return createDocuments(texts, metadata);
+	}
+
+	private List<Document> createDocuments(List<String> texts, Map<String, Object> metadata) {
+
+		// Process the data in a column oriented way and recreate the Document
+		List<Document> documents = new ArrayList<>();
+
+		for (int i = 0; i < texts.size(); i++) {
+			String text = texts.get(i);
+			List<String> chunks = splitText(text);
+			if (chunks.size() > 1) {
+				logger.info("Broke up document " + i + " into " + chunks.size() + " chunks.");
+			}
+			for (String chunk : chunks) {
+				// only primitive values are in here -
+				Map<String, Object> metadataCopy = metadata.entrySet()
+					.stream()
+					.collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue()));
+				Document newDoc = new Document(chunk, metadataCopy);
+				// TODO copy over other properties.
+				documents.add(newDoc);
+			}
+		}
+		return documents;
+	}
+
+	protected abstract List<String> splitText(String text);
+
+}

spring-ai-core/src/main/java/org/springframework/ai/core/loader/splitter/TokenTextSplitter.java

@@ -0,0 +1,93 @@
+package org.springframework.ai.core.loader.splitter;
+
+import com.knuddels.jtokkit.Encodings;
+import com.knuddels.jtokkit.api.Encoding;
+import com.knuddels.jtokkit.api.EncodingRegistry;
+import com.knuddels.jtokkit.api.EncodingType;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Raphael Yu
+ */
+public class TokenTextSplitter extends TextSplitter {
+
+	private static final int DEFAULT_CHUNK_SIZE = 800; // The target size of each text
+														// chunk in tokens
+
+	private static final int MIN_CHUNK_SIZE_CHARS = 350; // The minimum size of each text
+															// chunk in characters
+
+	private static final int MIN_CHUNK_LENGTH_TO_EMBED = 5; // Discard chunks shorter than
+															// this
+
+	private static final int MAX_NUM_CHUNKS = 10000; // The maximum number of chunks to
+														// generate from a text
+
+	private final EncodingRegistry registry = Encodings.newLazyEncodingRegistry();
+
+	private final Encoding encoding = registry.getEncoding(EncodingType.CL100K_BASE);
+
+	@Override
+	protected List<String> splitText(String text) {
+		return split(text, DEFAULT_CHUNK_SIZE);
+	}
+
+	public List<String> split(String text, int chunkSize) {
+		if (text == null || text.trim().isEmpty()) {
+			return new ArrayList<>();
+		}
+
+		List<Integer> tokens = getEncodedTokens(text);
+		List<String> chunks = new ArrayList<>();
+		int num_chunks = 0;
+		while (!tokens.isEmpty() && num_chunks < MAX_NUM_CHUNKS) {
+			List<Integer> chunk = tokens.subList(0, Math.min(chunkSize, tokens.size()));
+			String chunkText = decodeTokens(chunk);
+
+			// Skip the chunk if it is empty or whitespace
+			if (chunkText.trim().isEmpty()) {
+				tokens = tokens.subList(chunk.size(), tokens.size());
+				continue;
+			}
+
+			// Find the last period or punctuation mark in the chunk
+			int lastPunctuation = Math.max(chunkText.lastIndexOf('.'), Math.max(chunkText.lastIndexOf('?'),
+					Math.max(chunkText.lastIndexOf('!'), chunkText.lastIndexOf('\n'))));
+
+			if (lastPunctuation != -1 && lastPunctuation > MIN_CHUNK_SIZE_CHARS) {
+				// Truncate the chunk text at the punctuation mark
+				chunkText = chunkText.substring(0, lastPunctuation + 1);
+			}
+			String chunk_text_to_append = chunkText.replace("\n", " ").trim();
+			if (chunk_text_to_append.length() > MIN_CHUNK_LENGTH_TO_EMBED) {
+				chunks.add(chunk_text_to_append);
+			}
+
+			// Remove the tokens corresponding to the chunk text from the remaining tokens
+			tokens = tokens.subList(getEncodedTokens(chunkText).size(), tokens.size());
+
+			num_chunks++;
+		}
+
+		// Handle the remaining tokens
+		if (!tokens.isEmpty()) {
+			String remaining_text = decodeTokens(tokens).replace("\n", " ").trim();
+			if (remaining_text.length() > MIN_CHUNK_LENGTH_TO_EMBED) {
+				chunks.add(remaining_text);
+			}
+		}
+
+		return chunks;
+	}
+
+	private List<Integer> getEncodedTokens(String text) {
+		return encoding.encode(text);
+	}
+
+	private String decodeTokens(List<Integer> tokens) {
+		return encoding.decode(tokens);
+	}
+
+}

spring-ai-core/src/test/java/org/springframework/ai/core/TestConfiguration.java

@@ -0,0 +1,8 @@
+package org.springframework.ai.core;
+
+import org.springframework.boot.SpringBootConfiguration;
+
+@SpringBootConfiguration
+public class TestConfiguration {
+
+}

spring-ai-core/src/test/java/org/springframework/ai/core/loader/LoaderTests.java

@@ -0,0 +1,30 @@
+package org.springframework.ai.core.loader;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.ai.core.loader.impl.JsonLoader;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.core.io.Resource;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@SpringBootTest
+public class LoaderTests {
+
+	@Value("classpath:bikes.json")
+	private Resource resource;
+
+	@Test
+	void loadJson() {
+		assertThat(resource).isNotNull();
+		JsonLoader jsonLoader = new JsonLoader("description", resource);
+		List<Document> documents = jsonLoader.load();
+		assertThat(documents).isNotEmpty();
+		for (Document document : documents) {
+			System.out.println(document);
+		}
+	}
+
+}