diff --git a/pom.xml b/pom.xml
index 7a05da039..ef96813df 100644
--- a/pom.xml
+++ b/pom.xml
@@ -66,6 +66,7 @@
4.0.2
0.12.0
1.0.0-beta.3
+ 0.6.1
1.6.2
diff --git a/spring-ai-core/pom.xml b/spring-ai-core/pom.xml
index 0c1bf69d5..1bc08515a 100644
--- a/spring-ai-core/pom.xml
+++ b/spring-ai-core/pom.xml
@@ -27,6 +27,22 @@
${stringtemplate.version}
+
+ org.springframework
+ spring-core
+
+
+
+ org.springframework.boot
+ spring-boot-starter-json
+
+
+
+ com.knuddels
+ jtokkit
+ ${jtokkit.version}
+
+
org.springframework.boot
diff --git a/spring-ai-core/src/main/java/org/springframework/ai/core/loader/Document.java b/spring-ai-core/src/main/java/org/springframework/ai/core/loader/Document.java
new file mode 100644
index 000000000..0cc294fa9
--- /dev/null
+++ b/spring-ai-core/src/main/java/org/springframework/ai/core/loader/Document.java
@@ -0,0 +1,119 @@
+package org.springframework.ai.core.loader;
+
+import org.springframework.util.StringUtils;
+
+import java.util.*;
+
+public class Document {
+
+ private static String DEFAULT_TEXT_TEMPLATE = "{metadata_string}\n\n{text}";
+
+ private static String DEFAULT_METADATA_TEMPLATE = "{key}: {value}";
+
+ /**
+ * Unique ID, creates UUID by default
+ */
+ private String id;
+
+ // Embedding List
+
+ /**
+ * Metadata for the document. It should not be nested and values should be restricted
+ * to string, int, float, boolean for simple use with Vector Dbs.
+ */
+ private Map metadata = new HashMap<>();
+
+ // Type; introduce when support images, now only text.
+
+ private String text;
+
+ private MetadataMode metadataMode = MetadataMode.NONE;
+
+ private List excludedMetadataKeysForEmbedding;
+
+ private List excludedMetadataKeysForLlm;
+
+ private List relatedIds;
+
+ private String textTemplate = DEFAULT_TEXT_TEMPLATE;
+
+ private String metadataTemplate = DEFAULT_METADATA_TEMPLATE;
+
+ private String metadataSeperator;
+
+ public Document(String text) {
+ this.text = text;
+ this.metadata = metadata;
+ }
+
+ public Document(String text, Map metadata) {
+ this.text = text;
+ this.metadata = metadata;
+ }
+
+ public String getText() {
+ return this.text;
+ }
+
+ public String getContent(MetadataMode metadataMode) {
+ if (metadataMode == MetadataMode.NONE) {
+ return this.text;
+ }
+ String metadataString = getMetadataString(metadataMode);
+ if (!StringUtils.hasText(metadataString)) {
+ return this.text;
+ }
+ return getTextTemplate().replace("{metadata_string}", metadataString).replace("{text}", text);
+ }
+
+ public String getMetadataString() {
+ return getMetadataString(metadataMode);
+ }
+
+ public String getMetadataString(MetadataMode metadataMode) {
+ if (metadataMode == MetadataMode.NONE) {
+ return "";
+ }
+ Set usableMetadataKeys = new HashSet<>(metadata.keySet());
+ if (metadataMode == MetadataMode.LLM) {
+ usableMetadataKeys.removeAll(this.excludedMetadataKeysForLlm);
+ }
+ else if (metadataMode == MetadataMode.EMBED) {
+ usableMetadataKeys.removeAll(this.excludedMetadataKeysForLlm);
+ }
+
+ List metadataStringList = new ArrayList<>();
+
+ for (Map.Entry entry : metadata.entrySet()) {
+ String key = entry.getKey();
+ Object value = entry.getValue();
+ if (usableMetadataKeys.contains(key)) {
+ metadataStringList
+ .add(getMetadataTemplate().replace("{key}", key).replace("{value}", value.toString()));
+ }
+ }
+ return String.join(getMetadataSeperator(), metadataStringList);
+ }
+
+ public String getTextTemplate() {
+ return textTemplate;
+ }
+
+ public String getMetadataTemplate() {
+ return metadataTemplate;
+ }
+
+ public String getMetadataSeperator() {
+ return metadataSeperator;
+ }
+
+ public Map getMetadata() {
+ return metadata;
+ }
+
+ @Override
+ public String toString() {
+ return "Document{" + "id='" + id + '\'' + ", metadata=" + metadata + ", text='" + text + '\'' + '}';
+ }
+
+}
diff --git a/spring-ai-core/src/main/java/org/springframework/ai/core/loader/DocumentTransformer.java b/spring-ai-core/src/main/java/org/springframework/ai/core/loader/DocumentTransformer.java
new file mode 100644
index 000000000..89a2c53cc
--- /dev/null
+++ b/spring-ai-core/src/main/java/org/springframework/ai/core/loader/DocumentTransformer.java
@@ -0,0 +1,8 @@
+package org.springframework.ai.core.loader;
+
+import java.util.List;
+import java.util.function.Function;
+
+public interface DocumentTransformer extends Function, List> {
+
+}
diff --git a/spring-ai-core/src/main/java/org/springframework/ai/core/loader/Loader.java b/spring-ai-core/src/main/java/org/springframework/ai/core/loader/Loader.java
new file mode 100644
index 000000000..932cd662c
--- /dev/null
+++ b/spring-ai-core/src/main/java/org/springframework/ai/core/loader/Loader.java
@@ -0,0 +1,13 @@
+package org.springframework.ai.core.loader;
+
+import org.springframework.ai.core.loader.splitter.TextSplitter;
+
+import java.util.List;
+
+public interface Loader {
+
+ List load();
+
+ List load(TextSplitter textSplitter);
+
+}
diff --git a/spring-ai-core/src/main/java/org/springframework/ai/core/loader/MetadataMode.java b/spring-ai-core/src/main/java/org/springframework/ai/core/loader/MetadataMode.java
new file mode 100644
index 000000000..01c81164f
--- /dev/null
+++ b/spring-ai-core/src/main/java/org/springframework/ai/core/loader/MetadataMode.java
@@ -0,0 +1,7 @@
+package org.springframework.ai.core.loader;
+
+public enum MetadataMode {
+
+ ALL, EMBED, LLM, NONE;
+
+}
diff --git a/spring-ai-core/src/main/java/org/springframework/ai/core/loader/impl/JsonLoader.java b/spring-ai-core/src/main/java/org/springframework/ai/core/loader/impl/JsonLoader.java
new file mode 100644
index 000000000..a49d74778
--- /dev/null
+++ b/spring-ai-core/src/main/java/org/springframework/ai/core/loader/impl/JsonLoader.java
@@ -0,0 +1,67 @@
+package org.springframework.ai.core.loader.impl;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.ai.core.loader.Document;
+import org.springframework.ai.core.loader.Loader;
+import org.springframework.ai.core.loader.splitter.TextSplitter;
+import org.springframework.ai.core.loader.splitter.TokenTextSplitter;
+import org.springframework.core.io.Resource;
+
+import java.io.IOException;
+import java.util.*;
+
+public class JsonLoader implements Loader {
+
+ /**
+ * The key from the JSON that we will use as the text to parse into the Document text
+ */
+ private String textKey = "text";
+
+ private Resource resource;
+
+ public JsonLoader(Resource resource) {
+ Objects.requireNonNull(this.resource, "The Spring Resource must not be null");
+ this.resource = resource;
+ }
+
+ public JsonLoader(String textKey, Resource resource) {
+ Objects.requireNonNull(textKey, "textKey must not be null");
+ Objects.requireNonNull(resource, "The Spring Resource must not be null");
+ this.textKey = textKey;
+ this.resource = resource;
+ }
+
+ @Override
+ public List load() {
+ return load(new TokenTextSplitter());
+ }
+
+ @Override
+ public List load(TextSplitter textSplitter) {
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ TypeReference> typeRef = new TypeReference>() {
+ };
+
+ List documents = new ArrayList<>();
+ try {
+ // TODO, not all json will be an array
+ List