Add TextLoader implementation of Loader

- supports custom Charsets.
 - User metdata can be assigned to the generated documents.
 - Add tests.

spring-ai-core/src/main/java/org/springframework/ai/loader/impl/TextLoader.java

@@ -0,0 +1,90 @@
+package org.springframework.ai.loader.impl;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.loader.Loader;
+import org.springframework.ai.splitter.TextSplitter;
+import org.springframework.ai.splitter.TokenTextSplitter;
+import org.springframework.core.io.DefaultResourceLoader;
+import org.springframework.core.io.Resource;
+import org.springframework.util.StreamUtils;
+
+/**
+ * @author Craig Walls
+ * @author Christian Tzolov
+ */
+public class TextLoader implements Loader {
+
+	public static final String CHARSET_METADATA = "charset";
+
+	public static final String SOURCE_METADATA = "source";
+
+	/**
+	 * Input resource to load the text from.
+	 */
+	private final Resource resource;
+
+	/**
+	 * @return Character set to be used when loading data from the
+	 */
+	private Charset charset = StandardCharsets.UTF_8;
+
+	private Map<String, Object> customMetadata = new HashMap<>();
+
+	public TextLoader(String resourceUrl) {
+		this(new DefaultResourceLoader().getResource(resourceUrl));
+	}
+
+	public TextLoader(Resource resource) {
+		Objects.requireNonNull(resource, "The Spring Resource must not be null");
+		this.resource = resource;
+	}
+
+	public void setCharset(Charset charset) {
+		Objects.requireNonNull(charset, "The charset must not be null");
+		this.charset = charset;
+	}
+
+	public Charset getCharset() {
+		return this.charset;
+	}
+
+	/**
+	 * Metadata associated with all documents created by the loader.
+	 * @return Metadata to be assigned to the output Documents.
+	 */
+	public Map<String, Object> getCustomMetadata() {
+		return this.customMetadata;
+	}
+
+	@Override
+	public List<Document> load() {
+		return load(new TokenTextSplitter());
+	}
+
+	@Override
+	public List<Document> load(TextSplitter textSplitter) {
+		try {
+
+			String document = StreamUtils.copyToString(this.resource.getInputStream(), this.charset);
+
+			// Inject source information as a metadata.
+			this.customMetadata.put(CHARSET_METADATA, this.charset.name());
+			this.customMetadata.put(SOURCE_METADATA, this.resource.getFilename());
+
+			return textSplitter.apply(Collections.singletonList(new Document(document, this.customMetadata)));
+		}
+		catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+}

spring-ai-core/src/test/java/org/springframework/ai/loader/JsonLoaderTests.java

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2023-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.springframework.ai.loader;
 
 import org.junit.jupiter.api.Test;
@@ -12,7 +28,7 @@ import java.util.List;
 import static org.assertj.core.api.Assertions.assertThat;
 
 @SpringBootTest
-public class LoaderTests {
+public class JsonLoaderTests {
 
 	@Value("classpath:bikes.json")
 	private Resource resource;

spring-ai-core/src/test/java/org/springframework/ai/loader/TextLoaderTests.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2023-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.loader;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.loader.impl.TextLoader;
+import org.springframework.core.io.DefaultResourceLoader;
+import org.springframework.core.io.Resource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author Christian Tzolov
+ */
+public class TextLoaderTests {
+
+	private Resource resource = new DefaultResourceLoader().getResource("classpath:text_source.txt");
+
+	@Test
+	void loadText() {
+		assertThat(resource).isNotNull();
+		TextLoader textLoader = new TextLoader(resource);
+		textLoader.getCustomMetadata().put("customKey", "Value");
+
+		List<Document> documents = textLoader.load();
+
+		assertThat(documents.size()).isEqualTo(54);
+
+		for (Document document : documents) {
+			assertThat(document.getMetadata().get("customKey")).isEqualTo("Value");
+			assertThat(document.getMetadata().get(TextLoader.SOURCE_METADATA)).isEqualTo("text_source.txt");
+			assertThat(document.getMetadata().get(TextLoader.CHARSET_METADATA)).isEqualTo("UTF-8");
+			assertThat(document.getText()).isNotEmpty();
+		}
+	}
+
+}