Spring/spring-data-mongodb
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DATADOC-7 - Support for map-reduce operations in MongoTemplate

Browse Source
This commit is contained in:
Mark Pollack
2011-09-01 18:03:22 -04:00
parent 9bb42245bb
commit 756f886cef
5 changed files with 154 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java
View File

@@ -270,48 +270,52 @@ public interface MongoOperations {
/**
* Execute a map-reduce operation. The map-reduce operation will be formed with an output type of INLINE
* @param inputCollectionName the collection where the map-reduce will read from
* @param mapFunction The JavaScript map function
* @param reduceFunction The JavaScript reduce function
* @param mapReduceOptions Options that specify detailed map-reduce behavior
* @param entityClass The parameterized type of the returned list
* @return The results of the map reduce operation
*/
<T> MapReduceResults<T> mapReduce(String mapFunction, String reduceFunction, Class<T> entityClass );
<T> MapReduceResults<T> mapReduce(String inputCollectionName, String mapFunction, String reduceFunction, Class<T> entityClass );
/**
* Execute a map-reduce operation that takes additional map-reduce options.
* @param inputCollectionName the collection where the map-reduce will read from
* @param mapFunction The JavaScript map function
* @param reduceFunction The JavaScript reduce function
* @param mapReduceOptions Options that specify detailed map-reduce behavior
* @param entityClass The parameterized type of the returned list
* @return The results of the map reduce operation
*/
<T> MapReduceResults<T> mapReduce(String mapFunction, String reduceFunction, MapReduceOptions mapReduceOptions, Class<T> entityClass );
<T> MapReduceResults<T> mapReduce(String inputCollectionName, String mapFunction, String reduceFunction, MapReduceOptions mapReduceOptions, Class<T> entityClass );
/**
* Execute a map-reduce operation that takes a query. The map-reduce operation will be formed with an output type of INLINE
* @param query The query to use to select the data for the map phase
* @param inputCollectionName the collection where the map-reduce will read from
* @param mapFunction The JavaScript map function
* @param reduceFunction The JavaScript reduce function
* @param mapReduceOptions Options that specify detailed map-reduce behavior
* @param entityClass The parameterized type of the returned list
* @return The results of the map reduce operation
*/
<T> MapReduceResults<T> mapReduce(Query query, String mapFunction, String reduceFunction, Class<T> entityClass );
<T> MapReduceResults<T> mapReduce(Query query, String inputCollectionName, String mapFunction, String reduceFunction, Class<T> entityClass );
/**
* Execute a map-reduce operation that takes a query and additional map-reduce options
* @param query The query to use to select the data for the map phase
* @param inputCollectionName the collection where the map-reduce will read from
* @param mapFunction The JavaScript map function
* @param reduceFunction The JavaScript reduce function
* @param mapReduceOptions Options that specify detailed map-reduce behavior
* @param entityClass The parameterized type of the returned list
* @return The results of the map reduce operation
*/
<T> MapReduceResults<T> mapReduce(Query query, String mapFunction, String reduceFunction, MapReduceOptions mapReduceOptions, Class<T> entityClass );
<T> MapReduceResults<T> mapReduce(Query query, String inputCollectionName, String mapFunction, String reduceFunction, MapReduceOptions mapReduceOptions, Class<T> entityClass );
/**
* Returns {@link GeoResult} for all entities matching the given {@link NearQuery}. Will consider entity mapping

27
spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java
View File

@@ -832,24 +832,24 @@ public class MongoTemplate implements MongoOperations, ApplicationContextAware {
entityClass), collectionName);
}
public <T> MapReduceResults<T> mapReduce(String mapFunction, String reduceFunction, Class<T> entityClass) {
return mapReduce(null, mapFunction, reduceFunction, new MapReduceOptions().outputTypeInline(), entityClass);
public <T> MapReduceResults<T> mapReduce(String inputCollectionName, String mapFunction, String reduceFunction, Class<T> entityClass) {
return mapReduce(null, inputCollectionName, mapFunction, reduceFunction, new MapReduceOptions().outputTypeInline(), entityClass);
}
public <T> MapReduceResults<T> mapReduce(String mapFunction, String reduceFunction,
public <T> MapReduceResults<T> mapReduce(String inputCollectionName, String mapFunction, String reduceFunction,
MapReduceOptions mapReduceOptions, Class<T> entityClass) {
return mapReduce(null, mapFunction, reduceFunction, mapReduceOptions, entityClass);
return mapReduce(null, inputCollectionName, mapFunction, reduceFunction, mapReduceOptions, entityClass);
}
public <T> MapReduceResults<T> mapReduce(Query query, String mapFunction, String reduceFunction, Class<T> entityClass) {
return mapReduce(query, mapFunction, reduceFunction, new MapReduceOptions().outputTypeInline(), entityClass);
public <T> MapReduceResults<T> mapReduce(Query query, String inputCollectionName, String mapFunction, String reduceFunction, Class<T> entityClass) {
return mapReduce(query, inputCollectionName, mapFunction, reduceFunction, new MapReduceOptions().outputTypeInline(), entityClass);
}
public <T> MapReduceResults<T> mapReduce(Query query, String mapFunction, String reduceFunction,
public <T> MapReduceResults<T> mapReduce(Query query, String inputCollectionName, String mapFunction, String reduceFunction,
MapReduceOptions mapReduceOptions, Class<T> entityClass) {
String mapFunc = replaceWithResourceIfNecessary(mapFunction);
String reduceFunc = replaceWithResourceIfNecessary(reduceFunction);
DBCollection inputCollection = getCollection(this.determineCollectionName(entityClass));
DBCollection inputCollection = getCollection(inputCollectionName);
MapReduceCommand command = new MapReduceCommand(inputCollection, mapFunc, reduceFunc,
mapReduceOptions.getOutputCollection(), mapReduceOptions.getOutputType(), null);
@@ -866,11 +866,18 @@ public class MongoTemplate implements MongoOperations, ApplicationContextAware {
} else {
commandResult = executeCommand(commandObject);
}
commandResult.throwOnError();
commandResult.throwOnError();
} catch (RuntimeException ex) {
this.potentiallyConvertRuntimeException(ex);
}
String error = commandResult.getErrorMessage();
if (error != null) {
throw new InvalidDataAccessApiUsageException("Command execution failed: Error [" + error + "], Command = " + commandObject);
}
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("MapReduce command result = [" + commandResult + "]");
}
MapReduceOutput mapReduceOutput = new MapReduceOutput(inputCollection, commandObject, commandResult);
List<T> mappedResults = new ArrayList<T>();
DbObjectCallback<T> callback = new ReadDbObjectCallback<T>(mongoConverter, entityClass);

26
spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/mapreduce/MapReduceTests.java
View File

@@ -28,6 +28,7 @@ import java.util.Map;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
@@ -87,9 +88,18 @@ public class MapReduceTests {
protected void cleanDb() {
template.dropCollection(template.getCollectionName(ValueObject.class));
template.dropCollection("jmr1_out");
template.dropCollection("jmr1");
}
@Test
@Ignore
public void testForDocs() {
createMapReduceData();
MapReduceResults<ValueObject> results = mongoTemplate.mapReduce("jmr1", mapFunction, reduceFunction, ValueObject.class);
for (ValueObject valueObject : results) {
System.out.println(valueObject);
}
}
@Test
public void testMapReduce() {
performMapReduce(false, false);
@@ -114,7 +124,7 @@ public class MapReduceTests {
String mapWithExcludeFunction = "function(){ for ( var i=0; i<this.x.length; i++ ){ if(this.x[i] != exclude) emit( this.x[i] , 1 ); } }";
MapReduceResults<ValueObject> results = mongoTemplate.mapReduce(mapWithExcludeFunction, reduceFunction,
MapReduceResults<ValueObject> results = mongoTemplate.mapReduce("jmr1", mapWithExcludeFunction, reduceFunction,
new MapReduceOptions().scopeVariables(scopeVariables).outputTypeInline(), ValueObject.class);
Map<String, Float> m = copyToMap(results);
assertEquals(3, m.size());
@@ -128,7 +138,7 @@ public class MapReduceTests {
createMapReduceData();
Query query = new Query(where("x").ne(new String[] { "a", "b" }));
MapReduceResults<ValueObject> results = mongoTemplate.mapReduce(query, mapFunction, reduceFunction, ValueObject.class);
MapReduceResults<ValueObject> results = mongoTemplate.mapReduce(query, "jmr1", mapFunction, reduceFunction, ValueObject.class);
Map<String, Float> m = copyToMap(results);
assertEquals(3, m.size());
@@ -144,15 +154,15 @@ public class MapReduceTests {
MapReduceResults<ValueObject> results;
if (inline) {
if (withQuery) {
results = mongoTemplate.mapReduce(new Query(), "classpath:map.js", "classpath:reduce.js", ValueObject.class);
results = mongoTemplate.mapReduce(new Query(), "jmr1", "classpath:map.js", "classpath:reduce.js", ValueObject.class);
} else {
results = mongoTemplate.mapReduce(mapFunction, reduceFunction, ValueObject.class);
results = mongoTemplate.mapReduce("jmr1", mapFunction, reduceFunction, ValueObject.class);
}
} else {
if (withQuery) {
results = mongoTemplate.mapReduce(new Query(), mapFunction, reduceFunction, options().outputCollection("jmr1_out"), ValueObject.class);
results = mongoTemplate.mapReduce(new Query(), "jmr1", mapFunction, reduceFunction, options().outputCollection("jmr1_out"), ValueObject.class);
} else {
results = mongoTemplate.mapReduce(mapFunction, reduceFunction, new MapReduceOptions().outputCollection("jmr1_out"), ValueObject.class);
results = mongoTemplate.mapReduce("jmr1", mapFunction, reduceFunction, new MapReduceOptions().outputCollection("jmr1_out"), ValueObject.class);
}
}
Map<String, Float> m = copyToMap(results);
@@ -160,7 +170,7 @@ public class MapReduceTests {
}
private void createMapReduceData() {
DBCollection c = mongoTemplate.getDb().getCollection(template.getCollectionName(ValueObject.class));
DBCollection c = mongoTemplate.getDb().getCollection("jmr1");
c.save(new BasicDBObject("x", new String[] { "a", "b" }));
c.save(new BasicDBObject("x", new String[] { "b", "c" }));
c.save(new BasicDBObject("x", new String[] { "c", "d" }));

3
spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/mapreduce/ValueObject.java
View File

@@ -22,8 +22,5 @@ public class ValueObject {
public String toString() {
return "ValueObject [id=" + id + ", value=" + value + "]";
}
}

111
src/docbkx/reference/mongodb.xml
View File

@@ -1385,6 +1385,9 @@ import static org.springframework.data.document.mongodb.query.Update;
<para>GeoSpatial queries are also supported and are described more in the
section <link linkend="mongo.geospatial">GeoSpatial Queries</link>.</para>
<para>Map-Reduce operations are also supported and are described more in the
section <link linkend="mongo.mapreduce">Map-Reduce</link>.</para>
<section id="mongodb-template-query">
<title>Querying documents in a collection</title>
@@ -1835,6 +1838,114 @@ GeoResults&lt;Restaurant&gt; = operations.geoNear(query, Restaurant.class);</pro
</section>
</section>
<section id="mongo.mapreduce">
<title>Map-Reduce</title>
<para>You can query MongoDB using Map-Reduce which his useful for batch processing, data aggregation, and
for when the query language doesn't fulfill your needs. Spring provides integration with MongoDB's map reduce
by providing methods on MongoOperations to simplify the creation and execution of Map-Reduce operations.
It also integrates
with Spring's <ulink url="http://static.springsource.org/spring/docs/3.0.x/spring-framework-reference/html/resources.html">Resource abstraction</ulink>
abstraction. This will let you place your JavaScript files on the file system, classpath, http server or any other Spring Resource implementation and
then reference the JavaScript resources via an easy URI style syntax, e.g. 'classpath:reduce.js;.
Externalizing JavaScript code in files is preferable to embedding them as Java strings in your code.
</para>
<para>To understand how to perform Map-Reduce operations an example from the book 'MongoDB - The definitive guide' is used. In this example
we will create three documents, that have the values [a,b], [b,c], and [c,d] respectfully. The values in each document are associated with the key 'x' as shown below.
For the example assume it is in the collection named "jmr1".
<programlisting>
{ "_id" : ObjectId("4e5ff893c0277826074ec533"), "x" : [ "a", "b" ] }
{ "_id" : ObjectId("4e5ff893c0277826074ec534"), "x" : [ "b", "c" ] }
{ "_id" : ObjectId("4e5ff893c0277826074ec535"), "x" : [ "c", "d" ] }
</programlisting>
A map function that will count the occurance of each letter in the array for each document is shown below
<programlisting language="java">function () {
for (var i = 0; i &lt; this.x.length; i++) {
emit(this.x[i], 1);
}
}
</programlisting>
The reduce function that will sum up the occurance of each letter across all the documents is shown below
<programlisting language="java">function (key, values) {
var sum = 0;
for (var i = 0; i &lt; values.length; i++)
sum += values[i];
return sum;
}
</programlisting>
Executing this will result in a collection as shown below.
<programlisting>
{ "_id" : "a", "value" : 1 }
{ "_id" : "b", "value" : 2 }
{ "_id" : "c", "value" : 2 }
{ "_id" : "d", "value" : 1 }
</programlisting>
Assuming that the map and reduce functions are located in map.js and reduce.js and bundled in your jar so they are available on the classpath, you
can execute a map-reduce operation and obtain the results as shown below
<programlisting language="java">
MapReduceResults&lt;ValueObject> results = mongoOperations.mapReduce("jmr1", "classpath:map.js", "classpath:reduce.js", ValueObject.class);
for (ValueObject valueObject : results) {
System.out.println(valueObject);
}
</programlisting>
The output of the above code is
<programlisting>
ValueObject [id=a, value=1.0]
ValueObject [id=b, value=2.0]
ValueObject [id=c, value=2.0]
ValueObject [id=d, value=1.0]
</programlisting>
The MapReduceResults class implement <classname>Iterable</classname> and provides access to the raw output, as well as timing and count statisticas. The <classname>ValueObject</classname> class is simply
<programlisting language="java">
public class ValueObject {
private String id;
public String getId() {
return id;
}
private float value;
public float getValue() {
return value;
}
public void setValue(float value) {
this.value = value;
}
@Override
public String toString() {
return "ValueObject [id=" + id + ", value=" + value + "]";
}
}
</programlisting>
By default the output type of INLINE is used so you don't have to specify an output collection. To specify additional map-reduce options use an overloaded method
that takes an additional The class <classname>MapReduceOptions</classname> argument. <classname>MapReduceOptions</classname> has a fluent API so adding additional options can be done
in a very compact syntax. Here an example that sets the output collection to "jmr1_out". Note that setting only the output collection assumes a
default output type of REPLACE.
<programlisting language="java">
MapReduceResults&lt;ValueObject> results = mongoOperations.mapReduce("jmr1", "classpath:map.js", "classpath:reduce.js", new MapReduceOptions().outputCollection("jmr1_out"), ValueObject.class);
</programlisting>
There is also a static import <literal>import static org.springframework.data.mongodb.core.mapreduce.MapReduceOptions.options;</literal> that can be used to make the syntax slightly more compact
<programlisting language="java">
MapReduceResults&lt;ValueObject> results = mongoOperations.mapReduce("jmr1", "classpath:map.js", "classpath:reduce.js", options().outputCollection("jmr1_out"), ValueObject.class);
</programlisting>
You can also specify a query to reduce the set of data that will be used to feed into the map-reduce operation. This will remove the document that contains [a,b] from consideration for map-reduce operations.
<programlisting language="java">
Query query = new Query(where("x").ne(new String[] { "a", "b" }));
MapReduceResults&lt;ValueObject> results = mongoOperations.mapReduce(query, "jmr1", "classpath:map.js", "classpath:reduce.js", options().outputCollection("jmr1_out"), ValueObject.class);
</programlisting>
Note that you can specify additional limit and sort values as well on the query but not skip values.
</para>
</section>
<section>
<title>Overriding default mapping with custom converters</title>