diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java index d8e013e16..b3f150da5 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java @@ -270,48 +270,52 @@ public interface MongoOperations { /** * Execute a map-reduce operation. The map-reduce operation will be formed with an output type of INLINE + * @param inputCollectionName the collection where the map-reduce will read from * @param mapFunction The JavaScript map function * @param reduceFunction The JavaScript reduce function * @param mapReduceOptions Options that specify detailed map-reduce behavior * @param entityClass The parameterized type of the returned list * @return The results of the map reduce operation */ - MapReduceResults mapReduce(String mapFunction, String reduceFunction, Class entityClass ); + MapReduceResults mapReduce(String inputCollectionName, String mapFunction, String reduceFunction, Class entityClass ); /** * Execute a map-reduce operation that takes additional map-reduce options. + * @param inputCollectionName the collection where the map-reduce will read from * @param mapFunction The JavaScript map function * @param reduceFunction The JavaScript reduce function * @param mapReduceOptions Options that specify detailed map-reduce behavior * @param entityClass The parameterized type of the returned list * @return The results of the map reduce operation */ - MapReduceResults mapReduce(String mapFunction, String reduceFunction, MapReduceOptions mapReduceOptions, Class entityClass ); + MapReduceResults mapReduce(String inputCollectionName, String mapFunction, String reduceFunction, MapReduceOptions mapReduceOptions, Class entityClass ); /** * Execute a map-reduce operation that takes a query. The map-reduce operation will be formed with an output type of INLINE * @param query The query to use to select the data for the map phase + * @param inputCollectionName the collection where the map-reduce will read from * @param mapFunction The JavaScript map function * @param reduceFunction The JavaScript reduce function * @param mapReduceOptions Options that specify detailed map-reduce behavior * @param entityClass The parameterized type of the returned list * @return The results of the map reduce operation */ - MapReduceResults mapReduce(Query query, String mapFunction, String reduceFunction, Class entityClass ); + MapReduceResults mapReduce(Query query, String inputCollectionName, String mapFunction, String reduceFunction, Class entityClass ); /** * Execute a map-reduce operation that takes a query and additional map-reduce options * @param query The query to use to select the data for the map phase + * @param inputCollectionName the collection where the map-reduce will read from * @param mapFunction The JavaScript map function * @param reduceFunction The JavaScript reduce function * @param mapReduceOptions Options that specify detailed map-reduce behavior * @param entityClass The parameterized type of the returned list * @return The results of the map reduce operation */ - MapReduceResults mapReduce(Query query, String mapFunction, String reduceFunction, MapReduceOptions mapReduceOptions, Class entityClass ); + MapReduceResults mapReduce(Query query, String inputCollectionName, String mapFunction, String reduceFunction, MapReduceOptions mapReduceOptions, Class entityClass ); /** * Returns {@link GeoResult} for all entities matching the given {@link NearQuery}. Will consider entity mapping diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java index da0ec787c..3462ca471 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java @@ -832,24 +832,24 @@ public class MongoTemplate implements MongoOperations, ApplicationContextAware { entityClass), collectionName); } - public MapReduceResults mapReduce(String mapFunction, String reduceFunction, Class entityClass) { - return mapReduce(null, mapFunction, reduceFunction, new MapReduceOptions().outputTypeInline(), entityClass); + public MapReduceResults mapReduce(String inputCollectionName, String mapFunction, String reduceFunction, Class entityClass) { + return mapReduce(null, inputCollectionName, mapFunction, reduceFunction, new MapReduceOptions().outputTypeInline(), entityClass); } - public MapReduceResults mapReduce(String mapFunction, String reduceFunction, + public MapReduceResults mapReduce(String inputCollectionName, String mapFunction, String reduceFunction, MapReduceOptions mapReduceOptions, Class entityClass) { - return mapReduce(null, mapFunction, reduceFunction, mapReduceOptions, entityClass); + return mapReduce(null, inputCollectionName, mapFunction, reduceFunction, mapReduceOptions, entityClass); } - public MapReduceResults mapReduce(Query query, String mapFunction, String reduceFunction, Class entityClass) { - return mapReduce(query, mapFunction, reduceFunction, new MapReduceOptions().outputTypeInline(), entityClass); + public MapReduceResults mapReduce(Query query, String inputCollectionName, String mapFunction, String reduceFunction, Class entityClass) { + return mapReduce(query, inputCollectionName, mapFunction, reduceFunction, new MapReduceOptions().outputTypeInline(), entityClass); } - public MapReduceResults mapReduce(Query query, String mapFunction, String reduceFunction, + public MapReduceResults mapReduce(Query query, String inputCollectionName, String mapFunction, String reduceFunction, MapReduceOptions mapReduceOptions, Class entityClass) { String mapFunc = replaceWithResourceIfNecessary(mapFunction); String reduceFunc = replaceWithResourceIfNecessary(reduceFunction); - DBCollection inputCollection = getCollection(this.determineCollectionName(entityClass)); + DBCollection inputCollection = getCollection(inputCollectionName); MapReduceCommand command = new MapReduceCommand(inputCollection, mapFunc, reduceFunc, mapReduceOptions.getOutputCollection(), mapReduceOptions.getOutputType(), null); @@ -866,11 +866,18 @@ public class MongoTemplate implements MongoOperations, ApplicationContextAware { } else { commandResult = executeCommand(commandObject); } - commandResult.throwOnError(); + commandResult.throwOnError(); } catch (RuntimeException ex) { this.potentiallyConvertRuntimeException(ex); } - + String error = commandResult.getErrorMessage(); + if (error != null) { + throw new InvalidDataAccessApiUsageException("Command execution failed: Error [" + error + "], Command = " + commandObject); + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("MapReduce command result = [" + commandResult + "]"); + } MapReduceOutput mapReduceOutput = new MapReduceOutput(inputCollection, commandObject, commandResult); List mappedResults = new ArrayList(); DbObjectCallback callback = new ReadDbObjectCallback(mongoConverter, entityClass); diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/mapreduce/MapReduceTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/mapreduce/MapReduceTests.java index 90ec58560..01f17f12b 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/mapreduce/MapReduceTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/mapreduce/MapReduceTests.java @@ -28,6 +28,7 @@ import java.util.Map; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -87,9 +88,18 @@ public class MapReduceTests { protected void cleanDb() { template.dropCollection(template.getCollectionName(ValueObject.class)); template.dropCollection("jmr1_out"); - + template.dropCollection("jmr1"); } + @Test + @Ignore + public void testForDocs() { + createMapReduceData(); + MapReduceResults results = mongoTemplate.mapReduce("jmr1", mapFunction, reduceFunction, ValueObject.class); + for (ValueObject valueObject : results) { + System.out.println(valueObject); + } + } @Test public void testMapReduce() { performMapReduce(false, false); @@ -114,7 +124,7 @@ public class MapReduceTests { String mapWithExcludeFunction = "function(){ for ( var i=0; i results = mongoTemplate.mapReduce(mapWithExcludeFunction, reduceFunction, + MapReduceResults results = mongoTemplate.mapReduce("jmr1", mapWithExcludeFunction, reduceFunction, new MapReduceOptions().scopeVariables(scopeVariables).outputTypeInline(), ValueObject.class); Map m = copyToMap(results); assertEquals(3, m.size()); @@ -128,7 +138,7 @@ public class MapReduceTests { createMapReduceData(); Query query = new Query(where("x").ne(new String[] { "a", "b" })); - MapReduceResults results = mongoTemplate.mapReduce(query, mapFunction, reduceFunction, ValueObject.class); + MapReduceResults results = mongoTemplate.mapReduce(query, "jmr1", mapFunction, reduceFunction, ValueObject.class); Map m = copyToMap(results); assertEquals(3, m.size()); @@ -144,15 +154,15 @@ public class MapReduceTests { MapReduceResults results; if (inline) { if (withQuery) { - results = mongoTemplate.mapReduce(new Query(), "classpath:map.js", "classpath:reduce.js", ValueObject.class); + results = mongoTemplate.mapReduce(new Query(), "jmr1", "classpath:map.js", "classpath:reduce.js", ValueObject.class); } else { - results = mongoTemplate.mapReduce(mapFunction, reduceFunction, ValueObject.class); + results = mongoTemplate.mapReduce("jmr1", mapFunction, reduceFunction, ValueObject.class); } } else { if (withQuery) { - results = mongoTemplate.mapReduce(new Query(), mapFunction, reduceFunction, options().outputCollection("jmr1_out"), ValueObject.class); + results = mongoTemplate.mapReduce(new Query(), "jmr1", mapFunction, reduceFunction, options().outputCollection("jmr1_out"), ValueObject.class); } else { - results = mongoTemplate.mapReduce(mapFunction, reduceFunction, new MapReduceOptions().outputCollection("jmr1_out"), ValueObject.class); + results = mongoTemplate.mapReduce("jmr1", mapFunction, reduceFunction, new MapReduceOptions().outputCollection("jmr1_out"), ValueObject.class); } } Map m = copyToMap(results); @@ -160,7 +170,7 @@ public class MapReduceTests { } private void createMapReduceData() { - DBCollection c = mongoTemplate.getDb().getCollection(template.getCollectionName(ValueObject.class)); + DBCollection c = mongoTemplate.getDb().getCollection("jmr1"); c.save(new BasicDBObject("x", new String[] { "a", "b" })); c.save(new BasicDBObject("x", new String[] { "b", "c" })); c.save(new BasicDBObject("x", new String[] { "c", "d" })); diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/mapreduce/ValueObject.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/mapreduce/ValueObject.java index 7c992f83d..f8779da4c 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/mapreduce/ValueObject.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/mapreduce/ValueObject.java @@ -22,8 +22,5 @@ public class ValueObject { public String toString() { return "ValueObject [id=" + id + ", value=" + value + "]"; } - - - } diff --git a/src/docbkx/reference/mongodb.xml b/src/docbkx/reference/mongodb.xml index d0d853da7..e3847c517 100644 --- a/src/docbkx/reference/mongodb.xml +++ b/src/docbkx/reference/mongodb.xml @@ -1385,6 +1385,9 @@ import static org.springframework.data.document.mongodb.query.Update; GeoSpatial queries are also supported and are described more in the section GeoSpatial Queries. + Map-Reduce operations are also supported and are described more in the + section Map-Reduce. +
Querying documents in a collection @@ -1835,6 +1838,114 @@ GeoResults<Restaurant> = operations.geoNear(query, Restaurant.class);
+
+ Map-Reduce + + You can query MongoDB using Map-Reduce which his useful for batch processing, data aggregation, and + for when the query language doesn't fulfill your needs. Spring provides integration with MongoDB's map reduce + by providing methods on MongoOperations to simplify the creation and execution of Map-Reduce operations. + It also integrates + with Spring's Resource abstraction + abstraction. This will let you place your JavaScript files on the file system, classpath, http server or any other Spring Resource implementation and + then reference the JavaScript resources via an easy URI style syntax, e.g. 'classpath:reduce.js;. + Externalizing JavaScript code in files is preferable to embedding them as Java strings in your code. + + + To understand how to perform Map-Reduce operations an example from the book 'MongoDB - The definitive guide' is used. In this example + we will create three documents, that have the values [a,b], [b,c], and [c,d] respectfully. The values in each document are associated with the key 'x' as shown below. + For the example assume it is in the collection named "jmr1". + +{ "_id" : ObjectId("4e5ff893c0277826074ec533"), "x" : [ "a", "b" ] } +{ "_id" : ObjectId("4e5ff893c0277826074ec534"), "x" : [ "b", "c" ] } +{ "_id" : ObjectId("4e5ff893c0277826074ec535"), "x" : [ "c", "d" ] } + + A map function that will count the occurance of each letter in the array for each document is shown below + function () { + for (var i = 0; i < this.x.length; i++) { + emit(this.x[i], 1); + } +} + + The reduce function that will sum up the occurance of each letter across all the documents is shown below + function (key, values) { + var sum = 0; + for (var i = 0; i < values.length; i++) + sum += values[i]; + return sum; +} + + Executing this will result in a collection as shown below. + +{ "_id" : "a", "value" : 1 } +{ "_id" : "b", "value" : 2 } +{ "_id" : "c", "value" : 2 } +{ "_id" : "d", "value" : 1 } + + Assuming that the map and reduce functions are located in map.js and reduce.js and bundled in your jar so they are available on the classpath, you + can execute a map-reduce operation and obtain the results as shown below + + MapReduceResults<ValueObject> results = mongoOperations.mapReduce("jmr1", "classpath:map.js", "classpath:reduce.js", ValueObject.class); + for (ValueObject valueObject : results) { + System.out.println(valueObject); + } + + The output of the above code is + +ValueObject [id=a, value=1.0] +ValueObject [id=b, value=2.0] +ValueObject [id=c, value=2.0] +ValueObject [id=d, value=1.0] + + The MapReduceResults class implement Iterable and provides access to the raw output, as well as timing and count statisticas. The ValueObject class is simply + +public class ValueObject { + + private String id; + + public String getId() { + return id; + } + + private float value; + + public float getValue() { + return value; + } + + public void setValue(float value) { + this.value = value; + } + + @Override + public String toString() { + return "ValueObject [id=" + id + ", value=" + value + "]"; + } + +} + + By default the output type of INLINE is used so you don't have to specify an output collection. To specify additional map-reduce options use an overloaded method + that takes an additional The class MapReduceOptions argument. MapReduceOptions has a fluent API so adding additional options can be done + in a very compact syntax. Here an example that sets the output collection to "jmr1_out". Note that setting only the output collection assumes a + default output type of REPLACE. + + MapReduceResults<ValueObject> results = mongoOperations.mapReduce("jmr1", "classpath:map.js", "classpath:reduce.js", new MapReduceOptions().outputCollection("jmr1_out"), ValueObject.class); + + There is also a static import import static org.springframework.data.mongodb.core.mapreduce.MapReduceOptions.options; that can be used to make the syntax slightly more compact + + MapReduceResults<ValueObject> results = mongoOperations.mapReduce("jmr1", "classpath:map.js", "classpath:reduce.js", options().outputCollection("jmr1_out"), ValueObject.class); + + You can also specify a query to reduce the set of data that will be used to feed into the map-reduce operation. This will remove the document that contains [a,b] from consideration for map-reduce operations. + + Query query = new Query(where("x").ne(new String[] { "a", "b" })); + MapReduceResults<ValueObject> results = mongoOperations.mapReduce(query, "jmr1", "classpath:map.js", "classpath:reduce.js", options().outputCollection("jmr1_out"), ValueObject.class); + + + Note that you can specify additional limit and sort values as well on the query but not skip values. + + + +
+
Overriding default mapping with custom converters