From f292ec6fad715794bb03cac9edc214331b910ac4 Mon Sep 17 00:00:00 2001 From: John Blum Date: Mon, 31 Aug 2015 21:46:29 -0700 Subject: [PATCH] SGF-408 - Provide support in the SDG XML namespace to load a pre-defined data set using GemFire Snapshot Service for development and testing purposes. Renamed the '' and '' sub-elements to ' and '' respectively; updated the SnapshotServiceParser to appropriately handle the sub-element name change. Added a mini chapter in the Spring Data GemFire Reference Guide on how-to use the GemFire SnapshotService support provided by Spring Data GemFire. --- gradle.properties | 18 +- src/asciidoc/reference/bootstrap.adoc | 1 + src/asciidoc/reference/cache.adoc | 16 +- src/asciidoc/reference/snapshot.adoc | 234 ++++++++++++++++++ .../gemfire/config/SnapshotServiceParser.java | 4 +- .../config/spring-data-gemfire-1.7.xsd | 4 +- ...dImportsExportsIntegrationTest-context.xml | 14 +- ...iceImportExportIntegrationTest-context.xml | 4 +- 8 files changed, 265 insertions(+), 30 deletions(-) create mode 100644 src/asciidoc/reference/snapshot.adoc diff --git a/gradle.properties b/gradle.properties index 640a89ff..14242eac 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,14 +1,14 @@ antlrVersion=2.7.7 -slf4jVersion=1.7.12 -junitVersion=4.12 -gemfireVersion=8.1.0 -spring.range="[4.0.0, 5.0.0)" aspectjVersion=1.8.5 -springDataBuildVersion=1.7.0.BUILD-SNAPSHOT -springVersion=4.1.7.RELEASE -springDataCommonsVersion=1.11.0.BUILD-SNAPSHOT -log4jVersion=1.2.17 +gemfireVersion=8.1.0 hamcrestVersion=1.3 -version=1.7.0.BUILD-SNAPSHOT jacksonVersion=2.6.0 +junitVersion=4.12 +log4jVersion=1.2.17 mockitoVersion=1.10.19 +slf4jVersion=1.7.12 +spring.range="[4.0.0, 5.0.0)" +springDataBuildVersion=1.7.0.BUILD-SNAPSHOT +springDataCommonsVersion=1.11.0.BUILD-SNAPSHOT +springVersion=4.1.7.RELEASE +version=1.7.0.BUILD-SNAPSHOT diff --git a/src/asciidoc/reference/bootstrap.adoc b/src/asciidoc/reference/bootstrap.adoc index d16af798..d1865f07 100644 --- a/src/asciidoc/reference/bootstrap.adoc +++ b/src/asciidoc/reference/bootstrap.adoc @@ -119,6 +119,7 @@ but one can customize it accordingly or use a pool (if need be) - see the namesp :leveloffset: +1 include::diskstore.adoc[] +include::snapshot.adoc[] include::function.adoc[] include::gateway.adoc[] :leveloffset: -1 diff --git a/src/asciidoc/reference/cache.adoc b/src/asciidoc/reference/cache.adoc index 012c92ac..01ad9469 100644 --- a/src/asciidoc/reference/cache.adoc +++ b/src/asciidoc/reference/cache.adoc @@ -1,9 +1,9 @@ [[bootstrap:cache]] -= Configuring the GemFire Cache += Configuring a GemFire Cache In order to use GemFire, a developer needs to either create a new `Cache` or connect to an existing one. -In the current version of GemFire, there can be only one opened Cache per VM (or per `ClassLoader` to be -technically correct). In most cases the Cache is only created once. +In the current version of GemFire, there can be only one open Cache per VM (or per `ClassLoader` to be +technically correct). In most cases the Cache should only be created once. NOTE: This section describes the creation and configuration of a full Cache member, appropriate for peer-to-peer cache topologies and cache servers. A full cache is also commonly used for standalone applications, integration tests @@ -20,11 +20,11 @@ A cache with default configuration can be created with a very simple declaration Upon initialization, a Spring application context containing this cache definition will register a `CacheFactoryBean` to create a Spring bean named `gemfireCache` referencing a GemFire `Cache` instance. This will either be an -existing cache, or if one does not exist, a newly created one. Since no additional properties were specified, a -newly created cache will apply the default cache configuration. +existing cache, or if one does not already exist, a newly created one. Since no additional properties were specified, +a newly created cache will apply the default cache configuration. All Spring Data GemFire components that depend on the cache respect this naming convention so that there is no need -to explicitly declare the cache dependency. If you prefer, you can make the dependence explicit via the `cache-ref` +to explicitly declare the cache dependency. If you prefer, you can make the dependency explicit via the `cache-ref` attribute provided by various namespace elements. Also, you can easily override the cache's bean name: [source,xml] @@ -33,7 +33,7 @@ attribute provided by various namespace elements. Also, you can easily override ---- Starting with Spring Data GemFire 1.2.0, the GemFire `Cache` may be fully configured using Spring. However, GemFire's -native XML configuration file, `cache.xml`, is also supported. For scenarios in which the GemFire cache needs to be +native XML configuration file, `cache.xml`, is also supported. For scenarios in which the GemFire Cache needs to be configured natively, simply provide a reference to the GemFire configuration file using the `cache-xml-location` attribute: @@ -50,7 +50,7 @@ or the prefix specified (if any) in the resource location. In addition to referencing an external configuration file one can specify GemFire http://gemfire.docs.pivotal.io/latest/userguide/index.html#reference/topics/gemfire_properties.html[properties] using any of Spring's common properties support features. For example, one can use the `properties` element -defined in the `util` namespace to define properties directly or load properties from a properties files. The latter is +defined in the `util` namespace to define properties directly or load properties from a properties file. The latter is recommended for externalizing environment specific settings outside the application configuration: [source,xml] diff --git a/src/asciidoc/reference/snapshot.adoc b/src/asciidoc/reference/snapshot.adoc new file mode 100644 index 00000000..c6fcd499 --- /dev/null +++ b/src/asciidoc/reference/snapshot.adoc @@ -0,0 +1,234 @@ +[[bootstrap:snapshot]] += Using the GemFire Snapshot Service + +Spring Data GemFire supports `Cache` and `Region` snapshots using http://gemfire81.docs.pivotal.io/latest/userguide/index.html#managing/cache_snapshots/chapter_overview.html[GemFire's Snapshot Service]. +The out-of-the-box Snapshot Service support offers several convenient features to simply the use of GemFire's http://gemfire.docs.pivotal.io/latest/javadocs/japi/com/gemstone/gemfire/cache/snapshot/CacheSnapshotService.html[Cache] +and http://gemfire.docs.pivotal.io/latest/javadocs/japi/com/gemstone/gemfire/cache/snapshot/RegionSnapshotService.html[Region] Snapshot Service APIs. + +As http://gemfire81.docs.pivotal.io/latest/userguide/index.html#managing/cache_snapshots/chapter_overview.html[GemFire documentation] describes, +snapshots allow you to save and subsequently reload the data later, which can be useful for moving data between environments, +say from production to a staging or test environment in order to reproduce data-related issues in a controlled context. +You can imagine combining Spring Data GemFire's Snapshot Service support with http://docs.spring.io/spring/docs/current/spring-framework-reference/htmlsingle/#beans-definition-profiles[Spring's bean definition profiles] +to load snapshot data specific to the environment as necessary. + +Spring Data GemFire's support for GemFire's Snapshot Service begins with the `` element +from the GFE Data Access Namespace. For example, I might define Cache-wide snapshots to be loaded as well as saved +with a couple snapshot imports and a single data export definition as follows: + +[source,xml] +---- + + + + + +---- + +You can define as many imports and/or exports as you like. You can define just imports or just exports. The file locations +and directory paths can be absolute, or relative to the Spring Data GemFire application JVM process's working directory. + +This is a pretty simple example and the snapshot service defined in this case refers to the GemFire `Cache`, having a +default name of `gemfireCache` (as described in <>). If you name your cache bean definition something +different, than you can use the `cache-ref` attribute to refer to the cache bean by name: + +[source,xml] +---- + +... + +... + +---- + +It is also straightforward to define a snapshot service for a GemFire Region by specifying the `region-ref` attribute: + +[source,xml] +---- + +... + + + +---- + +When the `region-ref` attribute is specified the Spring Data GemFire `SnapshotServiceFactoryBean` resolves +the `region-ref` attribute to a Region bean defined in the Spring context and then proceeds to create a +http://gemfire.docs.pivotal.io/latest/javadocs/japi/com/gemstone/gemfire/cache/snapshot/RegionSnapshotService.html[RegionSnapshotService]. +Again, the snapshot import and export definitions function the same way, however, the `location` must refer to a file +on export. + +NOTE: GemFire is strict about imported snapshot files actually existing before they are referenced. For exports, +GemFire will create the snapshot file if it does not already exist. If the snapshot file for export already exists, +the data will be overwritten. + +NOTE: Spring Data GemFire includes a `suppress-import-on-init` attribute to the `` element +to suppress the configured snapshot service from trying to import data into the Cache or a Region on initialization. +This is useful when data exported from 1 Region is used to feed the import of another Region, for example. + +[[bootstrap:snapshot:location]] +== Snapshot Location + +For a `Cache`-based SnapshotService (i.e. a GemFire http://gemfire.docs.pivotal.io/latest/javadocs/japi/com/gemstone/gemfire/cache/snapshot/CacheSnapshotService.html[CacheSnapshotService]) +a developer would typically pass it a directory containing all the snapshot files to load rather than individual snapshot files, +as the overloaded http://gemfire.docs.pivotal.io/latest/javadocs/japi/com/gemstone/gemfire/cache/snapshot/CacheSnapshotService.html#load(java.io.File,%20com.gemstone.gemfire.cache.snapshot.SnapshotOptions.SnapshotFormat)[load] method +in the `CacheSnapshotService` API indicates. + +NOTE: Of course, a developer may use the other, overloaded `load(:File[], :SnapshotFormat, :SnapshotOptions)` method +variant to get specific about which snapshot files are to be loaded into the GemFire `Cache`. + +However, Spring Data GemFire recognizes that a typical developer workflow might be to extract and export data from one environment +into several snapshot files, zip all of them up, and then conveniently move the ZIP file to another environment for import. + +As such, Spring Data GemFire enables the developer to specify a JAR or ZIP file on import for a `Cache`-based SnapshotService +as follows: + +[source,xml] +---- + + + +---- + +Spring Data GemFire will conveniently extract the provided ZIP file and treat it like a directory import (load). + +[[bootstrap:snapshot:filters]] +== Snapshot Filters + +The real power of defining multiple snapshot imports and exports is realized through the use of snapshot filters. +Snapshot filters implement GemFire's http://gemfire.docs.pivotal.io/latest/javadocs/japi/com/gemstone/gemfire/cache/snapshot/SnapshotFilter.html[SnapshotFilter] interface +and are used to filter Region entries for inclusion into the Region on import and for inclusion into the snapshot on export. + +Spring Data GemFire makes it brain dead simple to utilize snapshot filters on import and export using the `filter-ref` +attribute or an anonymous, nested bean definition: + +[source,xml] +---- + + + + + + + + + + + + + + +---- + +In addition, more complex snapshot filters can be expressed with the `ComposableSnapshotFilter` Spring Data GemFire class. +This class implements GemFire's http://gemfire.docs.pivotal.io/latest/javadocs/japi/com/gemstone/gemfire/cache/snapshot/SnapshotFilter.html[SnapshotFilter] interface +as well as the https://en.wikipedia.org/wiki/Composite_pattern[Composite] software design pattern. In a nutshell, the +https://en.wikipedia.org/wiki/Composite_pattern[Composite] design pattern allows developers to compose multiple objects +of the same type and treat the conglomerate as single instance of the object type, a very powerful and useful abstraction +to be sure. + +The `ComposableSnapshotFilter` has two factory methods, `'and'` and `'or'`, allowing developers to logically combine individual +snapshot filters using the AND and OR logical operators, respectively. The factory methods just take a list of snapshot filters. + +One is only limited by his/her imagination to leverage this powerful construct, for instance: + +[source,xml] +---- + + + + + + + + +---- + +You could then go onto combine the `activesUsersSinceFilter` with another filter using `'or'` like so: + +[source,xml] +---- + + + + + + + + +---- + +[[bootstrap::snapshot::events]] +== Snapshot Events + +By default, Spring Data GemFire uses GemFire's Snapshot Services on startup to import data and shutdown to export data. +However, you may want to trigger periodic, event-based snapshots, for either import or export from within your application. + +For this purpose, Spring Data GemFire defines two additional Spring application events (extending Spring's http://docs.spring.io/spring/docs/current/javadoc-api/org/springframework/context/ApplicationEvent.html[ApplicationEvent] class) +for imports and exports, respectively: `ImportSnapshotApplicationEvent` and `ExportSnapshotApplicationEvent`. + +The two application events can be targeted at the entire GemFire Cache, or individual GemFire Regions. The constructors +of these `ApplicationEvent` classes accept an optional Region pathname (e.g. "/Example") as well as 0 or more +`SnapshotMetadata` instances. + +The array of `SnapshotMetadata` is used to override the snapshot meta-data defined by `` +and `` sub-elements in XML, which will be used in cases where snapshot application events +do not explicitly provide `SnapshotMetadata`. Each individual `SnapshotMetadata` instance can define it's own `location` +and `filters` properties. + +Import/export snapshot application events are received by all snapshot service beans defined in the Spring application context. +However, import/export events are only processed by "matching" snapshot service beans. + +A Region-based `[Import|Export]SnapshotApplicationEvent` matches if the snapshot service bean defined is a `RegionSnapshotService` +and it's Region reference (as determined by `region-ref`) matches the Region's pathname specified by the snapshot application event. +A Cache-based `[Import|Export]SnapshotApplicationEvent` (i.e. a snapshot application event without a Region pathname) triggers +all snapshot service beans, including any `RegionSnapshotService` beans, to perform either an import or export, respectively. + +It is very easy to use Spring's http://docs.spring.io/spring/docs/current/javadoc-api/org/springframework/context/ApplicationEventPublisher.html[ApplicationEventPublisher] interface +to fire import and/or export snapshot application events from your application like so: + +[source,java] +---- +@Component +public class ExampleApplicationComponent { + + @Autowired + private ApplicationEventPublisher eventPublisher; + + @Resource(name = "Example") + private Region example; + + public void someMethod() { + ... + + SnapshotFilter myFilter = ...; + + SnapshotMetadata exportSnapshotMetadata = new SnapshotMetadata(new File(System.getProperty("user.dir"), + "/path/to/export/data.snapshot"), myFilter, null); + + eventPublisher.publishEvent(new ExportSnapshotApplicationEvent(this, example.getFullPath(), exportSnapshotMetadata); + + ... + } +} +---- + +In this particular example, only the "/Example" Region's SnapshotService bean will pick up and handle the export event, +saving the filtered "/Example" Region's data to the "data.snapshot" file in a sub-direcrtory of the application's +working directory. + +Using Spring application events and messaging subsystem is a good way to keep your application loosely coupled. It is +also not difficult to imagine that the snapshot application events could be fired on a periodic basis using Spring's +http://docs.spring.io/spring/docs/current/spring-framework-reference/htmlsingle/#scheduling-task-scheduler[Scheduling] services. diff --git a/src/main/java/org/springframework/data/gemfire/config/SnapshotServiceParser.java b/src/main/java/org/springframework/data/gemfire/config/SnapshotServiceParser.java index 1c78be69..b1d25781 100644 --- a/src/main/java/org/springframework/data/gemfire/config/SnapshotServiceParser.java +++ b/src/main/java/org/springframework/data/gemfire/config/SnapshotServiceParser.java @@ -55,11 +55,11 @@ class SnapshotServiceParser extends AbstractSingleBeanDefinitionParser { } private ManagedList parseExports(Element element, ParserContext parserContext) { - return parseSnapshots(element, parserContext, "export-snapshot"); + return parseSnapshots(element, parserContext, "snapshot-export"); } private ManagedList parseImports(Element element, ParserContext parserContext) { - return parseSnapshots(element, parserContext, "import-snapshot"); + return parseSnapshots(element, parserContext, "snapshot-import"); } private ManagedList parseSnapshots(Element element, ParserContext parserContext, diff --git a/src/main/resources/org/springframework/data/gemfire/config/spring-data-gemfire-1.7.xsd b/src/main/resources/org/springframework/data/gemfire/config/spring-data-gemfire-1.7.xsd index 9a6768f0..836a7a20 100644 --- a/src/main/resources/org/springframework/data/gemfire/config/spring-data-gemfire-1.7.xsd +++ b/src/main/resources/org/springframework/data/gemfire/config/spring-data-gemfire-1.7.xsd @@ -183,14 +183,14 @@ Access to GemFire's Snapshot Service for taking snapshots of GemFire Cache and R - + - + - - + + - + - + - + - + - + diff --git a/src/test/resources/org/springframework/data/gemfire/snapshot/SnapshotServiceImportExportIntegrationTest-context.xml b/src/test/resources/org/springframework/data/gemfire/snapshot/SnapshotServiceImportExportIntegrationTest-context.xml index 58ba76ef..3b08c1e0 100644 --- a/src/test/resources/org/springframework/data/gemfire/snapshot/SnapshotServiceImportExportIntegrationTest-context.xml +++ b/src/test/resources/org/springframework/data/gemfire/snapshot/SnapshotServiceImportExportIntegrationTest-context.xml @@ -22,8 +22,8 @@ - - + +