From ab2277b3f78cc43bce0fbcb6fbb32bef60b3ac49 Mon Sep 17 00:00:00 2001
From: Janne Valkealahti <janne.valkealahti@gmail.com>
Date: Sat, 8 Aug 2015 17:13:50 +0100
Subject: [PATCH] Add initial set of jepsen tests

- Add simple tests for sending events via random machine,
  via all machines and also testing extended state variables.
- Relates to #80
---
 jepsen/spring-statemachine-jepsen/.gitignore  |   5 +
 jepsen/spring-statemachine-jepsen/README.md   |  49 +++
 jepsen/spring-statemachine-jepsen/project.clj |   8 +
 .../src/spring_statemachine_jepsen/core.clj   | 311 ++++++++++++++++++
 .../spring_statemachine_jepsen/core_test.clj  |  24 ++
 5 files changed, 397 insertions(+)
 create mode 100644 jepsen/spring-statemachine-jepsen/.gitignore
 create mode 100644 jepsen/spring-statemachine-jepsen/README.md
 create mode 100644 jepsen/spring-statemachine-jepsen/project.clj
 create mode 100644 jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/core.clj
 create mode 100644 jepsen/spring-statemachine-jepsen/test/spring_statemachine_jepsen/core_test.clj

diff --git a/jepsen/spring-statemachine-jepsen/.gitignore b/jepsen/spring-statemachine-jepsen/.gitignore
new file mode 100644
index 00000000..e5d0a6e8
--- /dev/null
+++ b/jepsen/spring-statemachine-jepsen/.gitignore
@@ -0,0 +1,5 @@
+.lein-failures
+report/
+store/
+pom.xml
+.lein-repl-history
diff --git a/jepsen/spring-statemachine-jepsen/README.md b/jepsen/spring-statemachine-jepsen/README.md
new file mode 100644
index 00000000..90ee16fa
--- /dev/null
+++ b/jepsen/spring-statemachine-jepsen/README.md
@@ -0,0 +1,49 @@
+# spring-statemachine-jepsen
+
+A Clojure project implementing jepsen tests for a Spring Statemachine
+
+## Usage
+
+* Setup nodes n1, n2, n3, n4 and n5 having a latest debian installation.
+* Install zookeeper for every node and create a cluster.
+* Build `spring-statemachine-samples-web-1.0.0.BUILD-SNAPSHOT.jar` and copy it to directory `/root`.
+
+Jepsen tests currently doesn't install `Zookeeper` or `spring-statemachine-samples-web` so this needs to be done manually.
+
+Zookeeper config `zoo.cfg` for each node looks like:
+
+```
+tickTime=2000
+initLimit=10
+syncLimit=5
+dataDir=/var/lib/zookeeper
+clientPort=2181
+server.1=0.0.0.0:2888:3888
+server.2=n2:2888:3888
+server.3=n3:2888:3888
+server.4=n4:2888:3888
+server.5=n5:2888:3888
+```
+
+I had to define `server.1` to use `0.0.0.0` and other node configs respectively.
+
+Run all tests:
+
+```
+# lein test
+```
+
+Run particular test:
+
+```
+lein test :only spring-statemachine-jepsen.core-test/send-isolated-event
+```
+
+## Setting up eclipse
+
+Easiest way to import clojure leiningen project into clipse is to first create a `pom.xml` and then create a project based on that pom.
+
+```
+# lein pom
+# mvn eclipse:eclipse
+```
diff --git a/jepsen/spring-statemachine-jepsen/project.clj b/jepsen/spring-statemachine-jepsen/project.clj
new file mode 100644
index 00000000..a77e7245
--- /dev/null
+++ b/jepsen/spring-statemachine-jepsen/project.clj
@@ -0,0 +1,8 @@
+(defproject spring-statemachine-jepsen "0.1.0-SNAPSHOT"
+  :description "FIXME: write description"
+  :url "http://example.com/FIXME"
+  :license {:name "Eclipse Public License"
+            :url "http://www.eclipse.org/legal/epl-v10.html"}
+  :dependencies [[org.clojure/clojure "1.6.0"]
+                 [clj-http "1.1.0"]
+                 [jepsen "0.0.5"]])
diff --git a/jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/core.clj b/jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/core.clj
new file mode 100644
index 00000000..200b8f9d
--- /dev/null
+++ b/jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/core.clj
@@ -0,0 +1,311 @@
+(ns spring-statemachine-jepsen.core
+  (:require [cheshire.core          :as json]
+            [clojure.java.io        :as io]
+            [clojure.string         :as str]
+            [clojure.tools.logging  :refer [info]]
+            [jepsen [core      :as jepsen]
+                    [db        :as db]
+                    [util      :as util :refer [meh timeout]]
+                    [control   :as c :refer [|]]
+                    [client    :as client]
+                    [checker   :as checker]
+                    [model     :as model]
+                    [generator :as gen]
+                    [nemesis   :as nemesis]
+                    [store     :as store]
+                    [report    :as report]
+                    [tests     :as tests]]
+            [jepsen.checker.timeline  :as timeline]
+            [jepsen.control.net       :as net]
+            [jepsen.os.debian         :as debian]
+            [clj-http.client          :as http]))
+
+(def pidfile      "/var/run/sm.pid")
+(def binary       "/usr/bin/java")
+(def jar          "/root/spring-statemachine-samples-web-1.0.0.BUILD-SNAPSHOT.jar")
+
+(defn sm-send-event
+  "Sends event to state machine"
+  [node event]
+  (info "Sending event" event (name node))
+  (http/post (str "http://" (name node) ":8080/event")
+    {:form-params {:id (str event)}}))
+
+(defn sm-send-event-with-variable
+  "Sends event to state machine"
+  [node event value]
+  (info "Sending event" event (name node))
+  (http/post (str "http://" (name node) ":8080/event")
+    {:form-params {:id (str event) :testVariable value}}))
+
+(defn sm-read-states
+  "Reading states from a state machine"
+  [node]
+  (let [response (http/get (str "http://" (name node) ":8080/states") {:as :json})]
+    (get response :body)))
+
+(defn sm-read-status-ok?
+  "Read status and check that there is no errors"
+  [node]
+  (let [response (http/get (str "http://" (name node) ":8080/status") {:as :json})]
+    (= (get (get response :body) :hasStateMachineError) false)))
+
+(defn sm-read-state-variable
+  "Read status and check that there is no errors"
+  [node key]
+  (let [response (http/get (str "http://" (name node) ":8080/status") {:as :json})]
+    (get (get (get response :body) :extendedStateVariables) (keyword key))))
+
+(defn running?
+  "Is the service running?"
+  []
+  (try
+    (c/exec :start-stop-daemon :--status
+            :--pidfile pidfile)
+    true
+    (catch RuntimeException _ false)))
+
+(defn wait
+  "Waits sm to become healthy"
+  [node timeout-secs]
+  (timeout (* 1000 timeout-secs)
+           (throw (RuntimeException.
+                    (str "Timed out after "
+                         timeout-secs
+                         " s waiting for state machine become healty")))
+    (loop []
+      (when
+        (try
+          (Thread/sleep 1000)
+          (if (sm-read-status-ok? node) false true)
+          (catch Exception e true))
+        (recur))))
+  )
+
+(defn start!
+  [node]
+  "Starts state machine."
+  (info node "starting state machine")
+  (c/su
+    (assert (not (running?)))
+    (meh (c/exec :rm :-rf "/tmp/spring.log"))
+    (c/exec :start-stop-daemon :--start
+            :--background
+            :--make-pidfile
+            :--pidfile  pidfile
+            :--exec     binary
+            :--
+            :-jar       jar
+            (c/lit "2>&1")))
+  (info node "waiting state machine to start")
+  (wait node 120)
+  (info node "state machine ready"))
+
+(defn stop!
+  "Stops state machine."
+  [node]
+  (info node "stopping state machine")
+  (c/exec :start-stop-daemon :--stop
+          :--pidfile pidfile)
+  (info node "waiting state machine to stop")
+  (Thread/sleep (* 10 1000)))
+
+(defn db
+  "Spring Statemachine for a particular version."
+  [version]
+  (reify db/DB
+    (setup! [_ test node]
+      (doto node
+        (start!)))
+
+    (teardown! [_ test node]
+      (stop! node)
+      ;; Leave system up, to collect logs, analyze post mortem, etc
+      )))
+
+(defrecord CreateEventClient [client]
+  client/Client
+  (setup! [_ test node]
+    (let []
+      (CreateEventClient. node)))
+
+  (invoke! [this test op]
+      (case (:f op)
+        :status (try
+                  (if (sm-read-status-ok? client)
+                    (assoc op :type :ok)
+                    (assoc op :type :fail :value "sm in error"))
+                (catch RuntimeException e
+                  (assoc op :type :fail :value (.getMessage e))))
+        :variable (try
+                    (let [variable (sm-read-state-variable client (:v op))]
+                      (if (= variable (:r op))
+                        (assoc op :type :ok)
+                        (assoc op :type :fail :value (str (:v op) " was " variable))))
+                  (catch RuntimeException e
+                    (assoc op :type :fail :value (.getMessage e))))
+        :states (try
+                  (if (= (sm-read-states client) (:s op))
+                    (assoc op :type :ok)
+                    (assoc op :type :fail :value "wrong states"))
+                (catch RuntimeException e
+                  (assoc op :type :fail :value (.getMessage e))))
+        :event (try
+                 (sm-send-event client (:e op))
+                 (assoc op :type :ok)
+               (catch RuntimeException e
+                 (assoc op :type :fail :value (.getMessage e))))
+        :eventvariable (try
+                         (sm-send-event-with-variable client (:e op) (:v op))
+                         (assoc op :type :ok)
+                       (catch RuntimeException e
+                         (assoc op :type :fail :value (.getMessage e))))
+        ))
+
+  (teardown! [_ test]
+    (.close client)))
+
+(defn create-event-client
+  "A client sendind events"
+  []
+  (CreateEventClient. nil))
+
+(defn event-gen-1
+  "Generates isolated event and checks states and status"
+  []
+
+  (gen/phases
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :status})))
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :states
+                   :s    ["S0","S1","S11"]})))
+    ; this picks random node for sending event
+    (gen/clients
+      (gen/once {:type :invoke
+                 :f    :event
+                 :e    "C"}))
+    (gen/sleep 2)
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :status})))
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :variable
+                   :v    "foo"
+                   :r    0})))
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :states
+                   :s    ["S0","S2","S21","S211"]})))
+    )
+  )
+
+(defn event-gen-2
+  "Generates parallel event and checks states and status"
+  []
+
+  (gen/phases
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :status})))
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :states
+                   :s    ["S0","S1","S11"]})))
+    ; this picks all nodes for sending event
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :event
+                   :e    "C"})))
+    (gen/sleep 2)
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :status})))
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :states
+                   :s    ["S0","S2","S21","S211"]})))
+    )
+  )
+
+(defn event-gen-3
+  "Generates event and checks states, status and variable"
+  []
+
+  (gen/phases
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :status})))
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :states
+                   :s    ["S0","S1","S11"]})))
+    ; this picks random node for sending event
+    (gen/clients
+      (gen/once {:type :invoke
+                 :f    :eventvariable
+                 :e    "J"
+                 :v    "x1"}))
+    (gen/sleep 2)
+    (gen/clients
+      (gen/each
+        (gen/once {:type :invoke
+                   :f    :variable
+                   :v    "testVariable"
+                   :r    "x1"})))
+
+    )
+  )
+
+(defn statemachine-test
+  "Defaults for testing state machine."
+  [name opts]
+  (merge tests/noop-test
+         {:name (str "statemachine-" name)
+          :os   debian/os
+          :db   (db "1.0.0")}
+         opts))
+
+(defn event-test
+  "A generic create test."
+  [name opts]
+  (statemachine-test (str "event-" name)
+           (merge {:client  (create-event-client)
+                   :model   model/noop}
+                  opts)))
+
+(defn send-isolated-event-test
+  "Sends simple events via random node."
+  []
+  (event-test "send-isolated-event"
+               {:nemesis   nemesis/noop
+                :generator (event-gen-1)}))
+
+(defn send-parallel-event-test
+  "Sends simple events via all nodes."
+  []
+  (event-test "send-parallel-event"
+               {:nemesis   nemesis/noop
+                :generator (event-gen-2)}))
+
+(defn send-isolated-event-with-variable-test
+  "Sends simple events via random node with variable."
+  []
+  (event-test "send-isolated-event-with-variable"
+               {:nemesis   nemesis/noop
+                :generator (event-gen-3)}))
diff --git a/jepsen/spring-statemachine-jepsen/test/spring_statemachine_jepsen/core_test.clj b/jepsen/spring-statemachine-jepsen/test/spring_statemachine_jepsen/core_test.clj
new file mode 100644
index 00000000..86229f54
--- /dev/null
+++ b/jepsen/spring-statemachine-jepsen/test/spring_statemachine_jepsen/core_test.clj
@@ -0,0 +1,24 @@
+(ns spring-statemachine-jepsen.core-test
+  (:require [clojure.test :refer :all]
+            [clojure.pprint :refer [pprint]]
+            [spring-statemachine-jepsen.core :refer :all]
+            [jepsen [core :as jepsen]
+                    [report :as report]]))
+
+(defn run-statemachine-test!
+  "Runs a test around a state machine and dumps some results to the report/ dir"
+  [t]
+  (let [test (jepsen/run! t)]
+    (or (is (:valid? (:results test)))
+        (println (:error (:results test))))
+    (report/to (str "report/" (:name test) "/history.edn")
+               (pprint (:history test)))))
+
+(deftest send-isolated-event
+  (run-statemachine-test! (send-isolated-event-test)))
+
+(deftest send-parallel-event
+  (run-statemachine-test! (send-parallel-event-test)))
+
+(deftest send-isolated-event-with-variable
+  (run-statemachine-test! (send-isolated-event-with-variable-test)))