Update jepsen tests

jepsen/spring-statemachine-jepsen/.gitignore

@@ -3,3 +3,4 @@ report/
 store/
 pom.xml
 .lein-repl-history
+.lein-env

jepsen/spring-statemachine-jepsen/project.clj

@@ -3,6 +3,9 @@
   :url "http://example.com/FIXME"
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}
+  :profiles {:plot {:env {:plot "plot"}}}
+  :plugins [[lein-environ "1.0.0"]]
   :dependencies [[org.clojure/clojure "1.6.0"]
                  [clj-http "1.1.0"]
+                 [environ "1.0.0"]
                  [jepsen "0.0.5"]])

jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/checker.clj

@@ -0,0 +1,144 @@
+(ns spring-statemachine-jepsen.checker
+  (:require [clj-time.core :as t]
+            [clj-time.format :as tf]
+            [clj-time.coerce :as tc]
+            [clojure.pprint :refer [pprint]]
+            [clojure.tools.logging  :refer [info]]
+            [environ.core :refer [env]]
+            [jepsen.util :as util :refer [meh]]
+            [jepsen.checker :refer [Checker]]
+            [jepsen.store :as store]
+            [knossos.op :as op]
+            [knossos.history :as history]
+            [gnuplot.core :as g]))
+
+(def statetovalue {"S11" 1 "S12" 2 "S211" 3 "S212" 4 })
+(def variabletovalue {"v1" 1 "v2" 2 "v3" 3 "v4" 4 "v5" 5 "v6" 6 "v7" 7 "v8" 8})
+
+(defn shiftvalue
+  [value process]
+  (double (+ value (/ (- process 2) 10))))
+
+(defn extract-plot-data
+  [history]
+  (let [data
+    (->> history
+         (filter #(= :ok (:type %)))
+         (filter #(= :states (:f %)))
+         (group-by :process))]
+    (vector
+      (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get statetovalue (last (get value :value))) (get value :process) ))) ) [] (get data 0))
+      (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get statetovalue (last (get value :value))) (get value :process) ))) ) [] (get data 1))
+      (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get statetovalue (last (get value :value))) (get value :process) ))) ) [] (get data 2))
+      (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get statetovalue (last (get value :value))) (get value :process) ))) ) [] (get data 3))
+      (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get statetovalue (last (get value :value))) (get value :process) ))) ) [] (get data 4)))))
+
+(defn extract-plot-data2
+  [history]
+  (let [data
+    (->> history
+         (filter #(= :ok (:type %)))
+         (filter #(= :event (:f %))))]
+    (vector
+      (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (+ (get value :process) 1) (get value :value))) ) [] (vec data)))))
+
+(defn extract-plot-data3
+  [history]
+  (let [data
+    (->> history
+         (filter #(= :ok (:type %)))
+         (filter #(= :variable (:f %)))
+         (group-by :process))]
+    (vector
+      (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get variabletovalue (get value :r)) (get value :process) ))) ) [] (get data 0))
+      (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get variabletovalue (get value :r)) (get value :process) ))) ) [] (get data 1))
+      (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get variabletovalue (get value :r)) (get value :process) ))) ) [] (get data 2))
+      (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get variabletovalue (get value :r)) (get value :process) ))) ) [] (get data 3))
+      (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get variabletovalue (get value :r)) (get value :process) ))) ) [] (get data 4)))))
+
+(defn extract-plot-data4
+  [history]
+  (let [data
+    (->> history
+         (filter #(= :ok (:type %)))
+         (filter #(= :eventvariable (:f %))))]
+    (vector
+      (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (+ (get value :process) 1) (get value :v))) ) [] (vec data)))))
+
+(defn plot1!
+  [test model history]
+
+  (let [output-path (.getCanonicalPath (store/path! test "states.png"))]
+    (g/raw-plot! [[:set :key :outside]
+                  [:set :style :textbox :opaque]
+                  [:set :terminal :qt :size (keyword "900,450")]
+                  [:set :yrange (keyword "[0.5:4.5]")]
+                  [:set :y2range (keyword "[0.5:5.5]")]
+                  [:set :xtics :format "%h\nns"]
+                  [:set :xlabel "elapsed time"]
+                  [:set :ylabel "states in nodes"]
+                  [:set :y2label "events via nodes"]
+                  [:set :ytics 1]
+                  [:set :ytics (keyword "('S21' 1, 'S22' 2, 'S211' 3, 'S212' 4)")]
+                  [:set :ytics :nomirror]
+                  [:set :y2tics 1]
+                  [:set :y2tics (keyword "('n1' 1, 'n2' 2, 'n3' 3, 'n4' 4, 'n5' 5)")]
+                  [:plot
+                   (g/list ["-" :title "states n1" :with :steps :lw :3]
+                           ["-" :title "states n2" :with :steps :lw :3]
+                           ["-" :title "states n3" :with :steps :lw :3]
+                           ["-" :title "states n4" :with :steps :lw :3]
+                           ["-" :title "states n5" :with :steps :lw :3]
+                           ["-" :title "events" :with :labels :center :boxed :font ",15" :axis :x1y2]
+                           )]]
+                   (into
+                     (extract-plot-data history)
+                     (extract-plot-data2 history)))
+    output-path)
+    {:valid? true})
+
+(defn plot2!
+  [test model history]
+
+  (let [output-path (.getCanonicalPath (store/path! test "states.png"))]
+    (g/raw-plot! [[:set :key :outside]
+                  [:set :style :textbox :opaque]
+                  [:set :terminal :qt :size (keyword "900,450")]
+                  [:set :yrange (keyword "[0.5:8.5]")]
+                  [:set :y2range (keyword "[0.5:5.5]")]
+                  [:set :xtics :format "%h\nns"]
+                  [:set :xlabel "elapsed time"]
+                  [:set :ylabel "variable in nodes"]
+                  [:set :y2label "variable via nodes"]
+                  [:set :ytics 1]
+                  [:set :ytics (keyword "('v1' 1, 'v2' 2, 'v3' 3, 'v4' 4, 'v5' 5, 'v6' 6, 'v7' 7, 'v8' 8)")]
+                  [:set :ytics :nomirror]
+                  [:set :y2tics 1]
+                  [:set :y2tics (keyword "('n1' 1, 'n2' 2, 'n3' 3, 'n4' 4, 'n5' 5)")]
+                  [:plot
+                   (g/list ["-" :title "variable n1" :with :steps :lw :3]
+                           ["-" :title "variable n2" :with :steps :lw :3]
+                           ["-" :title "variable n3" :with :steps :lw :3]
+                           ["-" :title "variable n4" :with :steps :lw :3]
+                           ["-" :title "variable n5" :with :steps :lw :3]
+                           ["-" :title "variables" :with :labels :center :boxed :font ",15" :axis :x1y2]
+                           )]]
+                   (into
+                     (extract-plot-data3 history)
+                     (extract-plot-data4 history)))
+    output-path)
+    {:valid? true})
+
+(defn checker1
+  "Constructs a Jepsen checker."
+  []
+  (reify Checker
+    (check [_ test model history]
+      (if (env :plot) (plot1! test model history) {:valid? true}))))
+
+(defn checker2
+  "Constructs a Jepsen checker."
+  []
+  (reify Checker
+    (check [_ test model history]
+      (if (env :plot) (plot2! test model history) {:valid? true}))))

jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/core.clj

@@ -15,6 +15,8 @@
                     [store     :as store]
                     [report    :as report]
                     [tests     :as tests]]
+            [spring-statemachine-jepsen.checker :refer [checker1]]
+            [spring-statemachine-jepsen.checker :refer [checker2]]
             [jepsen.checker.timeline  :as timeline]
             [jepsen.control.net       :as net]
             [jepsen.os.debian         :as debian]
@@ -145,14 +147,16 @@
                   (catch RuntimeException e
                     (assoc op :type :fail :value (.getMessage e))))
         :states (try
-                  (if (= (sm-read-states client) (:s op))
+                  (let [res (sm-read-states client)]
-                    (assoc op :type :ok)
+                    (if (= res (:s op))
-                    (assoc op :type :fail :value "wrong states"))
+                      (assoc op :type :ok :value (vec res))
+                      (assoc op :type :fail :value (str "wrong states " (pr-str res))))
+                    )
                 (catch RuntimeException e
                   (assoc op :type :fail :value (.getMessage e))))
         :event (try
                  (sm-send-event client (:e op))
-                 (assoc op :type :ok)
+                 (assoc op :type :ok :value (:e op))
                (catch RuntimeException e
                  (assoc op :type :fail :value (.getMessage e))))
         :eventvariable (try
@@ -170,150 +174,157 @@
   []
   (CreateEventClient. nil))
 
+(defn gen-read-states
+  "Read states n times and expect states."
+  [times expect]
+  (gen/clients
+    (gen/each
+      (gen/seq
+        (take (* times 2)
+          (cycle [(gen/sleep 1)
+                  {:type :invoke
+                   :f    :states
+                   :s    expect}]))))))
+
+(defn gen-send-event
+  "Send event one time to random node."
+  [event]
+  (gen/clients
+    (gen/once {:type :invoke
+               :f    :event
+               :e    event})))
+
+(defn gen-send-event-all
+  "Send event one time to all nodes."
+  [event]
+  (gen/clients
+    (gen/each
+      (gen/once {:type :invoke
+                 :f    :event
+                 :e    event}))))
+
+(defn gen-send-event-variable
+  "Send event with variable value to one node."
+  [event variable]
+  (gen/clients
+    (gen/once {:type :invoke
+               :f    :eventvariable
+               :e    event
+               :v    variable})))
+
+(defn gen-read-variable
+  "Read variable from all nodes and expect variable value."
+  [expect]
+  (gen/clients
+    (gen/each
+      (gen/seq
+        (take 10
+          (cycle [(gen/sleep 1)
+                  {:type :invoke
+                   :f    :variable
+                   :v    "testVariable"
+                   :r    expect}]))))))
+
+(defn gen-status
+  "Read states n times and expect states."
+  [times]
+  (gen/clients
+    (gen/each
+      (gen/seq
+        (take (* times 2)
+          (cycle [(gen/sleep 1)
+                  {:type :invoke
+                   :f    :status}]))))))
+
 (defn event-gen-1
   "Generates isolated event and checks states and status"
   []
-
   (gen/phases
-    ;get error status of all machines
+    (gen-read-states 5 ["S0","S1","S11"])
-    (gen/clients
+    (gen-send-event "I")
-      (gen/each
+    (gen-read-states 5 ["S0","S1","S12"])
-        (gen/once {:type :invoke
+    (gen-send-event "C")
-                   :f    :status})))
+    (gen-read-states 5 ["S0","S2","S21","S211"])
-    ;check states for all machines
+    (gen-send-event "I")
-    (gen/clients
+    (gen-read-states 5 ["S0","S2","S21","S212"])
-      (gen/each
+    (gen-send-event "K")
-        (gen/once {:type :invoke
+    (gen-read-states 5 ["S0","S1","S11"])
-                   :f    :states
+    (gen-send-event "I")
-                   :s    ["S0","S1","S11"]})))
+    (gen-read-states 5 ["S0","S1","S12"])
-    ;pick random node for sending event C
+    (gen-send-event "C")
-    (gen/clients
+    (gen-read-states 5 ["S0","S2","S21","S211"])
-      (gen/once {:type :invoke
+    (gen-send-event "I")
-                 :f    :event
+    (gen-read-states 5 ["S0","S2","S21","S212"])
-                 :e    "C"}))
+    (gen-send-event "K")
-    ;check states for all machines
+    (gen-read-states 5 ["S0","S1","S11"])))
-    (gen/clients
-      (gen/each
-        (gen/once {:type :invoke
-                   :f    :status})))
-    ;check variable foo=0 for all machines
-    (gen/clients
-      (gen/each
-        (gen/once {:type :invoke
-                   :f    :variable
-                   :v    "foo"
-                   :r    0})))
-    ;check states for all machines
-    (gen/clients
-      (gen/each
-        (gen/once {:type :invoke
-                   :f    :states
-                   :s    ["S0","S2","S21","S211"]})))))
 
 (defn event-gen-2
   "Generates parallel event and checks states and status"
   []
-
   (gen/phases
-    ;get error status of all machines
+    (gen-read-states 5 ["S0","S1","S11"])
-    (gen/clients
+    (gen-send-event-all "I")
-      (gen/each
+    (gen-read-states 5 ["S0","S1","S12"])
-        (gen/once {:type :invoke
+    (gen-send-event-all "C")
-                   :f    :status})))
+    (gen-read-states 5 ["S0","S2","S21","S211"])
-    ;check states for all machines
+    (gen-send-event-all "I")
-    (gen/clients
+    (gen-read-states 5 ["S0","S2","S21","S212"])
-      (gen/each
+    (gen-send-event-all "K")
-        (gen/once {:type :invoke
+    (gen-read-states 5 ["S0","S1","S11"])
-                   :f    :states
+    (gen-send-event-all "I")
-                   :s    ["S0","S1","S11"]})))
+    (gen-read-states 5 ["S0","S1","S12"])
-    ;pick all nodes for sending event C
+    (gen-send-event-all "C")
-    (gen/clients
+    (gen-read-states 5 ["S0","S2","S21","S211"])
-      (gen/each
+    (gen-send-event-all "I")
-        (gen/once {:type :invoke
+    (gen-read-states 5 ["S0","S2","S21","S212"])
-                   :f    :event
+    (gen-send-event-all "K")
-                   :e    "C"})))
+    (gen-read-states 5 ["S0","S1","S11"])))
-    (gen/sleep 2)
-    ;get error status of all machines
-    (gen/clients
-      (gen/each
-        (gen/once {:type :invoke
-                   :f    :status})))
-    ;check states for all machines
-    (gen/clients
-      (gen/each
-        (gen/once {:type :invoke
-                   :f    :states
-                   :s    ["S0","S2","S21","S211"]})))))
 
 (defn event-gen-3
   "Generates event and checks states, status and variable"
   []
-
   (gen/phases
-    ;get error status of all machines
+    (gen-read-states 5 ["S0","S1","S11"])
-    (gen/clients
+    (gen-send-event-variable "J" "v1")
-      (gen/each
+    (gen-read-variable "v1")
-        (gen/once {:type :invoke
+    (gen-send-event-variable "J" "v2")
-                   :f    :status})))
+    (gen-read-variable "v2")
-    ;check states for all machines
+    (gen-send-event-variable "J" "v3")
-    (gen/clients
+    (gen-read-variable "v3")
-      (gen/each
+    (gen-send-event-variable "J" "v4")
-        (gen/once {:type :invoke
+    (gen-read-variable "v4")
-                   :f    :states
+    (gen-send-event-variable "J" "v5")
-                   :s    ["S0","S1","S11"]})))
+    (gen-read-variable "v5")
-    ;pick random node for sending event J with variable x1
+    (gen-send-event-variable "J" "v6")
-    (gen/clients
+    (gen-read-variable "v6")
-      (gen/once {:type :invoke
+    (gen-send-event-variable "J" "v7")
-                 :f    :eventvariable
+    (gen-read-variable "v7")
-                 :e    "J"
+    (gen-send-event-variable "J" "v8")
-                 :v    "x1"}))
+    (gen-read-variable "v8")
-    (gen/sleep 2)
+))
-    ;check variable value x1 for all machines
-    (gen/clients
-      (gen/each
-        (gen/once {:type :invoke
-                   :f    :variable
-                   :v    "testVariable"
-                   :r    "x1"})))))
 
 (defn event-gen-4
   "Generates event and checks states while splitting network"
   []
-
   (gen/phases
-    ;get error status of all machines
+    (gen-read-states 5 ["S0","S1","S11"])
-    (gen/clients
+    (gen-send-event-all "C")
-      (gen/each
+    (gen-read-states 5 ["S0","S2","S21","S211"])
-        (gen/once {:type :invoke
+    (gen-status 2)
-                   :f    :status})))
     ;start nemesis, split network
     (gen/nemesis
       (gen/once {:type :info :f :start}))
-    (gen/sleep 30)
+    (gen-read-states 15 ["S0","S2","S21","S211"])
-    ;get error status of all machines
+    (gen-status 5)
-    (gen/clients
-      (gen/each
-        (gen/once {:type :invoke
-                   :f    :status})))
     ;stop nemesis, heal network
     (gen/nemesis
       (gen/once {:type :info :f :stop}))
-    ;get error status of all machines
+    (gen-status 5)
-    (gen/clients
+    (gen-read-states 15 ["S0","S2","S21","S211"])
-      (gen/each
+    (gen-send-event-all "K")
-        (gen/once {:type :invoke
+    (gen-read-states 10 ["S0","S1","S11"])
-                   :f    :status})))
+    (gen-status 30)
-    ;pick random node for sending event C
+    (gen-read-states 10 ["S0","S1","S11"])))
-    (gen/clients
-      (gen/once {:type :invoke
-                 :f    :event
-                 :e    "C"}))
-    ;check states for all machines
-    (gen/clients
-      (gen/each
-        (gen/once {:type :invoke
-                   :f    :states
-                   :s    ["S0","S2","S21","S211"]})))))
 
 (defn statemachine-test
   "Defaults for testing state machine."
@@ -337,25 +348,29 @@
   []
   (event-test "send-isolated-event"
                {:nemesis   nemesis/noop
-                :generator (event-gen-1)}))
+                :generator (event-gen-1)
+                :checker (checker1)}))
 
 (defn send-parallel-event-test
   "Sends simple events via all nodes."
   []
   (event-test "send-parallel-event"
                {:nemesis   nemesis/noop
-                :generator (event-gen-2)}))
+                :generator (event-gen-2)
+                :checker (checker1)}))
 
 (defn send-isolated-event-with-variable-test
   "Sends simple events via random node with variable."
   []
   (event-test "send-isolated-event-with-variable"
                {:nemesis   nemesis/noop
-                :generator (event-gen-3)}))
+                :generator (event-gen-3)
+                :checker (checker2)}))
 
 (defn partition-half-test
   "Does a half brain split and checks that machines are healing."
   []
   (event-test "partition-half"
                {:nemesis   (nemesis/partition-random-halves)
-                :generator (event-gen-4)}))
+                :generator (event-gen-4)
+                :checker (checker1)}))