From 1800616d4837f5c735bfb1d89d2c8ad25178df1d Mon Sep 17 00:00:00 2001 From: Janne Valkealahti Date: Fri, 21 Aug 2015 18:07:26 +0100 Subject: [PATCH] Update jepsen tests --- jepsen/spring-statemachine-jepsen/.gitignore | 1 + jepsen/spring-statemachine-jepsen/project.clj | 3 + .../spring_statemachine_jepsen/checker.clj | 144 ++++++++++ .../src/spring_statemachine_jepsen/core.clj | 269 +++++++++--------- 4 files changed, 290 insertions(+), 127 deletions(-) create mode 100644 jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/checker.clj diff --git a/jepsen/spring-statemachine-jepsen/.gitignore b/jepsen/spring-statemachine-jepsen/.gitignore index e5d0a6e8..93adbcb1 100644 --- a/jepsen/spring-statemachine-jepsen/.gitignore +++ b/jepsen/spring-statemachine-jepsen/.gitignore @@ -3,3 +3,4 @@ report/ store/ pom.xml .lein-repl-history +.lein-env diff --git a/jepsen/spring-statemachine-jepsen/project.clj b/jepsen/spring-statemachine-jepsen/project.clj index a77e7245..14a634a4 100644 --- a/jepsen/spring-statemachine-jepsen/project.clj +++ b/jepsen/spring-statemachine-jepsen/project.clj @@ -3,6 +3,9 @@ :url "http://example.com/FIXME" :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} + :profiles {:plot {:env {:plot "plot"}}} + :plugins [[lein-environ "1.0.0"]] :dependencies [[org.clojure/clojure "1.6.0"] [clj-http "1.1.0"] + [environ "1.0.0"] [jepsen "0.0.5"]]) diff --git a/jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/checker.clj b/jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/checker.clj new file mode 100644 index 00000000..85e89124 --- /dev/null +++ b/jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/checker.clj @@ -0,0 +1,144 @@ +(ns spring-statemachine-jepsen.checker + (:require [clj-time.core :as t] + [clj-time.format :as tf] + [clj-time.coerce :as tc] + [clojure.pprint :refer [pprint]] + [clojure.tools.logging :refer [info]] + [environ.core :refer [env]] + [jepsen.util :as util :refer [meh]] + [jepsen.checker :refer [Checker]] + [jepsen.store :as store] + [knossos.op :as op] + [knossos.history :as history] + [gnuplot.core :as g])) + +(def statetovalue {"S11" 1 "S12" 2 "S211" 3 "S212" 4 }) +(def variabletovalue {"v1" 1 "v2" 2 "v3" 3 "v4" 4 "v5" 5 "v6" 6 "v7" 7 "v8" 8}) + +(defn shiftvalue + [value process] + (double (+ value (/ (- process 2) 10)))) + +(defn extract-plot-data + [history] + (let [data + (->> history + (filter #(= :ok (:type %))) + (filter #(= :states (:f %))) + (group-by :process))] + (vector + (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get statetovalue (last (get value :value))) (get value :process) ))) ) [] (get data 0)) + (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get statetovalue (last (get value :value))) (get value :process) ))) ) [] (get data 1)) + (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get statetovalue (last (get value :value))) (get value :process) ))) ) [] (get data 2)) + (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get statetovalue (last (get value :value))) (get value :process) ))) ) [] (get data 3)) + (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get statetovalue (last (get value :value))) (get value :process) ))) ) [] (get data 4))))) + +(defn extract-plot-data2 + [history] + (let [data + (->> history + (filter #(= :ok (:type %))) + (filter #(= :event (:f %))))] + (vector + (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (+ (get value :process) 1) (get value :value))) ) [] (vec data))))) + +(defn extract-plot-data3 + [history] + (let [data + (->> history + (filter #(= :ok (:type %))) + (filter #(= :variable (:f %))) + (group-by :process))] + (vector + (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get variabletovalue (get value :r)) (get value :process) ))) ) [] (get data 0)) + (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get variabletovalue (get value :r)) (get value :process) ))) ) [] (get data 1)) + (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get variabletovalue (get value :r)) (get value :process) ))) ) [] (get data 2)) + (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get variabletovalue (get value :r)) (get value :process) ))) ) [] (get data 3)) + (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (shiftvalue (get variabletovalue (get value :r)) (get value :process) ))) ) [] (get data 4))))) + +(defn extract-plot-data4 + [history] + (let [data + (->> history + (filter #(= :ok (:type %))) + (filter #(= :eventvariable (:f %))))] + (vector + (reduce-kv (fn [vec key value] (conj vec (vector (get value :time) (+ (get value :process) 1) (get value :v))) ) [] (vec data))))) + +(defn plot1! + [test model history] + + (let [output-path (.getCanonicalPath (store/path! test "states.png"))] + (g/raw-plot! [[:set :key :outside] + [:set :style :textbox :opaque] + [:set :terminal :qt :size (keyword "900,450")] + [:set :yrange (keyword "[0.5:4.5]")] + [:set :y2range (keyword "[0.5:5.5]")] + [:set :xtics :format "%h\nns"] + [:set :xlabel "elapsed time"] + [:set :ylabel "states in nodes"] + [:set :y2label "events via nodes"] + [:set :ytics 1] + [:set :ytics (keyword "('S21' 1, 'S22' 2, 'S211' 3, 'S212' 4)")] + [:set :ytics :nomirror] + [:set :y2tics 1] + [:set :y2tics (keyword "('n1' 1, 'n2' 2, 'n3' 3, 'n4' 4, 'n5' 5)")] + [:plot + (g/list ["-" :title "states n1" :with :steps :lw :3] + ["-" :title "states n2" :with :steps :lw :3] + ["-" :title "states n3" :with :steps :lw :3] + ["-" :title "states n4" :with :steps :lw :3] + ["-" :title "states n5" :with :steps :lw :3] + ["-" :title "events" :with :labels :center :boxed :font ",15" :axis :x1y2] + )]] + (into + (extract-plot-data history) + (extract-plot-data2 history))) + output-path) + {:valid? true}) + +(defn plot2! + [test model history] + + (let [output-path (.getCanonicalPath (store/path! test "states.png"))] + (g/raw-plot! [[:set :key :outside] + [:set :style :textbox :opaque] + [:set :terminal :qt :size (keyword "900,450")] + [:set :yrange (keyword "[0.5:8.5]")] + [:set :y2range (keyword "[0.5:5.5]")] + [:set :xtics :format "%h\nns"] + [:set :xlabel "elapsed time"] + [:set :ylabel "variable in nodes"] + [:set :y2label "variable via nodes"] + [:set :ytics 1] + [:set :ytics (keyword "('v1' 1, 'v2' 2, 'v3' 3, 'v4' 4, 'v5' 5, 'v6' 6, 'v7' 7, 'v8' 8)")] + [:set :ytics :nomirror] + [:set :y2tics 1] + [:set :y2tics (keyword "('n1' 1, 'n2' 2, 'n3' 3, 'n4' 4, 'n5' 5)")] + [:plot + (g/list ["-" :title "variable n1" :with :steps :lw :3] + ["-" :title "variable n2" :with :steps :lw :3] + ["-" :title "variable n3" :with :steps :lw :3] + ["-" :title "variable n4" :with :steps :lw :3] + ["-" :title "variable n5" :with :steps :lw :3] + ["-" :title "variables" :with :labels :center :boxed :font ",15" :axis :x1y2] + )]] + (into + (extract-plot-data3 history) + (extract-plot-data4 history))) + output-path) + {:valid? true}) + +(defn checker1 + "Constructs a Jepsen checker." + [] + (reify Checker + (check [_ test model history] + (if (env :plot) (plot1! test model history) {:valid? true})))) + +(defn checker2 + "Constructs a Jepsen checker." + [] + (reify Checker + (check [_ test model history] + (if (env :plot) (plot2! test model history) {:valid? true})))) diff --git a/jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/core.clj b/jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/core.clj index 22b742d5..e8ebf1e5 100644 --- a/jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/core.clj +++ b/jepsen/spring-statemachine-jepsen/src/spring_statemachine_jepsen/core.clj @@ -15,6 +15,8 @@ [store :as store] [report :as report] [tests :as tests]] + [spring-statemachine-jepsen.checker :refer [checker1]] + [spring-statemachine-jepsen.checker :refer [checker2]] [jepsen.checker.timeline :as timeline] [jepsen.control.net :as net] [jepsen.os.debian :as debian] @@ -145,14 +147,16 @@ (catch RuntimeException e (assoc op :type :fail :value (.getMessage e)))) :states (try - (if (= (sm-read-states client) (:s op)) - (assoc op :type :ok) - (assoc op :type :fail :value "wrong states")) + (let [res (sm-read-states client)] + (if (= res (:s op)) + (assoc op :type :ok :value (vec res)) + (assoc op :type :fail :value (str "wrong states " (pr-str res)))) + ) (catch RuntimeException e (assoc op :type :fail :value (.getMessage e)))) :event (try (sm-send-event client (:e op)) - (assoc op :type :ok) + (assoc op :type :ok :value (:e op)) (catch RuntimeException e (assoc op :type :fail :value (.getMessage e)))) :eventvariable (try @@ -170,150 +174,157 @@ [] (CreateEventClient. nil)) +(defn gen-read-states + "Read states n times and expect states." + [times expect] + (gen/clients + (gen/each + (gen/seq + (take (* times 2) + (cycle [(gen/sleep 1) + {:type :invoke + :f :states + :s expect}])))))) + +(defn gen-send-event + "Send event one time to random node." + [event] + (gen/clients + (gen/once {:type :invoke + :f :event + :e event}))) + +(defn gen-send-event-all + "Send event one time to all nodes." + [event] + (gen/clients + (gen/each + (gen/once {:type :invoke + :f :event + :e event})))) + +(defn gen-send-event-variable + "Send event with variable value to one node." + [event variable] + (gen/clients + (gen/once {:type :invoke + :f :eventvariable + :e event + :v variable}))) + +(defn gen-read-variable + "Read variable from all nodes and expect variable value." + [expect] + (gen/clients + (gen/each + (gen/seq + (take 10 + (cycle [(gen/sleep 1) + {:type :invoke + :f :variable + :v "testVariable" + :r expect}])))))) + +(defn gen-status + "Read states n times and expect states." + [times] + (gen/clients + (gen/each + (gen/seq + (take (* times 2) + (cycle [(gen/sleep 1) + {:type :invoke + :f :status}])))))) + (defn event-gen-1 "Generates isolated event and checks states and status" [] - (gen/phases - ;get error status of all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :status}))) - ;check states for all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :states - :s ["S0","S1","S11"]}))) - ;pick random node for sending event C - (gen/clients - (gen/once {:type :invoke - :f :event - :e "C"})) - ;check states for all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :status}))) - ;check variable foo=0 for all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :variable - :v "foo" - :r 0}))) - ;check states for all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :states - :s ["S0","S2","S21","S211"]}))))) + (gen-read-states 5 ["S0","S1","S11"]) + (gen-send-event "I") + (gen-read-states 5 ["S0","S1","S12"]) + (gen-send-event "C") + (gen-read-states 5 ["S0","S2","S21","S211"]) + (gen-send-event "I") + (gen-read-states 5 ["S0","S2","S21","S212"]) + (gen-send-event "K") + (gen-read-states 5 ["S0","S1","S11"]) + (gen-send-event "I") + (gen-read-states 5 ["S0","S1","S12"]) + (gen-send-event "C") + (gen-read-states 5 ["S0","S2","S21","S211"]) + (gen-send-event "I") + (gen-read-states 5 ["S0","S2","S21","S212"]) + (gen-send-event "K") + (gen-read-states 5 ["S0","S1","S11"]))) (defn event-gen-2 "Generates parallel event and checks states and status" [] - (gen/phases - ;get error status of all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :status}))) - ;check states for all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :states - :s ["S0","S1","S11"]}))) - ;pick all nodes for sending event C - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :event - :e "C"}))) - (gen/sleep 2) - ;get error status of all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :status}))) - ;check states for all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :states - :s ["S0","S2","S21","S211"]}))))) + (gen-read-states 5 ["S0","S1","S11"]) + (gen-send-event-all "I") + (gen-read-states 5 ["S0","S1","S12"]) + (gen-send-event-all "C") + (gen-read-states 5 ["S0","S2","S21","S211"]) + (gen-send-event-all "I") + (gen-read-states 5 ["S0","S2","S21","S212"]) + (gen-send-event-all "K") + (gen-read-states 5 ["S0","S1","S11"]) + (gen-send-event-all "I") + (gen-read-states 5 ["S0","S1","S12"]) + (gen-send-event-all "C") + (gen-read-states 5 ["S0","S2","S21","S211"]) + (gen-send-event-all "I") + (gen-read-states 5 ["S0","S2","S21","S212"]) + (gen-send-event-all "K") + (gen-read-states 5 ["S0","S1","S11"]))) (defn event-gen-3 "Generates event and checks states, status and variable" [] - (gen/phases - ;get error status of all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :status}))) - ;check states for all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :states - :s ["S0","S1","S11"]}))) - ;pick random node for sending event J with variable x1 - (gen/clients - (gen/once {:type :invoke - :f :eventvariable - :e "J" - :v "x1"})) - (gen/sleep 2) - ;check variable value x1 for all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :variable - :v "testVariable" - :r "x1"}))))) + (gen-read-states 5 ["S0","S1","S11"]) + (gen-send-event-variable "J" "v1") + (gen-read-variable "v1") + (gen-send-event-variable "J" "v2") + (gen-read-variable "v2") + (gen-send-event-variable "J" "v3") + (gen-read-variable "v3") + (gen-send-event-variable "J" "v4") + (gen-read-variable "v4") + (gen-send-event-variable "J" "v5") + (gen-read-variable "v5") + (gen-send-event-variable "J" "v6") + (gen-read-variable "v6") + (gen-send-event-variable "J" "v7") + (gen-read-variable "v7") + (gen-send-event-variable "J" "v8") + (gen-read-variable "v8") +)) (defn event-gen-4 "Generates event and checks states while splitting network" [] - (gen/phases - ;get error status of all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :status}))) + (gen-read-states 5 ["S0","S1","S11"]) + (gen-send-event-all "C") + (gen-read-states 5 ["S0","S2","S21","S211"]) + (gen-status 2) ;start nemesis, split network (gen/nemesis (gen/once {:type :info :f :start})) - (gen/sleep 30) - ;get error status of all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :status}))) + (gen-read-states 15 ["S0","S2","S21","S211"]) + (gen-status 5) ;stop nemesis, heal network (gen/nemesis (gen/once {:type :info :f :stop})) - ;get error status of all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :status}))) - ;pick random node for sending event C - (gen/clients - (gen/once {:type :invoke - :f :event - :e "C"})) - ;check states for all machines - (gen/clients - (gen/each - (gen/once {:type :invoke - :f :states - :s ["S0","S2","S21","S211"]}))))) + (gen-status 5) + (gen-read-states 15 ["S0","S2","S21","S211"]) + (gen-send-event-all "K") + (gen-read-states 10 ["S0","S1","S11"]) + (gen-status 30) + (gen-read-states 10 ["S0","S1","S11"]))) (defn statemachine-test "Defaults for testing state machine." @@ -337,25 +348,29 @@ [] (event-test "send-isolated-event" {:nemesis nemesis/noop - :generator (event-gen-1)})) + :generator (event-gen-1) + :checker (checker1)})) (defn send-parallel-event-test "Sends simple events via all nodes." [] (event-test "send-parallel-event" {:nemesis nemesis/noop - :generator (event-gen-2)})) + :generator (event-gen-2) + :checker (checker1)})) (defn send-isolated-event-with-variable-test "Sends simple events via random node with variable." [] (event-test "send-isolated-event-with-variable" {:nemesis nemesis/noop - :generator (event-gen-3)})) + :generator (event-gen-3) + :checker (checker2)})) (defn partition-half-test "Does a half brain split and checks that machines are healing." [] (event-test "partition-half" {:nemesis (nemesis/partition-random-halves) - :generator (event-gen-4)})) + :generator (event-gen-4) + :checker (checker1)}))