+216
-31
@@ -1,6 +1,5 @@
|
||||
(ns tbd.ui.events-test
|
||||
(:require [cljs.test :refer [deftest is testing async use-fixtures]]
|
||||
[clojure.string :as str]
|
||||
[re-frame.core :as rf]
|
||||
[re-frame.db :as rf-db]
|
||||
[tbd.ui.db :as db]
|
||||
@@ -25,6 +24,7 @@
|
||||
:after model-context/clear!}
|
||||
(tu/stub-effects-fixture {::events/fetch (record ::events/fetch)
|
||||
::events/fetch-stream (record ::events/fetch-stream)
|
||||
::events/abort-stream (record ::events/abort-stream)
|
||||
::events/persist-chat (record ::events/persist-chat)
|
||||
::events/call-tools (record ::events/call-tools)
|
||||
::events/copy-to-clipboard (record ::events/copy-to-clipboard)
|
||||
@@ -334,6 +334,18 @@
|
||||
(dispatch! [::events/chat-toggle])
|
||||
(is (false? (:open? (chat)))))
|
||||
|
||||
(deftest chat-toggle-persists-across-a-reload-test
|
||||
(start!)
|
||||
(dispatch! [::events/chat-toggle])
|
||||
(is (true? (:open? (chat))))
|
||||
(testing "a fresh start picks the open panel back up"
|
||||
(dispatch! [::events/initialize])
|
||||
(is (true? (:open? (chat)))))
|
||||
(dispatch! [::events/chat-toggle])
|
||||
(testing "and its being closed again"
|
||||
(dispatch! [::events/initialize])
|
||||
(is (false? (:open? (chat))))))
|
||||
|
||||
(defn- without-ids
|
||||
"`messages`, as AG-UI has them on the wire, with the per-message `:id` (a counter, unpredictable
|
||||
from a test) dropped, so what is left can be compared."
|
||||
@@ -379,19 +391,22 @@
|
||||
(deftest chat-streaming-reply-test
|
||||
(logged-in! [])
|
||||
(say! "hello")
|
||||
(testing "the model's reasoning is not kept, and it keeps waiting"
|
||||
(testing "the model's reasoning joins the conversation of its own reply, and it keeps waiting"
|
||||
(event! {:thinking "Let me "})
|
||||
(event! {:thinking "think."})
|
||||
(is (= [{:role "user" :content "hello"}] (:messages (chat))))
|
||||
(is (= [{:role "user" :content "hello"}
|
||||
{:role "assistant" :content "" :thinking "Let me think."}]
|
||||
(:messages (chat))))
|
||||
(is (= :loading (:status (chat)))))
|
||||
(testing "the first piece of the reply, and the ones that follow, are added to that message"
|
||||
(event! {:content "Hel"})
|
||||
(is (= [{:role "user" :content "hello"} {:role "assistant" :content "Hel"}]
|
||||
(is (= [{:role "user" :content "hello"}
|
||||
{:role "assistant" :content "Hel" :thinking "Let me think."}]
|
||||
(:messages (chat))))
|
||||
(event! {:content "lo, "})
|
||||
(event! {:content "world"})
|
||||
(is (= [{:role "user" :content "hello"}
|
||||
{:role "assistant" :content "Hello, world"}]
|
||||
{:role "assistant" :content "Hello, world" :thinking "Let me think."}]
|
||||
(:messages (chat))))
|
||||
(is (= :loading (:status (chat))) "it isn't over until it says so"))
|
||||
(testing "another message can't be sent while the reply is coming in"
|
||||
@@ -401,12 +416,12 @@
|
||||
(event! {:done true})
|
||||
(is (= {:status :idle :error nil} (select-keys (chat) [:status :error])))
|
||||
(is (= [{:role "user" :content "hello"}
|
||||
{:role "assistant" :content "Hello, world"}]
|
||||
{:role "assistant" :content "Hello, world" :thinking "Let me think."}]
|
||||
(:messages (chat)))))
|
||||
(testing "events that come after the end are dropped"
|
||||
(event! {:content "late"})
|
||||
(is (= "Hello, world" (:content (last (:messages (chat)))))))
|
||||
(testing "the next message has the conversation so far"
|
||||
(testing "the next message has the conversation so far, without the reasoning (which stays only for display)"
|
||||
(say! "again")
|
||||
(is (= [{:role "user" :content "hello"} {:role "assistant" :content "Hello, world"}
|
||||
{:role "user" :content "again"}]
|
||||
@@ -448,15 +463,16 @@
|
||||
(say! "hello")
|
||||
(event! {:thinking "hm" :content "Hi"})
|
||||
(event! {:thinking "m" :content "!"})
|
||||
(is (= [{:role "user" :content "hello"} {:role "assistant" :content "Hi!"}]
|
||||
(is (= [{:role "user" :content "hello"} {:role "assistant" :content "Hi!" :thinking "hmm"}]
|
||||
(:messages (chat))))
|
||||
(testing "an empty piece changes nothing"
|
||||
(event! {:thinking "" :content ""})
|
||||
(is (= [{:role "user" :content "hello"} {:role "assistant" :content "Hi!"}]
|
||||
(is (= [{:role "user" :content "hello"} {:role "assistant" :content "Hi!" :thinking "hmm"}]
|
||||
(:messages (chat))))))
|
||||
|
||||
(deftest chat-thinking-is-never-kept-test
|
||||
;; The reasoning of each reply of a turn, before and after tools, leaves nothing in the conversation.
|
||||
(deftest chat-thinking-per-reply-test
|
||||
;; Each reply of a turn (before and after tools) keeps its own reasoning, on its own message, not
|
||||
;; sent to the LLM, but there for "Show reasoning" to reveal.
|
||||
(logged-in! [])
|
||||
(say! "hi")
|
||||
(event! {:thinking "first"})
|
||||
@@ -465,9 +481,9 @@
|
||||
(dispatch! [::events/chat-tool-results [{:id "c1" :name "ping" :content "pong" :error? false}]])
|
||||
(event! {:thinking "second"})
|
||||
(event! {:content "done"})
|
||||
(is (= [["user" nil] ["assistant" nil] ["tool" nil] ["assistant" nil]]
|
||||
(is (= [["user" nil] ["assistant" "first"] ["tool" nil] ["assistant" "second"]]
|
||||
(map (juxt :role :thinking) (:messages (chat)))))
|
||||
(is (not-any? #(re-find #"first|second" (pr-str %)) (:messages (chat)))))
|
||||
(is (not-any? #(re-find #"first|second" (pr-str %)) (sent-messages (last-request)))))
|
||||
|
||||
(deftest chat-blank-message-test
|
||||
(logged-in! [])
|
||||
@@ -595,6 +611,19 @@
|
||||
(dispatch! [::events/chat-toggle-tools])
|
||||
(is (false? (:tools-expanded? (chat)))))
|
||||
|
||||
(deftest chat-toggle-reasoning-test
|
||||
(logged-in! [])
|
||||
(is (false? (:reasoning-expanded? (chat))) "reasoning is hidden by default")
|
||||
(dispatch! [::events/chat-toggle-reasoning])
|
||||
(is (true? (:reasoning-expanded? (chat))))
|
||||
(testing "the choice stays when the conversation starts over"
|
||||
(say! "hello")
|
||||
(reply! "Hi!")
|
||||
(dispatch! [::events/chat-clear])
|
||||
(is (true? (:reasoning-expanded? (chat)))))
|
||||
(dispatch! [::events/chat-toggle-reasoning])
|
||||
(is (false? (:reasoning-expanded? (chat)))))
|
||||
|
||||
(deftest chat-resize-test
|
||||
(logged-in! [])
|
||||
(is (= {:width 320 :height 256} (:size (chat))) "the size it starts with")
|
||||
@@ -611,6 +640,12 @@
|
||||
(answer! 200 {:logged-out true})
|
||||
(is (= {:width 500 :height 400} (:size (chat))))))
|
||||
|
||||
(deftest chat-resize-persists-across-a-reload-test
|
||||
(logged-in! [])
|
||||
(dispatch! [::events/chat-resize 500 400])
|
||||
(dispatch! [::events/initialize])
|
||||
(is (= {:width 500 :height 400} (:size (chat)))))
|
||||
|
||||
(deftest chat-forgotten-on-logout-test
|
||||
(logged-in! [])
|
||||
(say! "hello")
|
||||
@@ -654,6 +689,62 @@
|
||||
{:id id :name tool-name :content content :error? error?})
|
||||
results))))
|
||||
|
||||
(defn- aborted-streams [] (count (filter (comp #{::events/abort-stream} first) @effects)))
|
||||
|
||||
(deftest chat-halt-test
|
||||
(testing "halting while it only reasons: the stream stops, and the reasoning stays on screen, marked"
|
||||
(logged-in! [])
|
||||
(say! "hello")
|
||||
(event! {:thinking "hmm, maybe"})
|
||||
(dispatch! [::events/chat-halt])
|
||||
(is (= 1 (aborted-streams)))
|
||||
(is (= {:messages [{:role "user" :content "hello"}
|
||||
{:role "assistant" :content "" :thinking "hmm, maybe" :halted? true}]
|
||||
:input "" :status :idle :error nil}
|
||||
(select-keys (chat) [:messages :input :status :error])))
|
||||
(testing "but isn't saved, nor sent: there is nothing in it for the server"
|
||||
(is (= {:messages [{:role "user" :content "hello"}]} (:body (last-persist-request))))
|
||||
(say! "no, do it this way")
|
||||
(is (= [{:role "user" :content "hello"} {:role "user" :content "no, do it this way"}]
|
||||
(sent-messages (last-request))))))
|
||||
(testing "halting mid-reply keeps what was said, and the next message steers from it"
|
||||
(logged-in! [])
|
||||
(say! "hello")
|
||||
(event! {:content "Hel"})
|
||||
(dispatch! [::events/chat-halt])
|
||||
(is (= [{:role "user" :content "hello"} {:role "assistant" :content "Hel" :halted? true}]
|
||||
(:messages (chat))))
|
||||
(testing "events that come after are dropped"
|
||||
(event! {:content "lo"})
|
||||
(event! {:done true})
|
||||
(is (= "Hel" (:content (last (:messages (chat)))))))
|
||||
(say! "shorter please")
|
||||
(is (= [{:role "user" :content "hello"} {:role "assistant" :content "Hel"}
|
||||
{:role "user" :content "shorter please"}]
|
||||
(sent-messages (last-request)))))
|
||||
(testing "halting before anything came"
|
||||
(logged-in! [])
|
||||
(say! "hello")
|
||||
(dispatch! [::events/chat-halt])
|
||||
(is (= [{:role "user" :content "hello"} {:role "assistant" :content "" :halted? true}]
|
||||
(:messages (chat)))))
|
||||
(testing "halting while the tools run: each call gets an answer, and their real results are dropped"
|
||||
(logged-in! [])
|
||||
(say! "hello")
|
||||
(asks! (call "c1" "add" "{}"))
|
||||
(dispatch! [::events/chat-halt])
|
||||
(is (= :idle (:status (chat))))
|
||||
(is (= {:role "tool" :tool-call-id "c1" :name "add" :error? true}
|
||||
(dissoc (last (:messages (chat))) :content)))
|
||||
(let [n (count (:messages (chat)))]
|
||||
(ran! ["c1" "add" "5"])
|
||||
(is (= n (count (:messages (chat))))))
|
||||
(is (= 1 (count (requests))) "nothing more went to the LLM"))
|
||||
(testing "halting when nothing is on its way does nothing"
|
||||
(logged-in! [])
|
||||
(dispatch! [::events/chat-halt])
|
||||
(is (zero? (aborted-streams)))))
|
||||
|
||||
(deftest chat-offers-the-tools-test
|
||||
(logged-in! [])
|
||||
(testing "no tools on the page, none offered"
|
||||
@@ -750,15 +841,17 @@
|
||||
(logged-in! [])
|
||||
(model-context/register! [{:name "add" :description "Adds" :input-schema add-schema :handler identity}])
|
||||
(say! "1+2?")
|
||||
(testing "while the model reasons or writes a call, nothing is done, and the conversation has neither"
|
||||
(testing "while the model reasons or writes a call, nothing is done, and no tools have run"
|
||||
(event! {:thinking "I should add."})
|
||||
(event! {:tool-calls [(call "c1" "add" "{\"x\":1,\"y\":2}")]})
|
||||
(is (= [{:role "user" :content "1+2?"}] (:messages (chat))))
|
||||
(is (= [{:role "user" :content "1+2?"}
|
||||
{:role "assistant" :content "" :thinking "I should add."}]
|
||||
(:messages (chat))))
|
||||
(is (= [] (tool-calls-asked))))
|
||||
(testing "when the reply is over, the calls join the conversation and are made"
|
||||
(event! {:done true})
|
||||
(is (= [{:role "user" :content "1+2?"}
|
||||
{:role "assistant" :content ""
|
||||
{:role "assistant" :content "" :thinking "I should add."
|
||||
:tool-calls [(call "c1" "add" "{\"x\":1,\"y\":2}")]}]
|
||||
(:messages (chat))))
|
||||
(is (= :loading (:status (chat))) "the LLM still has to answer")
|
||||
@@ -785,6 +878,57 @@
|
||||
(is (= {:status :idle :error nil} (select-keys (chat) [:status :error])))
|
||||
(is (nil? (:pending-calls (chat))))))
|
||||
|
||||
(deftest chat-tool-result-too-big-for-the-server-test
|
||||
;; A result over the LLM's limit is kept whole in the tab and in /api/chat/history (for "Expand
|
||||
;; tool use" to show all of it, even after a reload), but fit to that limit for the LLM.
|
||||
(logged-in! [])
|
||||
(model-context/register! [{:name "add" :description "Adds" :input-schema add-schema :handler identity}])
|
||||
(let [huge (apply str (repeat 9000 "x"))]
|
||||
(say! "1+2?")
|
||||
(event! {:tool-calls [(call "c1" "add" "{\"x\":1,\"y\":2}")] :done true})
|
||||
(ran! ["c1" "add" huge false])
|
||||
(testing "the tab's own copy is whole"
|
||||
(is (= huge (:content (last (:messages (chat)))))))
|
||||
(testing "what goes to the LLM is fit to the server's limit"
|
||||
(let [sent (:content (last (sent-messages (last-request))))]
|
||||
(is (= 8000 (count sent)))
|
||||
(is (= "xxx … (truncated)" (subs sent 7983)))))
|
||||
(reply! "It's 3.")
|
||||
(testing "what's saved for next login is whole, once the turn ends"
|
||||
(let [saved (:content (first (filter #(= "tool" (:role %)) (:messages (:body (last-persist-request))))))]
|
||||
(is (= huge saved))))))
|
||||
|
||||
(deftest chat-reasoning-too-big-for-history-test
|
||||
;; Reasoning, like a tool result, is kept whole in the tab and in what's saved (for "Show
|
||||
;; reasoning" to show all of it, even after a reload), up to the history's own, larger limit.
|
||||
(logged-in! [])
|
||||
(say! "hello")
|
||||
(let [huge (apply str (repeat 100010 "x"))]
|
||||
(event! {:thinking huge})
|
||||
(testing "the tab's own copy is whole"
|
||||
(is (= huge (:thinking (last (:messages (chat)))))))
|
||||
(reply! "Hi!")
|
||||
(testing "what's saved for next login is fit to the history's limit"
|
||||
(let [saved (:thinking (first (filter :thinking (:messages (:body (last-persist-request))))))]
|
||||
(is (= 100000 (count saved)))
|
||||
(is (= "xxx … (truncated)" (subs saved 99983)))))))
|
||||
|
||||
(deftest chat-history-too-big-drops-oldest-test
|
||||
;; Past the history's total limit, the oldest turns go, whole.
|
||||
(logged-in! [])
|
||||
(model-context/register! [{:name "add" :description "Adds" :input-schema add-schema :handler identity}])
|
||||
(let [big (apply str (repeat 90000 "x"))]
|
||||
(dotimes [i 12]
|
||||
(say! (str "q" i))
|
||||
(event! {:tool-calls [(call (str "c" i) "add" "{}")] :done true})
|
||||
(ran! [(str "c" i) "add" big false])
|
||||
(reply! (str "a" i)))
|
||||
(let [saved (:messages (:body (last-persist-request)))]
|
||||
(is (<= (count (pr-str saved)) 1000000))
|
||||
(is (= "user" (:role (first saved))))
|
||||
(is (= "a11" (:content (last saved))))
|
||||
(is (every? #(= big (:content %)) (filter #(= "tool" (:role %)) saved))))))
|
||||
|
||||
(deftest chat-tool-call-with-text-test
|
||||
(logged-in! [])
|
||||
(say! "hi")
|
||||
@@ -820,6 +964,41 @@
|
||||
(dispatch! [::events/chat-send])
|
||||
(is (zero? (:tool-calls-used (chat))))))
|
||||
|
||||
(deftest chat-once-per-turn-tools-test
|
||||
;; A once-per-turn tool (like tic-tac-toe's move_tic_tac_toe: the bug this guards against was
|
||||
;; the chatbot happily playing both sides of one game in answer to a single "make a move") may
|
||||
;; run once in a turn; a second attempt, whether the model asks again after the first result or
|
||||
;; in the very same reply, is refused without ever reaching the page.
|
||||
(logged-in! [])
|
||||
(model-context/register! [{:name "move" :once-per-turn? true :description "d" :input-schema add-schema
|
||||
:handler identity}
|
||||
{:name "ping" :description "d" :input-schema add-schema :handler identity}])
|
||||
(say! "make a move")
|
||||
(testing "offered like any other tool, the first time"
|
||||
(is (= ["move" "ping"] (map :name (:tools (:body (last-request)))))))
|
||||
(testing "once it has run, it is no longer offered for the rest of the turn"
|
||||
(asks! (call "a" "move" "{}"))
|
||||
(is (= [(call "a" "move" "{}")] (:calls (last (tool-calls-asked)))) "the first call goes through, unmarked")
|
||||
(ran! ["a" "move" "ok" false])
|
||||
(is (= ["ping"] (map :name (:tools (:body (last-request)))))))
|
||||
(testing "asking for it again anyway this turn is refused before it ever reaches the page"
|
||||
(asks! (call "b" "move" "{}"))
|
||||
(is (= [(assoc (call "b" "move" "{}") :blocked? true)] (:calls (last (tool-calls-asked))))))
|
||||
(testing "two calls to it in the very same reply: only the first of them runs"
|
||||
(ran! ["b" "move" "n/a" true])
|
||||
(reply! "Can't move again this turn.")
|
||||
(say! "go again")
|
||||
(asks! (call "c" "move" "{}") (call "d" "move" "{}"))
|
||||
(is (= [(call "c" "move" "{}") (assoc (call "d" "move" "{}") :blocked? true)]
|
||||
(:calls (last (tool-calls-asked))))))
|
||||
(testing "a new message starts a fresh turn: the tool is offered, and may run, again"
|
||||
(ran! ["c" "move" "n/a" false] ["d" "move" "n/a" true])
|
||||
(reply! "Moved.")
|
||||
(say! "make another")
|
||||
(is (= ["move" "ping"] (map :name (:tools (:body (last-request))))))
|
||||
(asks! (call "e" "move" "{}"))
|
||||
(is (= [(call "e" "move" "{}")] (:calls (last (tool-calls-asked)))))))
|
||||
|
||||
(deftest chat-tool-call-budget-test
|
||||
(logged-in! [])
|
||||
(say! "go wild")
|
||||
@@ -896,7 +1075,11 @@
|
||||
(select-keys (chat) [:messages :input :status :error :pending-calls])))
|
||||
(is (= [] (tool-calls-asked)))))
|
||||
|
||||
(deftest chat-sends-results-of-earlier-turns-as-out-of-date-test
|
||||
(deftest chat-sends-results-of-earlier-turns-as-they-are-test
|
||||
;; Nothing here marks an old result stale: the request's `state` (tbd.ui.model-context/
|
||||
;; current-state) is refreshed on every message, and the system prompt tells the model that is
|
||||
;; what to trust over anything said earlier, an old result included — so there is nothing left
|
||||
;; for a marker on the result itself to do.
|
||||
(logged-in! [])
|
||||
(swap! rf-db/app-db assoc-in [:chat :messages]
|
||||
[{:role "user" :content "play"}
|
||||
@@ -904,20 +1087,19 @@
|
||||
{:role "tool" :tool-call-id "old" :name "get_tic_tac_toe" :content "{\"turn\":\"x\"}" :error? false}
|
||||
{:role "assistant" :content "Your move."}])
|
||||
(say! "that's wrong, it's my turn")
|
||||
(testing "a result from before the user's last message is no longer sent, its call and reply are"
|
||||
(testing "an earlier result is sent as it was, like its call and reply"
|
||||
(let [sent (get-in (last-request) [:body :messages])]
|
||||
(is (= ["user" "assistant" "tool" "assistant" "user"] (map :role sent)))
|
||||
(is (= "old" (:toolCallId (nth sent 2))))
|
||||
(is (str/includes? (:content (nth sent 2)) "out of date"))
|
||||
(is (not (str/includes? (:content (nth sent 2)) "\"turn\"")) "what it read is gone")
|
||||
(is (= "{\"turn\":\"x\"}" (:content (nth sent 2))))
|
||||
(is (= "Your move." (:content (nth sent 3))))))
|
||||
(testing "the conversation the user sees keeps the result"
|
||||
(is (= "{\"turn\":\"x\"}" (:content (nth (:messages (chat)) 2)))))
|
||||
(testing "the results of the turn going on are sent as they are, for the chain to continue"
|
||||
(testing "the results of the turn going on are sent as they are too, for the chain to continue"
|
||||
(asks! (call "new" "get_tic_tac_toe" "{}"))
|
||||
(ran! ["new" "get_tic_tac_toe" "{\"turn\":\"o\"}" false])
|
||||
(let [sent (get-in (last-request) [:body :messages])]
|
||||
(is (str/includes? (:content (nth sent 2)) "out of date"))
|
||||
(is (= "{\"turn\":\"x\"}" (:content (nth sent 2))))
|
||||
(is (= "{\"turn\":\"o\"}" (:content (last sent)))))))
|
||||
|
||||
(deftest chat-sends-no-orphaned-result-test
|
||||
@@ -985,23 +1167,26 @@
|
||||
(call "4" "blank" "")
|
||||
(call "5" "fails" "{}")
|
||||
(call "6" "nope" "{}")
|
||||
(call "7" "fast" "oops")]
|
||||
(call "7" "fast" "oops")
|
||||
(assoc (call "8" "fast" "{}") :blocked? true)]
|
||||
:on-results [::ran]})))
|
||||
(fn [results]
|
||||
(testing "one after the other, in order: the tools work on the same page"
|
||||
(is (= [:slow :fast] @order)))
|
||||
(is (= [:slow :fast] @order)) "the blocked call never touches the tool, so \"fast\" runs only once")
|
||||
(testing "a result for each call, whatever became of it"
|
||||
(is (= ["1" "2" "3" "4" "5" "6" "7"] (map :id results)))
|
||||
(is (= ["slow" "fast" "huge" "blank" "fails" "nope" "fast"] (map :name results)))
|
||||
(is (= ["1" "2" "3" "4" "5" "6" "7" "8"] (map :id results)))
|
||||
(is (= ["slow" "fast" "huge" "blank" "fails" "nope" "fast" "fast"] (map :name results)))
|
||||
(is (= ["slow done" "fast {:x 1}"] (map :content (take 2 results)))))
|
||||
(testing "a result is never empty, and never longer than the server takes"
|
||||
(testing "a result is never empty, and kept whole (fit to the server's limit only once it leaves the tab)"
|
||||
(is (= "(no output)" (:content (nth results 3))))
|
||||
(is (= 8000 (count (:content (nth results 2)))))
|
||||
(is (= "xxx … (truncated)" (subs (:content (nth results 2)) 7983))))
|
||||
(is (= 9000 (count (:content (nth results 2))))))
|
||||
(testing "failures are results for the LLM to read"
|
||||
(is (= [false false false false true true true] (map (comp boolean :error?) results)))
|
||||
(is (= [false false false false true true true true] (map (comp boolean :error?) results)))
|
||||
(is (= ["it broke" "unknown tool: nope" "invalid arguments: they must be a JSON object"]
|
||||
(map :content (drop 4 results))))))
|
||||
(map :content (drop 4 (take 7 results))))))
|
||||
(testing "a blocked call is refused before it ever reaches the tool"
|
||||
(is (= "not run: this only runs once per message, and already has this turn — answer instead of calling it again"
|
||||
(:content (last results))))))
|
||||
#(js/Promise. (fn [resolve]
|
||||
(reset! answer resolve)
|
||||
(call-tools-fx {:calls [] :on-results [::ran]})))
|
||||
|
||||
@@ -55,10 +55,43 @@
|
||||
(tu/steps
|
||||
#(stream! (streaming asked (sse {:type "RUN_FINISHED"})) 1)
|
||||
(fn [_]
|
||||
(is (= ["/api/chat" {:method "POST"
|
||||
:headers {:Content-Type "application/edn"}
|
||||
:body "{:x 1}"}]
|
||||
@asked))))
|
||||
(let [[uri opts] @asked]
|
||||
(is (= ["/api/chat" {:method "POST"
|
||||
:headers {:Content-Type "application/edn"}
|
||||
:body "{:x 1}"}]
|
||||
[uri (dissoc opts :signal)]))
|
||||
(is (instance? js/AbortSignal (:signal opts)) "with a signal, for ::abort-stream"))))
|
||||
done))))
|
||||
|
||||
(deftest abort-test
|
||||
;; Halted (see ::events/chat-halt): nothing more is dispatched — not the rest of the stream, nor
|
||||
;; the \"connection lost\" its end would otherwise be.
|
||||
(async done
|
||||
(let [release (atom nil)
|
||||
fetch (fn [_ _]
|
||||
(js/Promise.resolve
|
||||
#js {:ok true :status 200
|
||||
:body #js {:getReader
|
||||
(fn []
|
||||
(let [reads (atom [(sse {:type "TEXT_MESSAGE_CONTENT" :delta "Hi"}) :wait])]
|
||||
#js {:read (fn []
|
||||
(let [r (first @reads)]
|
||||
(swap! reads rest)
|
||||
(cond
|
||||
(= :wait r) (js/Promise. #(reset! release %))
|
||||
(nil? r) (js/Promise.resolve #js {:done true})
|
||||
:else (js/Promise.resolve #js {:done false :value r}))))
|
||||
:cancel (fn [] (js/Promise.resolve nil))}))}}))]
|
||||
(tu/finish
|
||||
(tu/steps
|
||||
#(stream! fetch 1)
|
||||
(fn [got]
|
||||
(is (= [[:event {:type "TEXT_MESSAGE_CONTENT" :delta "Hi"}]] got))
|
||||
((tu/real-effect ::events/abort-stream) true)
|
||||
(@release #js {:done false :value (sse {:type "TEXT_MESSAGE_CONTENT" :delta "late"})})
|
||||
(js/Promise. (fn [resolve] (js/setTimeout resolve 30))))
|
||||
(fn [_]
|
||||
(is (= [[:event {:type "TEXT_MESSAGE_CONTENT" :delta "Hi"}]] @got) "nothing after the abort")))
|
||||
done))))
|
||||
|
||||
(deftest events-test
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
(ns tbd.ui.markdown-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[tbd.ui.markdown :as markdown]))
|
||||
|
||||
(deftest render-test
|
||||
(testing "plain text, wrapped in a paragraph"
|
||||
(is (= [:<> [:p "hello"]] (markdown/render "hello"))))
|
||||
(testing "inline formatting"
|
||||
(is (= [:<> [:p "a " [:strong "bold"] " " [:em "word"] " and " [:code "code"]]]
|
||||
(markdown/render "a **bold** *word* and `code`"))))
|
||||
(testing "a single newline is a line break, not a new paragraph"
|
||||
(is (= [:<> [:p "one" [:br] "two"]] (markdown/render "one\ntwo"))))
|
||||
(testing "a stray * or _ that isn't emphasis (padded with a space) is left alone"
|
||||
(is (= [:<> [:p "3 * 4 * 5"]] (markdown/render "3 * 4 * 5"))))
|
||||
(testing "headings, a list, a fenced code block"
|
||||
(is (= [:<>
|
||||
[:h1 "Title"]
|
||||
[:ul [:li "a"] [:li "b"]]
|
||||
[:pre [:code {:class "language-js"} "1"]]]
|
||||
(markdown/render "# Title\n\n- a\n- b\n\n```js\n1\n```"))))
|
||||
(testing "an ordered list and a blockquote"
|
||||
(is (= [:<> [:ol [:li "a"] [:li "b"]]] (markdown/render "1. a\n2. b")))
|
||||
(is (= [:<> [:blockquote [:p "hi"]]] (markdown/render "> hi"))))
|
||||
(testing "a table"
|
||||
(is (= [:<> [:table [:thead [:tr [:th "a"] [:th "b"]]]
|
||||
[:tbody [:tr [:td "1"] [:td "2"]]]]]
|
||||
(markdown/render "a | b\n-|-\n1 | 2"))))
|
||||
(testing "no markup leaks through as HTML: it's always a plain string, never dangerouslySetInnerHTML"
|
||||
(is (= [:<> [:p "hi<script>alert(1)</script>"]] (markdown/render "hi<script>alert(1)</script>"))))
|
||||
(testing "a link opens in its own tab, not the app's"
|
||||
(is (= [:<> [:p [:a {:href "https://example.com" :target "_blank" :rel "noopener noreferrer"} "x"]]]
|
||||
(markdown/render "[x](https://example.com)"))))
|
||||
(testing "a javascript: link loses its href (and so its target/rel), the text stays"
|
||||
(is (= [:<> [:p [:span "x"]]] (markdown/render "[x](javascript:evil)")))))
|
||||
@@ -174,10 +174,13 @@
|
||||
(aset js/globalThis "fetch" real-fetch))})
|
||||
|
||||
(def url-fixture
|
||||
"Each test starts on the page's own address, with no fragment (the page shown is the first)."
|
||||
(let [reset-hash! #(set! js/location.hash "")]
|
||||
{:before reset-hash!
|
||||
:after reset-hash!}))
|
||||
"Each test starts on the page's own address, with no fragment (the page shown is the first), and
|
||||
with nothing saved in localStorage (e.g. the chat widget's :open?/:size, see
|
||||
tbd.ui.events/persist-chat-ui) — jsdom's window, and so its localStorage, is shared by every
|
||||
test in the process."
|
||||
(let [reset! #(do (set! js/location.hash "") (.clear js/localStorage))]
|
||||
{:before reset!
|
||||
:after reset!}))
|
||||
|
||||
(defn log-in! [] (swap! server assoc :user {:login "alice" :id 7}))
|
||||
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
(ns tbd.ui.tictactoe-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
(:require [cljs.test :refer [deftest is testing use-fixtures]]
|
||||
[re-frame.core :as rf]
|
||||
[re-frame.db :as rf-db]
|
||||
[tbd.ui.events :as events]
|
||||
[tbd.ui.test-util :as tu]
|
||||
[tbd.ui.tictactoe :as ttt]))
|
||||
|
||||
(use-fixtures :each tu/url-fixture)
|
||||
|
||||
(defn- board
|
||||
"A board from a string of 9 characters: x, o or anything else for an empty cell."
|
||||
[s]
|
||||
@@ -70,3 +73,16 @@
|
||||
(testing "a draw"
|
||||
(move! 0 1 2 4 3 5 7 6 8)
|
||||
(is (= :draw (:outcome (game)))))))
|
||||
|
||||
(deftest persistence-test
|
||||
(rf/dispatch-sync [::events/initialize])
|
||||
(move! 0 3)
|
||||
(testing "a reload finds the game where it was left"
|
||||
(rf/dispatch-sync [::events/initialize])
|
||||
(let [{:keys [board turn]} (:ttt @rf-db/app-db)]
|
||||
(is (= [:x nil nil :o] (subvec board 0 4)))
|
||||
(is (= :x turn))))
|
||||
(testing "a reset is also remembered"
|
||||
(rf/dispatch-sync [::events/ttt-reset])
|
||||
(rf/dispatch-sync [::events/initialize])
|
||||
(is (= ttt/new-board (:board (:ttt @rf-db/app-db))))))
|
||||
|
||||
@@ -483,7 +483,9 @@
|
||||
(defn- chat-messages
|
||||
"What the conversation shows, message by message."
|
||||
[]
|
||||
(mapv #(.-textContent %) (q-all "[role=log] > div:not([role=status]):not([data-tool])")))
|
||||
(mapv #(.-textContent %) (q-all "[role=log] > div:not([data-pending]):not([data-tool]):not([data-halted])")))
|
||||
|
||||
(defn- chat-status [] (:status (:chat @rf-db/app-db)))
|
||||
|
||||
(defn- chat-tool-lines
|
||||
"The tool calls and results the conversation shows, line by line."
|
||||
@@ -500,6 +502,16 @@
|
||||
[]
|
||||
(mapv #(.-textContent %) (tool-toggle-buttons)))
|
||||
|
||||
(defn- reasoning-toggle-buttons []
|
||||
(filterv #(re-find #"reasoning$" (.-textContent %)) (q-all "[role=dialog] button")))
|
||||
|
||||
(defn- reasoning-toggle [] (first (reasoning-toggle-buttons)))
|
||||
|
||||
(defn- reasoning-toggle-labels
|
||||
"The label of the button that shows or hides the reasoning, if the panel has one."
|
||||
[]
|
||||
(mapv #(.-textContent %) (reasoning-toggle-buttons)))
|
||||
|
||||
(deftest chat-widget-test
|
||||
;; A green square in the bottom left corner, opposite WebMCP's blue one; it opens the panel.
|
||||
(async done
|
||||
@@ -549,7 +561,9 @@
|
||||
(fn [_]
|
||||
(is (= "" (.-value (q "input[name=message]"))) "the input is emptied when sent")
|
||||
(is (nil? (q "[role=status]")) "no longer thinking")
|
||||
(is (= ["Clear" "×"] (mapv #(.-textContent %) (q-all "[role=dialog] > div:first-child button"))))
|
||||
(is (= ["Show reasoning" "Clear" "×"]
|
||||
(mapv #(.-textContent %) (q-all "[role=dialog] > div:first-child button")))
|
||||
"the fake server reasons before it echoes, so the toggle shows too")
|
||||
(type! (q "input[name=message]") "again")
|
||||
(tu/eventually (fn [] (= "again" (.-value (q "input[name=message]"))))))
|
||||
(fn [_]
|
||||
@@ -589,7 +603,7 @@
|
||||
(fn [_]
|
||||
(is (= "Thinking…" (text "[role=status]")))
|
||||
(is (= ["hello"] (chat-messages)))
|
||||
(is (true? (.-readOnly (q "input[name=message]"))))
|
||||
(is (false? (.-readOnly (q "input[name=message]"))) "typing ahead is still possible")
|
||||
(is (true? (.-disabled (q "[role=dialog] button[type=submit]"))))
|
||||
(is (nil? (first (filter #(= "Clear" (.-textContent %)) (q-all "[role=dialog] button"))))
|
||||
"can't start over mid-reply")
|
||||
@@ -622,7 +636,8 @@
|
||||
|
||||
(deftest chat-streaming-test
|
||||
;; The reply shows as it arrives: "Thinking…" until the model's first word (however long it
|
||||
;; reasons, which is never shown), then the reply growing piece by piece, and only at its end can
|
||||
;; reasons, which is not shown unless "Show reasoning" is turned on — see
|
||||
;; chat-reasoning-toggle-test), then the reply growing piece by piece, and only at its end can
|
||||
;; another message be sent.
|
||||
(async done
|
||||
(swap! tu/server assoc :chat-script [:pause
|
||||
@@ -645,22 +660,52 @@
|
||||
"still thinking while the model reasons")
|
||||
(is (not (str/includes? (panel-text) "Let me think.")) "and what it reasons is not shown")
|
||||
(is (= ["hi"] (chat-messages)) "it hasn't said anything")
|
||||
(is (true? (.-readOnly (q "input[name=message]"))) "and it isn't over")
|
||||
(is (= :loading (chat-status)) "and it isn't over")
|
||||
(is (false? (.-readOnly (q "input[name=message]"))) "but typing ahead still works")
|
||||
(resume-stream!))
|
||||
#(tu/eventually (fn [] (and (:resume @tu/server) (= ["hi" "Hello"] (chat-messages)))))
|
||||
(fn [_]
|
||||
(is (nil? (q "[role=status]")) "the model has begun to answer")
|
||||
(is (true? (.-readOnly (q "input[name=message]"))) "but it isn't over")
|
||||
(is (= :loading (chat-status)) "but it isn't over")
|
||||
(is (true? (.-disabled (q "[role=dialog] button[type=submit]"))))
|
||||
(resume-stream!))
|
||||
#(tu/eventually (fn [] (= ["hi" "Hello, world"] (chat-messages))))
|
||||
#(tu/eventually (fn [] (false? (.-readOnly (q "input[name=message]")))))
|
||||
#(tu/eventually (fn [] (= :idle (chat-status))))
|
||||
(fn [_]
|
||||
(is (not (str/includes? (panel-text) "Let me think.")) "the reasoning is not shown after it either")
|
||||
(is (nil? (q "[role=status]")))
|
||||
(is (nil? (q "[role=dialog] [role=alert]")))))
|
||||
done)))
|
||||
|
||||
(deftest chat-halt-test
|
||||
;; The stop button beside "Thinking…" halts the reply mid-thought, so the next message can steer
|
||||
;; it; what the stream would have sent after is never shown.
|
||||
(async done
|
||||
(swap! tu/server assoc :chat-scripts [[{:thinking "Let me "} :pause {:content "late"} {:done true}]])
|
||||
(tu/finish
|
||||
(tu/steps
|
||||
#(log-in-page! [])
|
||||
(fn [] (click! (q "button[aria-label=Chat]")))
|
||||
(fn [] (is (nil? (q "button[aria-label=Stop]")) "nothing to stop yet"))
|
||||
#(send-chat! "hi")
|
||||
#(tu/eventually (fn [] (and (:resume @tu/server) (q "button[aria-label=Stop]"))))
|
||||
(fn [stop]
|
||||
(is (re-matches #"Thinking(\.{1,3}|…)" (text "[role=status]")))
|
||||
(click! stop))
|
||||
#(tu/eventually (fn [] (q "[data-halted]")))
|
||||
(fn [_]
|
||||
(is (nil? (q "button[aria-label=Stop]")))
|
||||
(is (nil? (q "[role=status]")))
|
||||
(is (some? (q "[data-halted]")) "the reply is marked as stopped")
|
||||
(resume-stream!))
|
||||
#(tu/eventually (fn [] true))
|
||||
(fn [_]
|
||||
(is (not (str/includes? (panel-text) "late")) "what came after the halt is dropped")
|
||||
(is (= ["hi"] (chat-messages))))
|
||||
#(send-chat! "try this instead")
|
||||
#(tu/eventually (fn [] (= ["hi" "try this instead" "echo: try this instead"] (chat-messages)))))
|
||||
done)))
|
||||
|
||||
(deftest chat-thinking-animation-test
|
||||
;; While pieces of the reasoning are coming in "Thinking" has one to three dots; once they stop
|
||||
;; (for a moment) it is "Thinking…" again.
|
||||
@@ -687,8 +732,10 @@
|
||||
#(tu/eventually (fn [] (nil? (q "[role=status]")))))
|
||||
done)))
|
||||
|
||||
(deftest chat-reasoning-is-never-shown-test
|
||||
;; Neither while it comes nor in the finished conversation, in any element of the panel.
|
||||
(deftest chat-reasoning-toggle-test
|
||||
;; The reasoning behind a reply is never shown by default, in any element of the panel, while it
|
||||
;; comes or after; "Show reasoning" reveals it, whole (nothing cut off), for every reply that
|
||||
;; reasoned; "Hide reasoning" hides it again.
|
||||
(async done
|
||||
(let [thought "secret reasoning about the answer"]
|
||||
(swap! tu/server assoc :chat-script [{:thinking thought} {:content "Done."} {:done true}])
|
||||
@@ -699,7 +746,16 @@
|
||||
#(send-chat! "hi")
|
||||
#(tu/eventually (fn [] (= ["hi" "Done."] (chat-messages))))
|
||||
(fn [_]
|
||||
(is (not (str/includes? (panel-text) "secret")))))
|
||||
(is (not (str/includes? (panel-text) "secret")) "hidden by default")
|
||||
(is (= ["Show reasoning"] (reasoning-toggle-labels)))
|
||||
(click! (reasoning-toggle)))
|
||||
#(tu/eventually (fn [] (= ["Hide reasoning"] (reasoning-toggle-labels))))
|
||||
(fn [_]
|
||||
(is (str/includes? (panel-text) thought) "shown, whole, once expanded")
|
||||
(click! (reasoning-toggle)))
|
||||
#(tu/eventually (fn [] (= ["Show reasoning"] (reasoning-toggle-labels))))
|
||||
(fn [_]
|
||||
(is (not (str/includes? (panel-text) "secret")) "hidden again")))
|
||||
done))))
|
||||
|
||||
(defn- pointer!
|
||||
@@ -798,7 +854,7 @@
|
||||
(is (= 1000 @@top) "back near the end, it follows again")
|
||||
(reset! @top 200)
|
||||
(.dispatchEvent (q "[role=log]") (js/window.Event. "scroll")))
|
||||
#(tu/eventually (fn [] (false? (.-readOnly (q "input[name=message]")))))
|
||||
#(tu/eventually (fn [] (= :idle (chat-status))))
|
||||
#(send-chat! "again")
|
||||
#(tu/eventually (fn [] (and (:resume @tu/server) (= 1000 @@top))))
|
||||
(fn [_]
|
||||
@@ -855,7 +911,7 @@
|
||||
{:role "tool" :toolCallId "call_1" :content "{\"x\":2,\"y\":3,\"sum\":5}"}]
|
||||
(mapv #(dissoc % :id) (:messages (second (:chat-requests @tu/server)))))))
|
||||
(testing "and it is over: the panel takes messages again"
|
||||
(is (false? (.-readOnly (q "input[name=message]")))))
|
||||
(is (= :idle (chat-status))))
|
||||
(click! (tool-toggle)))
|
||||
#(tu/eventually (fn [] (= ["Collapse tool use"] (tool-toggle-labels))))
|
||||
(fn [_]
|
||||
@@ -919,7 +975,7 @@
|
||||
(is (= ["hi" "Hel"] (chat-messages)))
|
||||
(is (nil? (q "[role=status]")))
|
||||
(is (= "" (.-value (q "input[name=message]"))))
|
||||
(is (false? (.-readOnly (q "input[name=message]"))) "another message can be sent")))
|
||||
(is (= :idle (chat-status)) "another message can be sent")))
|
||||
done)))
|
||||
|
||||
(deftest chat-stream-cut-test
|
||||
|
||||
@@ -188,10 +188,6 @@
|
||||
(.then (fn [_]
|
||||
(is (= :ai-setup (page)) "reading changes nothing")
|
||||
(call "move_tic_tac_toe" {:cell 4 :player "x"})))
|
||||
(.then (fn [_]
|
||||
(is (= :tic-tac-toe (page)))
|
||||
(rf/dispatch-sync [::events/navigate :ai-setup])
|
||||
(call "new_tic_tac_toe_game" {})))
|
||||
(.then (fn [_]
|
||||
(is (= :tic-tac-toe (page)))
|
||||
(rf/dispatch-sync [::events/navigate :ai-setup])
|
||||
@@ -402,6 +398,11 @@
|
||||
(is (empty? (get-in @rf-db/app-db [:todos :items])))
|
||||
(done))))))
|
||||
|
||||
(deftest tic-tac-toe-tools-are-once-per-turn
|
||||
;; A move is the person's to ask for again, not the model's to repeat on its own: tbd.ui.events
|
||||
;; (see its once-per-turn tests) only lets the chatbot's turn call it once.
|
||||
(is (true? (:once-per-turn? (first (filter #(= "move_tic_tac_toe" (:name %)) webmcp/tools))))))
|
||||
|
||||
(deftest tic-tac-toe
|
||||
(open-page!)
|
||||
(async done
|
||||
@@ -411,9 +412,11 @@
|
||||
(is (= {:board (vec (repeat 9 nil)) :status "in_progress" :turn "x"} (json (:text r))))
|
||||
(call "move_tic_tac_toe" {:cell 4 :player "x"})))
|
||||
(.then (fn [r]
|
||||
;; It pressed the cell, as a user would; there is nothing left to say (see
|
||||
;; It pressed the cell, as a user would; the result says the board it left (see
|
||||
;; provider-for's tic-tac-toe assertions for the shape it takes in the state).
|
||||
(is (= {:text "ok" :error? false} r))
|
||||
(is (false? (:error? r)))
|
||||
(is (= {:board [nil nil nil nil "x" nil nil nil nil] :status "in_progress" :turn "o"}
|
||||
(json (:text r))))
|
||||
(is (= :x (get-in @rf-db/app-db [:ttt :board 4])))
|
||||
(is (= :o (get-in @rf-db/app-db [:ttt :turn])) "it is O's turn now")
|
||||
(call "move_tic_tac_toe" {:cell 4 :player "o"})))
|
||||
@@ -443,23 +446,25 @@
|
||||
(is (:error? r))
|
||||
(call "move_tic_tac_toe" {:cell 0 :player "o"})))
|
||||
(.then (fn [r]
|
||||
(is (= {:text "ok" :error? false} r))
|
||||
(is (false? (:error? r)))
|
||||
(is (= {:board ["o" nil nil nil "x" nil nil nil nil] :status "in_progress" :turn "x"}
|
||||
(json (:text r))))
|
||||
(is (= :x (get-in @rf-db/app-db [:ttt :turn])))
|
||||
;; The person moves X to 3 and O to 1; the agent's X then completes the middle row.
|
||||
(rf/dispatch-sync [::events/ttt-move 3])
|
||||
(rf/dispatch-sync [::events/ttt-move 1])
|
||||
(call "move_tic_tac_toe" {:cell 5 :player "x"})))
|
||||
(.then (fn [r]
|
||||
(is (= {:text "ok" :error? false} r))
|
||||
(is (false? (:error? r)))
|
||||
(is (= {:board ["o" "o" nil "x" "x" "x" nil nil nil]
|
||||
:status "won" :winner "x" :line [3 4 5]}
|
||||
(json (:text r))))
|
||||
(is (= [:o :o nil :x :x :x nil nil nil] (get-in @rf-db/app-db [:ttt :board])) "X won the middle row")
|
||||
(call "move_tic_tac_toe" {:cell 8 :player "o"})))
|
||||
(.then (fn [r]
|
||||
(is (= {:text "the game is over: start a new game" :error? true} r))
|
||||
(call "new_tic_tac_toe_game" {})))
|
||||
(.then (fn [r]
|
||||
(is (= {:text "ok" :error? false} r))
|
||||
(is (= (vec (repeat 9 nil)) (get-in @rf-db/app-db [:ttt :board])))
|
||||
(is (= :x (get-in @rf-db/app-db [:ttt :turn])))
|
||||
(is (= {:text "the game is over: the person starts a new one, with the page's own New game button"
|
||||
:error? true}
|
||||
r))
|
||||
(done))))))
|
||||
|
||||
;; --- State (tbd.ui.model-context's registry, filled from webmcp/state-providers) ---
|
||||
@@ -561,19 +566,16 @@
|
||||
|
||||
(deftest every-tool-the-prompts-name-exists
|
||||
(let [tool-names (set (map :name webmcp/tools))]
|
||||
(doseq [text [prompt/guide (prompt/tic-tac-toe "o") (prompt/task nil)]
|
||||
(doseq [text [prompt/guide prompt/tic-tac-toe (prompt/task nil)]
|
||||
tool-name (re-seq #"\b(?:get|move|new|list|set|create|delete)_[a-z_]+|\badd\{" text)]
|
||||
(is (contains? tool-names (str/replace tool-name "{" "")) (str tool-name)))))
|
||||
|
||||
(deftest tic-tac-toe-prompt
|
||||
(webmcp/register-prompts! fake-mcp)
|
||||
(testing "the side is the one asked for, and O by default"
|
||||
(is (re-find #"as X\." (:text (prompt-text "play_tic_tac_toe" {:side "x"}))))
|
||||
(is (re-find #"as O\." (:text (prompt-text "play_tic_tac_toe" {:side "o"}))))
|
||||
(is (re-find #"as O\." (:text (prompt-text "play_tic_tac_toe" nil))))
|
||||
(is (re-find #"as O\." (:text (prompt-text "play_tic_tac_toe" {:side "z"})))))
|
||||
(testing "it says to move on the agent's own turn only"
|
||||
(is (re-find #"never move for the other side" (:text (prompt-text "play_tic_tac_toe" nil))))))
|
||||
(testing "no side is assigned by the prompt; the tools and state carry the game"
|
||||
(is (= prompt/tic-tac-toe (:text (prompt-text "play_tic_tac_toe" nil))))
|
||||
(is (re-find #"get_tic_tac_toe and move_tic_tac_toe"
|
||||
(:text (prompt-text "play_tic_tac_toe" {:side "x"}))))))
|
||||
|
||||
(deftest registers-the-resources
|
||||
(is (identical? fake-mcp (webmcp/register-resources! fake-mcp)))
|
||||
|
||||
Reference in New Issue
Block a user