fiddle

test/tbd/agent/core_test.clj

@@ -1,7 +1,13 @@
 (ns tbd.agent.core-test
   (:require [clojure.test :refer [deftest is testing]]
             [tbd.agent.core :as agent]
-            [tbd.agent.test-support :as ts]))
+            [tbd.agent.test-support :as ts]
+            [tbd.chat.trace :as trace])
+  (:import (dev.langchain4j.data.message AiMessage SystemMessage)
+           (dev.langchain4j.model.chat.request ChatRequest)
+           (dev.langchain4j.model.chat.response PartialResponse PartialResponseContext
+                                                 PartialThinking PartialThinkingContext)
+           (dev.langchain4j.model.output FinishReason)))
 
 (defn- turn! [config messages tools]
   (let [{:keys [ok error kind]} (agent/start config "system prompt" messages tools)]
@@ -53,6 +59,47 @@
       (is (= {:error "chat is not configured" :kind :unavailable} (turn! config [] nil)))
       (is (zero? @calls)))))
 
+(deftest a-loop-is-retried-once-then-recovers
+  (testing "a turn that repeats itself is cut short and sent again, nudged to answer; the retry's
+           reply is what the caller sees, not a failure"
+    (let [calls (atom 0)
+          reqs (atom [])
+          sentence "Wait, let me check the board again. "
+          model (ts/model-of
+                  (fn [req handler]
+                    (swap! reqs conj req)
+                    (let [h (ts/fake-handle)]
+                      (if (= 1 (swap! calls inc))
+                        (dotimes [_ trace/repeat-abort-count]
+                          (.onPartialThinking handler (PartialThinking. ^String sentence) (PartialThinkingContext. h)))
+                        (do (.onPartialResponse handler (PartialResponse. "ok") (PartialResponseContext. h))
+                            (ts/complete handler (AiMessage/from "ok") FinishReason/STOP [10 5]))))))
+          {:keys [events]} (turn! (ts/config model) [{:role "user" :content "hi"}] nil)]
+      (is (= 2 @calls) "the model is asked again, once, after the loop")
+      (is (= [{:content "ok"} {:done true}] (take-last 2 events))
+          "the retry's reply is what the caller ends up seeing")
+      (is (= trace/repeat-abort-count (count (filter #(= {:thinking sentence} %) events)))
+          "the caller still sees what the model was thinking before it was cut off")
+      (is (not-any? :error events) "a loop that recovers is not shown to the user as a failure")
+      (is (not (re-find #"Stop reasoning now" (.text ^SystemMessage (first (.messages ^ChatRequest (first @reqs))))))
+          "the first attempt's prompt is unchanged")
+      (is (re-find #"Stop reasoning now" (.text ^SystemMessage (first (.messages ^ChatRequest (second @reqs)))))
+          "the retry's prompt has the nudge appended"))))
+
+(deftest a-loop-that-does-not-recover-is-an-error
+  (testing "only one retry is made: a turn that loops again on the retry is finally an error"
+    (let [calls (atom 0)
+          sentence "Wait, let me check the board again. "
+          model (ts/model-of
+                  (fn [_ handler]
+                    (swap! calls inc)
+                    (let [h (ts/fake-handle)]
+                      (dotimes [_ trace/repeat-abort-count]
+                        (.onPartialThinking handler (PartialThinking. ^String sentence) (PartialThinkingContext. h))))))
+          {:keys [events]} (turn! (ts/config model) [{:role "user" :content "hi"}] nil)]
+      (is (= 2 @calls) "the retry itself is not retried again")
+      (is (= {:error "the model got stuck repeating itself, and the reply was stopped"} (last events))))))
+
 (deftest conversation-history-round-trips
   (testing "assistant messages with tool calls, and their tool-role results, reach the model
            without error (see tbd.agent.core/llm-message)"

test/tbd/agent/instructions_test.clj

@@ -15,7 +15,6 @@
     (is (str/includes? text "in this same turn"))
     (is (str/includes? text "Work it out yourself:"))
     (is (str/includes? text "Say something was done only if a tool result says it was"))
-    (is (str/includes? text "Tic-tac-toe: check whose turn it is"))
     (is (str/includes? text "Examples: "))))
 
 (deftest chat-is-framed-as-the-in-page-chatbot

test/tbd/agent/prompt_test.clj

@@ -14,10 +14,9 @@
     (is (str/includes? text "in this same turn"))
     (is (str/includes? text "Work it out yourself:"))
     (is (str/includes? text "Chain calls without stopping"))
-    (is (str/includes? text "results from an earlier message are marked out of date"))
+    (is (str/includes? text "an earlier one may no longer hold"))
     (is (str/includes? text "never go by an earlier result"))
     (is (str/includes? text "Say something was done only if a tool result says it was"))
-    (is (str/includes? text "Tic-tac-toe: check whose turn it is"))
     (is (str/includes? text "Examples: "))
     (is (not (str/includes? text "navigate there, then")) "an outside agent is offered every tool")))
 
@@ -31,19 +30,15 @@
 (deftest examples-differ-by-reads
   (testing "an outside agent (no state channel) is shown reading first"
     (is (str/includes? (prompt/examples true) "list_todos -> set_todo_done"))
-    (is (str/includes? (prompt/examples true) "list_todos -> ONE set_todo_done"))
-    (is (str/includes? (prompt/examples true) "get_tic_tac_toe -> it is X's turn")))
-  (testing "the chatbot (state already in view) is not, since it has no list_todos or get_tic_tac_toe"
-    (is (not (str/includes? (prompt/examples false) "list_todos")))
-    (is (not (str/includes? (prompt/examples false) "get_tic_tac_toe"))))
+    (is (str/includes? (prompt/examples true) "list_todos -> ONE set_todo_done")))
+  (testing "the chatbot (state already in view) is not, since it has no list_todos"
+    (is (not (str/includes? (prompt/examples false) "list_todos"))))
   (testing "the rest is the same either way"
     (doseq [text [(prompt/examples true) (prompt/examples false)]]
       (is (str/includes? text "create_todo{\"todos\":[{\"title\":\"buy milk\"}]} -> \"Added.\""))
       (is (str/includes? text "add{\"x\":19,\"y\":4} -> \"23.\"")))))
 
 (deftest tic-tac-toe-prompt
-  (is (str/includes? (prompt/tic-tac-toe "x") "as X."))
-  (is (str/includes? (prompt/tic-tac-toe "o") "as O."))
-  (doseq [other [nil "" "z" :x]]
-    (is (str/includes? (prompt/tic-tac-toe other) "as O.") (pr-str other)))
-  (is (str/includes? (prompt/tic-tac-toe "x") "never move for the other side")))
+  (is (str/includes? prompt/tic-tac-toe "get_tic_tac_toe and move_tic_tac_toe"))
+  (is (not (str/includes? prompt/tic-tac-toe "as X")) "no side is assigned by the prompt")
+  (is (not (str/includes? prompt/tic-tac-toe "as O")) "no side is assigned by the prompt"))

test/tbd/agent/test_support.clj

@@ -26,7 +26,11 @@
   (reify StreamingChatModel
     (doChat [_ req handler] (script req handler) nil)))
 
-(defn- complete [handler ai-message finish-reason usage]
+(defn complete
+  "Finishes a fake model's call: sends `handler` an onCompleteResponse of `ai-message`, with
+  `finish-reason` and `usage` ([prompt completion], or nil for neither) — for a test whose fake
+  model needs more control than text-model/tool-call-model/etc give."
+  [handler ai-message finish-reason usage]
   (.onCompleteResponse handler
                        (-> (ChatResponse/builder)
                            (.aiMessage ai-message)

test/tbd/chat/trace_test.clj

@@ -58,12 +58,14 @@
         (is (= {:event :turn-start :page "todos" :model "qwen" :messages 1 :tools 2 :prompt-chars 5 :step 0}
                (select-keys start [:event :page :model :messages :tools :prompt-chars :step])))
         (is (re-matches #"[0-9a-f]{8}" (:req start)))
-        (is (not (contains? start :input))))))
-  (testing "with the full text on, it has the run's messages"
+        (is (not (contains? start :input)))
+        (is (not (contains? start :system-text))))))
+  (testing "with the full text on, it has the run's messages and the system prompt it was sent"
     (let [entries (atom [])
           messages [{:role "user" :content "old"} {:role "assistant" :content "reply"} {:role "user" :content "new"}]]
-      (trace/begin! (collecting entries :full-text? true) {:messages messages})
-      (is (= [{:role "user" :content "new"}] (:input (first @entries))))))
+      (trace/begin! (collecting entries :full-text? true) {:messages messages :system-text "You are..."})
+      (is (= [{:role "user" :content "new"}] (:input (first @entries))))
+      (is (= "You are..." (:system-text (first @entries))))))
   (testing "every turn has its own req id"
     (let [entries (atom [])]
       (dotimes [_ 3] (trace/begin! (collecting entries) {:messages [{:role "user" :content "x"}]}))